feat(developer): serialize KMXPlus into XML 🍒

Cherry pick of #12969
> feat(developer): serialize KMXPlus (back) into XML 🗼

(cherry picked from commit f93c45c9e8 #12969)

Fixes: #12874

common/web/types/src/kmx/kmx-plus/kmx-plus.ts

@@ -385,15 +385,11 @@ export class UnicodeSetItem extends VarsItem {
 };
 
 export class SetVarItem extends VarsItem {
-  constructor(id: string, value: string[], sections: DependencySections, rawItems: string[]) {
+  constructor(id: string, value: string[], sections: DependencySections) {
     super(id, value.join(' '), sections);
     this.items = sections.elem.allocElementString(sections, value);
-    this.rawItems = rawItems;
   }
-  // element string array
-  items: ElementString;
-  // like items, but with unprocessed marker strings
-  rawItems: string[];
+  items: ElementString;  // element string array
   valid() : boolean {
     return !!this.items;
   }
@@ -409,10 +405,12 @@ export class StringVarItem extends VarsItem {
 // 'tran'
 
 export class TranTransform {
-  from: StrsItem;
-  to: StrsItem;
-  mapFrom: StrsItem; // var name
-  mapTo: StrsItem; // var name
+  from: StrsItem; // "from" computed regex string
+  to: StrsItem; // "to" (replacement) computed regex string
+  mapFrom: StrsItem; // var name for map
+  mapTo: StrsItem; // var name for map
+  _from?: string; // Not part of binary file: for use in the XML serializer. If present, sets the from= attribute for XML.
+  _to?: string; // Not part of binary file: for use in the XML serializer. If present, sets the to= attribute for XML.
 }
 
 export class TranGroup {
@@ -424,6 +422,9 @@ export class TranGroup {
 export class TranReorder {
   elements: ElementString;
   before: ElementString;
+  _before?: string; // Not part of binary file: for use in the XML serializer. If present, sets the before= attribute for XML.
+  _from?: string; // Not part of binary file: for use in the XML serializer. If present, sets the from= attribute for XML.
+  _order?: string; // Not part of binary file: for use in the XML serializer. If present, sets the order= attribute for XML.
 };
 
 export class Tran extends Section {

developer/src/common/web/utils/src/xml-utils.ts

@@ -114,6 +114,13 @@ const GENERATOR_OPTIONS: KeymanXMLOptionsBag = {
     textNodeName: '_',
     suppressEmptyNode: true,
   },
+  keyboard3: {
+    attributeNamePrefix: '$',
+    ignoreAttributes: false,
+    format: true,
+    textNodeName: '_',
+    suppressEmptyNode: true,
+  },
 };
 
 /** wrapper for XML parsing support */

developer/src/kmc-ldml/src/compiler/keys.ts

@@ -15,8 +15,9 @@ import { SubstitutionUse, Substitutions } from './substitution-tracker.js';
 /** reserved name for the special gap key. space is not allowed in key ids. */
 const reserved_gap = "gap (reserved)";
 
-
 export class KeysCompiler extends SectionCompiler {
+  /** keys that are of a reserved type */
+  public static RESERVED_KEY = Symbol('Reserved Key');
   static validateSubstitutions(
     keyboard: LDMLKeyboard.LKKeyboard,
     st: Substitutions
@@ -222,6 +223,19 @@ export class KeysCompiler extends SectionCompiler {
   /** count of reserved keys, for tests */
   public static readonly reserved_count = KeysCompiler.reserved_keys.length;
 
+  /** mark as reserved */
+  private static asReserved(k : KeysKeys) : KeysKeys {
+    const o = k as any;
+    o[KeysCompiler.RESERVED_KEY] = true;
+    return k;
+  }
+
+  /** true if a reserved key */
+  public static isReserved(k : KeysKeys) : boolean {
+    const o = k as any;
+    return !!o[KeysCompiler.RESERVED_KEY];
+  }
+
   /** load up all reserved keys */
   getReservedKeys(sections: KMXPlus.DependencySections) : Map<String, KeysKeys> {
     const r = new Map<String, KeysKeys>();
@@ -231,7 +245,7 @@ export class KeysCompiler extends SectionCompiler {
     const no_list = sections.list.allocList([], {}, sections);
 
     // now add the reserved key(s).
-    r.set(reserved_gap, {
+    r.set(reserved_gap, KeysCompiler.asReserved({
       flags: constants.keys_key_flags_gap | constants.keys_key_flags_extend,
       id: sections.strs.allocString(reserved_gap),
       flicks: '',
@@ -241,7 +255,7 @@ export class KeysCompiler extends SectionCompiler {
       switch: no_string,
       to: no_string,
       width: 10.0, // 10 * .1
-    });
+    }));
 
     if (r.size !== KeysCompiler.reserved_count) {
       throw Error(`Internal Error: KeysCompiler.reserved_count=${KeysCompiler.reserved_count} != ${r.size} actual reserved keys.`);
@@ -357,7 +371,7 @@ export class KeysCompiler extends SectionCompiler {
         flags |= constants.keys_key_flags_extend;
       }
       const width = Math.ceil((key.width || 1) * 10.0); // default, width=1
-      sect.keys.push({
+      sect.keys.push(SectionCompiler.copySymbols({
         flags,
         flicks: flickId,
         id,
@@ -367,7 +381,7 @@ export class KeysCompiler extends SectionCompiler {
         switch: keySwitch, // 'switch' is a reserved word
         to,
         width,
-      });
+      }, key));
     }
   }
 

developer/src/kmc-ldml/src/compiler/section-compiler.ts

@@ -55,4 +55,20 @@ export abstract class SectionCompiler {
     ]);
     return defaults;
   }
+
+  /**
+   * Copy symbols from 'from' onto 'onto'
+   * This is used to propagate special symbols such as ImportStatus
+   * and XML
+   * @param onto object to copy onto
+   * @param from source for symbols
+   * @returns the onto object
+   */
+  public static copySymbols<T>(onto: T, from: any) : T {
+    const o = onto as any;
+    for (const sym of Object.getOwnPropertySymbols(from)) {
+      o[sym] = from[sym];
+    }
+    return onto;
+  }
 }

developer/src/kmc-ldml/src/compiler/tran.ts

@@ -138,6 +138,9 @@ export abstract class TransformCompiler<T extends TransformCompilerType, TranBas
 
   private compileTransform(sections: DependencySections, transform: LKTransform) : TranTransform {
     let result = new TranTransform();
+    // setup for serializing
+    result._from = transform.from;
+    result._to = transform.to;
     let cookedFrom = transform.from;
 
     // check for incorrect \uXXXX escapes. Do this before substituting markers or sets.
@@ -262,6 +265,10 @@ export abstract class TransformCompiler<T extends TransformCompilerType, TranBas
 
   private compileReorder(sections: DependencySections, reorder: LKReorder): TranReorder {
     let result = new TranReorder();
+    // for serializing
+    result._before = reorder.before;
+    result._from = reorder.from;
+    result._order = reorder.order;
     if (reorder.from && this.checkEscapes(reorder.from) === null) {
       return null; // error'ed
     }

developer/src/kmc-ldml/src/compiler/vars.ts

@@ -285,7 +285,7 @@ export class VarsCompiler extends SectionCompiler {
     // this is not 'forMatch', all variables are to be assumed as string literals, not regex
     // content.
     const cookedItems: string[] = rawItems.map(v => result.substituteMarkerString(v, false));
-    result.sets.push(new SetVarItem(id, cookedItems, sections, rawItems));
+    result.sets.push(new SetVarItem(id, cookedItems, sections));
   }
   addUnicodeSet(result: Vars, e: LDMLKeyboard.LKUSet, sections: DependencySections): void {
     const { id } = e;

developer/src/kmc-ldml/src/util/serialize.ts

@@ -0,0 +1,273 @@
+/*
+ * Keyman is copyright (C) SIL Global. MIT License.
+ *
+ * This module contains routines for serializing from a KMXPlus file into XML.
+ */
+
+import { KMXPlus } from "@keymanapp/common-types";
+import { KeymanXMLWriter, LDMLKeyboard } from "@keymanapp/developer-utils";
+import { constants } from "@keymanapp/ldml-keyboard-constants";
+import { KeysCompiler } from "../compiler/keys.js";
+
+/**
+ * Serialize a KMXPlusFile back to XML.
+ * This is implemented for the LDML editor to be able to mutate LDML (XML) content by:
+ *  1. reading the original XML
+ *  2. compiling to KMXPlusFile
+ *  3. modifying the KMXPlusFile
+ *  4. serializing back to XML
+ *
+ * There are limitations:
+ *  - TODO-LDML-EDITOR: Transforms would not be serialized properly, due to
+ *    regex munging around markers and such.
+ *
+ *    To work around this, fields with underscores such as _from and _to are added to
+ *    the KMXPlusFile classes. These provide hints as to what the output XML should be,
+ *    and are populated by the tran compiler.
+ *
+ *  - TODO-LDML-EDITOR: Comments, whitespace, etc. are not preserved by this
+ *    approach. Updates to the XML parsing will support this, see #10622.
+ *
+ * @param kmx input KMXPlusFile
+ * @returns XML String
+ */
+export function kmxToXml(kmx: KMXPlus.KMXPlusFile): string {
+  const writer = new KeymanXMLWriter("keyboard3");
+  const { kmxplus } = kmx;
+  const {
+    //    sect,
+    bksp,
+    disp,
+    //    elem,
+    keys,
+    layr,
+    //    list,
+    loca,
+    meta,
+    //    strs,
+    tran,
+    //    uset,
+    vars,
+  } = kmxplus;
+  const data = {
+    keyboard3: {
+      ...getRootAttributes(),
+      ...getLocales(),
+      version: getVersion(),
+      info: getInfo(),
+      ...getDisplays(),
+      ...getKeys(),
+      ...getFlicks(),
+      ...getLayers(),
+      ...getVariables(),
+      ...getTransforms(),
+    }
+  };
+
+  return writer.write(data);
+
+  function getRootAttributes() {
+    return {
+      '$xmlns': `https://schemas.unicode.org/cldr/${constants.cldr_version_latest}/keyboard3`,
+      '$locale': kmx.kmxplus.loca.locales[0].value,
+      '$conformsTo': constants.cldr_version_latest,
+    };
+  }
+
+  function getLocales() {
+    if (loca?.locales?.length < 2) {
+      return {}; // no additional locales
+    } else {
+      return {
+        locales:
+          loca.locales.map(({ value }) => ({ '$id': value })),
+      }
+    }
+  }
+
+  function getInfo() {
+    return {
+      '$author': meta.author.value,
+      '$name': meta.name.value,
+      '$layout': meta.layout.value,
+      '$indicator': meta.indicator.value,
+    };
+  }
+
+  function getVersion() {
+    return { '$number': kmx.kmxplus.meta.version.value };
+  }
+
+  function getDisplays() {
+    const displays = {
+      display: disp?.disps.map(disp => getDisplay(disp)) || [],
+      ...getDisplaySettings(),
+    };
+    if (displays?.display?.length || displays?.displayOptions) {
+      return { displays }
+    } else {
+      return {};
+    }
+  }
+
+  function stringToAttr(attr: string, s?: KMXPlus.StrsItem, override?: string) {
+    if (override) return asAttr(attr, override);
+    if (!s || !s?.value?.length) return {};
+    return Object.fromEntries([[`\$${attr}`, s.value]]);
+  }
+
+  function asAttr(attr: string, s?: any) {
+    if (s === undefined) return {};
+    return Object.fromEntries([[`\$${attr}`, s]]);
+  }
+
+  function numberToAttr(attr: string, s?: number) {
+    if (s === undefined) return {};
+    return Object.fromEntries([[`\$${attr}`, s.toString()]]);
+  }
+
+  function getDisplay(disp: KMXPlus.DispItem) {
+    return {
+      ...stringToAttr('output', disp?.to),
+      ...stringToAttr('keyId', disp?.id),
+      ...stringToAttr('display', disp?.display),
+    };
+  }
+
+  function getDisplaySettings() {
+    if (!disp?.baseCharacter?.value) return {};
+    return {
+      displayOptions: {
+        '$baseCharacter': disp?.baseCharacter?.value,
+      }
+    };
+  }
+
+  function getKeys() {
+    if (!keys?.keys?.length) {
+      return {};
+    }
+    return {
+      keys: {
+        key: keys.keys
+          // skip reserved keys (gap)
+          .filter((key: KMXPlus.KeysKeys) =>
+            !KeysCompiler.isReserved(key) &&
+            !LDMLKeyboard.ImportStatus.isImpliedImport(key))
+          .map((key: KMXPlus.KeysKeys) => ({
+            ...stringToAttr('id', key.id),
+            ...stringToAttr('output', key.to),
+            ...asAttr('longPressKeyIds', key?.longPress?.join(' ') || undefined),
+          })),
+      },
+    };
+  }
+
+  function getFlicks() {
+    // skip the null flicks
+    if (keys?.flicks?.length < 2) {
+      return {};
+    }
+    return {
+      flicks: {
+        // keys.key..
+      }
+    };
+  }
+
+  function getLayers() {
+    if (!layr?.lists?.length) {
+      return {};
+    }
+    return {
+      layers: layr.lists.map(({ hardware, minDeviceWidth, layers }) => ({
+        ...stringToAttr('formId', hardware),
+        ...numberToAttr('minDeviceWidth', minDeviceWidth),
+        layer: layers.map(({ id, mod, rows }) => ({
+          ...stringToAttr('id', id),
+          ...asAttr('modifiers', modToString(mod)),
+          row: rows.map(({ keys }) => ({
+            ...asAttr('keys', keys.map(({ value }) => value).join(' ')),
+          })),
+        })),
+      })),
+    };
+  }
+
+  function getVariables() {
+    if (!vars?.strings.length && !vars?.sets.length && !vars?.usets.length) {
+      return {};
+    }
+    function varToObj(v: KMXPlus.VarsItem): any {
+      const { id, value } = v;
+      return {
+        ...stringToAttr('id', id),
+        ...stringToAttr('value', value),
+      };
+    }
+    function varsToArray(vars: KMXPlus.VarsItem[]): any[] {
+      return vars.map(varToObj);
+    }
+    const { strings, sets, usets } = vars;
+
+    return {
+      variables: {
+        string: varsToArray(strings),
+        set: varsToArray(sets),
+        uset: varsToArray(usets),
+      },
+    };
+  }
+
+  function getTransforms() {
+    return {
+      transforms: [
+      ...getTransformType("simple", tran),
+      ...getTransformType("backspace", bksp),
+      ],
+    };
+  }
+
+  /** NB: Bksp is a child class of Tran */
+  function getTransformType(type: string, t: KMXPlus.Tran) {
+    if (!t?.groups?.length) {
+      return [];
+    }
+    const { groups } = t;
+    return [{
+        ...asAttr('type', type),
+        transformGroup: groups.map((group) => {
+          if (group.type === constants.tran_group_type_transform) {
+            return {
+              transform: group.transforms.map(({from, to, _from, _to}) => ({
+                ...stringToAttr('from', from, _from),
+                ...stringToAttr('to', to, _to),
+              })),
+            };
+          } else if(group.type === constants.tran_group_type_reorder) {
+            return {
+              reorder: group.reorders.map(({before, elements, _before, _from, _order}) => ({
+                ...asAttr('before', _before || before.toString()),
+                ...asAttr('from', _from || elements.toString()),
+                ...asAttr('order', _order),
+              })),
+            };
+          } else {
+            throw Error(`Invalid tran.group.type ${group.type}`);
+          }
+        }),
+    }];
+  }
+}
+
+
+/** convert a keys_mod value to a space-separated string list */
+function modToString(mod: number) {
+  // first try exact match
+  const matches: string[] = [];
+  for (const [name, value] of constants.keys_mod_map.entries()) {
+    if (mod === value) return name; // exact match
+    if (mod & value) matches.push(name);
+  }
+  return matches.sort().join(' ');
+}

developer/src/kmc-ldml/test/compiler-e2e.tests.ts

@@ -3,7 +3,10 @@ import {assert} from 'chai';
 import hextobin from '@keymanapp/hextobin';
 import { KMXBuilder } from '@keymanapp/developer-utils';
 import {checkMessages, compileKeyboard, compilerTestCallbacks, compilerTestOptions, makePathToFixture} from './helpers/index.js';
+import { compareXml } from './helpers/compareXml.js';
 import { LdmlKeyboardCompiler } from '../src/compiler/compiler.js';
+import { kmxToXml } from '../src/util/serialize.js';
+import { writeFileSync } from 'node:fs';
 
 /** Overall compiler tests */
 describe('compiler-tests', function() {
@@ -35,8 +38,34 @@ describe('compiler-tests', function() {
     let expected = await hextobin(binaryFilename, undefined, {silent:true});
 
     assert.deepEqual<Uint8Array>(code, expected);
+
+    // now output it again as XML
+    const outputFilename = makePathToFixture('basic-serialized.xml');
+    const asXml = kmxToXml(kmx);
+    writeFileSync(outputFilename, asXml, 'utf-8');
+
   });
 
+  it('should-serialize-kmx', async function() {
+    this.timeout(4000);
+    // Let's build basic.xml
+    // It should match basic.kmx (built from basic.txt)
+
+    const inputFilename = makePathToFixture('basic.xml');
+
+    // Compile the keyboard
+    const kmx = await compileKeyboard(inputFilename, {...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false});
+    assert.isNotNull(kmx);
+
+    // now output it as XML
+    const outputFilename = makePathToFixture('basic-serialized.xml');
+    const asXml = kmxToXml(kmx);
+    writeFileSync(outputFilename, asXml, 'utf-8');
+
+    compareXml(outputFilename, inputFilename);
+  });
+
+
   it('should handle non existent files', async () => {
     const filename = 'DOES_NOT_EXIST.xml';
     const k = new LdmlKeyboardCompiler();

developer/src/kmc-ldml/test/fixtures/basic-serialized.xml

@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="utf-8"?>
+<keyboard3 xmlns="https://schemas.unicode.org/cldr/45/keyboard3" locale="mt" conformsTo="45">
+  <version number="1.0.0"/>
+  <info author="srl295" name="TestKbd" layout="qwerty" indicator="🙀"/>
+  <displays>
+    <display output="a" display="^"/>
+    <display keyId="e" display="^e"/>
+    <displayOptions baseCharacter="e"/>
+  </displays>
+  <keys>
+    <key id="hmaqtugha" output="ħ" longPressKeyIds="a e"/>
+    <key id="that" output="ថា"/>
+  </keys>
+  <layers formId="us" minDeviceWidth="123">
+    <layer modifiers="none">
+      <row keys="hmaqtugha that"/>
+    </layer>
+  </layers>
+  <variables>
+    <string id="a" value="\m{a}"/>
+    <string id="vst" value="abc"/>
+    <set id="vse" value="a b c"/>
+    <uset id="vus" value="[abc]"/>
+  </variables>
+  <transforms type="simple">
+    <transformGroup>
+      <transform from="^a" to="â"/>
+      <transform from="a" to="\m{a}"/>
+    </transformGroup>
+    <transformGroup>
+      <transform from="\m{a}"/>
+    </transformGroup>
+    <transformGroup>
+      <reorder before="\u{1A6B}" from="\u{1A60}[\u{1A75}-\u{1A79}]\u{1A45}" order="10 55 10"/>
+    </transformGroup>
+  </transforms>
+  <transforms type="backspace">
+    <transformGroup>
+      <transform from="^e"/>
+    </transformGroup>
+  </transforms>
+</keyboard3>

developer/src/kmc-ldml/test/helpers/compareXml.ts

@@ -0,0 +1,25 @@
+import {assert} from 'chai';
+import {readFileSync} from 'node:fs';
+import { KeymanXMLReader } from "@keymanapp/developer-utils";
+
+/**
+ *
+ * @param actual path to actual XML
+ * @param expect path to expected XML
+ * @param mutator optional function that will be applied to the parsed object
+ */
+export function compareXml(actual : string, expect: string, mutator?: (input: any) => any) {
+    if (!mutator) {
+        // no-op
+        mutator = (x: any) => x;
+    }
+    const reader = new KeymanXMLReader('keyboard3');
+
+    const actualStr = readFileSync(actual, 'utf-8');
+    const expectStr = readFileSync(expect, 'utf-8');
+
+    const actualParsed = mutator(reader.parse(actualStr));
+    const expectParsed = mutator(reader.parse(expectStr));
+
+    assert.deepEqual(actualParsed, expectParsed);
+}

developer/src/kmc-ldml/test/tsconfig.json

@@ -10,7 +10,7 @@
     },
   "include": [
       "**/*.tests.ts",
-      "./helpers/index.ts"
+      "./helpers/*.ts",
   ],
   "references": [
       { "path": "../../../../common/web/keyman-version" },