feat(common,developer): DRY out strings 🙀

- refactor string preprocessing pipeline into an options bag

common/web/types/src/kmx/element-string.ts

@@ -80,16 +80,16 @@ export class ElementString extends Array<ElemElement> {
           throw Error(`Could not parse uset ${item.segment}`);
         }
         elem.uset = sections.uset.allocUset(uset, sections);
-        elem.value = sections.strs.allocString('', true); // no string
+        elem.value = sections.strs.allocString('', {singleOk: true}); // no string
       } else if (item.type === ElementType.codepoint || item.type === ElementType.escaped || item.type === ElementType.string) {
         // some kind of a string
         let str = item.segment;
         if (item.type === ElementType.escaped && !MATCH_HEX_ESCAPE.test(str)) {
           str = unescapeOneQuadString(str);
           // TODO-LDML: any other escape forms here?
-          elem.value = sections.strs.allocString(str, true);
+          elem.value = sections.strs.allocString(str, { singleOk: true });
         } else {
-          elem.value = sections.strs.allocAndUnescapeString(str, true);
+          elem.value = sections.strs.allocString(str, { unescape: true, singleOk: true });
         }
         // Now did we end up with one char or no?
         if (elem.value.isOneChar) {

common/web/types/src/kmx/kmx-plus.ts

@@ -143,45 +143,67 @@ export class CharStrsItem extends StrsItem {
   }
 };
 
+/** class for string manipulation options. These are in order of the pipeline. */
+export interface StrsOptions {
+  /** apply string variables (requires sections) */
+  stringVariables?: boolean;
+  /** apply markers (requires sections) */
+  markers?: boolean;
+  /** unescape with unescapeString */
+  unescape?: boolean;
+  /** string can be stored as a single CharStrsItem, not in strs table. */
+  singleOk?: boolean;
+};
+
 export class Strs extends Section {
   strings: StrsItem[] = [ new StrsItem('') ]; // C7043: The null string is always requierd
   /**
    * Allocate a StrsItem given the string, unescaping if necessary.
    * @param s escaped string
-   * @param singleOk if true, allocate a CharStrsItem (not in strs table) if single-char capable.
-   * @returns
+   * @param opts options for allocation
+   * @param sections other sections, if needed
+   * @returns StrsItem
    */
-  allocAndUnescapeString(s?: string, singleOk?: boolean): StrsItem {
-    return this.allocString(unescapeString(s), singleOk);
-  }
-  /**
-   * Allocate a StrsItem given the string.
-   * @param s string
-   * @param singleOk if true, allocate a CharStrsItem (not in strs table) if single-char capable.
-   * @returns
-   */
-  allocString(s?: string, singleOk?: boolean): StrsItem {
-    if(s === undefined || s === null) {
-      // undefined or null are always equivalent to empty string, see C7043
-      s = '';
-    }
+  allocString(s?: string, opts?: StrsOptions, sections?: DependencySections): StrsItem {
+    // Run the string processing pipeline
+    s = Strs.processString(s, opts, sections);
 
-    if(typeof s !== 'string') {
-      throw new Error('alloc_string: s must be a string, undefined, or null.');
-    }
-
-    // if it's a single char, don't push it into the list
-    if (singleOk && isOneChar(s)) {
+    // if it's a single char, don't push it into the strs table
+    if (opts?.singleOk && isOneChar(s)) {
       return new CharStrsItem(s);
     }
 
+    // default: look to see if the string is already present
     let result = this.strings.find(item => item.value === s);
     if(result === undefined) {
+      // only add if not already present
       result = new StrsItem(s);
       this.strings.push(result);
     }
     return result;
   }
+
+  /** process everything according to opts */
+  static processString(s: string, opts: StrsOptions, sections: DependencySections) {
+    s = s ?? '';
+    // type check everything else
+    if (typeof s !== 'string') {
+      throw new Error('alloc_string: s must be a string, undefined, or null.');
+    }
+    // substitute variables
+    if (opts?.stringVariables) {
+      s = sections.vars.substituteStrings(s, sections);
+    }
+    // substitute markers
+    if (opts?.markers) {
+      s = sections.vars.substituteMarkerString(s);
+    }
+    // unescape \u{…}
+    if (opts?.unescape) {
+      s = unescapeString(s);
+    }
+    return s;
+  }
 };
 
 /**
@@ -310,7 +332,7 @@ export class VarsItem extends Section {
   constructor(id: string, value: string, sections: DependencySections) {
     super();
     this.id = sections.strs.allocString(id);
-    this.value = sections.strs.allocAndUnescapeString(value);
+    this.value = sections.strs.allocString(value, {unescape: true});
   }
 
   valid() : boolean {
@@ -500,31 +522,14 @@ export class List extends Section {
    * Allocate a list from a space-separated list of items.
    * Note that passing undefined or null or `''` will
    * end up being the same as the empty list `[]`
-   * @param strs Strs section for allocation
    * @param s space-separated list of items
+   * @param opts string options
+   * @param sections sections
    * @returns a List object
    */
-  allocListFromSpaces(strs: Strs, s?: string): ListItem {
+  allocListFromSpaces(s: string, opts: StrsOptions, sections: DependencySections): ListItem {
     s = s ?? '';
-    return this.allocList(strs, s.split(' '));
-  }
-  allocListFromEscapedSpaces(strs: Strs, s?: string): ListItem {
-    if(s === undefined || s === null) {
-      s = '';
-    }
-    return this.allocList(strs, s.split(' ').map(unescapeString));
-  }
-  /** perform string variable, marker, and unescaping */
-  allocListFromSubstitutedSpaces(s: string, sections: DependencySections): ListItem {
-    if(s === undefined || s === null) {
-      s = '';
-    }
-    return this.allocList(sections.strs, s.split(' ').map(s => {
-      s = sections.vars.substituteStrings(s, sections);
-      s = sections.vars.substituteMarkerString(s);
-      s = unescapeString(s);
-      return s;
-    }));
+    return this.allocList(s.split(' '), opts, sections);
   }
   /**
    * Return a List object referring to the string list.
@@ -534,7 +539,7 @@ export class List extends Section {
    * @param s string list to allocate
    * @returns
    */
-  allocList(strs: Strs, s?: string[]): ListItem {
+  allocList(s: string[], opts: StrsOptions, sections: DependencySections): ListItem {
     // Special case the 'null' list for [] or ['']
     if (!s || (s.length === 1 && s[0] === '')) {
       return this.lists[0];
@@ -542,14 +547,14 @@ export class List extends Section {
     let result = this.lists.find(item => item.isEqual(s));
     if(result === undefined) {
       // allocate a new ListItem
-      result = new ListItem(strs, s);
+      result = new ListItem(s, opts, sections);
       this.lists.push(result);
     }
     return result;
   }
   constructor(strs: Strs) {
     super();
-    this.lists.push(new ListItem(strs, [])); // C7043: null element string
+    this.lists.push(new ListItem([], {}, { strs })); // C7043: null element string
   }
   lists: ListItem[] = [];
 };

common/web/types/src/kmx/string-list.ts

@@ -1,5 +1,5 @@
 import { OrderedStringList } from 'src/ldml-keyboard/pattern-parser.js';
-import { Strs, StrsItem } from './kmx-plus.js';
+import { DependencySections, StrsItem, StrsOptions } from './kmx-plus.js';
 
 /**
  * A single entry in a ListItem.
@@ -31,14 +31,13 @@ export class ListItem extends Array<ListIndex> implements OrderedStringList {
    * @param source array of strings
    * @returns
    */
-  constructor(strs: Strs, source: Array<string>) {
+  constructor(source: Array<string>, opts: StrsOptions, sections: DependencySections) {
     super();
     if(!source) {
       return;
     }
-
     for (const str of source) {
-        let index = new ListIndex(strs.allocString(str));
+        let index = new ListIndex(sections.strs.allocString(str, opts, sections));
         this.push(index);
     }
   }

developer/src/kmc-ldml/src/compiler/disp.ts

@@ -57,16 +57,20 @@ export class DispCompiler extends SectionCompiler {
     let result = new Disp();
 
     // displayOptions
-    result.baseCharacter = sections.strs.allocAndUnescapeString(this.keyboard3.displays?.displayOptions?.baseCharacter);
+    result.baseCharacter = sections.strs.allocString(this.keyboard3.displays?.displayOptions?.baseCharacter, {unescape: true});
 
     // displays
     result.disps = this.keyboard3.displays?.display.map(display => ({
-      to: sections.strs.allocAndUnescapeString(
-        sections.vars.substituteMarkerString(
-          sections.vars.substituteStrings(display.to, sections))),
+      to: sections.strs.allocString(display.to, {
+        stringVariables: true,
+        markers: true,
+        unescape: true,
+      }, sections),
       id: sections.strs.allocString(display.id), // not escaped, not substituted
-      display: sections.strs.allocAndUnescapeString(
-        sections.vars.substituteStrings(display.display, sections)),
+      display: sections.strs.allocString(display.display, {
+        stringVariables: true,
+        unescape: true,
+      }, sections),
     })) || []; // TODO-LDML: need coverage for the []
 
     result.disps.sort((a: DispItem, b: DispItem) => {

developer/src/kmc-ldml/src/compiler/keys.ts

@@ -130,18 +130,18 @@ export class KeysCompiler extends SectionCompiler {
 
       for (let lkflick of lkflicks.flick) {
         let flags = 0;
-        let cookedTo = lkflick.to;
-        // pull in string variables and markers
-        cookedTo = sections.vars.substituteStrings(cookedTo, sections);
-        cookedTo = sections.vars.substituteMarkerString(cookedTo);
-        const to = sections.strs.allocAndUnescapeString(cookedTo, true);
+        const to = sections.strs.allocString(lkflick.to, {
+          stringVariables: true, markers: true, unescape: true, singleOk: true
+        }, sections);
         if (!to.isOneChar) {
           flags |= constants.keys_flick_flags_extend;
         }
         let directions: ListItem = sections.list.allocListFromSpaces(
-          sections.strs,
-          lkflick.directions
-        );
+          lkflick.directions,
+          {
+            stringVariables: true, markers: true, unescape: true
+          },
+          sections);
         flicks.flicks.push({
           directions,
           flags,
@@ -171,24 +171,44 @@ export class KeysCompiler extends SectionCompiler {
         flags |= constants.keys_key_flags_notransform;
       }
       const id = sections.strs.allocString(key.id);
-      const longPress: ListItem = sections.list.allocListFromSubstitutedSpaces(
-        key.longPress,
-        sections,
-      );
-      let cookedLongPressDefault = key.longPressDefault;
-      cookedLongPressDefault = sections.vars.substituteStrings(cookedLongPressDefault, sections);
-      cookedLongPressDefault = sections.vars.substituteMarkerString(cookedLongPressDefault)
-      const longPressDefault = sections.strs.allocAndUnescapeString(cookedLongPressDefault);
+      const longPress: ListItem = sections.list.allocListFromSpaces(
+        key.longPress, {
+          stringVariables: true,
+          markers: true,
+          unescape: true,
+        },
+        sections);
 
-      const multiTap: ListItem = sections.list.allocListFromSubstitutedSpaces(
+      const longPressDefault = sections.strs.allocString(key.longPressDefault,
+        {
+          stringVariables: true,
+          markers: true,
+          unescape: true,
+        },
+        sections);
+
+      const multiTap: ListItem = sections.list.allocListFromSpaces(
         key.multiTap,
-        sections,
-      );
+        {
+          stringVariables: true,
+          markers: true,
+          unescape: true,
+        },
+        sections);
       const keySwitch = sections.strs.allocString(key.switch); // 'switch' is a reserved word
+
       const toRaw = key.to;
+
       let toCooked = sections.vars.substituteStrings(toRaw, sections);
       toCooked = sections.vars.substituteMarkerString(toCooked);
-      const to = sections.strs.allocAndUnescapeString(toCooked, true);
+      const to = sections.strs.allocString(key.to,
+        {
+          stringVariables: true,
+          markers: true,
+          unescape: true,
+          singleOk: true
+        },
+        sections);
       if (!to.isOneChar) {
         flags |= constants.keys_key_flags_extend;
       }

developer/src/kmc-ldml/src/compiler/tran.ts

@@ -121,12 +121,11 @@ export class TransformCompiler<T extends TransformCompilerType, TranBase extends
   private compileTransform(sections: DependencySections, transform: LKTransform) : TranTransform {
     let result = new TranTransform();
     let cookedFrom = transform.from;
-    let cookedTo = transform.to;
 
     cookedFrom = sections.vars.substituteStrings(cookedFrom, sections);
     // TODO: handle 'map' case
     const mapFrom = VariableParser.CAPTURE_SET_REFERENCE.exec(cookedFrom);
-    const mapTo = VariableParser.MAPPED_SET_REFERENCE.exec(cookedTo || '');
+    const mapTo = VariableParser.MAPPED_SET_REFERENCE.exec(transform.to || '');
     if (mapFrom && mapTo) { // TODO-LDML: error cases
       result.mapFrom = sections.strs.allocString(mapFrom[1]); // var name
       result.mapTo = sections.strs.allocString(mapTo[1]); // var name
@@ -134,19 +133,21 @@ export class TransformCompiler<T extends TransformCompilerType, TranBase extends
       result.mapFrom = sections.strs.allocString(''); // TODO-LDML
       result.mapTo = sections.strs.allocString(''); // TODO-LDML
     }
-
     cookedFrom = sections.vars.substituteSetRegex(cookedFrom, sections);
 
-    if (cookedTo) {
-      cookedTo = sections.vars.substituteStrings(cookedTo, sections);
-    }
-
     // add in markers. idempotent if no markers.
-    cookedFrom = sections.vars.substituteMarkerString(cookedFrom, true); // TODO-LDML: need to support \m{.} here, maybe other edge cases
-    cookedTo = sections.vars.substituteMarkerString(cookedTo, false);
+    cookedFrom = sections.vars.substituteMarkerString(cookedFrom, true);
 
-    result.from = sections.strs.allocAndUnescapeString(cookedFrom); // TODO-LDML: not unescaped here, done previously
-    result.to = sections.strs.allocAndUnescapeString(cookedTo); // TODO-LDML: not unescaped here, done previously
+    // cookedFrom is cooked above, since there's some special treatment
+    result.from = sections.strs.allocString(cookedFrom, {
+      unescape: true
+    }, sections);
+    // 'to' is handled via allocString
+    result.to = sections.strs.allocString(transform.to, {
+      stringVariables: true,
+      markers: true,
+      unescape: true,
+    }, sections);
     return result;
   }
 

developer/src/kmc-ldml/src/compiler/vars.ts

@@ -216,7 +216,7 @@ export class VarsCompiler extends SectionCompiler {
 
     // collect all markers, excluding the match-all
     const allMarkers : string[] = Array.from(mt.all).filter(m => m !== MarkerParser.ANY_MARKER_ID).sort();
-    result.markers = sections.list.allocList(sections.strs, allMarkers);
+    result.markers = sections.list.allocList(allMarkers, {}, sections);
 
     return result.valid() ? result : null;
   }