Merge pull request #9364 from keymanapp/feat/developer/9119-marker-epic-ldml

feat(developer) marker steps 🙀

common/web/types/src/kmx/kmx-plus-builder/build-list.ts

@@ -86,6 +86,9 @@ export function build_list(source_list: List, sect_strs: BUILDER_STRS): BUILDER_
  * @returns
  */
 export function build_list_index(sect_list: BUILDER_LIST, value: ListItem) : BUILDER_LIST_REF {
+  if (!value) { 
+    return 0; // empty list
+  }
   if(!(value instanceof ListItem)) {
     throw new Error('unexpected value '+ value);
   }

common/web/types/src/kmx/kmx-plus-builder/build-vars.ts

@@ -2,7 +2,7 @@ import { constants } from "@keymanapp/ldml-keyboard-constants";
 import { KMXPlusData } from "../kmx-plus.js";
 import { build_strs_index, BUILDER_STR_REF, BUILDER_STRS } from "./build-strs.js";
 import { BUILDER_SECTION } from "./builder-section.js";
-import { BUILDER_LIST_REF } from "./build-list.js";
+import { build_list_index, BUILDER_LIST, BUILDER_LIST_REF } from "./build-list.js";
 import { build_elem_index, BUILDER_ELEM, BUILDER_ELEM_REF } from "./build-elem.js";
 
 
@@ -22,7 +22,7 @@ export interface BUILDER_VARS extends BUILDER_SECTION {
 /**
 * Builder for the 'vars' section
 */
-export function build_vars(kmxplus: KMXPlusData, sect_strs: BUILDER_STRS, sect_elem: BUILDER_ELEM) : BUILDER_VARS {
+export function build_vars(kmxplus: KMXPlusData, sect_strs: BUILDER_STRS, sect_elem: BUILDER_ELEM, sect_list: BUILDER_LIST) : BUILDER_VARS {
   if(!kmxplus.vars) {
     return null;
   }
@@ -49,7 +49,7 @@ export function build_vars(kmxplus: KMXPlusData, sect_strs: BUILDER_STRS, sect_e
     size: constants.length_vars +
       (constants.length_vars_item * kmxplus.vars.totalCount()),
     _offset: 0,
-    markers: 0,
+    markers: build_list_index(sect_list, kmxplus.vars.markers),
     varCount: kmxplus.vars.totalCount(),
     varEntries: [
       ...stringVars,

common/web/types/src/kmx/kmx-plus-builder/kmx-plus-builder.ts

@@ -99,7 +99,7 @@ export default class KMXPlusBuilder {
     this.sect.name = build_name(this.file.kmxplus, this.sect.strs);
     this.sect.tran = build_tran(this.file.kmxplus.tran, this.sect.strs, this.sect.elem);
     this.sect.uset = build_uset(this.file.kmxplus, this.sect.strs);
-    this.sect.vars = build_vars(this.file.kmxplus, this.sect.strs, this.sect.elem);
+    this.sect.vars = build_vars(this.file.kmxplus, this.sect.strs, this.sect.elem, this.sect.list);
     this.sect.vkey = build_vkey(this.file.kmxplus);
 
     // Finalize the sect (index) section

common/web/types/src/kmx/string-list.ts

@@ -68,7 +68,12 @@ export class ListItem extends Array<ListIndex> {
       return 0;
     }
   }
+  /** for debugging, print as single string */
   toString(): string {
-    return this.map(v => v.value.value).join(' ');
+    return this.toStringArray().join(' ');
+  }
+  /** for debugging, map to string array */
+  toStringArray(): string[] {
+    return this.map(v => v.value.value);
   }
 };

common/web/types/src/ldml-keyboard/pattern-parser.ts

@@ -51,6 +51,9 @@ export class MarkerParser {
    * @returns `[]` or an array of all markers referenced
    */
   public static allReferences(str: string): string[] {
+    if (!str) {
+      return [];
+    }
     return matchArray(str, this.REFERENCE);
   }
 }

core/src/ldml/C9134_ldml_markers.md

@@ -27,7 +27,6 @@ Markers can appear in both 'emitting' and 'matching-only' areas:
 #### Match only
 
 - `transform from=` to match markers
-- `transform after=` to match markers
 - `display to=` for matching keys which contain markers
 
 ## Theory / Encoding

developer/src/kmc-ldml/src/compiler/disp.ts

@@ -1,5 +1,5 @@
 import { constants } from "@keymanapp/ldml-keyboard-constants";
-import { KMXPlus } from '@keymanapp/common-types';
+import { KMXPlus, LDMLKeyboard, MarkerParser } from '@keymanapp/common-types';
 
 import { CompilerMessages } from "./messages.js";
 import { SectionCompiler } from "./section-compiler.js";
@@ -7,8 +7,14 @@ import { SectionCompiler } from "./section-compiler.js";
 import DependencySections = KMXPlus.DependencySections;
 import Disp = KMXPlus.Disp;
 import DispItem = KMXPlus.DispItem;
+import { MarkerTracker, MarkerUse } from "./marker-tracker.js";
 
 export class DispCompiler extends SectionCompiler {
+  static validateMarkers(keyboard: LDMLKeyboard.LKKeyboard, mt : MarkerTracker): boolean {
+    keyboard.displays?.display?.forEach(({ to }) =>
+      mt.add(MarkerUse.match, MarkerParser.allReferences(to)));
+    return true;
+  }
 
   public get id() {
     return constants.section.disp;

developer/src/kmc-ldml/src/compiler/keys.ts

@@ -1,5 +1,5 @@
 import { constants } from '@keymanapp/ldml-keyboard-constants';
-import { LDMLKeyboard, KMXPlus, Constants } from '@keymanapp/common-types';
+import { LDMLKeyboard, KMXPlus, Constants, MarkerParser } from '@keymanapp/common-types';
 import { CompilerMessages } from './messages.js';
 import { SectionCompiler } from "./section-compiler.js";
 
@@ -8,8 +8,18 @@ import Keys = KMXPlus.Keys;
 import ListItem = KMXPlus.ListItem;
 import KeysFlicks = KMXPlus.KeysFlicks;
 import { allUsedKeyIdsInLayers, calculateUniqueKeys, translateLayerAttrToModifier, validModifier } from '../util/util.js';
+import { MarkerTracker, MarkerUse } from './marker-tracker.js';
 
 export class KeysCompiler extends SectionCompiler {
+  static validateMarkers(
+    keyboard: LDMLKeyboard.LKKeyboard,
+    mt: MarkerTracker
+  ): boolean {
+    keyboard.keys?.key?.forEach(({ to }) =>
+      mt.add(MarkerUse.emit, MarkerParser.allReferences(to))
+    );
+    return true;
+  }
 
   public get id() {
     return constants.section.keys;
@@ -20,7 +30,7 @@ export class KeysCompiler extends SectionCompiler {
    * @returns just the non-touch layers.
    */
   public hardwareLayers() {
-    return this.keyboard.layers?.filter(({form}) => form !== 'touch');
+    return this.keyboard.layers?.filter(({ form }) => form !== "touch");
   }
 
   public validate() {
@@ -30,7 +40,7 @@ export class KeysCompiler extends SectionCompiler {
     const usedKeys = allUsedKeyIdsInLayers(this.keyboard?.layers);
     const uniqueKeys = calculateUniqueKeys([...this.keyboard.keys?.key]);
     for (let key of uniqueKeys) {
-      const {id, flicks} = key;
+      const { id, flicks } = key;
       if (!usedKeys.has(id)) {
         continue; // unused key, ignore
       }
@@ -38,10 +48,14 @@ export class KeysCompiler extends SectionCompiler {
       if (!flicks) {
         continue; // no flicks
       }
-      const flickEntry = this.keyboard.keys?.flicks?.find(x => x.id === flicks);
-      if (!flickEntry ) {
+      const flickEntry = this.keyboard.keys?.flicks?.find(
+        (x) => x.id === flicks
+      );
+      if (!flickEntry) {
         valid = false;
-        this.callbacks.reportMessage(CompilerMessages.Error_MissingFlicks({flicks, id}));
+        this.callbacks.reportMessage(
+          CompilerMessages.Error_MissingFlicks({ flicks, id })
+        );
       }
     }
 
@@ -53,8 +67,9 @@ export class KeysCompiler extends SectionCompiler {
     if (hardwareLayers.length >= 1) {
       // validate all errors
       for (let layers of hardwareLayers) {
-        for(let layer of layers.layer) {
-          valid = this.validateHardwareLayerForKmap(layers.form, layer) && valid; // note: always validate even if previously invalid results found
+        for (let layer of layers.layer) {
+          valid =
+            this.validateHardwareLayerForKmap(layers.form, layer) && valid; // note: always validate even if previously invalid results found
         }
       }
       // TODO-LDML: } else { touch?
@@ -84,11 +99,13 @@ export class KeysCompiler extends SectionCompiler {
     /* c8 ignore next 3 */
     if (hardwareLayers.length > 1) {
       // validation should have already caught this
-      throw Error(`Internal error: Expected 0 or 1 hardware layer, not ${hardwareLayers.length}`);
+      throw Error(
+        `Internal error: Expected 0 or 1 hardware layer, not ${hardwareLayers.length}`
+      );
     } else if (hardwareLayers.length === 1) {
       const theLayers = hardwareLayers[0];
       const { form } = theLayers;
-      for(let layer of theLayers.layer) {
+      for (let layer of theLayers.layer) {
         this.compileHardwareLayerToKmap(sections, layer, sect, form);
       }
     } // else: TODO-LDML do nothing if only touch layers
@@ -98,7 +115,9 @@ export class KeysCompiler extends SectionCompiler {
 
   public loadFlicks(sections: DependencySections, sect: Keys) {
     for (let lkflicks of this.keyboard.keys.flicks) {
-      let flicks: KeysFlicks = new KeysFlicks(sections.strs.allocString(lkflicks.id));
+      let flicks: KeysFlicks = new KeysFlicks(
+        sections.strs.allocString(lkflicks.id)
+      );
 
       for (let lkflick of lkflicks.flick) {
         let flags = 0;
@@ -106,7 +125,10 @@ export class KeysCompiler extends SectionCompiler {
         if (!to.isOneChar) {
           flags |= constants.keys_flick_flags_extend;
         }
-        let directions: ListItem = sections.list.allocListFromSpaces(sections.strs, lkflick.directions);
+        let directions: ListItem = sections.list.allocListFromSpaces(
+          sections.strs,
+          lkflick.directions
+        );
         flicks.flicks.push({
           directions,
           flags,
@@ -132,19 +154,27 @@ export class KeysCompiler extends SectionCompiler {
       if (!!key.gap) {
         flags |= constants.keys_key_flags_gap;
       }
-      if (key.transform === 'no') {
+      if (key.transform === "no") {
         flags |= constants.keys_key_flags_notransform;
       }
       const id = sections.strs.allocString(key.id);
-      const longPress: ListItem = sections.list.allocListFromEscapedSpaces(sections.strs, key.longPress);
-      const longPressDefault = sections.strs.allocAndUnescapeString(key.longPressDefault);
-      const multiTap: ListItem = sections.list.allocListFromEscapedSpaces(sections.strs, key.multiTap);
+      const longPress: ListItem = sections.list.allocListFromEscapedSpaces(
+        sections.strs,
+        key.longPress
+      );
+      const longPressDefault = sections.strs.allocAndUnescapeString(
+        key.longPressDefault
+      );
+      const multiTap: ListItem = sections.list.allocListFromEscapedSpaces(
+        sections.strs,
+        key.multiTap
+      );
       const keySwitch = sections.strs.allocString(key.switch); // 'switch' is a reserved word
       const to = sections.strs.allocAndUnescapeString(key.to, true);
       if (!to.isOneChar) {
         flags |= constants.keys_key_flags_extend;
       }
-      const width = Math.ceil((key.width || 1) * 10.0);  // default, width=1
+      const width = Math.ceil((key.width || 1) * 10.0); // default, width=1
       sect.keys.push({
         flags,
         flicks,
@@ -166,12 +196,17 @@ export class KeysCompiler extends SectionCompiler {
    * @param layer
    * @returns
    */
-  private validateHardwareLayerForKmap(hardware: string, layer: LDMLKeyboard.LKLayer) {
+  private validateHardwareLayerForKmap(
+    hardware: string,
+    layer: LDMLKeyboard.LKLayer
+  ) {
     let valid = true;
 
     const { modifier } = layer;
     if (!validModifier(modifier)) {
-      this.callbacks.reportMessage(CompilerMessages.Error_InvalidModifier({ modifier, layer: layer.id }));
+      this.callbacks.reportMessage(
+        CompilerMessages.Error_InvalidModifier({ modifier, layer: layer.id })
+      );
       valid = false;
     }
 
@@ -179,21 +214,31 @@ export class KeysCompiler extends SectionCompiler {
     /* c8 ignore next 5 */
     if (!keymap) {
       // not reached due to XML validation
-      this.callbacks.reportMessage(CompilerMessages.Error_InvalidHardware({ form: hardware }));
+      this.callbacks.reportMessage(
+        CompilerMessages.Error_InvalidHardware({ form: hardware })
+      );
       valid = false;
     }
 
     const uniqueKeys = calculateUniqueKeys([...this.keyboard.keys?.key]);
     if (layer.row.length > keymap.length) {
-      this.callbacks.reportMessage(CompilerMessages.Error_HardwareLayerHasTooManyRows());
+      this.callbacks.reportMessage(
+        CompilerMessages.Error_HardwareLayerHasTooManyRows()
+      );
       valid = false;
     }
 
     for (let y = 0; y < layer.row.length && y < keymap.length; y++) {
-      const keys = layer.row[y].keys.split(' ');
+      const keys = layer.row[y].keys.split(" ");
 
       if (keys.length > keymap[y].length) {
-        this.callbacks.reportMessage(CompilerMessages.Error_RowOnHardwareLayerHasTooManyKeys({ row: y + 1, hardware, modifier }));
+        this.callbacks.reportMessage(
+          CompilerMessages.Error_RowOnHardwareLayerHasTooManyKeys({
+            row: y + 1,
+            hardware,
+            modifier,
+          })
+        );
         valid = false;
       }
 
@@ -201,14 +246,24 @@ export class KeysCompiler extends SectionCompiler {
       for (let key of keys) {
         x++;
 
-        let keydef = uniqueKeys.find(x => x.id == key);
+        let keydef = uniqueKeys.find((x) => x.id == key);
         if (!keydef) {
-          this.callbacks.reportMessage(CompilerMessages.Error_KeyNotFoundInKeyBag({ keyId: key, col: x + 1, row: y + 1, layer: layer.id, form: 'hardware' }));
+          this.callbacks.reportMessage(
+            CompilerMessages.Error_KeyNotFoundInKeyBag({
+              keyId: key,
+              col: x + 1,
+              row: y + 1,
+              layer: layer.id,
+              form: "hardware",
+            })
+          );
           valid = false;
           continue;
         }
         if (!keydef.to && !keydef.gap && !keydef.switch) {
-          this.callbacks.reportMessage(CompilerMessages.Error_KeyMissingToGapOrSwitch({ keyId: key }));
+          this.callbacks.reportMessage(
+            CompilerMessages.Error_KeyMissingToGapOrSwitch({ keyId: key })
+          );
           valid = false;
           continue;
         }
@@ -222,7 +277,7 @@ export class KeysCompiler extends SectionCompiler {
     sections: DependencySections,
     layer: LDMLKeyboard.LKLayer,
     sect: Keys,
-    hardware: string,
+    hardware: string
   ): Keys {
     const mod = translateLayerAttrToModifier(layer);
     const keymap = Constants.HardwareToKeymap.get(hardware);
@@ -231,7 +286,7 @@ export class KeysCompiler extends SectionCompiler {
     for (let row of layer.row) {
       y++;
 
-      const keys = row.keys.split(' ');
+      const keys = row.keys.split(" ");
       let x = -1;
       for (let key of keys) {
         x++;

developer/src/kmc-ldml/src/compiler/marker-tracker.ts

@@ -0,0 +1,72 @@
+/**
+ * Verb for MarkerTracker.add()
+ */
+export enum MarkerUse {
+  /** outputs this marker into context (e.g. transform to= or key to=) */
+  emit,
+  /** consumes this marker out of the context (e.g. transform from=) */
+  consume,
+  /** matches the marker, but doesn't consume (e.g. display to=) */
+  match,
+  /** variable definition: might consume, emit, or match. */
+  variable,
+}
+
+type MarkerSet = Set<string>;
+
+/** Tracks usage of markers */
+export class MarkerTracker {
+  /** markers that were emitted */
+  emitted: MarkerSet;
+  /** markers that were consumed and removed from the context */
+  consumed: MarkerSet;
+  /** markers that were matched, but not necessarily consumed */
+  matched: MarkerSet;
+  /** all markers */
+  all: MarkerSet;
+
+  constructor() {
+    this.emitted = new Set<string>();
+    this.consumed = new Set<string>();
+    this.matched = new Set<string>();
+    this.all = new Set<string>();
+  }
+
+  /**
+   *
+   * @param verb what kind of use we are adding
+   * @param markers list of markers to add
+   */
+  add(verb: MarkerUse, markers: string[]) {
+    if (!markers.length) {
+      return; // skip if empty
+    }
+    if (verb == MarkerUse.emit) {
+      markers.forEach((m) => {
+        this.emitted.add(m);
+        this.all.add(m);
+      });
+    } else if (verb == MarkerUse.consume) {
+      markers.forEach((m) => {
+        this.consumed.add(m);
+        this.all.add(m);
+      });
+    } else if (verb == MarkerUse.match) {
+      markers.forEach((m) => {
+        this.matched.add(m);
+        this.all.add(m);
+      });
+    } else if (verb == MarkerUse.variable) {
+      markers.forEach((m) => {
+        // we don't know, so add it to all three
+        this.matched.add(m);
+        this.emitted.add(m);
+        this.consumed.add(m);
+        this.all.add(m);
+      });
+      /* c8 skip next 3 */
+    } else {
+      throw Error(`Internal error: unsupported verb ${verb} for match`);
+    }
+  }
+}

developer/src/kmc-ldml/src/compiler/messages.ts

@@ -130,5 +130,9 @@ export class CompilerMessages {
   static Error_CantReferenceSetFromUnicodeSet = (o:{id: string}) =>
   m(this.ERROR_CantReferenceSetFromUnicodeSet, `Illegal use of set variable from within UnicodeSet: \$[${o.id}]`);
   static ERROR_CantReferenceSetFromUnicodeSet = SevError | 0x0020;
+
+  static Error_MissingMarkers = (o: { ids: string[] }) =>
+  m(this.ERROR_MissingMarkers, `Markers used for matching but not defined: ${o.ids?.join(',')}`);
+  static ERROR_MissingMarkers = SevError | 0x0021;
 }
 

developer/src/kmc-ldml/src/compiler/tran.ts

@@ -1,5 +1,5 @@
 import { constants, SectionIdent } from "@keymanapp/ldml-keyboard-constants";
-import { KMXPlus, LDMLKeyboard, CompilerCallbacks, VariableParser } from '@keymanapp/common-types';
+import { KMXPlus, LDMLKeyboard, CompilerCallbacks, VariableParser, MarkerParser } from '@keymanapp/common-types';
 import { SectionCompiler } from "./section-compiler.js";
 
 import Bksp = KMXPlus.Bksp;
@@ -15,10 +15,21 @@ import LKTransform = LDMLKeyboard.LKTransform;
 import LKTransforms = LDMLKeyboard.LKTransforms;
 import { verifyValidAndUnique } from "../util/util.js";
 import { CompilerMessages } from "./messages.js";
+import { MarkerTracker, MarkerUse } from "./marker-tracker.js";
 
 type TransformCompilerType = 'simple' | 'backspace';
 
-class TransformCompiler<T extends TransformCompilerType, TranBase extends Tran> extends SectionCompiler {
+export class TransformCompiler<T extends TransformCompilerType, TranBase extends Tran> extends SectionCompiler {
+
+  static validateMarkers(keyboard: LDMLKeyboard.LKKeyboard, mt : MarkerTracker): boolean {
+    keyboard?.transforms?.forEach(transforms =>
+      transforms.transformGroup.forEach(transformGroup => {
+        transformGroup.transform?.forEach(({ to, from }) => {
+          mt.add(MarkerUse.emit, MarkerParser.allReferences(to));
+          mt.add(MarkerUse.consume, MarkerParser.allReferences(from));
+        })}));
+    return true;
+  }
 
   protected type: T;
 

developer/src/kmc-ldml/src/compiler/vars.ts

@@ -1,5 +1,5 @@
 import { SectionIdent, constants } from "@keymanapp/ldml-keyboard-constants";
-import { KMXPlus, LDMLKeyboard, CompilerCallbacks } from '@keymanapp/common-types';
+import { KMXPlus, LDMLKeyboard, CompilerCallbacks, MarkerParser } from '@keymanapp/common-types';
 import { VariableParser } from '@keymanapp/common-types';
 import { SectionCompiler } from "./section-compiler.js";
 import Vars = KMXPlus.Vars;
@@ -9,6 +9,10 @@ import UnicodeSetItem = KMXPlus.UnicodeSetItem;
 import DependencySections = KMXPlus.DependencySections;
 import LDMLKeyboardXMLSourceFile = LDMLKeyboard.LDMLKeyboardXMLSourceFile;
 import { CompilerMessages } from "./messages.js";
+import { KeysCompiler } from "./keys.js";
+import { TransformCompiler } from "./tran.js";
+import { DispCompiler } from "./disp.js";
+import { MarkerTracker, MarkerUse } from "./marker-tracker.js";
 export class VarsCompiler extends SectionCompiler {
   public get id() {
     return constants.section.vars;
@@ -17,7 +21,8 @@ export class VarsCompiler extends SectionCompiler {
   public get dependencies(): Set<SectionIdent> {
     const defaults = new Set(<SectionIdent[]>[
       constants.section.strs,
-      constants.section.elem
+      constants.section.elem,
+      constants.section.list,
     ]);
     defaults.delete(this.id);
     return defaults;
@@ -29,7 +34,6 @@ export class VarsCompiler extends SectionCompiler {
 
   public validate(): boolean {
     let valid = true;
-    // TODO-LDML scan for markers?
 
     // Check for duplicate ids
     const allIds = new Set();
@@ -130,9 +134,56 @@ export class VarsCompiler extends SectionCompiler {
       }));
       valid = false;
     }
+
+    valid = this.validateMarkers() && valid; // accumulate validity
+
     return valid;
   }
 
+  private collectMarkers(mt : MarkerTracker) : boolean {
+    let valid = true;
+
+    // call our friends to validate
+    valid = this.validateVarsMarkers(this.keyboard, mt) && valid; // accumulate validity
+    valid = KeysCompiler.validateMarkers(this.keyboard, mt) && valid; // accumulate validity
+    valid = TransformCompiler.validateMarkers(this.keyboard, mt) && valid; // accumulate validity
+    valid = DispCompiler.validateMarkers(this.keyboard, mt) && valid; // accumulate validity
+
+    return valid;
+  }
+
+  private validateMarkers(): boolean {
+    const mt = new MarkerTracker();
+    let valid = this.collectMarkers(mt);
+    // see if there are any matched-but-not-emitted
+    const matchedNotEmitted : Set<string> = new Set<string>();
+    for (const m of mt.matched.values()) {
+      if (m === MarkerParser.ANY_MARKER_ID) continue; // match-all marker
+      if (!mt.emitted.has(m)) {
+        matchedNotEmitted.add(m);
+      }
+    }
+    for (const m of mt.consumed.values()) {
+      if (m === MarkerParser.ANY_MARKER_ID) continue; // match-all marker
+      if (!mt.emitted.has(m)) {
+        matchedNotEmitted.add(m);
+      }
+    }
+
+    // report once
+    if (matchedNotEmitted.size > 0) {
+      this.callbacks.reportMessage(CompilerMessages.Error_MissingMarkers({ ids: Array.from(matchedNotEmitted.values()).sort() }));
+      valid = false;
+    }
+    return valid;
+  }
+
+  validateVarsMarkers(keyboard: LDMLKeyboard.LKKeyboard, mt : MarkerTracker) : boolean {
+    keyboard?.variables?.string?.forEach(({value}) =>
+          mt.add(MarkerUse.variable, MarkerParser.allReferences(value)));
+    return true;
+  }
+
   public compile(sections: DependencySections): Vars {
     const result =  new Vars();
 
@@ -153,6 +204,14 @@ export class VarsCompiler extends SectionCompiler {
     variables?.unicodeSet?.forEach((e) =>
       this.addUnicodeSet(result, e, sections));
 
+    // reload markers - TODO-LDML: double work!
+    const mt = new MarkerTracker();
+    this.collectMarkers(mt);
+
+    // collect all markers, excluding the match-all
+    const allMarkers : string[] = Array.from(mt.all).filter(m => m !== MarkerParser.ANY_MARKER_ID).sort();
+    result.markers = sections.list.allocList(sections.strs, allMarkers);
+
     return result.valid() ? result : null;
   }
 

developer/src/kmc-ldml/test/fixtures/basic.txt

@@ -257,7 +257,7 @@ block(keys)                         # struct COMP_KMXPLUS_KEYS {
   index(strNull,strHmaqtugha,2)     #   KMXPLUS_STR 'hmaqtugha'
   00 00 00 00                       #   KMXPLUS_STR switch
   0A 00 00 00                       #   KMX_DWORD width*10
-  01 00 00 00 # TODO: index(listNull,indexAe,4)         #   LIST longPress 'a e'
+  02 00 00 00 # TODO: index(listNull,indexAe,4)         #   LIST longPress 'a e'
   00 00 00 00                       #   STR longPressDefault
   00 00 00 00 # TODO: index(listNull,listNull,4)        #   LIST multiTap
   00 00 00 00                       #   flicks 0
@@ -324,21 +324,25 @@ block(layr)                         # struct COMP_KMXPLUS_LAYR {
 block(list)                         # struct COMP_KMXPLUS_LAYR_LIST {
   6c 69 73 74                       #   KMX_DWORD header.ident;   // 0000 Section name - list
   diff(list,endList)                #   KMX_DWORD header.size;    // 0004 Section length
-  02 00 00 00                       #   KMX_DWORD listCount (should be 2)
-  02 00 00 00                       #   KMX_DWORD indexCount (should be 2)
+  03 00 00 00                       #   KMX_DWORD listCount (should be 2)
+  03 00 00 00                       #   KMX_DWORD indexCount (should be 2)
   # list #0 the null list
   block(listNull)
   00 00 00 00 #index(indexNull,indexNull,2)      #   KMX_DWORD list index (0)
   00 00 00 00                       #   KMX_DWORD lists[0].count
-  # list #1 the ae list
+  block(listA)
+  00 00 00 00 # first index
+  01 00 00 00 #count
   block(listAe)
-  00 00 00 00 # index(indexAe,indexNull,2)        #   KMX_DWORD list index (also 0)
+  01 00 00 00 # index(indexAe,indexNull,2)        #   KMX_DWORD list index (also 0)
   02 00 00 00                       #   KMX_DWORD count
   block(endLists)
   # indices
   #block(indexNull)
   # No null index
   # index(strNull,strNull,2)         #   KMXPLUS_STR string index
+  block(indexA)
+  index(strNull,strA,2) # a
   block(indexAe)
   index(strNull,strA,2) #   KMXPLUS_STR a
   index(strNull,strElemBkspFrom2,2) #   KMXPLUS_STR e
@@ -401,6 +405,7 @@ block(strs)                         #  struct COMP_KMXPLUS_STRS {
   diff(strs,strName)       sizeof(strName,2)
   diff(strs,strFromSet)     sizeof(strFromSet,2)
   diff(strs,strUSet)       sizeof(strUSet,2)
+  diff(strs,strAmarker)     sizeof(strAmarker,2)
   diff(strs,strElemTranFrom1) sizeof(strElemTranFrom1,2)
   diff(strs,strElemTranFrom1a) sizeof(strElemTranFrom1a,2)
   diff(strs,strElemTranFrom1b) sizeof(strElemTranFrom1b,2)
@@ -433,6 +438,7 @@ block(strs)                         #  struct COMP_KMXPLUS_STRS {
   block(strName)             54 00 65 00 73 00 74 00 4b 00 62 00 64 00   block(x) 00 00 # 'TestKbd'
   block(strFromSet)          5B 00 5C 00 75 00 31 00 41 00 37 00 35 00 2D 00 5C 00 75 00 31 00 41 00 37 00 39 00 5D 00 block(x) 00 00 # [\u1a75-\u1a79]
   block(strUSet)             5b 00 61 00 62 00 63 00 5d 00 block(x) 00 00               # '[abc]'
+  block(strAmarker)          5C 00 6D 00 7B 00 61 00 7D 00 block(x) 00 00               # '\m{a}'
   block(strElemTranFrom1)    5E 00 block(x) 00 00                                       # '^'
   block(strElemTranFrom1a)   5E 00 61 00 block(x) 00 00                                       # '^a'
   block(strElemTranFrom1b)   5E 00 65 00 block(x) 00 00                                       # '^e'
@@ -520,23 +526,29 @@ block(uset)
 block(vars)                         # struct COMP_KMXPLUS_VARS {
   76 61 72 73                       #   KMX_DWORD header.ident;   // 0000 Section name - vars
   diff(vars,varsEnd)                #   KMX_DWORD header.size;    // 0004 Section length
-  00 00 00 00                       #   KMX_DWORD markers - list
+  01 00 00 00                       #   KMX_DWORD markers - list 1 ['a']
   diff(varsBegin,varsEnd,16)        #   KMX_DWORD varCount
 
-  # var 0
   block(varsBegin)
+  # var 0
+  00 00 00 00                       #   KMX_DWORD type = str
+  index(strNull,strA,2)             #   KMXPLUS_STR id  'a'
+  index(strNull,strAmarker,2)       #   KMXPLUS_STR value '\m{a}'
+  00 00 00 00                       #   KMXPLUS_ELEM 
+
+  # var 1
   01 00 00 00                       #   KMX_DWORD type = set
   index(strNull,strVse,2)           #   KMXPLUS_STR id  'vse'
   index(strNull,strSet,2)           #   KMXPLUS_STR value 'a b c'
   01 00 00 00                       #   KMXPLUS_ELEM elem 'a b c' see 'elemSet'
 
-  # var 1
+  # var 2
   00 00 00 00                       #   KMX_DWORD type = string
   index(strNull,strVst,2)           #   KMXPLUS_STR id  'vst'
   index(strNull,strSet2,2)          #   KMXPLUS_STR value 'abc'
   00 00 00 00                       #   KMXPLUS_ELEM elem
 
-  # var 2
+  # var 3
   02 00 00 00                       #   KMX_DWORD type = string
   index(strNull,strVus,2)           #   KMXPLUS_STR id  'vus'
   index(strNull,strUSet,2)          #   KMXPLUS_STR value '[abc]'

developer/src/kmc-ldml/test/fixtures/basic.xml

@@ -37,6 +37,7 @@
   </layers>
 
   <variables>
+    <string id="a" value="\m{a}"/>
     <string id="vst" value="abc"/>
     <set id="vse" value="a b c"/>
     <unicodeSet id="vus" value="[abc]"/>

developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml

@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+This will fail because the two markers given don't exist anywhere.
+
+-->
+<!DOCTYPE keyboard SYSTEM "../../../../../../../resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard.dtd">
+<keyboard locale="mt" conformsTo="techpreview">
+  <names>
+    <name value="vars-markers-badref-0" />
+  </names>
+
+  <displays>
+    <!-- Not ok, nobody emits this, so this display doesn't make sense -->
+    <display to="\m{doesnt_exist_1}" display="¬" />
+    <!-- OK - it is in the strings list.  Could warn that it matches no keys -->
+    <display to="\m{maybe_emitted}" display="‽" />
+  </displays>
+
+  <variables>
+    <!-- This may be emitted, so at present it does not cause an error. -->
+    <!-- A linter could, however, hint on an unused variable. -->
+    <!-- Removing this variable would cause \m{maybe-emitted}  to error in the transform group -->
+    <string id="unused_variable" value="\m{maybe_emitted}" />
+  </variables>
+
+  <transforms type="simple">
+    <transformGroup>
+      <!-- not OK, nobody emits this. -->
+      <transform from="\m{doesnt_exist_3}" to="3" />
+    </transformGroup>
+  </transforms>
+  <transforms type="backspace">
+    <transformGroup>
+      <!-- not OK, nobody emits this. -->
+      <transform from="\m{doesnt_exist_2}" />
+      <!-- OK - it's in the variable list -->
+      <transform from="\m{maybe_emitted}" />
+      <!-- OK - matches all markers -->
+      <transform from="\m{.}" />
+    </transformGroup>
+  </transforms>
+</keyboard>

developer/src/kmc-ldml/test/fixtures/sections/vars/markers-maximal.xml

@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+  This keyboard won't make a lot of sense, it's mostly
+  a test of validating markers.
+
+  When we get further on in implementation, we'll shape this up
+  into something testable for implementation.
+
+-->
+<!DOCTYPE keyboard SYSTEM "../../../../../../../resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard.dtd">
+<keyboard locale="mt" conformsTo="techpreview">
+  <names>
+    <name value="vars-markers-maximal" />
+  </names>
+
+  <displays>
+    <display to="\m{m}" display="Ⓜ️" />
+  </displays>
+
+  <keys>
+    <key id="m" to="\m{m}" />
+  </keys>
+
+  <!-- from spec -->
+  <variables>
+    <string id="x" value="\m{x}" />
+  </variables>
+
+  <transforms type="simple">
+    <transformGroup>
+      <transform from="m" to="\m{m}"/>
+      <transform from="x" to="\m{x}"/>
+    </transformGroup>
+    <transformGroup>
+      <transform from="e\m{x}" to="é" />
+    </transformGroup>
+  </transforms>
+  <transforms type="backspace">
+    <transformGroup>
+      <transform from="A\m{.}B" />
+    </transformGroup>
+  </transforms>
+</keyboard>

developer/src/kmc-ldml/test/test-vars.ts

@@ -183,4 +183,31 @@ describe('vars', function () {
       ],
     },
   ]);
+  describe('markers', function () {
+    this.slow(500); // 0.5 sec -- json schema validation takes a while
+
+    testCompilationCases(VarsCompiler, [
+      {
+        subpath: 'sections/vars/markers-maximal.xml',
+        callback(sect) {
+          const vars = <Vars> sect;
+          assert.ok(vars.markers);
+          assert.sameDeepOrderedMembers(vars.markers.toStringArray(),
+            ['m','x']);
+        },
+      },
+      {
+        subpath: 'sections/vars/fail-markers-badref-0.xml',
+        errors: [
+          CompilerMessages.Error_MissingMarkers({
+            ids: [
+              'doesnt_exist_1',
+              'doesnt_exist_2',
+              'doesnt_exist_3',
+            ]
+          }),
+        ],
+      },
+    ]);
+  });
 });