diff --git a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts index 8d6cd57003..767a221f9c 100644 --- a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts +++ b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts @@ -135,6 +135,18 @@ export class StrsItem { isEqual(a: StrsItem): boolean { return a.value === this.value && a.char === this.char; } + + private _context: any = undefined; + + /** add any context from the options to this strsitem */ + setContext(opts?: StrsOptions) { + // At present, there's only a single piece of context available + this._context = this._context || opts?.x; + } + + get context() : any { + return this._context; + } }; /** @@ -161,6 +173,8 @@ export interface StrsOptions { nfd?: boolean; /** string can be stored as a single CharStrsItem, not in strs table. */ singleOk?: boolean; + /** optional context */ + x?: any; }; export class Strs extends Section { @@ -192,6 +206,8 @@ export class Strs extends Section { result = new StrsItem(s); this.strings.push(result); } + // give an option to set the context + result.setContext(opts); return result; } diff --git a/developer/src/kmc-ldml/src/compiler/disp.ts b/developer/src/kmc-ldml/src/compiler/disp.ts index 5b63a99a8f..8121888fcf 100644 --- a/developer/src/kmc-ldml/src/compiler/disp.ts +++ b/developer/src/kmc-ldml/src/compiler/disp.ts @@ -61,7 +61,7 @@ export class DispCompiler extends SectionCompiler { const result = new Disp(); // displayOptions - result.baseCharacter = sections.strs.allocString(this.keyboard3.displays?.displayOptions?.baseCharacter, {unescape: true}); + result.baseCharacter = sections.strs.allocString(this.keyboard3.displays?.displayOptions?.baseCharacter, { unescape: true, x: this.keyboard3?.displays?.displayOptions }); // displays result.disps = this.keyboard3.displays?.display.map(display => ({ @@ -69,11 +69,13 @@ export class DispCompiler extends SectionCompiler { stringVariables: true, markers: true, unescape: true, + x: display, }, sections), - id: sections.strs.allocString(display.keyId), // not escaped, not substituted + id: sections.strs.allocString(display.keyId, { x: display }), // not escaped, not substituted display: sections.strs.allocString(display.display, { stringVariables: true, unescape: true, + x: display, }, sections), })) || []; // TODO-LDML: need coverage for the [] diff --git a/developer/src/kmc-ldml/src/compiler/empty-compiler.ts b/developer/src/kmc-ldml/src/compiler/empty-compiler.ts index 4b9aec7613..c0fe1d1ef6 100644 --- a/developer/src/kmc-ldml/src/compiler/empty-compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/empty-compiler.ts @@ -1,7 +1,7 @@ import { SectionIdent, constants } from '@keymanapp/ldml-keyboard-constants'; import { SectionCompiler } from "./section-compiler.js"; import { util, KMXPlus, LdmlKeyboardTypes } from "@keymanapp/common-types"; -import { CompilerCallbacks, LDMLKeyboard } from "@keymanapp/developer-utils"; +import { CompilerCallbacks, LDMLKeyboard, ObjectWithMetadata } from "@keymanapp/developer-utils"; import { VarsCompiler } from './vars.js'; import { LdmlCompilerMessages } from './ldml-compiler-messages.js'; @@ -33,13 +33,26 @@ export class StrsCompiler extends EmptyCompiler { public postValidate(section?: KMXPlus.Section): boolean { const strs = section; + /** attempt to find a context object for the string */ + function findContextForString(s: string): ObjectWithMetadata { + // try exact match + for(const str of strs.strings) { + if (str.value == s) return str.context; + } + // try substring match + for(const str of strs.strings) { + if (str.value.includes(s)) return str.context; + } + return null; + } + if (strs) { const badStringAnalyzer = new util.BadStringAnalyzer(); const CONTAINS_MARKER_REGEX = new RegExp(LdmlKeyboardTypes.MarkerParser.ANY_MARKER_MATCH); for (let s of strs.allProcessedStrings.values()) { // stop at the first denormalized string if (!util.isNormalized(s)) { - this.callbacks.reportMessage(LdmlCompilerMessages.Warn_StringDenorm({s})); + this.callbacks.reportMessage(LdmlCompilerMessages.Warn_StringDenorm({s}, findContextForString(s))); } // replace all \\uXXXX with the actual code point. // this lets us analyze whether there are PUA, unassigned, etc. @@ -63,16 +76,19 @@ export class StrsCompiler extends EmptyCompiler { const illegals = m.get(util.BadStringType.illegal); if (puas) { const [count, lowestCh] = [puas.size, Array.from(puas.values()).sort((a, b) => a - b)[0]]; - this.callbacks.reportMessage(LdmlCompilerMessages.Hint_PUACharacters({ count, lowestCh })) + this.callbacks.reportMessage(LdmlCompilerMessages.Hint_PUACharacters({ count, lowestCh }, + findContextForString(String.fromCodePoint(lowestCh)))); } if (unassigneds) { const [count, lowestCh] = [unassigneds.size, Array.from(unassigneds.values()).sort((a, b) => a - b)[0]]; - this.callbacks.reportMessage(LdmlCompilerMessages.Warn_UnassignedCharacters({ count, lowestCh })) + this.callbacks.reportMessage(LdmlCompilerMessages.Warn_UnassignedCharacters({ count, lowestCh }, + findContextForString(String.fromCodePoint(lowestCh)))); } if (illegals) { // do this last, because we will return false. const [count, lowestCh] = [illegals.size, Array.from(illegals.values()).sort((a, b) => a - b)[0]]; - this.callbacks.reportMessage(LdmlCompilerMessages.Error_IllegalCharacters({ count, lowestCh })) + this.callbacks.reportMessage(LdmlCompilerMessages.Error_IllegalCharacters({ count, lowestCh }, + findContextForString(String.fromCodePoint(lowestCh)))); return false; } } diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index 5ca61ceb48..01c4ffc675 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -324,12 +324,13 @@ export class KeysCompiler extends SectionCompiler { // allocate the in-memory const flicks: KeysFlicks = new KeysFlicks( - sections.strs.allocString(flickId) + sections.strs.allocString(flickId, { x: flick }) ); // add data from each segment - for (const { keyId, directions } of flick.flickSegment) { - const keyIdStr = sections.strs.allocString(keyId); + for (const flickSegment of flick.flickSegment) { + const { keyId, directions } = flickSegment; + const keyIdStr = sections.strs.allocString(keyId, { x: flickSegment }); const directionsList: ListItem = sections.list.allocListFromSpaces( directions, { }, @@ -369,20 +370,20 @@ export class KeysCompiler extends SectionCompiler { if (!!gap) { flags |= constants.keys_key_flags_gap; } - const id = sections.strs.allocString(key.id); + const id = sections.strs.allocString(key.id, { x: key }); const longPress: ListItem = sections.list.allocListFromSpaces( longPressKeyIds, {}, sections); const longPressDefault = sections.strs.allocString(longPressDefaultKeyId, - {}, + { x: key }, sections); const multiTap: ListItem = sections.list.allocListFromSpaces( multiTapKeyIds, {}, sections); - const keySwitch = sections.strs.allocString(layerId); // 'switch' is a reserved word + const keySwitch = sections.strs.allocString(layerId, { x: key }); // 'switch' is a reserved word const toRaw = output; @@ -395,6 +396,7 @@ export class KeysCompiler extends SectionCompiler { unescape: true, singleOk: true, nfd: true, + x: key, }, sections); if (!to.isOneChar) { diff --git a/developer/src/kmc-ldml/src/compiler/layr.ts b/developer/src/kmc-ldml/src/compiler/layr.ts index ec96a6bddd..b0c615f41b 100644 --- a/developer/src/kmc-ldml/src/compiler/layr.ts +++ b/developer/src/kmc-ldml/src/compiler/layr.ts @@ -69,13 +69,13 @@ export class LayrCompiler extends SectionCompiler { const sect = new Layr(); sect.lists = this.keyboard3.layers.map((layers) => { - const hardware = sections.strs.allocString(layers.formId); + const hardware = sections.strs.allocString(layers.formId, {x:layers}); // Already validated in validate const layerEntries = []; for (const layer of layers.layer) { const rows = layer.row.map((row) => { const erow: LayrRow = { - keys: row.keys.trim().split(/[ \t]+/).map((id) => sections.strs.allocString(id)), + keys: row.keys.trim().split(/[ \t]+/).map((id) => sections.strs.allocString(id, { x: row })), }; // include linenumber info for row return SectionCompiler.copySymbols(erow, row); @@ -84,7 +84,7 @@ export class LayrCompiler extends SectionCompiler { // push a layer entry for each modifier set for (const mod of mods) { layerEntries.push({ - id: sections.strs.allocString(layer.id), + id: sections.strs.allocString(layer.id, {x:layer}), mod, rows, }); diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 2955e1a95e..4e65138912 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -267,16 +267,22 @@ export class LdmlCompilerMessages { ); static HINT_PUACharacters = SevHint | 0x0023; - static Hint_PUACharacters = (o: { count: number, lowestCh: number }) => - m(this.HINT_PUACharacters, `File contains ${def(o.count)} PUA character(s), including ${util.describeCodepoint(o.lowestCh)}`); + static Hint_PUACharacters = (o: { count: number, lowestCh: number }, x?: ObjectWithMetadata) => mx( + this.HINT_PUACharacters, x, + `File contains ${def(o.count)} PUA character(s), including ${util.describeCodepoint(o.lowestCh)}`, + ); static WARN_UnassignedCharacters = SevWarn | 0x0024; - static Warn_UnassignedCharacters = (o: { count: number, lowestCh: number }) => - m(this.WARN_UnassignedCharacters, `File contains ${def(o.count)} unassigned character(s), including ${util.describeCodepoint(o.lowestCh)}`); + static Warn_UnassignedCharacters = (o: { count: number, lowestCh: number }, x?: ObjectWithMetadata) => mx( + this.WARN_UnassignedCharacters, x, + `File contains ${def(o.count)} unassigned character(s), including ${util.describeCodepoint(o.lowestCh)}`, + ); static ERROR_IllegalCharacters = SevError | 0x0025; - static Error_IllegalCharacters = (o: { count: number, lowestCh: number }) => - m(this.ERROR_IllegalCharacters, `File contains ${def(o.count)} illegal character(s), including ${util.describeCodepoint(o.lowestCh)}`); + static Error_IllegalCharacters = (o: { count: number, lowestCh: number }, x?: ObjectWithMetadata) => mx( + this.ERROR_IllegalCharacters, x, + `File contains ${def(o.count)} illegal character(s), including ${ util.describeCodepoint(o.lowestCh) }`, + ); static HINT_CharClassImplicitDenorm = SevHint | 0x0026; static Hint_CharClassImplicitDenorm = (o: { lowestCh: number }, x?: ObjectWithMetadata) => mx( @@ -307,8 +313,10 @@ export class LdmlCompilerMessages { ); static WARN_StringDenorm = SevWarn | 0x002B; - static Warn_StringDenorm = (o: { s: string }) => - m(this.WARN_StringDenorm, `File contains string "${def(o.s)}" that is neither NFC nor NFD.`); + static Warn_StringDenorm = (o: { s: string }, x?: ObjectWithMetadata) => mx( + this.WARN_StringDenorm, x, + `File contains string "${def(o.s)}" that is neither NFC nor NFD.`, + ); static ERROR_DuplicateLayerWidth = SevError | 0x002C; static Error_DuplicateLayerWidth = (o: { minDeviceWidth: number }, x?: ObjectWithMetadata) => mx( diff --git a/developer/src/kmc-ldml/src/compiler/loca.ts b/developer/src/kmc-ldml/src/compiler/loca.ts index 48be2d032f..3cc222e613 100644 --- a/developer/src/kmc-ldml/src/compiler/loca.ts +++ b/developer/src/kmc-ldml/src/compiler/loca.ts @@ -71,7 +71,7 @@ export class LocaCompiler extends SectionCompiler { // yet include `getCanonicalLocales` but node 16 does include it so we can // safely use it. Also well supported in modern browsers. const canonicalLocales = (Intl as any).getCanonicalLocales(locales) as string[]; - result.locales = canonicalLocales.map(locale => sections.strs.allocString(locale)); + result.locales = canonicalLocales.map(locale => sections.strs.allocString(locale, {x: this.contextForLocale(locale)})); if(result.locales.length < locales.length) { this.callbacks.reportMessage(LdmlCompilerMessages.Hint_OneOrMoreRepeatedLocales(this.keyboard3?.locales)); diff --git a/developer/src/kmc-ldml/src/compiler/meta.ts b/developer/src/kmc-ldml/src/compiler/meta.ts index 57f0cb36c0..e6cd6371ee 100644 --- a/developer/src/kmc-ldml/src/compiler/meta.ts +++ b/developer/src/kmc-ldml/src/compiler/meta.ts @@ -53,12 +53,18 @@ export class MetaCompiler extends SectionCompiler { public compile(sections: DependencySections): Meta { const result = new Meta(); - result.author = sections.strs.allocString(this.keyboard3.info?.author); - result.conform = sections.strs.allocString(this.keyboard3.conformsTo); - result.layout = sections.strs.allocString(this.keyboard3.info?.layout); - result.name = sections.strs.allocString(this.keyboard3.info?.name); - result.indicator = sections.strs.allocString(this.keyboard3.info?.indicator); - result.version = sections.strs.allocString(this.keyboard3.version?.number ?? "0.0.0"); + result.author = sections.strs.allocString(this.keyboard3.info?.author, + {x: this.keyboard3.info}); + result.conform = sections.strs.allocString(this.keyboard3.conformsTo, + {x: this.keyboard3}); + result.layout = sections.strs.allocString(this.keyboard3.info?.layout, + {x: this.keyboard3.info}); + result.name = sections.strs.allocString(this.keyboard3.info?.name, + {x: this.keyboard3.info}); + result.indicator = sections.strs.allocString(this.keyboard3.info?.indicator, + {x: this.keyboard3.info}); + result.version = sections.strs.allocString(this.keyboard3.version?.number ?? "0.0.0", + {x: this.keyboard3.version}); result.settings = (this.keyboard3.settings?.normalization == "disabled" ? KeyboardSettings.normalizationDisabled : 0); return result; diff --git a/developer/src/kmc-ldml/src/compiler/tran.ts b/developer/src/kmc-ldml/src/compiler/tran.ts index 32ebbca3dd..875da13406 100644 --- a/developer/src/kmc-ldml/src/compiler/tran.ts +++ b/developer/src/kmc-ldml/src/compiler/tran.ts @@ -18,6 +18,7 @@ import { verifyValidAndUnique } from "../util/util.js"; import { LdmlCompilerMessages } from "./ldml-compiler-messages.js"; import { Substitutions, SubstitutionUse } from "./substitution-tracker.js"; import { transform_from_parse, transform_to_parse } from "../util/abnf/abnf.js"; +import { StrsOptions } from "../../../../../common/web/types/src/kmx/kmx-plus/kmx-plus.js"; type TransformCompilerType = 'simple' | 'backspace'; @@ -138,6 +139,8 @@ export abstract class TransformCompiler { + const scrubbed = Object.assign({}, m); + // Turn this on once all messages have offsets, see messages.tests.ts + // if (!scrubbed.offset) { + // throw Error(`Error, no offset detected in message ${CompilerError.formatEvent(m)}`); + // } + delete scrubbed.offset; + delete scrubbed.line; + delete scrubbed.filename; + delete scrubbed.column; + return scrubbed; + }); +} + /** * Run a bunch of cases * @param cases cases to run @@ -283,15 +315,7 @@ export function testCompilationCases(compiler: SectionCompilerNew, cases : Compi let messagesToCheck = callbacks.messages; // scrub offsets from messages to reduce churn in the test casws if (!testcase.retainOffsetInMessages && callbacks.messages) { - messagesToCheck = callbacks.messages.map(m => { - const scrubbed = Object.assign({}, m); - // Turn this on once all messages have offsets, see messages.tests.ts - // if (!scrubbed.offset) { - // throw Error(`Error, no offset detected in message ${CompilerError.formatEvent(m)}`); - // } - delete scrubbed.offset; - return scrubbed; - }); + messagesToCheck = scrubContextFromMessages(callbacks.messages); } const testcaseErrors = matchCompilerEventsOrBoolean(messagesToCheck, testcase.errors); const testcaseWarnings = matchCompilerEvents(messagesToCheck, testcase.warnings);