From ec68e74bde8418472b9641ee33e9a88b63845ad9 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 27 Feb 2025 16:24:21 -0600 Subject: [PATCH 01/27] feat(developer): add experimental char-offset data Depends on PR https://github.com/NaturalIntelligence/fast-xml-parser/pull/728 so uses srl repository "temporarily" Fixes: #10622 --- developer/src/common/web/utils/package.json | 2 +- .../src/common/web/utils/src/xml-utils.ts | 24 ++++++++++-- .../common/web/utils/test/xml-utils.tests.ts | 24 +++++++++++- package-lock.json | 39 ++++++++----------- 4 files changed, 61 insertions(+), 28 deletions(-) diff --git a/developer/src/common/web/utils/package.json b/developer/src/common/web/utils/package.json index 4f066313f6..571123ec68 100644 --- a/developer/src/common/web/utils/package.json +++ b/developer/src/common/web/utils/package.json @@ -12,7 +12,7 @@ "@keymanapp/common-types": "*", "@sentry/node": "^7.57.0", "eventemitter3": "^5.0.0", - "fast-xml-parser": "^4.5.0", + "fast-xml-parser": "github:srl295/fast-xml-parser#srl295/line-number-593", "path-browserify": "^1.0.1", "restructure": "^3.0.1", "sax": ">=0.6.0", diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 69a039aa00..626233dd17 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -8,6 +8,8 @@ import { XMLParser, XMLBuilder } from 'fast-xml-parser'; +export const START_INDEX = new XMLParser().getStartIndexSymbol(); + export type KeymanXMLType = 'keyboard3' // LDML | 'keyboardTest3' // LDML @@ -128,6 +130,22 @@ export class KeymanXMLReader { public constructor(public type: KeymanXMLType) { } + /** + * Copy symbols from 'from' onto 'onto' + * This is used to propagate special symbols + * and XML + * @param onto object to copy onto + * @param from source for symbols + * @returns the onto object + */ + public static copySymbols(onto: T, from: any): T { + const o = onto as any; + for (const sym of Object.getOwnPropertySymbols(from)) { + o[sym] = from[sym]; + } + return onto; + } + /** move `{ $abc: 4 }` into `{ $: { abc: 4 } }` */ private static fixupDollarAttributes(data: any) : any { if (typeof data === 'object') { @@ -146,9 +164,9 @@ export class KeymanXMLReader { } }); if (attrs.length) { - e.push(['$', Object.fromEntries(attrs)]); + e.push(['$', this.copySymbols(Object.fromEntries(attrs), data)]); } - return Object.fromEntries(e); + return this.copySymbols(Object.fromEntries(e), data); } else { return data; } @@ -184,7 +202,7 @@ export class KeymanXMLReader { } } }); - return Object.fromEntries(e); + return this.copySymbols(Object.fromEntries(e), data); } else { return data; } diff --git a/developer/src/common/web/utils/test/xml-utils.tests.ts b/developer/src/common/web/utils/test/xml-utils.tests.ts index 1cc9724b27..65ee826f57 100644 --- a/developer/src/common/web/utils/test/xml-utils.tests.ts +++ b/developer/src/common/web/utils/test/xml-utils.tests.ts @@ -12,7 +12,7 @@ import { env } from 'node:process'; import { readFileSync, writeFileSync } from 'node:fs'; -import { KeymanXMLType, KeymanXMLReader, KeymanXMLWriter } from '../src/xml-utils.js'; +import { KeymanXMLType, KeymanXMLReader, KeymanXMLWriter, START_INDEX } from '../src/xml-utils.js'; import { makePathToFixture } from './helpers/index.js'; // if true, attempt to WRITE the fixtures @@ -160,3 +160,25 @@ describe(`XML Writer Test ${GEN_XML_FIXTURES && '(update mode!)' || ''}`, () => }); } }); + +describe(`XML Reader line number test`, () => { + const path = 'tran_fail-empty.xml'; + const xmlPath = makePathToFixture('xml', `${path}`); + const type: KeymanXMLType = 'keyboard3'; + it(`Should report line numbers on parse of ${type} ${path}`, () => { + const xml = readData(xmlPath); + assert.ok(xml, `Could not read ${xmlPath}`); + + const reader = new KeymanXMLReader(type); + assert.ok(reader); + + // now, parse. subsitute endings for Win + const actual = reader.parse(xml.replace(/\r\n/g, '\n')); + assert.ok(actual, `Parser failed on ${xmlPath}`); + + // now, assert char offset + assert.equal(actual.keyboard3[START_INDEX as any], 40); // index of element + assert.equal(actual.keyboard3.info[START_INDEX as any], 136); // index of etc + assert.equal(actual.keyboard3.transforms[START_INDEX as any], 186); + }); +}); diff --git a/package-lock.json b/package-lock.json index 4e340f8af4..0f62509ac5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -342,7 +342,7 @@ "@keymanapp/common-types": "*", "@sentry/node": "^7.57.0", "eventemitter3": "^5.0.0", - "fast-xml-parser": "^4.5.0", + "fast-xml-parser": "github:srl295/fast-xml-parser#srl295/line-number-593", "path-browserify": "^1.0.1", "restructure": "^3.0.1", "sax": ">=0.6.0", @@ -372,6 +372,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "developer/src/common/web/utils/node_modules/fast-xml-parser": { + "version": "4.1.3", + "resolved": "git+ssh://git@github.com/srl295/fast-xml-parser.git#d7601c31f39fe69e6612af79f9d778dee5dfcb42", + "license": "MIT", + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + }, + "funding": { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + }, "developer/src/common/web/utils/node_modules/js-yaml": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", @@ -9016,28 +9031,6 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, - "node_modules/fast-xml-parser": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.5.0.tgz", - "integrity": "sha512-/PlTQCI96+fZMAOLMZK4CWG1ItCbfZ/0jx7UIJFChPNrx7tcEgerUgWbeieCM9MfHInUDyK8DWYZ+YrywDJuTg==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - }, - { - "type": "paypal", - "url": "https://paypal.me/naturalintelligence" - } - ], - "license": "MIT", - "dependencies": { - "strnum": "^1.0.5" - }, - "bin": { - "fxparser": "src/cli/cli.js" - } - }, "node_modules/fastq": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", From b60c64c3d3f1b4ec072090215be95c5d0d7e8464 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 27 Feb 2025 16:42:41 -0600 Subject: [PATCH 02/27] feat(developer): test for line number extension to xml-utils Fixes: #10622 --- .../src/common/web/utils/src/xml-utils.ts | 30 +++++++++++++++---- .../common/web/utils/test/xml-utils.tests.ts | 10 +++++-- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 626233dd17..9fbebdf397 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -125,11 +125,29 @@ const GENERATOR_OPTIONS: KeymanXMLOptionsBag = { }, }; +export interface LineColumn { + line: number; + column?: number; +} + /** wrapper for XML parsing support */ export class KeymanXMLReader { public constructor(public type: KeymanXMLType) { } + public static offsetToLineColumn(offset: number, text: string) : LineColumn { + const lines: string[] = text.split("\n"); + for (let line = 1; line < lines.length + 1; line++) { // 1-based + const linestr = lines[line - 1]; + if (linestr.length < offset) { + offset = offset - (linestr.length + 1); // count newline at end + continue; + } + return { line, column: offset }; + } + return { line: lines.length + 1 }; + } + /** * Copy symbols from 'from' onto 'onto' * This is used to propagate special symbols @@ -138,13 +156,13 @@ export class KeymanXMLReader { * @param from source for symbols * @returns the onto object */ - public static copySymbols(onto: T, from: any): T { - const o = onto as any; - for (const sym of Object.getOwnPropertySymbols(from)) { - o[sym] = from[sym]; - } - return onto; + private static copySymbols(onto: T, from: any): T { + const o = onto as any; + for (const sym of Object.getOwnPropertySymbols(from)) { + o[sym] = from[sym]; } + return onto; + } /** move `{ $abc: 4 }` into `{ $: { abc: 4 } }` */ private static fixupDollarAttributes(data: any) : any { diff --git a/developer/src/common/web/utils/test/xml-utils.tests.ts b/developer/src/common/web/utils/test/xml-utils.tests.ts index 65ee826f57..bdcd18817d 100644 --- a/developer/src/common/web/utils/test/xml-utils.tests.ts +++ b/developer/src/common/web/utils/test/xml-utils.tests.ts @@ -166,19 +166,23 @@ describe(`XML Reader line number test`, () => { const xmlPath = makePathToFixture('xml', `${path}`); const type: KeymanXMLType = 'keyboard3'; it(`Should report line numbers on parse of ${type} ${path}`, () => { - const xml = readData(xmlPath); + let xml = readData(xmlPath); assert.ok(xml, `Could not read ${xmlPath}`); - + xml = xml.replace(/\r\n/g, '\n'); const reader = new KeymanXMLReader(type); assert.ok(reader); // now, parse. subsitute endings for Win - const actual = reader.parse(xml.replace(/\r\n/g, '\n')); + const actual = reader.parse(xml); assert.ok(actual, `Parser failed on ${xmlPath}`); // now, assert char offset assert.equal(actual.keyboard3[START_INDEX as any], 40); // index of element assert.equal(actual.keyboard3.info[START_INDEX as any], 136); // index of etc assert.equal(actual.keyboard3.transforms[START_INDEX as any], 186); + + assert.equal(KeymanXMLReader.offsetToLineColumn(actual.keyboard3[START_INDEX as any], xml), { line: 3, column: 0 }); + assert.equal(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.info[START_INDEX as any], xml), { line: 4, column: 2 }); + assert.equal(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.transforms[START_INDEX as any], xml), { line: 8, column: 2 }); }); }); From c4ca359d182d28cf900b15a9c0c13fb63e54bb04 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 28 Feb 2025 09:27:25 -0600 Subject: [PATCH 03/27] feat(developer): API for resolving line numbers for messages - add 'column' to CompilerEvent - Add a LdmlCompilerMessages.col() function to set the column number - add a LdmlCompilerMessages.resolveLineNumber() to map offsets to line number Fixes: #10622 --- .../web/utils/src/compiler-interfaces.ts | 6 +++++ developer/src/common/web/utils/src/index.ts | 2 +- developer/src/kmc-ldml/src/compiler/keys.ts | 5 +++- .../src/compiler/ldml-compiler-messages.ts | 26 ++++++++++++++++++- .../src/kmc/src/util/NodeCompilerCallbacks.ts | 11 ++++++++ 5 files changed, 47 insertions(+), 3 deletions(-) diff --git a/developer/src/common/web/utils/src/compiler-interfaces.ts b/developer/src/common/web/utils/src/compiler-interfaces.ts index 869e0165da..390eea2f7e 100644 --- a/developer/src/common/web/utils/src/compiler-interfaces.ts +++ b/developer/src/common/web/utils/src/compiler-interfaces.ts @@ -5,7 +5,13 @@ import { CompilerCallbacks } from "./compiler-callbacks.js"; */ export interface CompilerEvent { filename?: string; + /** line where a message applies */ line?: number; + /** + * column where a message applies. + * where line is unset, column is a file offset + */ + column?: number; code: number; message: string; /** diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index f3e23e79f2..1a56a735e8 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -65,7 +65,7 @@ export { UrlSubpathCompilerCallback } from './utils/UrlSubpathCompilerCallback.j export { CommonTypesMessages } from './common-messages.js'; export * as SourceFilenamePatterns from './source-filename-patterns.js'; -export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader } from './xml-utils.js'; +export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, START_INDEX } from './xml-utils.js'; export * as GitHubUrls from './github-urls.js'; export * as CloudUrls from './cloud-urls.js'; diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index d86d7e2bf5..372bbea891 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -446,7 +446,10 @@ export class KeysCompiler extends SectionCompiler { if (layer.row.length > keymap.length) { this.callbacks.reportMessage( - LdmlCompilerMessages.Error_HardwareLayerHasTooManyRows() + LdmlCompilerMessages.col( + LdmlCompilerMessages.Error_HardwareLayerHasTooManyRows(), + layer + ) ); valid = false; } diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 1a84b10937..b6293d0158 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -1,5 +1,5 @@ import { util } from "@keymanapp/common-types"; -import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def } from '@keymanapp/developer-utils'; +import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, START_INDEX, CompilerEvent, KeymanXMLReader } from '@keymanapp/developer-utils'; // const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.LdmlKeyboardCompiler; @@ -264,5 +264,29 @@ export class LdmlCompilerMessages { `Invalid transform to="${def(o.to)}": "${def(o.message)}"`, ); + /** + * Get a column number from o and set e's column field + * @param event a compiler event, such as from functions in this class + * @param o any object parsed from XML or with the START_INDEX symbol copied over + * @returns modified event object + */ + static col(event: CompilerEvent, o: any): CompilerEvent { + event.column = (o as any)[START_INDEX as any]; + return event; + } + /** + * Given an event with a column number but no line, resolve it into line:column + * @param event event to modify + * @param xml XML source + * @returns the modified event + */ + static resolveLineNumber(event: CompilerEvent, xml: string) : CompilerEvent { + if (event.column && !event.line && xml) { + const loc = KeymanXMLReader.offsetToLineColumn(event.column, xml); + event.line = loc.line; + event.column = loc.column; + } + return event; + } } diff --git a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts index f3cdf1e7b2..5b340633fa 100644 --- a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts +++ b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts @@ -15,6 +15,7 @@ import chalk from 'chalk'; import supportsColor from 'supports-color'; import { KeymanSentry } from '@keymanapp/developer-utils'; import { fileURLToPath } from 'url'; +import { LdmlCompilerMessages } from '@keymanapp/kmc-ldml'; const color = chalk.default; const severityColors: {[value in CompilerErrorSeverity]: chalk.Chalk} = { @@ -150,6 +151,16 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { event.filename = this.messageFilename; } + if (event.column && !event.line && event.filename) { + try { + // TODO: rereads XML every single time. + // Should cache unless the filename changes (this.messageFilename) + LdmlCompilerMessages.resolveLineNumber(event, this.fs.readFileSync(event.filename, "utf-8")); + } catch(e) { + console.error(e); // resolving line numbers + } + } + if(this.messageFilename != event.filename) { // Reset max message limit when a new file is being processed this.messageFilename = event.filename; From d38025b4a19418e80931059ea7767895842dba83 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 28 Feb 2025 10:32:05 -0600 Subject: [PATCH 04/27] feat(developer): API for resolving line numbers for messages - Only read the xml file once Fixes: #10622 --- .../src/kmc/src/util/NodeCompilerCallbacks.ts | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts index 5b340633fa..7c184c1417 100644 --- a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts +++ b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts @@ -46,6 +46,8 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { messages: CompilerEvent[] = []; messageCount = 0; messageFilename: string = ''; + /** cache of the contentes of the text */ + messageFiletext: string = ''; maxLogMessages = MaxMessagesDefault; constructor(private options: CompilerCallbackOptions) { @@ -56,6 +58,7 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { this.messages = []; this.messageCount = 0; this.messageFilename = ''; + this.messageFiletext = ''; } /** @@ -151,20 +154,29 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { event.filename = this.messageFilename; } - if (event.column && !event.line && event.filename) { - try { - // TODO: rereads XML every single time. - // Should cache unless the filename changes (this.messageFilename) - LdmlCompilerMessages.resolveLineNumber(event, this.fs.readFileSync(event.filename, "utf-8")); - } catch(e) { - console.error(e); // resolving line numbers - } - } - if(this.messageFilename != event.filename) { // Reset max message limit when a new file is being processed this.messageFilename = event.filename; this.messageCount = 0; + this.messageFiletext = ''; + } + + if (event.column && !event.line && event.filename) { + try { + let text = ''; + if (this.messageFilename == event.filename) { + text = this.messageFiletext; + } + if (text == '') { + text = this.fs.readFileSync(event.filename, "utf-8"); + if (this.messageFilename == event.filename) { + this.messageFiletext = text; + } + } + LdmlCompilerMessages.resolveLineNumber(event, text); + } catch(e) { + console.error(e); // resolving line numbers + } } const disable = CompilerFileCallbacks.applyMessageOverridesToEvent(event, this.options.messageOverrides); From 650042fb6e7ce831d7e14bd0c74d5fd886a88588 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 28 Feb 2025 10:53:21 -0600 Subject: [PATCH 05/27] feat(developer): API for resolving line numbers for messages - simplify the error call site again Fixes: #10622 --- developer/src/kmc-ldml/src/compiler/keys.ts | 5 +---- .../src/compiler/ldml-compiler-messages.ts | 14 ++++++++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index 372bbea891..35a931dd94 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -446,10 +446,7 @@ export class KeysCompiler extends SectionCompiler { if (layer.row.length > keymap.length) { this.callbacks.reportMessage( - LdmlCompilerMessages.col( - LdmlCompilerMessages.Error_HardwareLayerHasTooManyRows(), - layer - ) + LdmlCompilerMessages.Error_HardwareLayerHasTooManyRows(layer) ); valid = false; } diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index b6293d0158..6f8cc5fa3d 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -9,6 +9,9 @@ const SevError = CompilerErrorSeverity.Error | CompilerErrorNamespace.LdmlKeyboa // sub-numberspace for transform errors const SevErrorTransform = SevError | 0xF00; +// like m() but takes an XML object for line numbers +const mx = (x: any, code: number, message: string, detail?: string): CompilerEvent => LdmlCompilerMessages.col(m(code, message, detail), x); + /** * @internal */ @@ -20,7 +23,8 @@ export class LdmlCompilerMessages { static Error_InvalidLocale = (o:{tag: string}) => m(this.ERROR_InvalidLocale, `Invalid BCP 47 locale form '${def(o.tag)}'`); static ERROR_HardwareLayerHasTooManyRows = SevError | 0x0003; - static Error_HardwareLayerHasTooManyRows = () => m(this.ERROR_HardwareLayerHasTooManyRows, `'hardware' layer has too many rows`); + static Error_HardwareLayerHasTooManyRows = (x: any) => mx(x, + this.ERROR_HardwareLayerHasTooManyRows, `'hardware' layer has too many rows`); static ERROR_RowOnHardwareLayerHasTooManyKeys = SevError | 0x0004; static Error_RowOnHardwareLayerHasTooManyKeys = (o:{row: number, hardware: string, modifiers: string}) => m(this.ERROR_RowOnHardwareLayerHasTooManyKeys, `Row #${def(o.row)} on 'hardware' ${def(o.hardware)} layer for modifier ${o.modifiers || 'none'} has too many keys`); @@ -267,11 +271,13 @@ export class LdmlCompilerMessages { /** * Get a column number from o and set e's column field * @param event a compiler event, such as from functions in this class - * @param o any object parsed from XML or with the START_INDEX symbol copied over + * @param x any object parsed from XML or with the START_INDEX symbol copied over * @returns modified event object */ - static col(event: CompilerEvent, o: any): CompilerEvent { - event.column = (o as any)[START_INDEX as any]; + static col(event: CompilerEvent, x?: any): CompilerEvent { + if(x) { + event.column = (x as any)[START_INDEX as any]; + } return event; } From 819d9bf5620a65ae9ef9d7a0832bd17303480e72 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 28 Feb 2025 12:08:31 -0600 Subject: [PATCH 06/27] feat(developer): API for resolving line numbers for messages - change to use a different fast-xml-parser PR, https://github.com/NaturalIntelligence/fast-xml-parser/pull/729 Fixes: #10622 --- developer/src/common/web/utils/package.json | 2 +- .../src/common/web/utils/src/xml-utils.ts | 2 +- .../utils/test/kpj/kpj-file-writer.tests.ts | 1 - .../common/web/utils/test/xml-utils.tests.ts | 7 ++- developer/src/kmc-ldml/test/helpers/index.ts | 9 +++- developer/src/kmc-ldml/test/keys.tests.ts | 4 +- package-lock.json | 46 +++++++++++-------- 7 files changed, 42 insertions(+), 29 deletions(-) diff --git a/developer/src/common/web/utils/package.json b/developer/src/common/web/utils/package.json index 571123ec68..a56fdc386a 100644 --- a/developer/src/common/web/utils/package.json +++ b/developer/src/common/web/utils/package.json @@ -12,7 +12,7 @@ "@keymanapp/common-types": "*", "@sentry/node": "^7.57.0", "eventemitter3": "^5.0.0", - "fast-xml-parser": "github:srl295/fast-xml-parser#srl295/line-number-593", + "fast-xml-parser": "github:srl295/fast-xml-parser#line-number-593bis", "path-browserify": "^1.0.1", "restructure": "^3.0.1", "sax": ">=0.6.0", diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 9fbebdf397..6365134db7 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -8,7 +8,7 @@ import { XMLParser, XMLBuilder } from 'fast-xml-parser'; -export const START_INDEX = new XMLParser().getStartIndexSymbol(); +export const START_INDEX = XMLParser.getStartIndexSymbol(); export type KeymanXMLType = 'keyboard3' // LDML diff --git a/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts b/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts index 700d223406..899527edbb 100644 --- a/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts +++ b/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts @@ -26,7 +26,6 @@ describe('kpj-file-writer', function () { // The outputKpj may not contain all the fields from the inputKpj, only the // essential fields. Many of the fields in .kpj are deprecated, when they // relate to file content (e.g. parented files, file details) - assert.deepEqual(outputKpj.KeymanDeveloperProject.Options, { "BuildPath": "$PROJECTPATH\\build", "CompilerWarningsAsErrors": "True", diff --git a/developer/src/common/web/utils/test/xml-utils.tests.ts b/developer/src/common/web/utils/test/xml-utils.tests.ts index bdcd18817d..44b4d95f21 100644 --- a/developer/src/common/web/utils/test/xml-utils.tests.ts +++ b/developer/src/common/web/utils/test/xml-utils.tests.ts @@ -180,9 +180,8 @@ describe(`XML Reader line number test`, () => { assert.equal(actual.keyboard3[START_INDEX as any], 40); // index of element assert.equal(actual.keyboard3.info[START_INDEX as any], 136); // index of etc assert.equal(actual.keyboard3.transforms[START_INDEX as any], 186); - - assert.equal(KeymanXMLReader.offsetToLineColumn(actual.keyboard3[START_INDEX as any], xml), { line: 3, column: 0 }); - assert.equal(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.info[START_INDEX as any], xml), { line: 4, column: 2 }); - assert.equal(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.transforms[START_INDEX as any], xml), { line: 8, column: 2 }); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3[START_INDEX as any], xml), { line: 3, column: 0 }); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.info[START_INDEX as any], xml), { line: 4, column: 2 }); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.transforms[START_INDEX as any], xml), { line: 8, column: 2 }); }); }); diff --git a/developer/src/kmc-ldml/test/helpers/index.ts b/developer/src/kmc-ldml/test/helpers/index.ts index 7b67ee4e4c..5599f695f6 100644 --- a/developer/src/kmc-ldml/test/helpers/index.ts +++ b/developer/src/kmc-ldml/test/helpers/index.ts @@ -8,7 +8,7 @@ import * as path from 'path'; import { fileURLToPath } from 'url'; import { SectionCompiler, SectionCompilerNew } from '../../src/compiler/section-compiler.js'; import { util, KMXPlus, LdmlKeyboardTypes } from '@keymanapp/common-types'; -import { CompilerEvent, compilerEventFormat, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboard, } from "@keymanapp/developer-utils"; +import { CompilerEvent, compilerEventFormat, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboard, START_INDEX, } from "@keymanapp/developer-utils"; import { LdmlKeyboardCompiler } from '../../src/main.js'; // make sure main.js compiles import { assert } from 'chai'; import { KMXPlusMetadataCompiler } from '../../src/compiler/metadata-compiler.js'; @@ -327,3 +327,10 @@ const dontEscape = /[a-zA-Z0-9\.${}\[\]-]/; export function hex_str(s?: string) : string { return [...s].map(ch => dontEscape.test(ch) ? ch : util.escapeRegexChar(ch)).join(''); } + +/** return an object simulating an XML object with a column number */ +export function withColumn(c: number) : any { + return { + [START_INDEX as any]: c + }; +} diff --git a/developer/src/kmc-ldml/test/keys.tests.ts b/developer/src/kmc-ldml/test/keys.tests.ts index cfb37e6baf..1b74763646 100644 --- a/developer/src/kmc-ldml/test/keys.tests.ts +++ b/developer/src/kmc-ldml/test/keys.tests.ts @@ -1,7 +1,7 @@ import 'mocha'; import { assert } from 'chai'; import { KeysCompiler } from '../src/compiler/keys.js'; -import { assertCodePoints, compilerTestCallbacks, loadSectionFixture, testCompilationCases } from './helpers/index.js'; +import { assertCodePoints, compilerTestCallbacks, loadSectionFixture, testCompilationCases, withColumn } from './helpers/index.js'; import { KMXPlus, Constants, LdmlKeyboardTypes } from '@keymanapp/common-types'; import { LdmlCompilerMessages } from '../src/compiler/ldml-compiler-messages.js'; import { constants } from '@keymanapp/ldml-keyboard-constants'; @@ -442,7 +442,7 @@ describe('keys.kmap', function () { assert.isNull(keys); assert.equal(compilerTestCallbacks.messages.length, 1); - assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_HardwareLayerHasTooManyRows()); + assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_HardwareLayerHasTooManyRows(withColumn(276))); }); it('should reject layouts with too many hardware keys', async function() { diff --git a/package-lock.json b/package-lock.json index 0f62509ac5..c44fd00061 100644 --- a/package-lock.json +++ b/package-lock.json @@ -342,7 +342,7 @@ "@keymanapp/common-types": "*", "@sentry/node": "^7.57.0", "eventemitter3": "^5.0.0", - "fast-xml-parser": "github:srl295/fast-xml-parser#srl295/line-number-593", + "fast-xml-parser": "github:srl295/fast-xml-parser#line-number-593bis", "path-browserify": "^1.0.1", "restructure": "^3.0.1", "sax": ">=0.6.0", @@ -372,21 +372,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "developer/src/common/web/utils/node_modules/fast-xml-parser": { - "version": "4.1.3", - "resolved": "git+ssh://git@github.com/srl295/fast-xml-parser.git#d7601c31f39fe69e6612af79f9d778dee5dfcb42", - "license": "MIT", - "dependencies": { - "strnum": "^1.0.5" - }, - "bin": { - "fxparser": "src/cli/cli.js" - }, - "funding": { - "type": "paypal", - "url": "https://paypal.me/naturalintelligence" - } - }, "developer/src/common/web/utils/node_modules/js-yaml": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", @@ -9031,6 +9016,23 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "node_modules/fast-xml-parser": { + "version": "5.0.8", + "resolved": "git+ssh://git@github.com/srl295/fast-xml-parser.git#72f20480e8591aca819fac684d46f8d678fcb7f3", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "strnum": "^2.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fastq": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", @@ -13888,9 +13890,15 @@ "link": true }, "node_modules/strnum": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", - "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==", + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.0.5.tgz", + "integrity": "sha512-YAT3K/sgpCUxhxNMrrdhtod3jckkpYwH6JAuwmUdXZsmzH1wUyzTMrrK2wYCEEqlKwrWDd35NeuUkbBy/1iK+Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], "license": "MIT" }, "node_modules/supports-color": { From 4d9ea2db2d1a1768225a074ad58d91da77941ebc Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 28 Feb 2025 14:24:12 -0600 Subject: [PATCH 07/27] feat(developer): work around line number symbols - add KeymanXMLReader.removeSymbols() and call it from tests Fixes: #10622 --- developer/src/common/web/utils/src/xml-utils.ts | 11 ++++++++++- .../web/utils/test/kpj/kpj-file-writer.tests.ts | 5 +++-- .../web/utils/test/kps/kps-file-reader.tests.ts | 5 +++-- .../src/common/web/utils/test/kvks/kvks-file.tests.ts | 5 +++-- .../src/common/web/utils/test/xml-utils.tests.ts | 2 +- developer/src/kmc-ldml/test/helpers/compareXml.ts | 2 +- 6 files changed, 21 insertions(+), 9 deletions(-) diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 6365134db7..98af170790 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -156,13 +156,22 @@ export class KeymanXMLReader { * @param from source for symbols * @returns the onto object */ - private static copySymbols(onto: T, from: any): T { + public static copySymbols(onto: T, from: any): T { const o = onto as any; for (const sym of Object.getOwnPropertySymbols(from)) { o[sym] = from[sym]; } return onto; } + + /** use Object.entries to remove all symbols */ + public static removeSymbols(from: T): T { + if (Array.isArray(from)) { + return from.map(o => KeymanXMLReader.removeSymbols(o)) as T; + } + if (typeof from !== "object") return from; + return Object.fromEntries(Object.entries(from).map(([k, v]) => ([k, KeymanXMLReader.removeSymbols(v)]))) as T; + } /** move `{ $abc: 4 }` into `{ $: { abc: 4 } }` */ private static fixupDollarAttributes(data: any) : any { diff --git a/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts b/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts index 899527edbb..5fc223b240 100644 --- a/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts +++ b/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts @@ -6,6 +6,7 @@ import { KPJFileReader } from "../../src/types/kpj/kpj-file-reader.js"; import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; import { KPJFileWriter } from '../../src/types/kpj/kpj-file-writer.js'; import { KeymanDeveloperProjectOptions } from '../../src/types/kpj/keyman-developer-project.js'; +import { KeymanXMLReader } from '../../src/xml-utils.js'; const callbacks = new TestCompilerCallbacks(); @@ -21,7 +22,7 @@ describe('kpj-file-writer', function () { const writer = new KPJFileWriter(); const output = writer.write(project); - const outputKpj = reader.read(new TextEncoder().encode(output)); + const outputKpj = KeymanXMLReader.removeSymbols(reader.read(new TextEncoder().encode(output))); // The outputKpj may not contain all the fields from the inputKpj, only the // essential fields. Many of the fields in .kpj are deprecated, when they @@ -60,7 +61,7 @@ describe('kpj-file-writer', function () { const writer = new KPJFileWriter(); const output = writer.write(project); - const outputKpj = reader.read(new TextEncoder().encode(output)); + const outputKpj = KeymanXMLReader.removeSymbols(reader.read(new TextEncoder().encode(output))); // The outputKpj may not contain all the fields from the inputKpj, only the // essential fields. Many of the fields in .kpj are deprecated, when they diff --git a/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts b/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts index 1b809f39d0..94a1628bd9 100644 --- a/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts +++ b/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts @@ -15,6 +15,7 @@ import { makePathToFixture } from '../helpers/index.js'; import { KpsFileReader } from "../../src/types/kps/kps-file-reader.js"; import { KpsFileWriter } from '../../src/types/kps/kps-file-writer.js'; import { CommonTypesMessages } from '../../src/common-messages.js'; +import { KeymanXMLReader } from '../../src/xml-utils.js'; const callbacks = new TestCompilerCallbacks(); @@ -61,13 +62,13 @@ describe('kps-file-reader', function () { it('kps-file-reader should round-trip with kps-file-writer', function() { const input = fs.readFileSync(makePathToFixture('kps', 'khmer_angkor.kps')); const reader = new KpsFileReader(callbacks); - const kps = reader.read(input); + const kps = KeymanXMLReader.removeSymbols(reader.read(input)); const writer = new KpsFileWriter(); const output = writer.write(kps); // Round Trip - const kps2 = reader.read(new TextEncoder().encode(output)); + const kps2 = KeymanXMLReader.removeSymbols(reader.read(new TextEncoder().encode(output))); assert.deepEqual(kps2, kps); }); diff --git a/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts b/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts index 4c009e9d5c..96be2283b8 100644 --- a/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts +++ b/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts @@ -5,6 +5,7 @@ import KvksFileReader from "../../src/types/kvks/kvks-file-reader.js"; import KvksFileWriter from "../../src/types/kvks/kvks-file-writer.js"; import { verify_khmer_angkor, verify_balochi_inpage } from './kvk-utils.tests.js'; import { assert } from 'chai'; +import { KeymanXMLReader } from '../../src/xml-utils.js'; describe('kvks-file-reader', function() { it('should read a valid file', function() { @@ -48,7 +49,7 @@ describe('kvks-file-writer', function() { const input = fs.readFileSync(path); const reader = new KvksFileReader(); - const kvksExpected = reader.read(input); + const kvksExpected = KeymanXMLReader.removeSymbols(reader.read(input)); const invalidVkeys: string[] = []; const vk = reader.transform(kvksExpected, invalidVkeys); assert.isEmpty(invalidVkeys); @@ -59,6 +60,6 @@ describe('kvks-file-writer', function() { // We compare the (re)loaded data, because there may be // minor, irrelevant formatting differences in the emitted xml const kvks = reader.read(Buffer.from(output, 'utf8')); - assert.deepEqual(kvks, kvksExpected); + assert.deepEqual(KeymanXMLReader.removeSymbols(kvks), kvksExpected); }); }); diff --git a/developer/src/common/web/utils/test/xml-utils.tests.ts b/developer/src/common/web/utils/test/xml-utils.tests.ts index 44b4d95f21..9039760913 100644 --- a/developer/src/common/web/utils/test/xml-utils.tests.ts +++ b/developer/src/common/web/utils/test/xml-utils.tests.ts @@ -119,7 +119,7 @@ describe(`XML Reader Test ${GEN_XML_FIXTURES && '(update mode!)' || ''}`, () => writeJson(jsonPath, actual); } else { assert.ok(expect, `Could not read ${jsonPath} - run with env GEN_XML_FIXTURES=1 to update.`); - assert.deepEqual(actual, expect, `Mismatch of ${xmlPath} vs ${jsonPath}`); + assert.deepEqual(KeymanXMLReader.removeSymbols(actual), expect, `Mismatch of ${xmlPath} vs ${jsonPath}`); } }); } diff --git a/developer/src/kmc-ldml/test/helpers/compareXml.ts b/developer/src/kmc-ldml/test/helpers/compareXml.ts index f245da51a9..1503d4c606 100644 --- a/developer/src/kmc-ldml/test/helpers/compareXml.ts +++ b/developer/src/kmc-ldml/test/helpers/compareXml.ts @@ -21,5 +21,5 @@ export function compareXml(actual : string, expect: string, mutator?: (input: an const actualParsed = mutator(reader.parse(actualStr)); const expectParsed = mutator(reader.parse(expectStr)); - assert.deepEqual(actualParsed, expectParsed); + assert.deepEqual(KeymanXMLReader.removeSymbols(actualParsed), KeymanXMLReader.removeSymbols(expectParsed)); } From 3e73a3f0819ffa5a38cf4075f0c6bdab2b573137 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 3 Mar 2025 12:36:42 -0600 Subject: [PATCH 08/27] feat(developer): add CompilerEvent.offset - per review, keep this as a separate field. Fixes: #10622 --- .../common/web/utils/src/compiler-interfaces.ts | 8 ++++++-- .../src/compiler/ldml-compiler-messages.ts | 14 +++++++------- .../src/kmc/src/util/NodeCompilerCallbacks.ts | 2 +- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/developer/src/common/web/utils/src/compiler-interfaces.ts b/developer/src/common/web/utils/src/compiler-interfaces.ts index 390eea2f7e..dc24a772e9 100644 --- a/developer/src/common/web/utils/src/compiler-interfaces.ts +++ b/developer/src/common/web/utils/src/compiler-interfaces.ts @@ -7,11 +7,15 @@ export interface CompilerEvent { filename?: string; /** line where a message applies */ line?: number; - /** + /** * column where a message applies. - * where line is unset, column is a file offset */ column?: number; + /** + * offset where a message applies. + * If set, encompasses line and column. + */ + offset?: number; code: number; message: string; /** diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 6f8cc5fa3d..27f587b04f 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -10,7 +10,7 @@ const SevError = CompilerErrorSeverity.Error | CompilerErrorNamespace.LdmlKeyboa const SevErrorTransform = SevError | 0xF00; // like m() but takes an XML object for line numbers -const mx = (x: any, code: number, message: string, detail?: string): CompilerEvent => LdmlCompilerMessages.col(m(code, message, detail), x); +const mx = (x: any, code: number, message: string, detail?: string): CompilerEvent => LdmlCompilerMessages.offset(m(code, message, detail), x); /** * @internal @@ -269,14 +269,14 @@ export class LdmlCompilerMessages { ); /** - * Get a column number from o and set e's column field + * Get an offset from o and set e's offset field * @param event a compiler event, such as from functions in this class * @param x any object parsed from XML or with the START_INDEX symbol copied over * @returns modified event object */ - static col(event: CompilerEvent, x?: any): CompilerEvent { + static offset(event: CompilerEvent, x?: any): CompilerEvent { if(x) { - event.column = (x as any)[START_INDEX as any]; + event.offset = (x as any)[START_INDEX as any]; } return event; } @@ -287,9 +287,9 @@ export class LdmlCompilerMessages { * @param xml XML source * @returns the modified event */ - static resolveLineNumber(event: CompilerEvent, xml: string) : CompilerEvent { - if (event.column && !event.line && xml) { - const loc = KeymanXMLReader.offsetToLineColumn(event.column, xml); + static resolveLineNumber(event: CompilerEvent, xml: string) : CompilerEvent { + if (event.offset && !event.line && xml) { + const loc = KeymanXMLReader.offsetToLineColumn(event.offset, xml); event.line = loc.line; event.column = loc.column; } diff --git a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts index 7c184c1417..8186f29d70 100644 --- a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts +++ b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts @@ -161,7 +161,7 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { this.messageFiletext = ''; } - if (event.column && !event.line && event.filename) { + if (event.offset && !event.line && event.filename) { try { let text = ''; if (this.messageFilename == event.filename) { From d7cb7b63a3db987229e42a3b3fcbf78ff906eb1e Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 3 Mar 2025 13:14:41 -0600 Subject: [PATCH 09/27] feat(developer): add an EventResolver - it's a hook to the hook to allow the LDML compiler to resolve error message line numbers - TODO still not extra performant, but maybe the right separation of concerns Fixes: #10622 --- .../web/test-helpers/TestCompilerCallbacks.ts | 13 +++++++- .../web/utils/src/compiler-callbacks.ts | 28 +++++++++++++++- developer/src/common/web/utils/src/index.ts | 2 ++ .../src/kmc-ldml/src/compiler/compiler.ts | 4 +++ .../kmc-ldml/src/compiler/eventresolver.ts | 21 ++++++++++++ .../src/kmc/src/util/NodeCompilerCallbacks.ts | 32 +++++++------------ 6 files changed, 78 insertions(+), 22 deletions(-) create mode 100644 developer/src/kmc-ldml/src/compiler/eventresolver.ts diff --git a/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts b/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts index fad1dd646f..4771b16417 100644 --- a/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts +++ b/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts @@ -2,7 +2,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { CompilerEvent, CompilerCallbacks, CompilerPathCallbacks, CompilerFileSystemCallbacks, CompilerError, CompilerNetAsyncCallbacks, DefaultCompilerFileSystemAsyncCallbacks, - CompilerFileSystemAsyncCallbacks, + CompilerFileSystemAsyncCallbacks, EventResolver, NullEventResolver, CompilerErrorSeverity} from '@keymanapp/developer-utils'; import { fileURLToPath } from 'url'; @@ -111,7 +111,18 @@ export class TestCompilerCallbacks implements CompilerCallbacks { return filename; } + private eventResolver = new NullEventResolver(); + + setEventResolver(eventResolver: EventResolver): void { + this.eventResolver = eventResolver; + } + reportMessage(event: CompilerEvent): void { + this.eventResolver.resolve(event); + this.handleReportMessage(event); + } + + handleReportMessage(event: CompilerEvent): void { // console.log(event.message); this.messages.push(event); } diff --git a/developer/src/common/web/utils/src/compiler-callbacks.ts b/developer/src/common/web/utils/src/compiler-callbacks.ts index a7163175e8..bebfe5fd55 100644 --- a/developer/src/common/web/utils/src/compiler-callbacks.ts +++ b/developer/src/common/web/utils/src/compiler-callbacks.ts @@ -7,6 +7,21 @@ import { CompilerEvent, CompilerCallbackOptions, CompilerErrorSeverity, CompilerError, CompilerMessageOverrideMap, CompilerErrorMask } from "./compiler-interfaces.js"; +/** + * The EventResolver implementation is able to expand error messages, + * particularly to change a byte offset into a line number. + */ +export interface EventResolver { + resolve(event: CompilerEvent): void; +} + +/** A do-nothing EventResolver. Implementations can use this as their default EventResolver. */ +export class NullEventResolver implements EventResolver { + resolve(event: CompilerEvent): void { + // do nothing, this is the null event resolver. + } +} + /** * Abstract interface for callbacks, to abstract out file i/o */ @@ -34,12 +49,20 @@ export interface CompilerCallbacks { get net(): CompilerNetAsyncCallbacks; /** - * Resolves a file path relative to the baseFilename + * Resolves a file path relative to the baseFilename. * @param baseFilename * @param filename */ resolveFilename(baseFilename: string, filename: string): string; + /** change the EventResolver to use. If called more than once, the last resolver replaces a prior one. */ + setEventResolver(eventResolver: EventResolver): void; + + /** + * Report a message from the compiler back to the caller. + * It is expected that eventResolver.resolve() will be called on the message before processing. + * @param event + */ reportMessage(event: CompilerEvent): void; debug(msg: string): void; @@ -115,6 +138,9 @@ export class CompilerFileCallbacks implements CompilerCallbacks { constructor(private filename: string, private options: CompilerCallbackOptions, private parent: CompilerCallbacks) { } + setEventResolver(eventResolver: EventResolver): void { + throw new Error("Method not implemented."); + } /** * Returns `true` if any message in the `messages` array is a Fatal or Error diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index 1a56a735e8..812d503e96 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -42,6 +42,8 @@ export { CompilerPathCallbacks, CompilerFileSystemCallbacksFolderEntry as FileSystemFolderEntry, DefaultCompilerFileSystemAsyncCallbacks, + EventResolver, + NullEventResolver, } from './compiler-callbacks.js'; export { defaultCompilerOptions, CompilerBaseOptions, CompilerOptions, CompilerEvent, CompilerErrorNamespace, diff --git a/developer/src/kmc-ldml/src/compiler/compiler.ts b/developer/src/kmc-ldml/src/compiler/compiler.ts index e8b38184e9..e0b86dc712 100644 --- a/developer/src/kmc-ldml/src/compiler/compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/compiler.ts @@ -30,6 +30,7 @@ import { KmnCompiler } from '@keymanapp/kmc-kmn'; import { KMXPlusMetadataCompiler } from './metadata-compiler.js'; import { LdmlKeyboardVisualKeyboardCompiler } from './visual-keyboard-compiler.js'; import { LinterKeycaps } from './linter-keycaps.js'; +import { LdmlEventResolver } from './eventresolver.js'; export const SECTION_COMPILERS = [ // These are in dependency order. @@ -94,6 +95,7 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { // uset parser private usetparser?: LdmlKeyboardTypes.UnicodeSetParser = undefined; + private static eventResolver: LdmlEventResolver = new LdmlEventResolver(); /** * Initialize the compiler, including loading the WASM host for uset parsing. @@ -106,6 +108,7 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { async init(callbacks: CompilerCallbacks, options: LdmlCompilerOptions): Promise { this.options = { ...options }; this.callbacks = callbacks; + this.callbacks.setEventResolver(LdmlKeyboardCompiler.eventResolver); return true; } @@ -245,6 +248,7 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidFile({ errorText: 'Unable to read XML file' })); return null; } + LdmlKeyboardCompiler.eventResolver.addFile(filename, new TextDecoder().decode(data)); // TODO: double decode // parse (load) the string into an object tree const source = reader.load(data); if (!source) { diff --git a/developer/src/kmc-ldml/src/compiler/eventresolver.ts b/developer/src/kmc-ldml/src/compiler/eventresolver.ts new file mode 100644 index 0000000000..491e4a3423 --- /dev/null +++ b/developer/src/kmc-ldml/src/compiler/eventresolver.ts @@ -0,0 +1,21 @@ +import { CompilerEvent, EventResolver } from "@keymanapp/developer-utils"; +import { LdmlCompilerMessages } from "./ldml-compiler-messages.js"; + +export class LdmlEventResolver implements EventResolver { + + contentsCache: Map = new Map(); + + /** add or update a file */ + addFile(filename: string, xml: any) { + if (typeof xml !== 'string') { + xml = xml.toString(); + } + this.contentsCache.set(filename, xml); + } + + resolve(event: CompilerEvent) { + if (event.offset && !event.line && event.filename) { + LdmlCompilerMessages.resolveLineNumber(event, this.contentsCache.get(event.filename)); + } + } +} diff --git a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts index 8186f29d70..5f4d0e9018 100644 --- a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts +++ b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { platform } from 'os'; -import { CompilerCallbacks, CompilerEvent, +import { CompilerCallbacks, CompilerEvent, EventResolver, NullEventResolver, CompilerPathCallbacks, CompilerFileSystemCallbacks, compilerLogLevelToSeverity, CompilerErrorSeverity, CompilerError, @@ -15,7 +15,6 @@ import chalk from 'chalk'; import supportsColor from 'supports-color'; import { KeymanSentry } from '@keymanapp/developer-utils'; import { fileURLToPath } from 'url'; -import { LdmlCompilerMessages } from '@keymanapp/kmc-ldml'; const color = chalk.default; const severityColors: {[value in CompilerErrorSeverity]: chalk.Chalk} = { @@ -149,7 +148,18 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { return fileURLToPath(url); } + private eventResolver = new NullEventResolver(); + + setEventResolver(eventResolver: EventResolver): void { + this.eventResolver = eventResolver; + } + reportMessage(event: CompilerEvent): void { + this.eventResolver.resolve(event); + this.handleReportMessage(event); + } + + handleReportMessage(event: CompilerEvent): void { if(!event.filename) { event.filename = this.messageFilename; } @@ -161,24 +171,6 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { this.messageFiletext = ''; } - if (event.offset && !event.line && event.filename) { - try { - let text = ''; - if (this.messageFilename == event.filename) { - text = this.messageFiletext; - } - if (text == '') { - text = this.fs.readFileSync(event.filename, "utf-8"); - if (this.messageFilename == event.filename) { - this.messageFiletext = text; - } - } - LdmlCompilerMessages.resolveLineNumber(event, text); - } catch(e) { - console.error(e); // resolving line numbers - } - } - const disable = CompilerFileCallbacks.applyMessageOverridesToEvent(event, this.options.messageOverrides); this.messages.push({...event}); From 1febf9a0de36c2e979642291f91904d337c3c17c Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 3 Mar 2025 13:18:02 -0600 Subject: [PATCH 10/27] Update developer/src/common/web/utils/src/xml-utils.ts Co-authored-by: Marc Durdin --- developer/src/common/web/utils/src/xml-utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 98af170790..8c084231fe 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -8,7 +8,7 @@ import { XMLParser, XMLBuilder } from 'fast-xml-parser'; -export const START_INDEX = XMLParser.getStartIndexSymbol(); +export const XML_START_INDEX_SYMBOL = XMLParser.getStartIndexSymbol(); export type KeymanXMLType = 'keyboard3' // LDML From b015221814f83463908d88a5209376dc14916ff5 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 3 Mar 2025 17:08:40 -0600 Subject: [PATCH 11/27] feat(developer): line number symbols - change special symbol to XML_START_INDEX_SYMBOL - use an array of line lengths rather than an array of strings for the file info - add an EventResolver interface to the callbacks, such that it has an opportunity to resolve events Fixes: #10622 --- .../web/utils/src/compiler-callbacks.ts | 14 ++++++++++--- developer/src/common/web/utils/src/index.ts | 2 +- .../src/common/web/utils/src/xml-utils.ts | 19 ++++++++++------- .../common/web/utils/test/xml-utils.tests.ts | 14 ++++++------- .../kmc-ldml/src/compiler/eventresolver.ts | 16 +++++++------- .../src/compiler/ldml-compiler-messages.ts | 21 +++---------------- developer/src/kmc-ldml/test/helpers/index.ts | 6 +++--- 7 files changed, 44 insertions(+), 48 deletions(-) diff --git a/developer/src/common/web/utils/src/compiler-callbacks.ts b/developer/src/common/web/utils/src/compiler-callbacks.ts index bebfe5fd55..bce9d9f395 100644 --- a/developer/src/common/web/utils/src/compiler-callbacks.ts +++ b/developer/src/common/web/utils/src/compiler-callbacks.ts @@ -138,9 +138,6 @@ export class CompilerFileCallbacks implements CompilerCallbacks { constructor(private filename: string, private options: CompilerCallbackOptions, private parent: CompilerCallbacks) { } - setEventResolver(eventResolver: EventResolver): void { - throw new Error("Method not implemented."); - } /** * Returns `true` if any message in the `messages` array is a Fatal or Error @@ -228,7 +225,18 @@ export class CompilerFileCallbacks implements CompilerCallbacks { return this.parent.resolveFilename(baseFilename, filename); } + private eventResolver = new NullEventResolver(); + + setEventResolver(eventResolver: EventResolver): void { + this.eventResolver = eventResolver; + } + reportMessage(event: CompilerEvent): void { + this.eventResolver.resolve(event); + this.handleReportMessage(event); + } + + handleReportMessage(event: CompilerEvent): void { const disable = CompilerFileCallbacks.applyMessageOverridesToEvent(event, this.options.messageOverrides); this.messages.push(event); if (!disable) { diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index 812d503e96..c703e48aa2 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -67,7 +67,7 @@ export { UrlSubpathCompilerCallback } from './utils/UrlSubpathCompilerCallback.j export { CommonTypesMessages } from './common-messages.js'; export * as SourceFilenamePatterns from './source-filename-patterns.js'; -export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, START_INDEX } from './xml-utils.js'; +export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, XML_START_INDEX_SYMBOL } from './xml-utils.js'; export * as GitHubUrls from './github-urls.js'; export * as CloudUrls from './cloud-urls.js'; diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 8c084231fe..de78fa5b54 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -134,18 +134,21 @@ export interface LineColumn { export class KeymanXMLReader { public constructor(public type: KeymanXMLType) { } - - public static offsetToLineColumn(offset: number, text: string) : LineColumn { - const lines: string[] = text.split("\n"); + /** preprocess text to turn it into arrays of line lengths */ + public static textToLines(text: string) : number[] { + return text.replaceAll("\r\n", "\n").split("\n") + .map(l => l.length + 1); // line length (counting the trailing newline) + } + public static offsetToLineColumn(offset: number, lines: number[]) : LineColumn { for (let line = 1; line < lines.length + 1; line++) { // 1-based - const linestr = lines[line - 1]; - if (linestr.length < offset) { - offset = offset - (linestr.length + 1); // count newline at end + if (lines[line-1] < offset) { + offset = offset - (lines[line-1]); // count newline at end continue; } return { line, column: offset }; } - return { line: lines.length + 1 }; + // default: line 0, error + return { line: 0 } } /** @@ -172,7 +175,7 @@ export class KeymanXMLReader { if (typeof from !== "object") return from; return Object.fromEntries(Object.entries(from).map(([k, v]) => ([k, KeymanXMLReader.removeSymbols(v)]))) as T; } - + /** move `{ $abc: 4 }` into `{ $: { abc: 4 } }` */ private static fixupDollarAttributes(data: any) : any { if (typeof data === 'object') { diff --git a/developer/src/common/web/utils/test/xml-utils.tests.ts b/developer/src/common/web/utils/test/xml-utils.tests.ts index 9039760913..f487573724 100644 --- a/developer/src/common/web/utils/test/xml-utils.tests.ts +++ b/developer/src/common/web/utils/test/xml-utils.tests.ts @@ -12,7 +12,7 @@ import { env } from 'node:process'; import { readFileSync, writeFileSync } from 'node:fs'; -import { KeymanXMLType, KeymanXMLReader, KeymanXMLWriter, START_INDEX } from '../src/xml-utils.js'; +import { KeymanXMLType, KeymanXMLReader, KeymanXMLWriter, XML_START_INDEX_SYMBOL } from '../src/xml-utils.js'; import { makePathToFixture } from './helpers/index.js'; // if true, attempt to WRITE the fixtures @@ -177,11 +177,11 @@ describe(`XML Reader line number test`, () => { assert.ok(actual, `Parser failed on ${xmlPath}`); // now, assert char offset - assert.equal(actual.keyboard3[START_INDEX as any], 40); // index of element - assert.equal(actual.keyboard3.info[START_INDEX as any], 136); // index of etc - assert.equal(actual.keyboard3.transforms[START_INDEX as any], 186); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3[START_INDEX as any], xml), { line: 3, column: 0 }); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.info[START_INDEX as any], xml), { line: 4, column: 2 }); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.transforms[START_INDEX as any], xml), { line: 8, column: 2 }); + assert.equal(actual.keyboard3[XML_START_INDEX_SYMBOL as any], 40); // index of element + assert.equal(actual.keyboard3.info[XML_START_INDEX_SYMBOL as any], 136); // index of etc + assert.equal(actual.keyboard3.transforms[XML_START_INDEX_SYMBOL as any], 186); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3[XML_START_INDEX_SYMBOL as any], xml), { line: 3, column: 0 }); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.info[XML_START_INDEX_SYMBOL as any], xml), { line: 4, column: 2 }); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.transforms[XML_START_INDEX_SYMBOL as any], xml), { line: 8, column: 2 }); }); }); diff --git a/developer/src/kmc-ldml/src/compiler/eventresolver.ts b/developer/src/kmc-ldml/src/compiler/eventresolver.ts index 491e4a3423..7e1dd87fc3 100644 --- a/developer/src/kmc-ldml/src/compiler/eventresolver.ts +++ b/developer/src/kmc-ldml/src/compiler/eventresolver.ts @@ -1,21 +1,21 @@ -import { CompilerEvent, EventResolver } from "@keymanapp/developer-utils"; -import { LdmlCompilerMessages } from "./ldml-compiler-messages.js"; +import { CompilerEvent, EventResolver, KeymanXMLReader } from "@keymanapp/developer-utils"; export class LdmlEventResolver implements EventResolver { - contentsCache: Map = new Map(); + contentsCache: Map = new Map(); /** add or update a file */ addFile(filename: string, xml: any) { - if (typeof xml !== 'string') { - xml = xml.toString(); - } - this.contentsCache.set(filename, xml); + this.contentsCache.set(filename, KeymanXMLReader.textToLines(xml)); } resolve(event: CompilerEvent) { if (event.offset && !event.line && event.filename) { - LdmlCompilerMessages.resolveLineNumber(event, this.contentsCache.get(event.filename)); + const lines = this.contentsCache.get(event.filename); + const loc = KeymanXMLReader.offsetToLineColumn(event.offset, lines); + event.line = loc.line; + event.column = loc.column; } + return event; } } diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 27f587b04f..1298d0f0cd 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -1,5 +1,5 @@ import { util } from "@keymanapp/common-types"; -import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, START_INDEX, CompilerEvent, KeymanXMLReader } from '@keymanapp/developer-utils'; +import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_START_INDEX_SYMBOL, CompilerEvent } from '@keymanapp/developer-utils'; // const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.LdmlKeyboardCompiler; @@ -271,27 +271,12 @@ export class LdmlCompilerMessages { /** * Get an offset from o and set e's offset field * @param event a compiler event, such as from functions in this class - * @param x any object parsed from XML or with the START_INDEX symbol copied over + * @param x any object parsed from XML or with the XML_START_INDEX_SYMBOL symbol copied over * @returns modified event object */ static offset(event: CompilerEvent, x?: any): CompilerEvent { if(x) { - event.offset = (x as any)[START_INDEX as any]; - } - return event; - } - - /** - * Given an event with a column number but no line, resolve it into line:column - * @param event event to modify - * @param xml XML source - * @returns the modified event - */ - static resolveLineNumber(event: CompilerEvent, xml: string) : CompilerEvent { - if (event.offset && !event.line && xml) { - const loc = KeymanXMLReader.offsetToLineColumn(event.offset, xml); - event.line = loc.line; - event.column = loc.column; + event.offset = (x as any)[XML_START_INDEX_SYMBOL as any]; } return event; } diff --git a/developer/src/kmc-ldml/test/helpers/index.ts b/developer/src/kmc-ldml/test/helpers/index.ts index 5599f695f6..497f8ef919 100644 --- a/developer/src/kmc-ldml/test/helpers/index.ts +++ b/developer/src/kmc-ldml/test/helpers/index.ts @@ -8,7 +8,7 @@ import * as path from 'path'; import { fileURLToPath } from 'url'; import { SectionCompiler, SectionCompilerNew } from '../../src/compiler/section-compiler.js'; import { util, KMXPlus, LdmlKeyboardTypes } from '@keymanapp/common-types'; -import { CompilerEvent, compilerEventFormat, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboard, START_INDEX, } from "@keymanapp/developer-utils"; +import { CompilerEvent, compilerEventFormat, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboard, XML_START_INDEX_SYMBOL, } from "@keymanapp/developer-utils"; import { LdmlKeyboardCompiler } from '../../src/main.js'; // make sure main.js compiles import { assert } from 'chai'; import { KMXPlusMetadataCompiler } from '../../src/compiler/metadata-compiler.js'; @@ -290,7 +290,7 @@ export function testCompilationCases(compiler: SectionCompilerNew, cases : Compi // no warnings, so expect zero messages assert.sameDeepMembers(callbacks.messages, [], 'expected zero messages but got ' + callbacks.messages); } - + if (expectFailure) { assert.isNull(section, 'expected compilation result failure (null)'); } else { @@ -331,6 +331,6 @@ export function hex_str(s?: string) : string { /** return an object simulating an XML object with a column number */ export function withColumn(c: number) : any { return { - [START_INDEX as any]: c + [XML_START_INDEX_SYMBOL as any]: c }; } From 92e16b90a033da5e3d9133e9521fc9b6df5088db Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 3 Mar 2025 19:12:34 -0600 Subject: [PATCH 12/27] feat(developer): line number symbols - move file read bottleneck out of compiler.ts into the LDML XML reader - move the eventresolver into the LDML XML reader - add XML_FILENAME_SYMBOL - for tagging when an object is from a different file than the default (such as with an import) Fixes: #10622 --- .../web/utils/src/compiler-callbacks.ts | 1 + developer/src/common/web/utils/src/index.ts | 2 +- .../src/types/ldml-keyboard}/eventresolver.ts | 2 +- .../ldml-keyboard/ldml-keyboard-xml-reader.ts | 40 ++++++++++++++----- .../src/common/web/utils/src/xml-utils.ts | 3 ++ .../src/kmc-ldml/src/compiler/compiler.ts | 6 +-- developer/src/kmc-ldml/src/compiler/keys.ts | 4 +- .../src/compiler/ldml-compiler-messages.ts | 7 ++-- 8 files changed, 44 insertions(+), 21 deletions(-) rename developer/src/{kmc-ldml/src/compiler => common/web/utils/src/types/ldml-keyboard}/eventresolver.ts (86%) diff --git a/developer/src/common/web/utils/src/compiler-callbacks.ts b/developer/src/common/web/utils/src/compiler-callbacks.ts index bce9d9f395..f8b42afb7a 100644 --- a/developer/src/common/web/utils/src/compiler-callbacks.ts +++ b/developer/src/common/web/utils/src/compiler-callbacks.ts @@ -229,6 +229,7 @@ export class CompilerFileCallbacks implements CompilerCallbacks { setEventResolver(eventResolver: EventResolver): void { this.eventResolver = eventResolver; + this.parent.setEventResolver(eventResolver); } reportMessage(event: CompilerEvent): void { diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index c703e48aa2..b722a11b21 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -67,7 +67,7 @@ export { UrlSubpathCompilerCallback } from './utils/UrlSubpathCompilerCallback.j export { CommonTypesMessages } from './common-messages.js'; export * as SourceFilenamePatterns from './source-filename-patterns.js'; -export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, XML_START_INDEX_SYMBOL } from './xml-utils.js'; +export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, XML_START_INDEX_SYMBOL, XML_FILENAME_SYMBOL } from './xml-utils.js'; export * as GitHubUrls from './github-urls.js'; export * as CloudUrls from './cloud-urls.js'; diff --git a/developer/src/kmc-ldml/src/compiler/eventresolver.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts similarity index 86% rename from developer/src/kmc-ldml/src/compiler/eventresolver.ts rename to developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts index 7e1dd87fc3..3299169642 100644 --- a/developer/src/kmc-ldml/src/compiler/eventresolver.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts @@ -1,4 +1,4 @@ -import { CompilerEvent, EventResolver, KeymanXMLReader } from "@keymanapp/developer-utils"; +import { CompilerEvent, EventResolver, KeymanXMLReader } from "../../index.js"; export class LdmlEventResolver implements EventResolver { diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index 0a5abf6355..ca32fe4905 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -11,6 +11,8 @@ import { constants } from '@keymanapp/ldml-keyboard-constants'; import { LDMLKeyboardTestDataXMLSourceFile, LKTTest, LKTTests } from './ldml-keyboard-testdata-xml.js'; import { KeymanXMLReader } from '@keymanapp/developer-utils'; import boxXmlArray = util.boxXmlArray; +import { LdmlEventResolver } from './eventresolver.js'; +import { XML_FILENAME_SYMBOL } from 'src/xml-utils.js'; interface NameAndProps { '$'?: any; // content @@ -26,27 +28,39 @@ export class LDMLKeyboardXMLSourceFileReaderOptions { }; export class LDMLKeyboardXMLSourceFileReader { - constructor(private options: LDMLKeyboardXMLSourceFileReaderOptions, private callbacks : CompilerCallbacks) { + static eventResolver: LdmlEventResolver = new LdmlEventResolver(); + constructor(private options: LDMLKeyboardXMLSourceFileReaderOptions, private callbacks : CompilerCallbacks) { + callbacks.setEventResolver(LDMLKeyboardXMLSourceFileReader.eventResolver); } static get defaultImportsURL(): [string,string] { return ['../import/', import.meta.url]; } - readImportFile(version: string, subpath: string): Uint8Array { - const importPath = this.callbacks.resolveFilename(this.options.cldrImportsPath, `${version}/${subpath}`); - return this.callbacks.loadFile(importPath); + /** bottleneck for reading keyboard XML files */ + readFile(path: string): Uint8Array { + const data = this.callbacks.loadFile(path); + if (data) { + LDMLKeyboardXMLSourceFileReader.eventResolver.addFile(path, new TextDecoder().decode(data)); + } + return data; } - readLocalImportFile(path: string): Uint8Array { + /** @returns [data, filename] */ + readImportFile(version: string, subpath: string): [Uint8Array, string] { + const importPath = this.callbacks.resolveFilename(this.options.cldrImportsPath, `${version}/${subpath}`); + return [this.readFile(importPath), importPath]; + } + + readLocalImportFile(path: string): [Uint8Array, string] { // try each of the local imports paths for (const localPath of this.options.localImportsPaths) { const importPath = this.callbacks.path.join(localPath, path); if(this.callbacks.fs.existsSync(importPath)) { - return this.callbacks.loadFile(importPath); + return [this.readFile(importPath), importPath]; } } - return null; // was not able to load from any of the paths + return [null, null]; // was not able to load from any of the paths } /** @@ -223,6 +237,7 @@ export class LDMLKeyboardXMLSourceFileReader { return false; } let importData: Uint8Array; + let importPath: string; if (base === constants.cldr_import_base) { // CLDR import @@ -235,10 +250,10 @@ export class LDMLKeyboardXMLSourceFileReader { /** There's no data or DTD change in 45, 46, 46.1, 47 so map them all to 46 at present. */ paths[0] = constants.cldr_version_latest; } - importData = this.readImportFile(paths[0], paths[1]); + [importData, importPath] = this.readImportFile(paths[0], paths[1]); } else { // local import - importData = this.readLocalImportFile(path); + [importData, importPath] = this.readLocalImportFile(path); } if (!importData || !importData.length) { this.callbacks.reportMessage(CommonTypesMessages.Error_ImportReadFail({base, path, subtag})); @@ -246,6 +261,7 @@ export class LDMLKeyboardXMLSourceFileReader { } const importXml: any = this.loadUnboxed(importData); // TODO-LDML: have to load as any because it is an arbitrary part const importRootNode = importXml[subtag]; // e.g. + LDMLKeyboardXMLSourceFileReader.eventResolver.addFile(importPath, new TextDecoder().decode(importData)); // TODO: double decode // importXml will have one property: the root element. if (!importRootNode) { @@ -263,7 +279,11 @@ export class LDMLKeyboardXMLSourceFileReader { return false; } // Mark all children as an import - subsubval.forEach(o => o[ImportStatus.import] = basePath); + subsubval.forEach(o => { + o[ImportStatus.import] = basePath; + o[XML_FILENAME_SYMBOL] = importPath; // mark overriding import path + }); + if (implied) { // mark all children as an implied import subsubval.forEach(o => o[ImportStatus.impliedImport] = basePath); diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index de78fa5b54..ee33b40563 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -8,7 +8,10 @@ import { XMLParser, XMLBuilder } from 'fast-xml-parser'; +/** Symbol giving the start offset, in chars, of the node */ export const XML_START_INDEX_SYMBOL = XMLParser.getStartIndexSymbol(); +/** Symbol giving an override which file a node came from */ +export const XML_FILENAME_SYMBOL = Symbol("XML Filename"); export type KeymanXMLType = 'keyboard3' // LDML diff --git a/developer/src/kmc-ldml/src/compiler/compiler.ts b/developer/src/kmc-ldml/src/compiler/compiler.ts index e0b86dc712..5488afb188 100644 --- a/developer/src/kmc-ldml/src/compiler/compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/compiler.ts @@ -30,7 +30,6 @@ import { KmnCompiler } from '@keymanapp/kmc-kmn'; import { KMXPlusMetadataCompiler } from './metadata-compiler.js'; import { LdmlKeyboardVisualKeyboardCompiler } from './visual-keyboard-compiler.js'; import { LinterKeycaps } from './linter-keycaps.js'; -import { LdmlEventResolver } from './eventresolver.js'; export const SECTION_COMPILERS = [ // These are in dependency order. @@ -95,7 +94,6 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { // uset parser private usetparser?: LdmlKeyboardTypes.UnicodeSetParser = undefined; - private static eventResolver: LdmlEventResolver = new LdmlEventResolver(); /** * Initialize the compiler, including loading the WASM host for uset parsing. @@ -108,7 +106,6 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { async init(callbacks: CompilerCallbacks, options: LdmlCompilerOptions): Promise { this.options = { ...options }; this.callbacks = callbacks; - this.callbacks.setEventResolver(LdmlKeyboardCompiler.eventResolver); return true; } @@ -243,12 +240,11 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { public load(filename: string): LDMLKeyboardXMLSourceFile | null { const reader = new LDMLKeyboardXMLSourceFileReader(this.options.readerOptions, this.callbacks); // load the file from disk into a string - const data = this.callbacks.loadFile(filename); + const data = reader.readFile(filename); if (!data) { this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidFile({ errorText: 'Unable to read XML file' })); return null; } - LdmlKeyboardCompiler.eventResolver.addFile(filename, new TextDecoder().decode(data)); // TODO: double decode // parse (load) the string into an object tree const source = reader.load(data); if (!source) { diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index 35a931dd94..5906646587 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -485,7 +485,9 @@ export class KeysCompiler extends SectionCompiler { } if (!keydef.output && !keydef.gap && !keydef.layerId) { this.callbacks.reportMessage( - LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch({ keyId: key }) + LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch({ keyId: key }, + keydef + ) ); valid = false; continue; diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 1298d0f0cd..1fcb82193b 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -1,5 +1,5 @@ import { util } from "@keymanapp/common-types"; -import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_START_INDEX_SYMBOL, CompilerEvent } from '@keymanapp/developer-utils'; +import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_START_INDEX_SYMBOL, XML_FILENAME_SYMBOL, CompilerEvent } from '@keymanapp/developer-utils'; // const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.LdmlKeyboardCompiler; @@ -91,8 +91,8 @@ export class LdmlCompilerMessages { m(this.ERROR_DisplayIsRepeated, `display ${LdmlCompilerMessages.outputOrKeyId(o)} has more than one display entry.`); static ERROR_KeyMissingToGapOrSwitch = SevError | 0x0011; - static Error_KeyMissingToGapOrSwitch = (o:{keyId: string}) => - m(this.ERROR_KeyMissingToGapOrSwitch, `key id='${def(o.keyId)}' must have either output=, gap=, or layerId=.`); + static Error_KeyMissingToGapOrSwitch = (o:{keyId: string}, x: any) => mx(x, + this.ERROR_KeyMissingToGapOrSwitch, `key id='${def(o.keyId)}' must have either output=, gap=, or layerId=.`); static ERROR_ExcessHardware = SevError | 0x0012; static Error_ExcessHardware = (o:{formId: string}) => m(this.ERROR_ExcessHardware, @@ -277,6 +277,7 @@ export class LdmlCompilerMessages { static offset(event: CompilerEvent, x?: any): CompilerEvent { if(x) { event.offset = (x as any)[XML_START_INDEX_SYMBOL as any]; + event.filename = event.filename || (x as any)[XML_FILENAME_SYMBOL as any]; } return event; } From 0ba005a55b21b728a92ad11171c788025155eb99 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Tue, 4 Mar 2025 09:34:47 -0600 Subject: [PATCH 13/27] feat(developer): line number symbols - improve and simplify lookup algorithm - add test for override filename on imports Fixes: #10622 --- .../src/common/web/utils/src/xml-utils.ts | 22 ++++++++++++------- .../kmx/ldml-keyboard-xml-reader.tests.ts | 13 ++++++++--- .../common/web/utils/test/xml-utils.tests.ts | 7 +++--- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index ee33b40563..7d7dc10011 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -137,18 +137,24 @@ export interface LineColumn { export class KeymanXMLReader { public constructor(public type: KeymanXMLType) { } - /** preprocess text to turn it into arrays of line lengths */ + /** + * preprocess text to turn it into arrays of line lengths. + * This is effectively a 1-based line length, since line 0 has length of + * 0. + */ public static textToLines(text: string) : number[] { - return text.replaceAll("\r\n", "\n").split("\n") - .map(l => l.length + 1); // line length (counting the trailing newline) + return [ + 0, // "line 0" is empty + ...text.replaceAll("\r\n", "\n").split("\n") + .map(l => l.length + 1) // line length (counting the trailing newline) + ]; } public static offsetToLineColumn(offset: number, lines: number[]) : LineColumn { - for (let line = 1; line < lines.length + 1; line++) { // 1-based - if (lines[line-1] < offset) { - offset = offset - (lines[line-1]); // count newline at end - continue; + for (let line = 1; line < lines.length; line++) { // 1-based (assume the first row is 0) + if (lines[line] > offset) { + return { line, column: offset }; } - return { line, column: offset }; + offset = offset - (lines[line]); // count newline at end } // default: line 0, error return { line: 0 } diff --git a/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts b/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts index fcb3628144..56a70e3ea0 100644 --- a/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts +++ b/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts @@ -8,6 +8,7 @@ import { Constants } from '@keymanapp/common-types'; import CLDRScanToVkey = Constants.CLDRScanToVkey; import CLDRScanToKeyMap = Constants.CLDRScanToKeyMap; import USVirtualKeyCodes = Constants.USVirtualKeyCodes; +import { XML_FILENAME_SYMBOL } from '../../src/xml-utils.js'; function pluckKeysFromKeybag(keys: LKKey[], ids: string[]) { return keys.filter(({id}) => ids.indexOf(id) !== -1); @@ -140,10 +141,16 @@ describe('ldml keyboard xml reader tests', function () { { id: 'interrobang', output: '‽' }, { id: 'snail', output: '@' }, ]); + const snailKey = source?.keyboard3?.keys.key.find(({ id }) => id === 'snail'); // all of the keys are implied imports here - assert.isFalse(ImportStatus.isImpliedImport(source?.keyboard3?.keys.key.find(({id}) => id === 'snail'))); - assert.isTrue(ImportStatus.isImport(source?.keyboard3?.keys.key.find(({id}) => id === 'snail'))); - assert.isTrue(ImportStatus.isLocalImport(source?.keyboard3?.keys.key.find(({id}) => id === 'snail'))); + assert.isFalse(ImportStatus.isImpliedImport(snailKey)); + assert.isTrue(ImportStatus.isImport(snailKey)); + assert.isTrue(ImportStatus.isLocalImport(snailKey)); + // get the actual filename of where the import was located + const snailFilename = (snailKey as any)[XML_FILENAME_SYMBOL as any]; + assert.ok(snailFilename); + assert.ok(/\/keys-Zyyy-morepunctuation.xml$/.test(snailFilename) + , `snail key filename is ${snailFilename}`); }, }, { diff --git a/developer/src/common/web/utils/test/xml-utils.tests.ts b/developer/src/common/web/utils/test/xml-utils.tests.ts index f487573724..5045bcfcb3 100644 --- a/developer/src/common/web/utils/test/xml-utils.tests.ts +++ b/developer/src/common/web/utils/test/xml-utils.tests.ts @@ -174,14 +174,15 @@ describe(`XML Reader line number test`, () => { // now, parse. subsitute endings for Win const actual = reader.parse(xml); + const lines = KeymanXMLReader.textToLines(xml); assert.ok(actual, `Parser failed on ${xmlPath}`); // now, assert char offset assert.equal(actual.keyboard3[XML_START_INDEX_SYMBOL as any], 40); // index of element assert.equal(actual.keyboard3.info[XML_START_INDEX_SYMBOL as any], 136); // index of etc assert.equal(actual.keyboard3.transforms[XML_START_INDEX_SYMBOL as any], 186); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3[XML_START_INDEX_SYMBOL as any], xml), { line: 3, column: 0 }); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.info[XML_START_INDEX_SYMBOL as any], xml), { line: 4, column: 2 }); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.transforms[XML_START_INDEX_SYMBOL as any], xml), { line: 8, column: 2 }); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3[XML_START_INDEX_SYMBOL as any], lines), { line: 3, column: 0 }); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.info[XML_START_INDEX_SYMBOL as any], lines), { line: 4, column: 2 }); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.transforms[XML_START_INDEX_SYMBOL as any], lines), { line: 8, column: 2 }); }); }); From 0c290bd37ad8b7454783f01d6a9b3043251d3af2 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 31 Mar 2025 16:11:19 -0500 Subject: [PATCH 14/27] feat(developer): update for line numbers - update to metadata branch of fast-xml-parser - restructure metadata handling - use type safety for fast-xml-parser options Fixes: #10622 --- developer/src/common/web/utils/src/index.ts | 4 +- .../src/common/web/utils/src/symbol-utils.ts | 33 ++++++++ .../ldml-keyboard/ldml-keyboard-xml-reader.ts | 4 +- .../src/common/web/utils/src/xml-utils.ts | 84 ++++++++++--------- .../kmx/ldml-keyboard-xml-reader.tests.ts | 6 +- .../utils/test/kpj/kpj-file-writer.tests.ts | 6 +- .../utils/test/kps/kps-file-reader.tests.ts | 6 +- .../web/utils/test/kvks/kvks-file.tests.ts | 6 +- .../common/web/utils/test/xml-utils.tests.ts | 28 +++++-- .../src/compiler/ldml-compiler-messages.ts | 9 +- .../src/kmc-ldml/test/helpers/compareXml.ts | 4 +- developer/src/kmc-ldml/test/helpers/index.ts | 13 +-- developer/src/kmc-ldml/test/keys.tests.ts | 2 +- package-lock.json | 4 +- 14 files changed, 132 insertions(+), 77 deletions(-) create mode 100644 developer/src/common/web/utils/src/symbol-utils.ts diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index b722a11b21..95ef39ffcb 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -67,8 +67,8 @@ export { UrlSubpathCompilerCallback } from './utils/UrlSubpathCompilerCallback.j export { CommonTypesMessages } from './common-messages.js'; export * as SourceFilenamePatterns from './source-filename-patterns.js'; -export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, XML_START_INDEX_SYMBOL, XML_FILENAME_SYMBOL } from './xml-utils.js'; - +export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, KeymanXMLMetadata, XML_FILENAME_SYMBOL } from './xml-utils.js'; +export { SymbolUtils } from './symbol-utils.js'; export * as GitHubUrls from './github-urls.js'; export * as CloudUrls from './cloud-urls.js'; diff --git a/developer/src/common/web/utils/src/symbol-utils.ts b/developer/src/common/web/utils/src/symbol-utils.ts new file mode 100644 index 0000000000..971d02cc16 --- /dev/null +++ b/developer/src/common/web/utils/src/symbol-utils.ts @@ -0,0 +1,33 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by srl on 2025-03-27 + * + * Utilities for manipulating Symbol properties + */ + +export class SymbolUtils { + /** + * Copy symbols from 'from' onto 'onto' + * @param onto object to copy onto + * @param from source for symbols + * @returns the onto object + */ + public static copySymbols(onto: T, from: any): T { + const o = onto as any; + for (const sym of Object.getOwnPropertySymbols(from)) { + o[sym] = from[sym]; + } + return onto; + } + + /** use Object.entries to remove all symbols */ + public static removeSymbols(from: T): T { + if (Array.isArray(from)) { + return from.map(o => SymbolUtils.removeSymbols(o)) as T; + } + if (typeof from !== "object") return from; + return Object.fromEntries(Object.entries(from).map(([k, v]) => ([k, SymbolUtils.removeSymbols(v)]))) as T; + } + +} diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index ca32fe4905..84e82c62ea 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -12,7 +12,7 @@ import { LDMLKeyboardTestDataXMLSourceFile, LKTTest, LKTTests } from './ldml-key import { KeymanXMLReader } from '@keymanapp/developer-utils'; import boxXmlArray = util.boxXmlArray; import { LdmlEventResolver } from './eventresolver.js'; -import { XML_FILENAME_SYMBOL } from 'src/xml-utils.js'; +import { XML_FILENAME_SYMBOL } from '../../xml-utils.js'; interface NameAndProps { '$'?: any; // content @@ -281,7 +281,7 @@ export class LDMLKeyboardXMLSourceFileReader { // Mark all children as an import subsubval.forEach(o => { o[ImportStatus.import] = basePath; - o[XML_FILENAME_SYMBOL] = importPath; // mark overriding import path + KeymanXMLReader.setMetaData(o, {[XML_FILENAME_SYMBOL]: importPath}); // mark overriding import path }); if (implied) { diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 7d7dc10011..eadeb8659d 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -6,10 +6,11 @@ * Abstraction for XML reading and writing */ -import { XMLParser, XMLBuilder } from 'fast-xml-parser'; +import { XMLParser, XMLBuilder, XMLMetaData, X2jOptions, XmlBuilderOptions } from 'fast-xml-parser'; +import { SymbolUtils } from "./symbol-utils.js"; /** Symbol giving the start offset, in chars, of the node */ -export const XML_START_INDEX_SYMBOL = XMLParser.getStartIndexSymbol(); +const XML_META_DATA_SYMBOL = XMLParser.getMetaDataSymbol(); /** Symbol giving an override which file a node came from */ export const XML_FILENAME_SYMBOL = Symbol("XML Filename"); @@ -22,12 +23,12 @@ export type KeymanXMLType = ; /** Bag of options, maximally one for each KeymanXMLType */ -type KeymanXMLOptionsBag = { - [key in KeymanXMLType]?: any +type KeymanXMLParserOptionsBag = { + [key in KeymanXMLType]?: X2jOptions }; /** map of options for the XML parser */ -const PARSER_OPTIONS: KeymanXMLOptionsBag = { +const PARSER_OPTIONS: KeymanXMLParserOptionsBag = { 'keyboard3': { ignoreAttributes: false, // We'd like attributes, please attributeNamePrefix: '@__', // We'll use this to convert attributes to strings and subobjects to arrays, when empty. @@ -40,6 +41,7 @@ const PARSER_OPTIONS: KeymanXMLOptionsBag = { // if we do need elements in the future, we'd check the preserve-space attribute here. return tagValue?.trim(); }, + captureMetaData: true, }, 'keyboardTest3': { ignorePiTags: true, @@ -87,7 +89,7 @@ const PARSER_OPTIONS: KeymanXMLOptionsBag = { eNotation: null, }, trimValues: false, // preserve spaces, but: - tagValueProcessor: (tagName: string, tagValue: string, jPath: string, hasAttributes: string, isLeafNode: boolean) : string | undefined => { + tagValueProcessor: (tagName: string, tagValue: string, jPath: string, hasAttributes: boolean, isLeafNode: boolean) : string | undefined => { if (!isLeafNode) { return tagValue?.trim(); // trimmed value } else { @@ -97,7 +99,11 @@ const PARSER_OPTIONS: KeymanXMLOptionsBag = { }, }; -const GENERATOR_OPTIONS: KeymanXMLOptionsBag = { +type KeymanXMLGeneratorOptionsBag = { + [key in KeymanXMLType]?: XmlBuilderOptions +}; + +const GENERATOR_OPTIONS: KeymanXMLGeneratorOptionsBag = { kvks: { attributeNamePrefix: '$', ignoreAttributes: false, @@ -133,10 +139,37 @@ export interface LineColumn { column?: number; } +export interface KeymanXMLMetadata extends XMLMetaData { + /** override of name of XML file */ + [XML_FILENAME_SYMBOL]?: string; +} + /** wrapper for XML parsing support */ export class KeymanXMLReader { public constructor(public type: KeymanXMLType) { } + + /** Get metadata on a node if not already set */ + static getMetaData(o: any) : KeymanXMLMetadata { + if(!o) return o; + const metadata : KeymanXMLMetadata = o[XML_META_DATA_SYMBOL as any]; + return metadata; + } + + /** Set metadata if not already set */ + public static setMetaData(o: any, metadata: KeymanXMLMetadata) : KeymanXMLMetadata { + let m : KeymanXMLMetadata = KeymanXMLReader.getMetaData(o); + if (!m) { + m = {}; + } + // copy non-symbols + m = {...metadata, ...m}; + // copy symbols + SymbolUtils.copySymbols(m, metadata); + o[XML_META_DATA_SYMBOL as any] = m; + return m; + } + /** * preprocess text to turn it into arrays of line lengths. * This is effectively a 1-based line length, since line 0 has length of @@ -160,31 +193,6 @@ export class KeymanXMLReader { return { line: 0 } } - /** - * Copy symbols from 'from' onto 'onto' - * This is used to propagate special symbols - * and XML - * @param onto object to copy onto - * @param from source for symbols - * @returns the onto object - */ - public static copySymbols(onto: T, from: any): T { - const o = onto as any; - for (const sym of Object.getOwnPropertySymbols(from)) { - o[sym] = from[sym]; - } - return onto; - } - - /** use Object.entries to remove all symbols */ - public static removeSymbols(from: T): T { - if (Array.isArray(from)) { - return from.map(o => KeymanXMLReader.removeSymbols(o)) as T; - } - if (typeof from !== "object") return from; - return Object.fromEntries(Object.entries(from).map(([k, v]) => ([k, KeymanXMLReader.removeSymbols(v)]))) as T; - } - /** move `{ $abc: 4 }` into `{ $: { abc: 4 } }` */ private static fixupDollarAttributes(data: any) : any { if (typeof data === 'object') { @@ -203,9 +211,9 @@ export class KeymanXMLReader { } }); if (attrs.length) { - e.push(['$', this.copySymbols(Object.fromEntries(attrs), data)]); + e.push(['$', SymbolUtils.copySymbols(Object.fromEntries(attrs), data)]); } - return this.copySymbols(Object.fromEntries(e), data); + return SymbolUtils.copySymbols(Object.fromEntries(e), data); } else { return data; } @@ -241,7 +249,7 @@ export class KeymanXMLReader { } } }); - return this.copySymbols(Object.fromEntries(e), data); + return SymbolUtils.copySymbols(Object.fromEntries(e), data); } else { return data; } @@ -321,15 +329,11 @@ export class KeymanXMLReader { } public parser() { - let options = PARSER_OPTIONS[this.type]; + const options = PARSER_OPTIONS[this.type]; if (!options) { /* c8 ignore next 1 */ throw Error(`Internal error: unhandled XML type ${this.type}`); } - options = Object.assign({}, options); // TODO: xml2js likes to mutate the options here. Shallow clone the object. - if (options.emptyTag) { - options.emptyTag = {}; // TODO: xml2js likes to mutate the options here. Reset it. - } return new XMLParser(options); } } diff --git a/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts b/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts index 56a70e3ea0..bfd076a34d 100644 --- a/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts +++ b/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts @@ -8,7 +8,7 @@ import { Constants } from '@keymanapp/common-types'; import CLDRScanToVkey = Constants.CLDRScanToVkey; import CLDRScanToKeyMap = Constants.CLDRScanToKeyMap; import USVirtualKeyCodes = Constants.USVirtualKeyCodes; -import { XML_FILENAME_SYMBOL } from '../../src/xml-utils.js'; +import { KeymanXMLReader, XML_FILENAME_SYMBOL } from '../../src/xml-utils.js'; function pluckKeysFromKeybag(keys: LKKey[], ids: string[]) { return keys.filter(({id}) => ids.indexOf(id) !== -1); @@ -147,7 +147,9 @@ describe('ldml keyboard xml reader tests', function () { assert.isTrue(ImportStatus.isImport(snailKey)); assert.isTrue(ImportStatus.isLocalImport(snailKey)); // get the actual filename of where the import was located - const snailFilename = (snailKey as any)[XML_FILENAME_SYMBOL as any]; + const metadata = KeymanXMLReader.getMetaData(snailKey); + assert.ok(metadata); + const snailFilename = (metadata)[XML_FILENAME_SYMBOL]; assert.ok(snailFilename); assert.ok(/\/keys-Zyyy-morepunctuation.xml$/.test(snailFilename) , `snail key filename is ${snailFilename}`); diff --git a/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts b/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts index 5fc223b240..425d76a6ef 100644 --- a/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts +++ b/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts @@ -6,7 +6,7 @@ import { KPJFileReader } from "../../src/types/kpj/kpj-file-reader.js"; import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; import { KPJFileWriter } from '../../src/types/kpj/kpj-file-writer.js'; import { KeymanDeveloperProjectOptions } from '../../src/types/kpj/keyman-developer-project.js'; -import { KeymanXMLReader } from '../../src/xml-utils.js'; +import { SymbolUtils } from '../../src/symbol-utils.js'; const callbacks = new TestCompilerCallbacks(); @@ -22,7 +22,7 @@ describe('kpj-file-writer', function () { const writer = new KPJFileWriter(); const output = writer.write(project); - const outputKpj = KeymanXMLReader.removeSymbols(reader.read(new TextEncoder().encode(output))); + const outputKpj = SymbolUtils.removeSymbols(reader.read(new TextEncoder().encode(output))); // The outputKpj may not contain all the fields from the inputKpj, only the // essential fields. Many of the fields in .kpj are deprecated, when they @@ -61,7 +61,7 @@ describe('kpj-file-writer', function () { const writer = new KPJFileWriter(); const output = writer.write(project); - const outputKpj = KeymanXMLReader.removeSymbols(reader.read(new TextEncoder().encode(output))); + const outputKpj = SymbolUtils.removeSymbols(reader.read(new TextEncoder().encode(output))); // The outputKpj may not contain all the fields from the inputKpj, only the // essential fields. Many of the fields in .kpj are deprecated, when they diff --git a/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts b/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts index 94a1628bd9..9b7955819d 100644 --- a/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts +++ b/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts @@ -15,7 +15,7 @@ import { makePathToFixture } from '../helpers/index.js'; import { KpsFileReader } from "../../src/types/kps/kps-file-reader.js"; import { KpsFileWriter } from '../../src/types/kps/kps-file-writer.js'; import { CommonTypesMessages } from '../../src/common-messages.js'; -import { KeymanXMLReader } from '../../src/xml-utils.js'; +import { SymbolUtils } from '../../src/symbol-utils.js'; const callbacks = new TestCompilerCallbacks(); @@ -62,13 +62,13 @@ describe('kps-file-reader', function () { it('kps-file-reader should round-trip with kps-file-writer', function() { const input = fs.readFileSync(makePathToFixture('kps', 'khmer_angkor.kps')); const reader = new KpsFileReader(callbacks); - const kps = KeymanXMLReader.removeSymbols(reader.read(input)); + const kps = SymbolUtils.removeSymbols(reader.read(input)); const writer = new KpsFileWriter(); const output = writer.write(kps); // Round Trip - const kps2 = KeymanXMLReader.removeSymbols(reader.read(new TextEncoder().encode(output))); + const kps2 = SymbolUtils.removeSymbols(reader.read(new TextEncoder().encode(output))); assert.deepEqual(kps2, kps); }); diff --git a/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts b/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts index 96be2283b8..d264dc86aa 100644 --- a/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts +++ b/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts @@ -5,7 +5,7 @@ import KvksFileReader from "../../src/types/kvks/kvks-file-reader.js"; import KvksFileWriter from "../../src/types/kvks/kvks-file-writer.js"; import { verify_khmer_angkor, verify_balochi_inpage } from './kvk-utils.tests.js'; import { assert } from 'chai'; -import { KeymanXMLReader } from '../../src/xml-utils.js'; +import { SymbolUtils } from '../../src/symbol-utils.js'; describe('kvks-file-reader', function() { it('should read a valid file', function() { @@ -49,7 +49,7 @@ describe('kvks-file-writer', function() { const input = fs.readFileSync(path); const reader = new KvksFileReader(); - const kvksExpected = KeymanXMLReader.removeSymbols(reader.read(input)); + const kvksExpected = SymbolUtils.removeSymbols(reader.read(input)); const invalidVkeys: string[] = []; const vk = reader.transform(kvksExpected, invalidVkeys); assert.isEmpty(invalidVkeys); @@ -60,6 +60,6 @@ describe('kvks-file-writer', function() { // We compare the (re)loaded data, because there may be // minor, irrelevant formatting differences in the emitted xml const kvks = reader.read(Buffer.from(output, 'utf8')); - assert.deepEqual(KeymanXMLReader.removeSymbols(kvks), kvksExpected); + assert.deepEqual(SymbolUtils.removeSymbols(kvks), kvksExpected); }); }); diff --git a/developer/src/common/web/utils/test/xml-utils.tests.ts b/developer/src/common/web/utils/test/xml-utils.tests.ts index 5045bcfcb3..8728eec207 100644 --- a/developer/src/common/web/utils/test/xml-utils.tests.ts +++ b/developer/src/common/web/utils/test/xml-utils.tests.ts @@ -12,8 +12,9 @@ import { env } from 'node:process'; import { readFileSync, writeFileSync } from 'node:fs'; -import { KeymanXMLType, KeymanXMLReader, KeymanXMLWriter, XML_START_INDEX_SYMBOL } from '../src/xml-utils.js'; +import { KeymanXMLType, KeymanXMLReader, KeymanXMLWriter } from '../src/xml-utils.js'; import { makePathToFixture } from './helpers/index.js'; +import { SymbolUtils } from '../src/symbol-utils.js'; // if true, attempt to WRITE the fixtures const { GEN_XML_FIXTURES } = env; @@ -119,7 +120,7 @@ describe(`XML Reader Test ${GEN_XML_FIXTURES && '(update mode!)' || ''}`, () => writeJson(jsonPath, actual); } else { assert.ok(expect, `Could not read ${jsonPath} - run with env GEN_XML_FIXTURES=1 to update.`); - assert.deepEqual(KeymanXMLReader.removeSymbols(actual), expect, `Mismatch of ${xmlPath} vs ${jsonPath}`); + assert.deepEqual(SymbolUtils.removeSymbols(actual), expect, `Mismatch of ${xmlPath} vs ${jsonPath}`); } }); } @@ -178,11 +179,22 @@ describe(`XML Reader line number test`, () => { assert.ok(actual, `Parser failed on ${xmlPath}`); // now, assert char offset - assert.equal(actual.keyboard3[XML_START_INDEX_SYMBOL as any], 40); // index of element - assert.equal(actual.keyboard3.info[XML_START_INDEX_SYMBOL as any], 136); // index of etc - assert.equal(actual.keyboard3.transforms[XML_START_INDEX_SYMBOL as any], 186); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3[XML_START_INDEX_SYMBOL as any], lines), { line: 3, column: 0 }); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.info[XML_START_INDEX_SYMBOL as any], lines), { line: 4, column: 2 }); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.transforms[XML_START_INDEX_SYMBOL as any], lines), { line: 8, column: 2 }); + const getMetaData = KeymanXMLReader.getMetaData; + assert.ok(getMetaData(actual.keyboard3)); + assert.equal( + getMetaData(actual.keyboard3)?.startIndex, 40); // index of element + assert.equal( + getMetaData(actual.keyboard3.info)?.startIndex, 136); // index of etc + assert.equal( + getMetaData(actual.keyboard3.transforms)?.startIndex, 186); + assert.deepEqual( + KeymanXMLReader.offsetToLineColumn( + getMetaData(actual.keyboard3).startIndex, lines), { line: 3, column: 0 }); + assert.deepEqual( + KeymanXMLReader.offsetToLineColumn( + getMetaData(actual.keyboard3.info).startIndex, lines), { line: 4, column: 2 }); + assert.deepEqual( + KeymanXMLReader.offsetToLineColumn( + getMetaData(actual.keyboard3.transforms).startIndex, lines), { line: 8, column: 2 }); }); }); diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 1fcb82193b..d363fa03db 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -1,5 +1,5 @@ import { util } from "@keymanapp/common-types"; -import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_START_INDEX_SYMBOL, XML_FILENAME_SYMBOL, CompilerEvent } from '@keymanapp/developer-utils'; +import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_FILENAME_SYMBOL, CompilerEvent, KeymanXMLReader } from '@keymanapp/developer-utils'; // const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.LdmlKeyboardCompiler; @@ -271,13 +271,14 @@ export class LdmlCompilerMessages { /** * Get an offset from o and set e's offset field * @param event a compiler event, such as from functions in this class - * @param x any object parsed from XML or with the XML_START_INDEX_SYMBOL symbol copied over + * @param x any object parsed from XML or with the XML_META_DATA_SYMBOL symbol copied over * @returns modified event object */ static offset(event: CompilerEvent, x?: any): CompilerEvent { if(x) { - event.offset = (x as any)[XML_START_INDEX_SYMBOL as any]; - event.filename = event.filename || (x as any)[XML_FILENAME_SYMBOL as any]; + const metadata = KeymanXMLReader.getMetaData(x) || {}; + event.offset = metadata?.startIndex; + event.filename = event.filename || metadata[XML_FILENAME_SYMBOL]; } return event; } diff --git a/developer/src/kmc-ldml/test/helpers/compareXml.ts b/developer/src/kmc-ldml/test/helpers/compareXml.ts index 1503d4c606..057986bdb2 100644 --- a/developer/src/kmc-ldml/test/helpers/compareXml.ts +++ b/developer/src/kmc-ldml/test/helpers/compareXml.ts @@ -1,6 +1,6 @@ import {assert} from 'chai'; import {readFileSync} from 'node:fs'; -import { KeymanXMLReader } from "@keymanapp/developer-utils"; +import { KeymanXMLReader, SymbolUtils } from "@keymanapp/developer-utils"; /** * @@ -21,5 +21,5 @@ export function compareXml(actual : string, expect: string, mutator?: (input: an const actualParsed = mutator(reader.parse(actualStr)); const expectParsed = mutator(reader.parse(expectStr)); - assert.deepEqual(KeymanXMLReader.removeSymbols(actualParsed), KeymanXMLReader.removeSymbols(expectParsed)); + assert.deepEqual(SymbolUtils.removeSymbols(actualParsed), SymbolUtils.removeSymbols(expectParsed)); } diff --git a/developer/src/kmc-ldml/test/helpers/index.ts b/developer/src/kmc-ldml/test/helpers/index.ts index 497f8ef919..bb10825315 100644 --- a/developer/src/kmc-ldml/test/helpers/index.ts +++ b/developer/src/kmc-ldml/test/helpers/index.ts @@ -8,7 +8,7 @@ import * as path from 'path'; import { fileURLToPath } from 'url'; import { SectionCompiler, SectionCompilerNew } from '../../src/compiler/section-compiler.js'; import { util, KMXPlus, LdmlKeyboardTypes } from '@keymanapp/common-types'; -import { CompilerEvent, compilerEventFormat, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboard, XML_START_INDEX_SYMBOL, } from "@keymanapp/developer-utils"; +import { CompilerEvent, compilerEventFormat, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboard, KeymanXMLMetadata, KeymanXMLReader } from "@keymanapp/developer-utils"; import { LdmlKeyboardCompiler } from '../../src/main.js'; // make sure main.js compiles import { assert } from 'chai'; import { KMXPlusMetadataCompiler } from '../../src/compiler/metadata-compiler.js'; @@ -329,8 +329,11 @@ export function hex_str(s?: string) : string { } /** return an object simulating an XML object with a column number */ -export function withColumn(c: number) : any { - return { - [XML_START_INDEX_SYMBOL as any]: c - }; +export function withColumn(c: number) : KeymanXMLMetadata { + // set metadata on an empty object + const o = {}; + KeymanXMLReader.setMetaData(o, { + startIndex: c + }); + return o; } diff --git a/developer/src/kmc-ldml/test/keys.tests.ts b/developer/src/kmc-ldml/test/keys.tests.ts index 1b74763646..3d87caaa7a 100644 --- a/developer/src/kmc-ldml/test/keys.tests.ts +++ b/developer/src/kmc-ldml/test/keys.tests.ts @@ -464,7 +464,7 @@ describe('keys.kmap', function () { let keys = await loadSectionFixture(KeysCompiler, 'sections/keys/invalid-key-missing-attrs.xml', compilerTestCallbacks, keysDependencies) as Keys; assert.isNull(keys); assert.equal(compilerTestCallbacks.messages.length, 1); - assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch({keyId: 'Q'})); + assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch({keyId: 'Q'}, withColumn(188))); }); it('should accept layouts with gap/switch keys', async function() { let keys = await loadSectionFixture(KeysCompiler, 'sections/keys/gap-switch.xml', compilerTestCallbacks, keysDependencies) as Keys; diff --git a/package-lock.json b/package-lock.json index 192f383713..10a5e5d9d6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9094,8 +9094,8 @@ "dev": true }, "node_modules/fast-xml-parser": { - "version": "5.0.8", - "resolved": "git+ssh://git@github.com/srl295/fast-xml-parser.git#72f20480e8591aca819fac684d46f8d678fcb7f3", + "version": "5.0.9", + "resolved": "git+ssh://git@github.com/srl295/fast-xml-parser.git#985271d7c731a6777456c362849d106073feaddd", "funding": [ { "type": "github", From 1f2b8463f5c368bd5fd79584b4249e00a78041ad Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 3 Apr 2025 12:15:20 -0500 Subject: [PATCH 15/27] chore(developer): deps: update to fast-xml-parser@5.1.0 (Metadata branch) Fixes: #10622 --- developer/src/common/web/utils/package.json | 2 +- package-lock.json | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/developer/src/common/web/utils/package.json b/developer/src/common/web/utils/package.json index a56fdc386a..4bceb5d6e4 100644 --- a/developer/src/common/web/utils/package.json +++ b/developer/src/common/web/utils/package.json @@ -12,7 +12,7 @@ "@keymanapp/common-types": "*", "@sentry/node": "^7.57.0", "eventemitter3": "^5.0.0", - "fast-xml-parser": "github:srl295/fast-xml-parser#line-number-593bis", + "fast-xml-parser": "github:NaturalIntelligence/fast-xml-parser#metadata", "path-browserify": "^1.0.1", "restructure": "^3.0.1", "sax": ">=0.6.0", diff --git a/package-lock.json b/package-lock.json index 10a5e5d9d6..05ed59e500 100644 --- a/package-lock.json +++ b/package-lock.json @@ -342,7 +342,7 @@ "@keymanapp/common-types": "*", "@sentry/node": "^7.57.0", "eventemitter3": "^5.0.0", - "fast-xml-parser": "github:srl295/fast-xml-parser#line-number-593bis", + "fast-xml-parser": "github:NaturalIntelligence/fast-xml-parser#metadata", "path-browserify": "^1.0.1", "restructure": "^3.0.1", "sax": ">=0.6.0", @@ -9094,15 +9094,14 @@ "dev": true }, "node_modules/fast-xml-parser": { - "version": "5.0.9", - "resolved": "git+ssh://git@github.com/srl295/fast-xml-parser.git#985271d7c731a6777456c362849d106073feaddd", + "version": "5.1.0", + "resolved": "git+ssh://git@github.com/NaturalIntelligence/fast-xml-parser.git#ab8d40a3077f109f34d02d05653146b8621bd255", "funding": [ { "type": "github", "url": "https://github.com/sponsors/NaturalIntelligence" } ], - "license": "MIT", "dependencies": { "strnum": "^2.0.5" }, From 8d438283c61afb6f26a5a70d106ed41ec91d3a8c Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 3 Apr 2025 16:40:53 -0500 Subject: [PATCH 16/27] feat(developer): update for line numbers - split out LineFinder to separate class Fixes: #10622 --- developer/src/common/web/utils/src/index.ts | 1 + .../src/common/web/utils/src/line-utils.ts | 61 +++++++++++++++++++ .../src/types/ldml-keyboard/eventresolver.ts | 20 ++++-- .../src/common/web/utils/src/xml-utils.ts | 28 --------- .../common/web/utils/test/line-utils.tests.ts | 28 +++++++++ .../common/web/utils/test/xml-utils.tests.ts | 10 +-- 6 files changed, 110 insertions(+), 38 deletions(-) create mode 100644 developer/src/common/web/utils/src/line-utils.ts create mode 100644 developer/src/common/web/utils/test/line-utils.tests.ts diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index 95ef39ffcb..dc51720b25 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -69,6 +69,7 @@ export { CommonTypesMessages } from './common-messages.js'; export * as SourceFilenamePatterns from './source-filename-patterns.js'; export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, KeymanXMLMetadata, XML_FILENAME_SYMBOL } from './xml-utils.js'; export { SymbolUtils } from './symbol-utils.js'; +export { LineFinder, LineColumn } from './line-utils.js'; export * as GitHubUrls from './github-urls.js'; export * as CloudUrls from './cloud-urls.js'; diff --git a/developer/src/common/web/utils/src/line-utils.ts b/developer/src/common/web/utils/src/line-utils.ts new file mode 100644 index 0000000000..5c9a4dec31 --- /dev/null +++ b/developer/src/common/web/utils/src/line-utils.ts @@ -0,0 +1,61 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by srl on 2025-04-03 + * + * Abstraction for line number processing + */ + +/** line number with an optional column number */ +export interface LineColumn { + line: number; + column?: number; +} + +export type LineList = number[]; + +/** a LineFinder is able to convert from an offset to a line/column */ +export class LineFinder { + private list: LineList; + constructor(public text: string) { + this.list = LineFinder.textToLines(text); + } + + /** + * Convert an offset into line:column + * @param offset input offset into the text + * @returns line:column information + */ + public findOffset(offset: number): LineColumn { + return LineFinder.offsetToLineColumn(offset, this.list); + } + + /** + * preprocess text to turn it into arrays of line lengths. + * This is effectively a 1-based line length, since line 0 has length of + * 0. + */ + public static textToLines(text: string): LineList { + return [ + 0, // "line 0" is empty + ...text.replaceAll("\r\n", "\n").split("\n") + .map(l => l.length + 1) // line length (counting the trailing newline) + ]; + } + + /** + * convert a line number array to a line/column + */ + public static offsetToLineColumn(offset: number, list: LineList): LineColumn { + for (let line = 1; line < list.length; line++) { // 1-based (assume the first row is 0) + if (list[line] > offset) { + return { line, column: offset }; + } + offset = offset - (list[line]); // count newline at end + } + // default: line 0, error + return { line: 0 } + } +} + + diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts index 3299169642..d155cf27d2 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts @@ -1,18 +1,26 @@ -import { CompilerEvent, EventResolver, KeymanXMLReader } from "../../index.js"; +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by srl on 2025-03-03 + * + * Implementation of a hook on LDML events + */ +import { CompilerEvent, EventResolver, LineFinder } from "../../index.js"; export class LdmlEventResolver implements EventResolver { - contentsCache: Map = new Map(); + contentsCache: Map = new Map(); - /** add or update a file */ + /** add or update a source file */ addFile(filename: string, xml: any) { - this.contentsCache.set(filename, KeymanXMLReader.textToLines(xml)); + this.contentsCache.set(filename, new LineFinder(xml)); } + /** resolve a CompilerEvent by expanding the line numbers */ resolve(event: CompilerEvent) { if (event.offset && !event.line && event.filename) { - const lines = this.contentsCache.get(event.filename); - const loc = KeymanXMLReader.offsetToLineColumn(event.offset, lines); + const lf = this.contentsCache.get(event.filename); + const loc = lf.findOffset(event.offset); event.line = loc.line; event.column = loc.column; } diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 435fbb4c20..9ec8872975 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -115,11 +115,6 @@ const GENERATOR_OPTIONS: KeymanXMLGeneratorOptionsBag = { }, }; -export interface LineColumn { - line: number; - column?: number; -} - export interface KeymanXMLMetadata extends XMLMetaData { /** override of name of XML file */ [XML_FILENAME_SYMBOL]?: string; @@ -151,29 +146,6 @@ export class KeymanXMLReader { return m; } - /** - * preprocess text to turn it into arrays of line lengths. - * This is effectively a 1-based line length, since line 0 has length of - * 0. - */ - public static textToLines(text: string) : number[] { - return [ - 0, // "line 0" is empty - ...text.replaceAll("\r\n", "\n").split("\n") - .map(l => l.length + 1) // line length (counting the trailing newline) - ]; - } - public static offsetToLineColumn(offset: number, lines: number[]) : LineColumn { - for (let line = 1; line < lines.length; line++) { // 1-based (assume the first row is 0) - if (lines[line] > offset) { - return { line, column: offset }; - } - offset = offset - (lines[line]); // count newline at end - } - // default: line 0, error - return { line: 0 } - } - /** move `{ $abc: 4 }` into `{ $: { abc: 4 } }` */ private static fixupDollarAttributes(data: any) : any { if (typeof data === 'object') { diff --git a/developer/src/common/web/utils/test/line-utils.tests.ts b/developer/src/common/web/utils/test/line-utils.tests.ts new file mode 100644 index 0000000000..9c7b5da01c --- /dev/null +++ b/developer/src/common/web/utils/test/line-utils.tests.ts @@ -0,0 +1,28 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by srl on 2025-04-03 + * + * Test for LineFinder + */ + +import { assert } from 'chai'; +import { readFileSync } from 'node:fs'; +import 'mocha'; +import { LineFinder } from '../src/line-utils.js'; +import { makePathToFixture } from './helpers/index.js'; + +describe(`LineFinder test`, () => { + const path = 'tran_fail-empty.xml'; + + it(`Should be able to process ${path}`, () => { + const xmlPath = makePathToFixture('xml', `${path}`); + const data = readFileSync(xmlPath, 'utf-8'); + assert.ok(data); + const lf = new LineFinder(data); + assert.ok(lf); + assert.deepEqual(lf.findOffset(40), {line: 3, column: 0}); + assert.deepEqual(lf.findOffset(136), {line: 4, column: 2}); + assert.deepEqual(lf.findOffset(186), {line: 8, column: 2}); + }); +}); diff --git a/developer/src/common/web/utils/test/xml-utils.tests.ts b/developer/src/common/web/utils/test/xml-utils.tests.ts index 8728eec207..4b3df736b6 100644 --- a/developer/src/common/web/utils/test/xml-utils.tests.ts +++ b/developer/src/common/web/utils/test/xml-utils.tests.ts @@ -13,6 +13,7 @@ import { readFileSync, writeFileSync } from 'node:fs'; import { KeymanXMLType, KeymanXMLReader, KeymanXMLWriter } from '../src/xml-utils.js'; +import { LineFinder } from '../src/line-utils.js'; import { makePathToFixture } from './helpers/index.js'; import { SymbolUtils } from '../src/symbol-utils.js'; @@ -175,7 +176,7 @@ describe(`XML Reader line number test`, () => { // now, parse. subsitute endings for Win const actual = reader.parse(xml); - const lines = KeymanXMLReader.textToLines(xml); + const lines = LineFinder.textToLines(xml); assert.ok(actual, `Parser failed on ${xmlPath}`); // now, assert char offset @@ -188,13 +189,14 @@ describe(`XML Reader line number test`, () => { assert.equal( getMetaData(actual.keyboard3.transforms)?.startIndex, 186); assert.deepEqual( - KeymanXMLReader.offsetToLineColumn( + LineFinder.offsetToLineColumn( getMetaData(actual.keyboard3).startIndex, lines), { line: 3, column: 0 }); assert.deepEqual( - KeymanXMLReader.offsetToLineColumn( + LineFinder.offsetToLineColumn( getMetaData(actual.keyboard3.info).startIndex, lines), { line: 4, column: 2 }); assert.deepEqual( - KeymanXMLReader.offsetToLineColumn( + LineFinder.offsetToLineColumn( getMetaData(actual.keyboard3.transforms).startIndex, lines), { line: 8, column: 2 }); }); }); + From 11f77ed87647ff4e1e32c90388d664a939168092 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 3 Apr 2025 16:47:04 -0500 Subject: [PATCH 17/27] feat(developer): update for line numbers - fix ts issue on reader Fixes: #10622 --- .../utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index 84e82c62ea..2b5b0fc746 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -281,7 +281,7 @@ export class LDMLKeyboardXMLSourceFileReader { // Mark all children as an import subsubval.forEach(o => { o[ImportStatus.import] = basePath; - KeymanXMLReader.setMetaData(o, {[XML_FILENAME_SYMBOL]: importPath}); // mark overriding import path + KeymanXMLReader.setMetaData(o, {[XML_FILENAME_SYMBOL as any]: importPath}); // mark overriding import path }); if (implied) { From 53fd228d0889da1fb1f3aabf791af3fae9bb9d2b Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 3 Apr 2025 17:24:57 -0500 Subject: [PATCH 18/27] feat(developer): update for line numbers - fix windows path specific issue in test Fixes: #10622 --- .../web/utils/test/ldml/ldml-keyboard-xml-reader.tests.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/src/common/web/utils/test/ldml/ldml-keyboard-xml-reader.tests.ts b/developer/src/common/web/utils/test/ldml/ldml-keyboard-xml-reader.tests.ts index 6da13e0645..af0005cb6d 100644 --- a/developer/src/common/web/utils/test/ldml/ldml-keyboard-xml-reader.tests.ts +++ b/developer/src/common/web/utils/test/ldml/ldml-keyboard-xml-reader.tests.ts @@ -151,7 +151,7 @@ describe('ldml keyboard xml reader tests', function () { assert.ok(metadata); const snailFilename = (metadata)[XML_FILENAME_SYMBOL]; assert.ok(snailFilename); - assert.ok(/\/keys-Zyyy-morepunctuation.xml$/.test(snailFilename) + assert.ok(/keys-Zyyy-morepunctuation.xml$/.test(snailFilename) , `snail key filename is ${snailFilename}`); }, }, From e003612ea05b5bdcb048868b7900250088e82ca7 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 10 Apr 2025 17:22:51 -0500 Subject: [PATCH 19/27] feat(developer): update for line numbers - remove resolver complexity in callback subclasses - call LDMLKeyboardXMLSourceFileReader.eventResolver.resolve() statically from ldml-compiler-messages Fixes: #10622 --- .../web/test-helpers/TestCompilerCallbacks.ts | 13 +------------ .../common/web/utils/src/compiler-callbacks.ts | 16 ---------------- .../ldml-keyboard/ldml-keyboard-xml-reader.ts | 1 - .../src/compiler/ldml-compiler-messages.ts | 9 +++++++-- .../src/kmc/src/util/NodeCompilerCallbacks.ts | 13 +------------ 5 files changed, 9 insertions(+), 43 deletions(-) diff --git a/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts b/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts index cd1c816aaa..893e3363ed 100644 --- a/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts +++ b/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts @@ -2,7 +2,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { CompilerEvent, CompilerCallbacks, CompilerPathCallbacks, CompilerFileSystemCallbacks, CompilerError, CompilerNetAsyncCallbacks, DefaultCompilerFileSystemAsyncCallbacks, - CompilerFileSystemAsyncCallbacks, EventResolver, NullEventResolver, + CompilerFileSystemAsyncCallbacks, CompilerErrorSeverity} from '@keymanapp/developer-utils'; import { fileURLToPath } from 'url'; @@ -111,18 +111,7 @@ export class TestCompilerCallbacks implements CompilerCallbacks { return filename; } - private eventResolver = new NullEventResolver(); - - setEventResolver(eventResolver: EventResolver): void { - this.eventResolver = eventResolver; - } - reportMessage(event: CompilerEvent): void { - this.eventResolver.resolve(event); - this.handleReportMessage(event); - } - - handleReportMessage(event: CompilerEvent): void { // console.log(event.message); this.messages.push(event); } diff --git a/developer/src/common/web/utils/src/compiler-callbacks.ts b/developer/src/common/web/utils/src/compiler-callbacks.ts index f8b42afb7a..aabf1794e5 100644 --- a/developer/src/common/web/utils/src/compiler-callbacks.ts +++ b/developer/src/common/web/utils/src/compiler-callbacks.ts @@ -55,12 +55,8 @@ export interface CompilerCallbacks { */ resolveFilename(baseFilename: string, filename: string): string; - /** change the EventResolver to use. If called more than once, the last resolver replaces a prior one. */ - setEventResolver(eventResolver: EventResolver): void; - /** * Report a message from the compiler back to the caller. - * It is expected that eventResolver.resolve() will be called on the message before processing. * @param event */ reportMessage(event: CompilerEvent): void; @@ -225,19 +221,7 @@ export class CompilerFileCallbacks implements CompilerCallbacks { return this.parent.resolveFilename(baseFilename, filename); } - private eventResolver = new NullEventResolver(); - - setEventResolver(eventResolver: EventResolver): void { - this.eventResolver = eventResolver; - this.parent.setEventResolver(eventResolver); - } - reportMessage(event: CompilerEvent): void { - this.eventResolver.resolve(event); - this.handleReportMessage(event); - } - - handleReportMessage(event: CompilerEvent): void { const disable = CompilerFileCallbacks.applyMessageOverridesToEvent(event, this.options.messageOverrides); this.messages.push(event); if (!disable) { diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index 36dab45202..2d099849cd 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -30,7 +30,6 @@ export class LDMLKeyboardXMLSourceFileReaderOptions { export class LDMLKeyboardXMLSourceFileReader { static eventResolver: LdmlEventResolver = new LdmlEventResolver(); constructor(private options: LDMLKeyboardXMLSourceFileReaderOptions, private callbacks : CompilerCallbacks) { - callbacks.setEventResolver(LDMLKeyboardXMLSourceFileReader.eventResolver); } static get defaultImportsURL(): [string,string] { diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 3a26c9a5fe..6deeedfceb 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -1,5 +1,5 @@ import { util } from "@keymanapp/common-types"; -import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_FILENAME_SYMBOL, CompilerEvent, KeymanXMLReader } from '@keymanapp/developer-utils'; +import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_FILENAME_SYMBOL, CompilerEvent, KeymanXMLReader, LDMLKeyboardXMLSourceFileReader } from '@keymanapp/developer-utils'; // const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.LdmlKeyboardCompiler; @@ -10,7 +10,12 @@ const SevError = CompilerErrorSeverity.Error | CompilerErrorNamespace.LdmlKeyboa const SevErrorTransform = SevError | 0xF00; // like m() but takes an XML object for line numbers -const mx = (x: any, code: number, message: string, detail?: string): CompilerEvent => LdmlCompilerMessages.offset(m(code, message, detail), x); +const mx = (x: any, code: number, message: string, detail?: string): CompilerEvent => { + let evt = m(code, message, detail); // raw message + evt = LdmlCompilerMessages.offset(evt, x); // with offset + evt = LDMLKeyboardXMLSourceFileReader.eventResolver.resolve(evt); // offset expanded to line number + return evt; +}; /** * @internal diff --git a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts index d42e3b6545..2ec3854c3e 100644 --- a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts +++ b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { platform } from 'os'; -import { CompilerCallbacks, CompilerEvent, EventResolver, NullEventResolver, +import { CompilerCallbacks, CompilerEvent, CompilerPathCallbacks, CompilerFileSystemCallbacks, compilerLogLevelToSeverity, CompilerErrorSeverity, CompilerError, @@ -148,18 +148,7 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { return fileURLToPath(url); } - private eventResolver = new NullEventResolver(); - - setEventResolver(eventResolver: EventResolver): void { - this.eventResolver = eventResolver; - } - reportMessage(event: CompilerEvent): void { - this.eventResolver.resolve(event); - this.handleReportMessage(event); - } - - handleReportMessage(event: CompilerEvent): void { if(!event.filename) { event.filename = this.messageFilename; } From a401d7eec34d1321249cea060243257ce6b6518b Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 5 May 2025 09:17:36 -0500 Subject: [PATCH 20/27] feat(developer): update for line numbers - make line finder cache lazy loading - rename LDMLResolver to LineFinderResolver Fixes: #10622 --- .../src/common/web/utils/src/line-utils.ts | 36 +++++++++++++++++-- .../src/types/ldml-keyboard/eventresolver.ts | 15 ++++---- .../ldml-keyboard/ldml-keyboard-xml-reader.ts | 4 +-- 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/developer/src/common/web/utils/src/line-utils.ts b/developer/src/common/web/utils/src/line-utils.ts index 5c9a4dec31..e496f9b6ac 100644 --- a/developer/src/common/web/utils/src/line-utils.ts +++ b/developer/src/common/web/utils/src/line-utils.ts @@ -18,7 +18,13 @@ export type LineList = number[]; export class LineFinder { private list: LineList; constructor(public text: string) { - this.list = LineFinder.textToLines(text); + } + + public getLineList(): LineList { + if (!this.list) { + this.list = LineFinder.textToLines(this.text); + } + return this.list; } /** @@ -27,7 +33,7 @@ export class LineFinder { * @returns line:column information */ public findOffset(offset: number): LineColumn { - return LineFinder.offsetToLineColumn(offset, this.list); + return LineFinder.offsetToLineColumn(offset, this.getLineList()); } /** @@ -58,4 +64,30 @@ export class LineFinder { } } +/** + * Interface for a class which can receive file contents, + * keyed by the filename + */ +export interface FileConsumer { + /** + * @param filename name of the file + * @param contents string contents of the file + */ + addFile(filename: string, contents: string): void; +} + +/** Cache of LineFinder elements, organized by filename */ +export class LineFinderCache implements FileConsumer { + contentsCache: Map = new Map(); + + /** add or update a source file */ + addFile(filename: string, contents: string): void { + this.contentsCache.set(filename, new LineFinder(contents)); + } + + getByFilename(filename: string) : LineFinder | undefined { + const lf = this.contentsCache.get(filename); + return lf; + } +} diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts index d155cf27d2..b97e8908fd 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts @@ -6,20 +6,19 @@ * Implementation of a hook on LDML events */ -import { CompilerEvent, EventResolver, LineFinder } from "../../index.js"; -export class LdmlEventResolver implements EventResolver { +import { LineFinderCache, FileConsumer } from "../../line-utils.js"; +import { CompilerEvent, EventResolver } from "../../index.js"; +export class LineFinderEventResolver implements EventResolver, FileConsumer { + private lfcache = new LineFinderCache(); - contentsCache: Map = new Map(); - - /** add or update a source file */ - addFile(filename: string, xml: any) { - this.contentsCache.set(filename, new LineFinder(xml)); + addFile(filename: string, contents: string): void { + this.lfcache.addFile(filename, contents); } /** resolve a CompilerEvent by expanding the line numbers */ resolve(event: CompilerEvent) { if (event.offset && !event.line && event.filename) { - const lf = this.contentsCache.get(event.filename); + const lf = this.lfcache.getByFilename(event.filename); const loc = lf.findOffset(event.offset); event.line = loc.line; event.column = loc.column; diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index 2d099849cd..9e1d1e2e06 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -11,7 +11,7 @@ import { constants } from '@keymanapp/ldml-keyboard-constants'; import { LDMLKeyboardTestDataXMLSourceFile, LKTTest, LKTTests } from './ldml-keyboard-testdata-xml.js'; import { KeymanXMLReader } from '@keymanapp/developer-utils'; import boxXmlArray = util.boxXmlArray; -import { LdmlEventResolver } from './eventresolver.js'; +import { LineFinderEventResolver } from './eventresolver.js'; import { XML_FILENAME_SYMBOL } from '../../xml-utils.js'; interface NameAndProps { @@ -28,7 +28,7 @@ export class LDMLKeyboardXMLSourceFileReaderOptions { }; export class LDMLKeyboardXMLSourceFileReader { - static eventResolver: LdmlEventResolver = new LdmlEventResolver(); + static eventResolver: LineFinderEventResolver = new LineFinderEventResolver(); constructor(private options: LDMLKeyboardXMLSourceFileReaderOptions, private callbacks : CompilerCallbacks) { } From ad0ea8678b9337f3823eaa04d992347184dc8b53 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 8 May 2025 17:29:30 -0500 Subject: [PATCH 21/27] feat(developer): update for line numbers - added new DelegatingCompilerCallbacks class - added ResolvingCompilerCallbacks which does event resolution - added the XML file name metadata to *every object* after XML parse. Fixes: #10622 --- .../web/utils/src/compiler-callbacks.ts | 106 +++++++++++------- developer/src/common/web/utils/src/index.ts | 6 +- .../src/common/web/utils/src/line-utils.ts | 19 ++++ .../src/types/ldml-keyboard/eventresolver.ts | 28 ----- .../ldml-keyboard/ldml-keyboard-xml-reader.ts | 10 +- .../src/common/web/utils/src/xml-utils.ts | 17 +++ .../src/kmc-ldml/src/compiler/compiler.ts | 16 ++- .../src/compiler/ldml-compiler-messages.ts | 3 +- .../src/commands/buildClasses/BuildProject.ts | 2 +- 9 files changed, 124 insertions(+), 83 deletions(-) delete mode 100644 developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts diff --git a/developer/src/common/web/utils/src/compiler-callbacks.ts b/developer/src/common/web/utils/src/compiler-callbacks.ts index aabf1794e5..78187b9a58 100644 --- a/developer/src/common/web/utils/src/compiler-callbacks.ts +++ b/developer/src/common/web/utils/src/compiler-callbacks.ts @@ -125,14 +125,64 @@ export interface CompilerFileSystemAsyncCallbacks { resolveFilename(baseFilename: string, filename: string): string; } +/** a CompilerCallbacks implementation that delegates to a parent */ +export class DelegatingCompilerCallbacks implements CompilerCallbacks { + constructor(protected options: CompilerCallbackOptions, protected parent: CompilerCallbacks) { + } + + loadFile(filename: string): Uint8Array { + return this.parent.loadFile(filename); + } + + fileSize(filename: string): number { + return this.parent.fileSize(filename); + } + + isDirectory(filename: string): boolean { + return this.parent.isDirectory(filename); + } + + get path(): CompilerPathCallbacks { + return this.parent.path; + } + + get fs(): CompilerFileSystemCallbacks { + return this.parent.fs; + } + + get net(): CompilerNetAsyncCallbacks { + return this.parent.net; + } + + get fsAsync(): CompilerFileSystemAsyncCallbacks { + return this.parent.fsAsync; + } + + resolveFilename(baseFilename: string, filename: string): string { + return this.parent.resolveFilename(baseFilename, filename); + } + + debug(msg: string): void { + return this.parent.debug(msg); + } + + fileURLToPath(url: string | URL): string { + return this.parent.fileURLToPath(url); + } + + reportMessage(event: CompilerEvent) { + this.parent.reportMessage(event); + } +} /** * Wrapper class for CompilerCallbacks for a given input file */ -export class CompilerFileCallbacks implements CompilerCallbacks { +export class CompilerFileCallbacks extends DelegatingCompilerCallbacks { messages: CompilerEvent[] = []; - constructor(private filename: string, private options: CompilerCallbackOptions, private parent: CompilerCallbacks) { + constructor(private filename: string, options: CompilerCallbackOptions, parent: CompilerCallbacks) { + super(options, parent); } /** @@ -189,38 +239,6 @@ export class CompilerFileCallbacks implements CompilerCallbacks { this.messages = []; } - loadFile(filename: string): Uint8Array { - return this.parent.loadFile(filename); - } - - fileSize(filename: string): number { - return this.parent.fileSize(filename); - } - - isDirectory(filename: string): boolean { - return this.parent.isDirectory(filename); - } - - get path(): CompilerPathCallbacks { - return this.parent.path; - } - - get fs(): CompilerFileSystemCallbacks { - return this.parent.fs; - } - - get net(): CompilerNetAsyncCallbacks { - return this.parent.net; - } - - get fsAsync(): CompilerFileSystemAsyncCallbacks { - return this.parent.fsAsync; - } - - resolveFilename(baseFilename: string, filename: string): string { - return this.parent.resolveFilename(baseFilename, filename); - } - reportMessage(event: CompilerEvent): void { const disable = CompilerFileCallbacks.applyMessageOverridesToEvent(event, this.options.messageOverrides); this.messages.push(event); @@ -228,14 +246,6 @@ export class CompilerFileCallbacks implements CompilerCallbacks { this.parent.reportMessage({ filename: this.filename, ...event }); } } - - debug(msg: string): void { - return this.parent.debug(msg); - } - - fileURLToPath(url: string | URL): string { - return this.parent.fileURLToPath(url); - } } export class DefaultCompilerFileSystemAsyncCallbacks implements CompilerFileSystemAsyncCallbacks { @@ -261,3 +271,15 @@ export class DefaultCompilerFileSystemAsyncCallbacks implements CompilerFileSyst return this.owner.resolveFilename(baseFilename, filename); } } + +/** a CompilerCallbacks that applies the EventResolver to any reported message */ +export class ResolvingCompilerCallbacks extends DelegatingCompilerCallbacks { + constructor(private eventResolver: EventResolver, options: CompilerCallbackOptions, parent: CompilerCallbacks) { + super(options, parent); + } + + reportMessage(event: CompilerEvent) { + this.eventResolver.resolve(event); + this.parent.reportMessage(event); + } +} diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index 822dbffed1..2224463408 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -42,6 +42,8 @@ export { DefaultCompilerFileSystemAsyncCallbacks, EventResolver, NullEventResolver, + DelegatingCompilerCallbacks, + ResolvingCompilerCallbacks, } from './compiler-callbacks.js'; export { defaultCompilerOptions, CompilerBaseOptions, CompilerOptions, CompilerEvent, CompilerErrorNamespace, @@ -67,7 +69,7 @@ export { DeveloperUtilsMessages } from './developer-utils-messages.js'; export * as SourceFilenamePatterns from './source-filename-patterns.js'; export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, KeymanXMLMetadata, XML_FILENAME_SYMBOL } from './xml-utils.js'; export { SymbolUtils } from './symbol-utils.js'; -export { LineFinder, LineColumn } from './line-utils.js'; +export { LineFinder, LineColumn, LineFinderEventResolver } from './line-utils.js'; export * as GitHubUrls from './github-urls.js'; export * as CloudUrls from './cloud-urls.js'; @@ -75,4 +77,4 @@ export { getFontFamily, getFontFamilySync } from './font-family.js'; export * as ValidIds from './valid-ids.js'; -export * as ProjectLoader from './project-loader.js'; \ No newline at end of file +export * as ProjectLoader from './project-loader.js'; diff --git a/developer/src/common/web/utils/src/line-utils.ts b/developer/src/common/web/utils/src/line-utils.ts index e496f9b6ac..95f5744c55 100644 --- a/developer/src/common/web/utils/src/line-utils.ts +++ b/developer/src/common/web/utils/src/line-utils.ts @@ -6,6 +6,8 @@ * Abstraction for line number processing */ +import { CompilerEvent, EventResolver } from "./index.js"; + /** line number with an optional column number */ export interface LineColumn { line: number; @@ -90,4 +92,21 @@ export class LineFinderCache implements FileConsumer { return lf; } } +export class LineFinderEventResolver implements EventResolver, FileConsumer { + private lfcache = new LineFinderCache(); + addFile(filename: string, contents: string): void { + this.lfcache.addFile(filename, contents); + } + + /** resolve a CompilerEvent by expanding the line numbers */ + resolve(event: CompilerEvent) { + if (event.offset && !event.line && event.filename) { + const lf = this.lfcache.getByFilename(event.filename); + const loc = lf.findOffset(event.offset); + event.line = loc.line; + event.column = loc.column; + } + return event; + } +} diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts deleted file mode 100644 index b97e8908fd..0000000000 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Keyman is copyright (C) SIL Global. MIT License. - * - * Created by srl on 2025-03-03 - * - * Implementation of a hook on LDML events - */ - -import { LineFinderCache, FileConsumer } from "../../line-utils.js"; -import { CompilerEvent, EventResolver } from "../../index.js"; -export class LineFinderEventResolver implements EventResolver, FileConsumer { - private lfcache = new LineFinderCache(); - - addFile(filename: string, contents: string): void { - this.lfcache.addFile(filename, contents); - } - - /** resolve a CompilerEvent by expanding the line numbers */ - resolve(event: CompilerEvent) { - if (event.offset && !event.line && event.filename) { - const lf = this.lfcache.getByFilename(event.filename); - const loc = lf.findOffset(event.offset); - event.line = loc.line; - event.column = loc.column; - } - return event; - } -} diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index 9e1d1e2e06..f581d661cb 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -11,7 +11,7 @@ import { constants } from '@keymanapp/ldml-keyboard-constants'; import { LDMLKeyboardTestDataXMLSourceFile, LKTTest, LKTTests } from './ldml-keyboard-testdata-xml.js'; import { KeymanXMLReader } from '@keymanapp/developer-utils'; import boxXmlArray = util.boxXmlArray; -import { LineFinderEventResolver } from './eventresolver.js'; +import { LineFinderEventResolver } from '../../index.js'; import { XML_FILENAME_SYMBOL } from '../../xml-utils.js'; interface NameAndProps { @@ -28,7 +28,9 @@ export class LDMLKeyboardXMLSourceFileReaderOptions { }; export class LDMLKeyboardXMLSourceFileReader { - static eventResolver: LineFinderEventResolver = new LineFinderEventResolver(); + /** for resolving messages involving line numbers */ + eventResolver: LineFinderEventResolver = new LineFinderEventResolver(); + constructor(private options: LDMLKeyboardXMLSourceFileReaderOptions, private callbacks : CompilerCallbacks) { } @@ -40,7 +42,7 @@ export class LDMLKeyboardXMLSourceFileReader { readFile(path: string): Uint8Array { const data = this.callbacks.loadFile(path); if (data) { - LDMLKeyboardXMLSourceFileReader.eventResolver.addFile(path, new TextDecoder().decode(data)); + this.eventResolver.addFile(path, new TextDecoder().decode(data)); } return data; } @@ -260,7 +262,7 @@ export class LDMLKeyboardXMLSourceFileReader { } const importXml: any = this.loadUnboxed(importData); // TODO-LDML: have to load as any because it is an arbitrary part const importRootNode = importXml[subtag]; // e.g. - LDMLKeyboardXMLSourceFileReader.eventResolver.addFile(importPath, new TextDecoder().decode(importData)); // TODO: double decode + this.eventResolver.addFile(importPath, new TextDecoder().decode(importData)); // TODO: double decode // importXml will have one property: the root element. if (!importRootNode) { diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 9ec8872975..f83ca75dc7 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -146,6 +146,23 @@ export class KeymanXMLReader { return m; } + /** set metadata on this and children with the default filename - if not already set */ + public static setDefaultFilename(data: any, filename: string) { + if (!data || !filename) return; + if (typeof data === 'object') { + const m = KeymanXMLReader.getMetaData(data) || {}; + if (!m[XML_FILENAME_SYMBOL]) { + (m as any)[XML_FILENAME_SYMBOL] = filename; + KeymanXMLReader.setMetaData(data, m); + } + if (Array.isArray(data)) { + data.forEach(e => KeymanXMLReader.setDefaultFilename(e, filename)); + } else for(const k of Object.keys(data)) { + KeymanXMLReader.setDefaultFilename(data[k], filename); + } + } + } + /** move `{ $abc: 4 }` into `{ $: { abc: 4 } }` */ private static fixupDollarAttributes(data: any) : any { if (typeof data === 'object') { diff --git a/developer/src/kmc-ldml/src/compiler/compiler.ts b/developer/src/kmc-ldml/src/compiler/compiler.ts index 1163bb77d8..4564230e04 100644 --- a/developer/src/kmc-ldml/src/compiler/compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/compiler.ts @@ -8,7 +8,9 @@ import { CompilerCallbacks, KeymanCompiler, KeymanCompilerResult, KeymanCompilerArtifacts, defaultCompilerOptions, LDMLKeyboardXMLSourceFileReader, LDMLKeyboard, LDMLKeyboardTestDataXMLSourceFile, KMXBuilder, - KeymanCompilerArtifactOptional + KeymanCompilerArtifactOptional, + ResolvingCompilerCallbacks, + KeymanXMLReader, } from "@keymanapp/developer-utils"; import { LdmlCompilerOptions } from './ldml-compiler-options.js'; import { LdmlCompilerMessages } from './ldml-compiler-messages.js'; @@ -94,6 +96,7 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { // uset parser private usetparser?: LdmlKeyboardTypes.UnicodeSetParser = undefined; + private reader?: LDMLKeyboardXMLSourceFileReader; /** * Initialize the compiler, including loading the WASM host for uset parsing. @@ -105,7 +108,9 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { */ async init(callbacks: CompilerCallbacks, options: LdmlCompilerOptions): Promise { this.options = { ...options }; - this.callbacks = callbacks; + this.reader = new LDMLKeyboardXMLSourceFileReader(this.options.readerOptions, callbacks); + // wrap the callbacks so that the eventresolver is called + this.callbacks = new ResolvingCompilerCallbacks(this.reader.eventResolver, this.options, callbacks); return true; } @@ -238,7 +243,7 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { * @returns the source file, or null if invalid */ public load(filename: string): LDMLKeyboardXMLSourceFile | null { - const reader = new LDMLKeyboardXMLSourceFileReader(this.options.readerOptions, this.callbacks); + const reader = this.reader; // load the file from disk into a string const data = reader.readFile(filename); if (!data) { @@ -261,6 +266,9 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { return null; } + // record the default filename - for error reporting. + KeymanXMLReader.setDefaultFilename(source, filename); + return source; } @@ -272,7 +280,7 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { * @returns the source file, or null if invalid */ public loadTestData(filename: string): LDMLKeyboardTestDataXMLSourceFile | null { - const reader = new LDMLKeyboardXMLSourceFileReader(this.options.readerOptions, this.callbacks); + const reader = this.reader; const data = this.callbacks.loadFile(filename); if (!data) { this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidFile({ errorText: 'Unable to read XML file' })); diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 6deeedfceb..56111be865 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -1,5 +1,5 @@ import { util } from "@keymanapp/common-types"; -import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_FILENAME_SYMBOL, CompilerEvent, KeymanXMLReader, LDMLKeyboardXMLSourceFileReader } from '@keymanapp/developer-utils'; +import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_FILENAME_SYMBOL, CompilerEvent, KeymanXMLReader } from '@keymanapp/developer-utils'; // const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.LdmlKeyboardCompiler; @@ -13,7 +13,6 @@ const SevErrorTransform = SevError | 0xF00; const mx = (x: any, code: number, message: string, detail?: string): CompilerEvent => { let evt = m(code, message, detail); // raw message evt = LdmlCompilerMessages.offset(evt, x); // with offset - evt = LDMLKeyboardXMLSourceFileReader.eventResolver.resolve(evt); // offset expanded to line number return evt; }; diff --git a/developer/src/kmc/src/commands/buildClasses/BuildProject.ts b/developer/src/kmc/src/commands/buildClasses/BuildProject.ts index 82e1347aa5..434325e5cf 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildProject.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildProject.ts @@ -132,4 +132,4 @@ class ProjectBuilder { return result; } -} \ No newline at end of file +} From 307288dc3b043a78c5859e50b80259b5f6a4262c Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 8 May 2025 17:43:19 -0500 Subject: [PATCH 22/27] chore(developer): deps: fast-xml-parser@5.2.2 My line number changes were merged into the main branch Fixes: #10622 --- developer/src/common/web/utils/package.json | 2 +- package-lock.json | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/developer/src/common/web/utils/package.json b/developer/src/common/web/utils/package.json index d0fff064d7..20f71e6e9a 100644 --- a/developer/src/common/web/utils/package.json +++ b/developer/src/common/web/utils/package.json @@ -11,7 +11,7 @@ "dependencies": { "@keymanapp/common-types": "*", "eventemitter3": "^5.0.0", - "fast-xml-parser": "github:NaturalIntelligence/fast-xml-parser#metadata", + "fast-xml-parser": "^5.2.2", "path-browserify": "^1.0.1", "restructure": "^3.0.1", "sax": ">=0.6.0", diff --git a/package-lock.json b/package-lock.json index 5f407f851f..4c811b578e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -341,7 +341,7 @@ "dependencies": { "@keymanapp/common-types": "*", "eventemitter3": "^5.0.0", - "fast-xml-parser": "github:NaturalIntelligence/fast-xml-parser#metadata", + "fast-xml-parser": "^5.2.2", "path-browserify": "^1.0.1", "restructure": "^3.0.1", "sax": ">=0.6.0", @@ -9124,16 +9124,18 @@ "dev": true }, "node_modules/fast-xml-parser": { - "version": "5.1.0", - "resolved": "git+ssh://git@github.com/NaturalIntelligence/fast-xml-parser.git#ab8d40a3077f109f34d02d05653146b8621bd255", + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.2.2.tgz", + "integrity": "sha512-ZaCmslH75Jkfowo/x44Uq8KT5SutC5BFxHmY61nmTXPccw11PVuIXKUqC2hembMkJ3nPwTkQESXiUlsKutCbMg==", "funding": [ { "type": "github", "url": "https://github.com/sponsors/NaturalIntelligence" } ], + "license": "MIT", "dependencies": { - "strnum": "^2.0.5" + "strnum": "^2.1.0" }, "bin": { "fxparser": "src/cli/cli.js" @@ -13991,9 +13993,9 @@ "link": true }, "node_modules/strnum": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.0.5.tgz", - "integrity": "sha512-YAT3K/sgpCUxhxNMrrdhtod3jckkpYwH6JAuwmUdXZsmzH1wUyzTMrrK2wYCEEqlKwrWDd35NeuUkbBy/1iK+Q==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.1.0.tgz", + "integrity": "sha512-w0S//9BqZZGw0L0Y8uLSelFGnDJgTyyNQLmSlPnVz43zPAiqu3w4t8J8sDqqANOGeZIZ/9jWuPguYcEnsoHv4A==", "funding": [ { "type": "github", From 8e2eb498e3b0b9a7030fb7aa1cfcd23550affd20 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 9 May 2025 11:41:23 -0500 Subject: [PATCH 23/27] feat(developer): update for line numbers - reduce whitespace churn Fixes: #10622 --- developer/src/common/web/test-helpers/TestCompilerCallbacks.ts | 2 +- developer/src/common/web/utils/src/compiler-callbacks.ts | 2 +- developer/src/common/web/utils/src/index.ts | 2 +- developer/src/common/web/utils/src/line-utils.ts | 2 +- .../utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts | 2 +- developer/src/kmc/src/commands/buildClasses/BuildProject.ts | 2 +- developer/src/kmc/src/util/NodeCompilerCallbacks.ts | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts b/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts index 893e3363ed..04e5734f18 100644 --- a/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts +++ b/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts @@ -2,7 +2,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { CompilerEvent, CompilerCallbacks, CompilerPathCallbacks, CompilerFileSystemCallbacks, CompilerError, CompilerNetAsyncCallbacks, DefaultCompilerFileSystemAsyncCallbacks, - CompilerFileSystemAsyncCallbacks, + CompilerFileSystemAsyncCallbacks, CompilerErrorSeverity} from '@keymanapp/developer-utils'; import { fileURLToPath } from 'url'; diff --git a/developer/src/common/web/utils/src/compiler-callbacks.ts b/developer/src/common/web/utils/src/compiler-callbacks.ts index 78187b9a58..dda2c1485b 100644 --- a/developer/src/common/web/utils/src/compiler-callbacks.ts +++ b/developer/src/common/web/utils/src/compiler-callbacks.ts @@ -49,7 +49,7 @@ export interface CompilerCallbacks { get net(): CompilerNetAsyncCallbacks; /** - * Resolves a file path relative to the baseFilename. + * Resolves a file path relative to the baseFilename * @param baseFilename * @param filename */ diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index 2224463408..fd442356a7 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -77,4 +77,4 @@ export { getFontFamily, getFontFamilySync } from './font-family.js'; export * as ValidIds from './valid-ids.js'; -export * as ProjectLoader from './project-loader.js'; +export * as ProjectLoader from './project-loader.js'; \ No newline at end of file diff --git a/developer/src/common/web/utils/src/line-utils.ts b/developer/src/common/web/utils/src/line-utils.ts index 95f5744c55..2d2282db41 100644 --- a/developer/src/common/web/utils/src/line-utils.ts +++ b/developer/src/common/web/utils/src/line-utils.ts @@ -38,7 +38,7 @@ export class LineFinder { return LineFinder.offsetToLineColumn(offset, this.getLineList()); } - /** + /** * preprocess text to turn it into arrays of line lengths. * This is effectively a 1-based line length, since line 0 has length of * 0. diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index f581d661cb..8569e617b9 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -31,7 +31,7 @@ export class LDMLKeyboardXMLSourceFileReader { /** for resolving messages involving line numbers */ eventResolver: LineFinderEventResolver = new LineFinderEventResolver(); - constructor(private options: LDMLKeyboardXMLSourceFileReaderOptions, private callbacks : CompilerCallbacks) { + constructor(private options: LDMLKeyboardXMLSourceFileReaderOptions, private callbacks : CompilerCallbacks) { } static get defaultImportsURL(): [string,string] { diff --git a/developer/src/kmc/src/commands/buildClasses/BuildProject.ts b/developer/src/kmc/src/commands/buildClasses/BuildProject.ts index 434325e5cf..82e1347aa5 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildProject.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildProject.ts @@ -132,4 +132,4 @@ class ProjectBuilder { return result; } -} +} \ No newline at end of file diff --git a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts index 2ec3854c3e..a92314aa5f 100644 --- a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts +++ b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { platform } from 'os'; -import { CompilerCallbacks, CompilerEvent, +import { CompilerCallbacks, CompilerEvent, CompilerPathCallbacks, CompilerFileSystemCallbacks, compilerLogLevelToSeverity, CompilerErrorSeverity, CompilerError, From 61943923100c67beef99cae33099b429b2860732 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 9 May 2025 13:07:51 -0500 Subject: [PATCH 24/27] feat(developer): update for line numbers - reduce whitespace churn - other review comments Fixes: #10622 Co-authored-by: Marc Durdin --- developer/src/common/web/utils/src/index.ts | 4 +- .../src/common/web/utils/src/line-utils.ts | 124 ++++++++++-------- .../src/common/web/utils/src/symbol-utils.ts | 38 +++--- .../ldml-keyboard/ldml-keyboard-xml-reader.ts | 11 +- .../common/web/utils/test/line-utils.tests.ts | 22 ++-- .../src/kmc-ldml/src/compiler/compiler.ts | 2 +- developer/src/kmc-ldml/src/compiler/keys.ts | 4 +- .../src/compiler/ldml-compiler-messages.ts | 32 ++++- developer/src/kmc-ldml/test/keys.tests.ts | 4 +- 9 files changed, 140 insertions(+), 101 deletions(-) diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index fd442356a7..937cd98642 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -69,7 +69,7 @@ export { DeveloperUtilsMessages } from './developer-utils-messages.js'; export * as SourceFilenamePatterns from './source-filename-patterns.js'; export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, KeymanXMLMetadata, XML_FILENAME_SYMBOL } from './xml-utils.js'; export { SymbolUtils } from './symbol-utils.js'; -export { LineFinder, LineColumn, LineFinderEventResolver } from './line-utils.js'; +export * as LineUtils from './line-utils.js'; export * as GitHubUrls from './github-urls.js'; export * as CloudUrls from './cloud-urls.js'; @@ -77,4 +77,4 @@ export { getFontFamily, getFontFamilySync } from './font-family.js'; export * as ValidIds from './valid-ids.js'; -export * as ProjectLoader from './project-loader.js'; \ No newline at end of file +export * as ProjectLoader from './project-loader.js'; diff --git a/developer/src/common/web/utils/src/line-utils.ts b/developer/src/common/web/utils/src/line-utils.ts index 2d2282db41..2e1cced292 100644 --- a/developer/src/common/web/utils/src/line-utils.ts +++ b/developer/src/common/web/utils/src/line-utils.ts @@ -10,60 +10,66 @@ import { CompilerEvent, EventResolver } from "./index.js"; /** line number with an optional column number */ export interface LineColumn { - line: number; - column?: number; + line: number; + column?: number; } export type LineList = number[]; /** a LineFinder is able to convert from an offset to a line/column */ export class LineFinder { - private list: LineList; - constructor(public text: string) { - } + private list: LineList; + constructor(public text: string) { + } - public getLineList(): LineList { - if (!this.list) { - this.list = LineFinder.textToLines(this.text); - } - return this.list; + public getLineList(): LineList { + if (!this.list) { + this.list = LineFinder.textToLines(this.text); } + return this.list; + } - /** - * Convert an offset into line:column - * @param offset input offset into the text - * @returns line:column information - */ - public findOffset(offset: number): LineColumn { - return LineFinder.offsetToLineColumn(offset, this.getLineList()); - } + /** + * Convert an offset into line:column + * @param offset input offset into the text + * @returns line:column information + */ + public findOffset(offset: number): LineColumn { + return LineFinder.offsetToLineColumn(offset, this.getLineList()); + } - /** - * preprocess text to turn it into arrays of line lengths. - * This is effectively a 1-based line length, since line 0 has length of - * 0. - */ - public static textToLines(text: string): LineList { - return [ - 0, // "line 0" is empty - ...text.replaceAll("\r\n", "\n").split("\n") - .map(l => l.length + 1) // line length (counting the trailing newline) - ]; - } + /** + * Preprocess text to turn it into arrays of line lengths. + * This is effectively a 1-based line length, since line 0 has length of + * 0. + * Note that the fast-xml-parser converts \r\n to \n before processing, + * and this function does as well. So, any offsets passed in will need to + * have that taken into consideration. + */ + public static textToLines(text: string): LineList { + return [ + 0, // "line 0" is empty + ...text.replaceAll("\r\n", "\n").split("\n") + .map(l => l.length + 1) // line length (counting the trailing newline) + ]; + } - /** - * convert a line number array to a line/column - */ - public static offsetToLineColumn(offset: number, list: LineList): LineColumn { - for (let line = 1; line < list.length; line++) { // 1-based (assume the first row is 0) - if (list[line] > offset) { - return { line, column: offset }; - } - offset = offset - (list[line]); // count newline at end - } - // default: line 0, error - return { line: 0 } + /** + * convert a line number array to a line/column. + * Note that the fast-xml-parser converts \r\n to \n before processing, + * and this function does as well. So, any offsets passed in will need to + * have that taken into consideration. + */ + public static offsetToLineColumn(offset: number, list: LineList): LineColumn { + for (let line = 1; line < list.length; line++) { // 1-based (assume the first row is 0) + if (list[line] > offset) { + return { line, column: offset }; + } + offset = offset - (list[line]); // count newline at end } + // default: line 0, error + return { line: 0 } + } } /** @@ -71,27 +77,33 @@ export class LineFinder { * keyed by the filename */ export interface FileConsumer { - /** - * @param filename name of the file - * @param contents string contents of the file - */ - addFile(filename: string, contents: string): void; + /** + * @param filename name of the file + * @param contents string contents of the file + */ + addFile(filename: string, contents: string): void; } /** Cache of LineFinder elements, organized by filename */ export class LineFinderCache implements FileConsumer { - contentsCache: Map = new Map(); + contentsCache: Map = new Map(); - /** add or update a source file */ - addFile(filename: string, contents: string): void { - this.contentsCache.set(filename, new LineFinder(contents)); - } + /** add or update a source file */ + addFile(filename: string, contents: string): void { + this.contentsCache.set(filename, new LineFinder(contents)); + } - getByFilename(filename: string) : LineFinder | undefined { - const lf = this.contentsCache.get(filename); - return lf; - } + getByFilename(filename: string): LineFinder | undefined { + const lf = this.contentsCache.get(filename); + return lf; + } } + +/** + * This EventResolver expands events that have a filename and offset set, + * by looking up the event.offset in the LineFinder list. + * The expanded event has the event.line and event.column fields set. + */ export class LineFinderEventResolver implements EventResolver, FileConsumer { private lfcache = new LineFinderCache(); diff --git a/developer/src/common/web/utils/src/symbol-utils.ts b/developer/src/common/web/utils/src/symbol-utils.ts index 971d02cc16..92cf07dd02 100644 --- a/developer/src/common/web/utils/src/symbol-utils.ts +++ b/developer/src/common/web/utils/src/symbol-utils.ts @@ -7,27 +7,27 @@ */ export class SymbolUtils { - /** - * Copy symbols from 'from' onto 'onto' - * @param onto object to copy onto - * @param from source for symbols - * @returns the onto object - */ - public static copySymbols(onto: T, from: any): T { - const o = onto as any; - for (const sym of Object.getOwnPropertySymbols(from)) { - o[sym] = from[sym]; - } - return onto; + /** + * Copy symbols shallowly from 'from' onto 'onto' + * @param onto object to copy onto + * @param from source for symbols + * @returns the onto object + */ + public static copySymbols(onto: T, from: any): T { + const o = onto as any; + for (const sym of Object.getOwnPropertySymbols(from)) { + o[sym] = from[sym]; } + return onto; + } - /** use Object.entries to remove all symbols */ - public static removeSymbols(from: T): T { - if (Array.isArray(from)) { - return from.map(o => SymbolUtils.removeSymbols(o)) as T; - } - if (typeof from !== "object") return from; - return Object.fromEntries(Object.entries(from).map(([k, v]) => ([k, SymbolUtils.removeSymbols(v)]))) as T; + /** use Object.entries to remove all symbols, recursively. */ + public static removeSymbols(from: T): T { + if (Array.isArray(from)) { + return from.map(o => SymbolUtils.removeSymbols(o)) as T; } + if (typeof from !== "object") return from; + return Object.fromEntries(Object.entries(from).map(([k, v]) => ([k, SymbolUtils.removeSymbols(v)]))) as T; + } } diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index 8569e617b9..6d06bd2546 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -9,9 +9,9 @@ import { CompilerCallbacks } from "../../compiler-callbacks.js"; import { LDMLKeyboardXMLSourceFile, LKImport, ImportStatus } from './ldml-keyboard-xml.js'; import { constants } from '@keymanapp/ldml-keyboard-constants'; import { LDMLKeyboardTestDataXMLSourceFile, LKTTest, LKTTests } from './ldml-keyboard-testdata-xml.js'; -import { KeymanXMLReader } from '@keymanapp/developer-utils'; +import { CompilerEvent, EventResolver, KeymanXMLReader } from '@keymanapp/developer-utils'; import boxXmlArray = util.boxXmlArray; -import { LineFinderEventResolver } from '../../index.js'; +import { LineFinderEventResolver } from '../../line-utils.js'; import { XML_FILENAME_SYMBOL } from '../../xml-utils.js'; interface NameAndProps { @@ -27,12 +27,15 @@ export class LDMLKeyboardXMLSourceFileReaderOptions { localImportsPaths: string[]; }; -export class LDMLKeyboardXMLSourceFileReader { +export class LDMLKeyboardXMLSourceFileReader implements EventResolver { /** for resolving messages involving line numbers */ - eventResolver: LineFinderEventResolver = new LineFinderEventResolver(); + private eventResolver: LineFinderEventResolver = new LineFinderEventResolver(); constructor(private options: LDMLKeyboardXMLSourceFileReaderOptions, private callbacks : CompilerCallbacks) { } + resolve(event: CompilerEvent): void { + this.eventResolver.resolve(event); + } static get defaultImportsURL(): [string,string] { return ['../import/', import.meta.url]; diff --git a/developer/src/common/web/utils/test/line-utils.tests.ts b/developer/src/common/web/utils/test/line-utils.tests.ts index 9c7b5da01c..ffbdba5b0e 100644 --- a/developer/src/common/web/utils/test/line-utils.tests.ts +++ b/developer/src/common/web/utils/test/line-utils.tests.ts @@ -13,16 +13,16 @@ import { LineFinder } from '../src/line-utils.js'; import { makePathToFixture } from './helpers/index.js'; describe(`LineFinder test`, () => { - const path = 'tran_fail-empty.xml'; + const path = 'tran_fail-empty.xml'; - it(`Should be able to process ${path}`, () => { - const xmlPath = makePathToFixture('xml', `${path}`); - const data = readFileSync(xmlPath, 'utf-8'); - assert.ok(data); - const lf = new LineFinder(data); - assert.ok(lf); - assert.deepEqual(lf.findOffset(40), {line: 3, column: 0}); - assert.deepEqual(lf.findOffset(136), {line: 4, column: 2}); - assert.deepEqual(lf.findOffset(186), {line: 8, column: 2}); - }); + it(`Should be able to process ${path}`, () => { + const xmlPath = makePathToFixture('xml', `${path}`); + const data = readFileSync(xmlPath, 'utf-8'); + assert.ok(data); + const lf = new LineFinder(data); + assert.ok(lf); + assert.deepEqual(lf.findOffset(40), { line: 3, column: 0 }); + assert.deepEqual(lf.findOffset(136), { line: 4, column: 2 }); + assert.deepEqual(lf.findOffset(186), { line: 8, column: 2 }); + }); }); diff --git a/developer/src/kmc-ldml/src/compiler/compiler.ts b/developer/src/kmc-ldml/src/compiler/compiler.ts index 4564230e04..b69e984728 100644 --- a/developer/src/kmc-ldml/src/compiler/compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/compiler.ts @@ -110,7 +110,7 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { this.options = { ...options }; this.reader = new LDMLKeyboardXMLSourceFileReader(this.options.readerOptions, callbacks); // wrap the callbacks so that the eventresolver is called - this.callbacks = new ResolvingCompilerCallbacks(this.reader.eventResolver, this.options, callbacks); + this.callbacks = new ResolvingCompilerCallbacks(this.reader, this.options, callbacks); return true; } diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index 6776057756..3b0d228e34 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -485,8 +485,8 @@ export class KeysCompiler extends SectionCompiler { } if (!keydef.output && !keydef.gap && !keydef.layerId) { this.callbacks.reportMessage( - LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch({ keyId: key }, - keydef + LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch(keydef, + { keyId: key }, ) ); valid = false; diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 56111be865..fba40669fb 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -9,13 +9,32 @@ const SevError = CompilerErrorSeverity.Error | CompilerErrorNamespace.LdmlKeyboa // sub-numberspace for transform errors const SevErrorTransform = SevError | 0xF00; -// like m() but takes an XML object for line numbers -const mx = (x: any, code: number, message: string, detail?: string): CompilerEvent => { +/** + * Any object with metadata, for line number errs. + * Could be for example an LKKeys or KMXPlus.KeysKeys object. + * Defined as 'any' here to reduce noise on the client side. + * @see {@link KeymanXMLReader.getMetaData()} + */ +type ObjectWithMetadata = any; + +/** + * Convenience function for constructing CompilerEvents with line numbers + * @param x Object to be used as a source for line number information + * @param code Unique numeric value of the event + * @param message A short description of the error presented to the user + * @param detail Detailed Markdown-formatted description of the error + * including references to documentation, remediation options. + * @see CompilerMessageSpec + * @returns + */ +function CompilerMessageObjectSpec(x: ObjectWithMetadata, code: number, message: string, detail?: string): CompilerEvent { let evt = m(code, message, detail); // raw message evt = LdmlCompilerMessages.offset(evt, x); // with offset return evt; }; +const mx = CompilerMessageObjectSpec; + /** * @internal */ @@ -28,7 +47,8 @@ export class LdmlCompilerMessages { static ERROR_HardwareLayerHasTooManyRows = SevError | 0x0003; static Error_HardwareLayerHasTooManyRows = (x: any) => mx(x, - this.ERROR_HardwareLayerHasTooManyRows, `'hardware' layer has too many rows`); + this.ERROR_HardwareLayerHasTooManyRows, `'hardware' layer has too many rows`, + ); static ERROR_RowOnHardwareLayerHasTooManyKeys = SevError | 0x0004; static Error_RowOnHardwareLayerHasTooManyKeys = (o:{row: number, hardware: string, modifiers: string}) => m(this.ERROR_RowOnHardwareLayerHasTooManyKeys, `Row #${def(o.row)} on 'hardware' ${def(o.hardware)} layer for modifier ${o.modifiers || 'none'} has too many keys`); @@ -105,8 +125,10 @@ export class LdmlCompilerMessages { m(this.ERROR_DisplayIsRepeated, `display ${LdmlCompilerMessages.outputOrKeyId(o)} has more than one display entry.`); static ERROR_KeyMissingToGapOrSwitch = SevError | 0x0011; - static Error_KeyMissingToGapOrSwitch = (o:{keyId: string}, x: any) => mx(x, - this.ERROR_KeyMissingToGapOrSwitch, `key id='${def(o.keyId)}' must have either output=, gap=, or layerId=.`); + static Error_KeyMissingToGapOrSwitch = (x: any, o:{keyId: string}) => mx(x, + this.ERROR_KeyMissingToGapOrSwitch, + `key id='${def(o.keyId)}' must have either output=, gap=, or layerId=.`, + ); static ERROR_ExcessHardware = SevError | 0x0012; static Error_ExcessHardware = (o:{formId: string}) => m(this.ERROR_ExcessHardware, diff --git a/developer/src/kmc-ldml/test/keys.tests.ts b/developer/src/kmc-ldml/test/keys.tests.ts index 73036be877..cd90def1cd 100644 --- a/developer/src/kmc-ldml/test/keys.tests.ts +++ b/developer/src/kmc-ldml/test/keys.tests.ts @@ -464,7 +464,9 @@ describe('keys.kmap', function () { const keys = await loadSectionFixture(KeysCompiler, 'sections/keys/invalid-key-missing-attrs.xml', compilerTestCallbacks, keysDependencies) as Keys; assert.isNull(keys); assert.equal(compilerTestCallbacks.messages.length, 1); - assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch({keyId: 'Q'}, withColumn(188))); + assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch(withColumn(188), + {keyId: 'Q'} + )); }); it('should accept layouts with gap/switch keys', async function() { const keys = await loadSectionFixture(KeysCompiler, 'sections/keys/gap-switch.xml', compilerTestCallbacks, keysDependencies) as Keys; From 8ec8fb189096559f21039ea0a6ba520e6aa1a865 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 9 May 2025 13:09:53 -0500 Subject: [PATCH 25/27] feat(developer): update for line numbers - annotate why we're calling SymbolUtils.removeSymbols() Fixes: #10622 --- .../src/common/web/utils/test/kpj/kpj-file-writer.tests.ts | 2 ++ .../src/common/web/utils/test/kps/kps-file-reader.tests.ts | 2 ++ developer/src/common/web/utils/test/kvks/kvks-file.tests.ts | 2 ++ 3 files changed, 6 insertions(+) diff --git a/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts b/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts index 425d76a6ef..d1a7cdddb1 100644 --- a/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts +++ b/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts @@ -22,6 +22,7 @@ describe('kpj-file-writer', function () { const writer = new KPJFileWriter(); const output = writer.write(project); + // Remove XML metadata symbols to reduce clutter for testing purposes const outputKpj = SymbolUtils.removeSymbols(reader.read(new TextEncoder().encode(output))); // The outputKpj may not contain all the fields from the inputKpj, only the @@ -61,6 +62,7 @@ describe('kpj-file-writer', function () { const writer = new KPJFileWriter(); const output = writer.write(project); + // Remove XML metadata symbols to reduce clutter for testing purposes const outputKpj = SymbolUtils.removeSymbols(reader.read(new TextEncoder().encode(output))); // The outputKpj may not contain all the fields from the inputKpj, only the diff --git a/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts b/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts index 78e69c6ef6..117a48a281 100644 --- a/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts +++ b/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts @@ -60,12 +60,14 @@ describe('kps-file-reader', function () { it('kps-file-reader should round-trip with kps-file-writer', function() { const input = fs.readFileSync(makePathToFixture('kps', 'khmer_angkor.kps')); const reader = new KpsFileReader(callbacks); + // Remove XML metadata symbols to reduce clutter for testing purposes const kps = SymbolUtils.removeSymbols(reader.read(input)); const writer = new KpsFileWriter(); const output = writer.write(kps); // Round Trip + // Remove XML metadata symbols to reduce clutter for testing purposes const kps2 = SymbolUtils.removeSymbols(reader.read(new TextEncoder().encode(output))); assert.deepEqual(kps2, kps); }); diff --git a/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts b/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts index 9f7cca504e..7e60119d28 100644 --- a/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts +++ b/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts @@ -64,6 +64,7 @@ describe('kvks-file-writer', function() { const input = fs.readFileSync(path); const reader = new KvksFileReader(); + // Remove XML metadata symbols to reduce clutter for testing purposes const kvksExpected = SymbolUtils.removeSymbols(reader.read(input)); const invalidVkeys: string[] = []; const vk = reader.transform(kvksExpected, invalidVkeys); @@ -75,6 +76,7 @@ describe('kvks-file-writer', function() { // We compare the (re)loaded data, because there may be // minor, irrelevant formatting differences in the emitted xml const kvks = reader.read(Buffer.from(output, 'utf8')); + // Remove XML metadata symbols to reduce clutter for testing purposes assert.deepEqual(SymbolUtils.removeSymbols(kvks), kvksExpected); }); }); From b23275d7e17d3b52f27de8e5aa17b9a8e034cec4 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 9 May 2025 13:32:15 -0500 Subject: [PATCH 26/27] feat(developer): fix imports in ldml-keyboard-xml-reader.ts Fixes: #10622 --- .../src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index 6d06bd2546..0a31bc051b 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -5,14 +5,14 @@ */ import { SchemaValidators, util } from '@keymanapp/common-types'; import { DeveloperUtilsMessages } from '../../developer-utils-messages.js'; -import { CompilerCallbacks } from "../../compiler-callbacks.js"; +import { CompilerCallbacks, EventResolver } from "../../compiler-callbacks.js"; +import { CompilerEvent } from "../../compiler-interfaces.js"; import { LDMLKeyboardXMLSourceFile, LKImport, ImportStatus } from './ldml-keyboard-xml.js'; import { constants } from '@keymanapp/ldml-keyboard-constants'; import { LDMLKeyboardTestDataXMLSourceFile, LKTTest, LKTTests } from './ldml-keyboard-testdata-xml.js'; -import { CompilerEvent, EventResolver, KeymanXMLReader } from '@keymanapp/developer-utils'; import boxXmlArray = util.boxXmlArray; import { LineFinderEventResolver } from '../../line-utils.js'; -import { XML_FILENAME_SYMBOL } from '../../xml-utils.js'; +import { XML_FILENAME_SYMBOL, KeymanXMLReader } from '../../xml-utils.js'; interface NameAndProps { '$'?: any; // content From 42d9c68e2b31daf02745937443df7a3cb8040e93 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 9 May 2025 16:43:59 -0500 Subject: [PATCH 27/27] feat(developer): fix Error() function - swap x, o because the 'api' call getMessageDetail() is going to call with {} Fixes: #10622 --- developer/src/kmc-ldml/src/compiler/keys.ts | 3 ++- .../kmc-ldml/src/compiler/ldml-compiler-messages.ts | 13 ++++++++----- developer/src/kmc-ldml/test/keys.tests.ts | 5 +++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index 3b0d228e34..7b79a3be83 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -485,8 +485,9 @@ export class KeysCompiler extends SectionCompiler { } if (!keydef.output && !keydef.gap && !keydef.layerId) { this.callbacks.reportMessage( - LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch(keydef, + LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch( { keyId: key }, + keydef, ) ); valid = false; diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index fba40669fb..28c5b52e1f 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -19,15 +19,15 @@ type ObjectWithMetadata = any; /** * Convenience function for constructing CompilerEvents with line numbers - * @param x Object to be used as a source for line number information * @param code Unique numeric value of the event * @param message A short description of the error presented to the user + * @param x Object to be used as a source for line number information * @param detail Detailed Markdown-formatted description of the error * including references to documentation, remediation options. * @see CompilerMessageSpec * @returns */ -function CompilerMessageObjectSpec(x: ObjectWithMetadata, code: number, message: string, detail?: string): CompilerEvent { +function CompilerMessageObjectSpec(code: number, message: string, x: ObjectWithMetadata, detail?: string): CompilerEvent { let evt = m(code, message, detail); // raw message evt = LdmlCompilerMessages.offset(evt, x); // with offset return evt; @@ -46,8 +46,10 @@ export class LdmlCompilerMessages { static Error_InvalidLocale = (o:{tag: string}) => m(this.ERROR_InvalidLocale, `Invalid BCP 47 locale form '${def(o.tag)}'`); static ERROR_HardwareLayerHasTooManyRows = SevError | 0x0003; - static Error_HardwareLayerHasTooManyRows = (x: any) => mx(x, - this.ERROR_HardwareLayerHasTooManyRows, `'hardware' layer has too many rows`, + static Error_HardwareLayerHasTooManyRows = (x: any) => mx( + this.ERROR_HardwareLayerHasTooManyRows, + `'hardware' layer has too many rows`, + x, ); static ERROR_RowOnHardwareLayerHasTooManyKeys = SevError | 0x0004; @@ -125,9 +127,10 @@ export class LdmlCompilerMessages { m(this.ERROR_DisplayIsRepeated, `display ${LdmlCompilerMessages.outputOrKeyId(o)} has more than one display entry.`); static ERROR_KeyMissingToGapOrSwitch = SevError | 0x0011; - static Error_KeyMissingToGapOrSwitch = (x: any, o:{keyId: string}) => mx(x, + static Error_KeyMissingToGapOrSwitch = (o:{keyId: string}, x: ObjectWithMetadata) => mx( this.ERROR_KeyMissingToGapOrSwitch, `key id='${def(o.keyId)}' must have either output=, gap=, or layerId=.`, + x, ); static ERROR_ExcessHardware = SevError | 0x0012; diff --git a/developer/src/kmc-ldml/test/keys.tests.ts b/developer/src/kmc-ldml/test/keys.tests.ts index cd90def1cd..3db4388a35 100644 --- a/developer/src/kmc-ldml/test/keys.tests.ts +++ b/developer/src/kmc-ldml/test/keys.tests.ts @@ -464,8 +464,9 @@ describe('keys.kmap', function () { const keys = await loadSectionFixture(KeysCompiler, 'sections/keys/invalid-key-missing-attrs.xml', compilerTestCallbacks, keysDependencies) as Keys; assert.isNull(keys); assert.equal(compilerTestCallbacks.messages.length, 1); - assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch(withColumn(188), - {keyId: 'Q'} + assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch( + {keyId: 'Q'}, + withColumn(188) )); }); it('should accept layouts with gap/switch keys', async function() {