From ec68e74bde8418472b9641ee33e9a88b63845ad9 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 27 Feb 2025 16:24:21 -0600 Subject: [PATCH 01/46] feat(developer): add experimental char-offset data Depends on PR https://github.com/NaturalIntelligence/fast-xml-parser/pull/728 so uses srl repository "temporarily" Fixes: #10622 --- developer/src/common/web/utils/package.json | 2 +- .../src/common/web/utils/src/xml-utils.ts | 24 ++++++++++-- .../common/web/utils/test/xml-utils.tests.ts | 24 +++++++++++- package-lock.json | 39 ++++++++----------- 4 files changed, 61 insertions(+), 28 deletions(-) diff --git a/developer/src/common/web/utils/package.json b/developer/src/common/web/utils/package.json index 4f066313f6..571123ec68 100644 --- a/developer/src/common/web/utils/package.json +++ b/developer/src/common/web/utils/package.json @@ -12,7 +12,7 @@ "@keymanapp/common-types": "*", "@sentry/node": "^7.57.0", "eventemitter3": "^5.0.0", - "fast-xml-parser": "^4.5.0", + "fast-xml-parser": "github:srl295/fast-xml-parser#srl295/line-number-593", "path-browserify": "^1.0.1", "restructure": "^3.0.1", "sax": ">=0.6.0", diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 69a039aa00..626233dd17 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -8,6 +8,8 @@ import { XMLParser, XMLBuilder } from 'fast-xml-parser'; +export const START_INDEX = new XMLParser().getStartIndexSymbol(); + export type KeymanXMLType = 'keyboard3' // LDML | 'keyboardTest3' // LDML @@ -128,6 +130,22 @@ export class KeymanXMLReader { public constructor(public type: KeymanXMLType) { } + /** + * Copy symbols from 'from' onto 'onto' + * This is used to propagate special symbols + * and XML + * @param onto object to copy onto + * @param from source for symbols + * @returns the onto object + */ + public static copySymbols(onto: T, from: any): T { + const o = onto as any; + for (const sym of Object.getOwnPropertySymbols(from)) { + o[sym] = from[sym]; + } + return onto; + } + /** move `{ $abc: 4 }` into `{ $: { abc: 4 } }` */ private static fixupDollarAttributes(data: any) : any { if (typeof data === 'object') { @@ -146,9 +164,9 @@ export class KeymanXMLReader { } }); if (attrs.length) { - e.push(['$', Object.fromEntries(attrs)]); + e.push(['$', this.copySymbols(Object.fromEntries(attrs), data)]); } - return Object.fromEntries(e); + return this.copySymbols(Object.fromEntries(e), data); } else { return data; } @@ -184,7 +202,7 @@ export class KeymanXMLReader { } } }); - return Object.fromEntries(e); + return this.copySymbols(Object.fromEntries(e), data); } else { return data; } diff --git a/developer/src/common/web/utils/test/xml-utils.tests.ts b/developer/src/common/web/utils/test/xml-utils.tests.ts index 1cc9724b27..65ee826f57 100644 --- a/developer/src/common/web/utils/test/xml-utils.tests.ts +++ b/developer/src/common/web/utils/test/xml-utils.tests.ts @@ -12,7 +12,7 @@ import { env } from 'node:process'; import { readFileSync, writeFileSync } from 'node:fs'; -import { KeymanXMLType, KeymanXMLReader, KeymanXMLWriter } from '../src/xml-utils.js'; +import { KeymanXMLType, KeymanXMLReader, KeymanXMLWriter, START_INDEX } from '../src/xml-utils.js'; import { makePathToFixture } from './helpers/index.js'; // if true, attempt to WRITE the fixtures @@ -160,3 +160,25 @@ describe(`XML Writer Test ${GEN_XML_FIXTURES && '(update mode!)' || ''}`, () => }); } }); + +describe(`XML Reader line number test`, () => { + const path = 'tran_fail-empty.xml'; + const xmlPath = makePathToFixture('xml', `${path}`); + const type: KeymanXMLType = 'keyboard3'; + it(`Should report line numbers on parse of ${type} ${path}`, () => { + const xml = readData(xmlPath); + assert.ok(xml, `Could not read ${xmlPath}`); + + const reader = new KeymanXMLReader(type); + assert.ok(reader); + + // now, parse. subsitute endings for Win + const actual = reader.parse(xml.replace(/\r\n/g, '\n')); + assert.ok(actual, `Parser failed on ${xmlPath}`); + + // now, assert char offset + assert.equal(actual.keyboard3[START_INDEX as any], 40); // index of element + assert.equal(actual.keyboard3.info[START_INDEX as any], 136); // index of etc + assert.equal(actual.keyboard3.transforms[START_INDEX as any], 186); + }); +}); diff --git a/package-lock.json b/package-lock.json index 4e340f8af4..0f62509ac5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -342,7 +342,7 @@ "@keymanapp/common-types": "*", "@sentry/node": "^7.57.0", "eventemitter3": "^5.0.0", - "fast-xml-parser": "^4.5.0", + "fast-xml-parser": "github:srl295/fast-xml-parser#srl295/line-number-593", "path-browserify": "^1.0.1", "restructure": "^3.0.1", "sax": ">=0.6.0", @@ -372,6 +372,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "developer/src/common/web/utils/node_modules/fast-xml-parser": { + "version": "4.1.3", + "resolved": "git+ssh://git@github.com/srl295/fast-xml-parser.git#d7601c31f39fe69e6612af79f9d778dee5dfcb42", + "license": "MIT", + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + }, + "funding": { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + }, "developer/src/common/web/utils/node_modules/js-yaml": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", @@ -9016,28 +9031,6 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, - "node_modules/fast-xml-parser": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.5.0.tgz", - "integrity": "sha512-/PlTQCI96+fZMAOLMZK4CWG1ItCbfZ/0jx7UIJFChPNrx7tcEgerUgWbeieCM9MfHInUDyK8DWYZ+YrywDJuTg==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - }, - { - "type": "paypal", - "url": "https://paypal.me/naturalintelligence" - } - ], - "license": "MIT", - "dependencies": { - "strnum": "^1.0.5" - }, - "bin": { - "fxparser": "src/cli/cli.js" - } - }, "node_modules/fastq": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", From b60c64c3d3f1b4ec072090215be95c5d0d7e8464 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 27 Feb 2025 16:42:41 -0600 Subject: [PATCH 02/46] feat(developer): test for line number extension to xml-utils Fixes: #10622 --- .../src/common/web/utils/src/xml-utils.ts | 30 +++++++++++++++---- .../common/web/utils/test/xml-utils.tests.ts | 10 +++++-- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 626233dd17..9fbebdf397 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -125,11 +125,29 @@ const GENERATOR_OPTIONS: KeymanXMLOptionsBag = { }, }; +export interface LineColumn { + line: number; + column?: number; +} + /** wrapper for XML parsing support */ export class KeymanXMLReader { public constructor(public type: KeymanXMLType) { } + public static offsetToLineColumn(offset: number, text: string) : LineColumn { + const lines: string[] = text.split("\n"); + for (let line = 1; line < lines.length + 1; line++) { // 1-based + const linestr = lines[line - 1]; + if (linestr.length < offset) { + offset = offset - (linestr.length + 1); // count newline at end + continue; + } + return { line, column: offset }; + } + return { line: lines.length + 1 }; + } + /** * Copy symbols from 'from' onto 'onto' * This is used to propagate special symbols @@ -138,13 +156,13 @@ export class KeymanXMLReader { * @param from source for symbols * @returns the onto object */ - public static copySymbols(onto: T, from: any): T { - const o = onto as any; - for (const sym of Object.getOwnPropertySymbols(from)) { - o[sym] = from[sym]; - } - return onto; + private static copySymbols(onto: T, from: any): T { + const o = onto as any; + for (const sym of Object.getOwnPropertySymbols(from)) { + o[sym] = from[sym]; } + return onto; + } /** move `{ $abc: 4 }` into `{ $: { abc: 4 } }` */ private static fixupDollarAttributes(data: any) : any { diff --git a/developer/src/common/web/utils/test/xml-utils.tests.ts b/developer/src/common/web/utils/test/xml-utils.tests.ts index 65ee826f57..bdcd18817d 100644 --- a/developer/src/common/web/utils/test/xml-utils.tests.ts +++ b/developer/src/common/web/utils/test/xml-utils.tests.ts @@ -166,19 +166,23 @@ describe(`XML Reader line number test`, () => { const xmlPath = makePathToFixture('xml', `${path}`); const type: KeymanXMLType = 'keyboard3'; it(`Should report line numbers on parse of ${type} ${path}`, () => { - const xml = readData(xmlPath); + let xml = readData(xmlPath); assert.ok(xml, `Could not read ${xmlPath}`); - + xml = xml.replace(/\r\n/g, '\n'); const reader = new KeymanXMLReader(type); assert.ok(reader); // now, parse. subsitute endings for Win - const actual = reader.parse(xml.replace(/\r\n/g, '\n')); + const actual = reader.parse(xml); assert.ok(actual, `Parser failed on ${xmlPath}`); // now, assert char offset assert.equal(actual.keyboard3[START_INDEX as any], 40); // index of element assert.equal(actual.keyboard3.info[START_INDEX as any], 136); // index of etc assert.equal(actual.keyboard3.transforms[START_INDEX as any], 186); + + assert.equal(KeymanXMLReader.offsetToLineColumn(actual.keyboard3[START_INDEX as any], xml), { line: 3, column: 0 }); + assert.equal(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.info[START_INDEX as any], xml), { line: 4, column: 2 }); + assert.equal(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.transforms[START_INDEX as any], xml), { line: 8, column: 2 }); }); }); From c4ca359d182d28cf900b15a9c0c13fb63e54bb04 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 28 Feb 2025 09:27:25 -0600 Subject: [PATCH 03/46] feat(developer): API for resolving line numbers for messages - add 'column' to CompilerEvent - Add a LdmlCompilerMessages.col() function to set the column number - add a LdmlCompilerMessages.resolveLineNumber() to map offsets to line number Fixes: #10622 --- .../web/utils/src/compiler-interfaces.ts | 6 +++++ developer/src/common/web/utils/src/index.ts | 2 +- developer/src/kmc-ldml/src/compiler/keys.ts | 5 +++- .../src/compiler/ldml-compiler-messages.ts | 26 ++++++++++++++++++- .../src/kmc/src/util/NodeCompilerCallbacks.ts | 11 ++++++++ 5 files changed, 47 insertions(+), 3 deletions(-) diff --git a/developer/src/common/web/utils/src/compiler-interfaces.ts b/developer/src/common/web/utils/src/compiler-interfaces.ts index 869e0165da..390eea2f7e 100644 --- a/developer/src/common/web/utils/src/compiler-interfaces.ts +++ b/developer/src/common/web/utils/src/compiler-interfaces.ts @@ -5,7 +5,13 @@ import { CompilerCallbacks } from "./compiler-callbacks.js"; */ export interface CompilerEvent { filename?: string; + /** line where a message applies */ line?: number; + /** + * column where a message applies. + * where line is unset, column is a file offset + */ + column?: number; code: number; message: string; /** diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index f3e23e79f2..1a56a735e8 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -65,7 +65,7 @@ export { UrlSubpathCompilerCallback } from './utils/UrlSubpathCompilerCallback.j export { CommonTypesMessages } from './common-messages.js'; export * as SourceFilenamePatterns from './source-filename-patterns.js'; -export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader } from './xml-utils.js'; +export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, START_INDEX } from './xml-utils.js'; export * as GitHubUrls from './github-urls.js'; export * as CloudUrls from './cloud-urls.js'; diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index d86d7e2bf5..372bbea891 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -446,7 +446,10 @@ export class KeysCompiler extends SectionCompiler { if (layer.row.length > keymap.length) { this.callbacks.reportMessage( - LdmlCompilerMessages.Error_HardwareLayerHasTooManyRows() + LdmlCompilerMessages.col( + LdmlCompilerMessages.Error_HardwareLayerHasTooManyRows(), + layer + ) ); valid = false; } diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 1a84b10937..b6293d0158 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -1,5 +1,5 @@ import { util } from "@keymanapp/common-types"; -import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def } from '@keymanapp/developer-utils'; +import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, START_INDEX, CompilerEvent, KeymanXMLReader } from '@keymanapp/developer-utils'; // const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.LdmlKeyboardCompiler; @@ -264,5 +264,29 @@ export class LdmlCompilerMessages { `Invalid transform to="${def(o.to)}": "${def(o.message)}"`, ); + /** + * Get a column number from o and set e's column field + * @param event a compiler event, such as from functions in this class + * @param o any object parsed from XML or with the START_INDEX symbol copied over + * @returns modified event object + */ + static col(event: CompilerEvent, o: any): CompilerEvent { + event.column = (o as any)[START_INDEX as any]; + return event; + } + /** + * Given an event with a column number but no line, resolve it into line:column + * @param event event to modify + * @param xml XML source + * @returns the modified event + */ + static resolveLineNumber(event: CompilerEvent, xml: string) : CompilerEvent { + if (event.column && !event.line && xml) { + const loc = KeymanXMLReader.offsetToLineColumn(event.column, xml); + event.line = loc.line; + event.column = loc.column; + } + return event; + } } diff --git a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts index f3cdf1e7b2..5b340633fa 100644 --- a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts +++ b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts @@ -15,6 +15,7 @@ import chalk from 'chalk'; import supportsColor from 'supports-color'; import { KeymanSentry } from '@keymanapp/developer-utils'; import { fileURLToPath } from 'url'; +import { LdmlCompilerMessages } from '@keymanapp/kmc-ldml'; const color = chalk.default; const severityColors: {[value in CompilerErrorSeverity]: chalk.Chalk} = { @@ -150,6 +151,16 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { event.filename = this.messageFilename; } + if (event.column && !event.line && event.filename) { + try { + // TODO: rereads XML every single time. + // Should cache unless the filename changes (this.messageFilename) + LdmlCompilerMessages.resolveLineNumber(event, this.fs.readFileSync(event.filename, "utf-8")); + } catch(e) { + console.error(e); // resolving line numbers + } + } + if(this.messageFilename != event.filename) { // Reset max message limit when a new file is being processed this.messageFilename = event.filename; From d38025b4a19418e80931059ea7767895842dba83 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 28 Feb 2025 10:32:05 -0600 Subject: [PATCH 04/46] feat(developer): API for resolving line numbers for messages - Only read the xml file once Fixes: #10622 --- .../src/kmc/src/util/NodeCompilerCallbacks.ts | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts index 5b340633fa..7c184c1417 100644 --- a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts +++ b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts @@ -46,6 +46,8 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { messages: CompilerEvent[] = []; messageCount = 0; messageFilename: string = ''; + /** cache of the contentes of the text */ + messageFiletext: string = ''; maxLogMessages = MaxMessagesDefault; constructor(private options: CompilerCallbackOptions) { @@ -56,6 +58,7 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { this.messages = []; this.messageCount = 0; this.messageFilename = ''; + this.messageFiletext = ''; } /** @@ -151,20 +154,29 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { event.filename = this.messageFilename; } - if (event.column && !event.line && event.filename) { - try { - // TODO: rereads XML every single time. - // Should cache unless the filename changes (this.messageFilename) - LdmlCompilerMessages.resolveLineNumber(event, this.fs.readFileSync(event.filename, "utf-8")); - } catch(e) { - console.error(e); // resolving line numbers - } - } - if(this.messageFilename != event.filename) { // Reset max message limit when a new file is being processed this.messageFilename = event.filename; this.messageCount = 0; + this.messageFiletext = ''; + } + + if (event.column && !event.line && event.filename) { + try { + let text = ''; + if (this.messageFilename == event.filename) { + text = this.messageFiletext; + } + if (text == '') { + text = this.fs.readFileSync(event.filename, "utf-8"); + if (this.messageFilename == event.filename) { + this.messageFiletext = text; + } + } + LdmlCompilerMessages.resolveLineNumber(event, text); + } catch(e) { + console.error(e); // resolving line numbers + } } const disable = CompilerFileCallbacks.applyMessageOverridesToEvent(event, this.options.messageOverrides); From 650042fb6e7ce831d7e14bd0c74d5fd886a88588 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 28 Feb 2025 10:53:21 -0600 Subject: [PATCH 05/46] feat(developer): API for resolving line numbers for messages - simplify the error call site again Fixes: #10622 --- developer/src/kmc-ldml/src/compiler/keys.ts | 5 +---- .../src/compiler/ldml-compiler-messages.ts | 14 ++++++++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index 372bbea891..35a931dd94 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -446,10 +446,7 @@ export class KeysCompiler extends SectionCompiler { if (layer.row.length > keymap.length) { this.callbacks.reportMessage( - LdmlCompilerMessages.col( - LdmlCompilerMessages.Error_HardwareLayerHasTooManyRows(), - layer - ) + LdmlCompilerMessages.Error_HardwareLayerHasTooManyRows(layer) ); valid = false; } diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index b6293d0158..6f8cc5fa3d 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -9,6 +9,9 @@ const SevError = CompilerErrorSeverity.Error | CompilerErrorNamespace.LdmlKeyboa // sub-numberspace for transform errors const SevErrorTransform = SevError | 0xF00; +// like m() but takes an XML object for line numbers +const mx = (x: any, code: number, message: string, detail?: string): CompilerEvent => LdmlCompilerMessages.col(m(code, message, detail), x); + /** * @internal */ @@ -20,7 +23,8 @@ export class LdmlCompilerMessages { static Error_InvalidLocale = (o:{tag: string}) => m(this.ERROR_InvalidLocale, `Invalid BCP 47 locale form '${def(o.tag)}'`); static ERROR_HardwareLayerHasTooManyRows = SevError | 0x0003; - static Error_HardwareLayerHasTooManyRows = () => m(this.ERROR_HardwareLayerHasTooManyRows, `'hardware' layer has too many rows`); + static Error_HardwareLayerHasTooManyRows = (x: any) => mx(x, + this.ERROR_HardwareLayerHasTooManyRows, `'hardware' layer has too many rows`); static ERROR_RowOnHardwareLayerHasTooManyKeys = SevError | 0x0004; static Error_RowOnHardwareLayerHasTooManyKeys = (o:{row: number, hardware: string, modifiers: string}) => m(this.ERROR_RowOnHardwareLayerHasTooManyKeys, `Row #${def(o.row)} on 'hardware' ${def(o.hardware)} layer for modifier ${o.modifiers || 'none'} has too many keys`); @@ -267,11 +271,13 @@ export class LdmlCompilerMessages { /** * Get a column number from o and set e's column field * @param event a compiler event, such as from functions in this class - * @param o any object parsed from XML or with the START_INDEX symbol copied over + * @param x any object parsed from XML or with the START_INDEX symbol copied over * @returns modified event object */ - static col(event: CompilerEvent, o: any): CompilerEvent { - event.column = (o as any)[START_INDEX as any]; + static col(event: CompilerEvent, x?: any): CompilerEvent { + if(x) { + event.column = (x as any)[START_INDEX as any]; + } return event; } From 819d9bf5620a65ae9ef9d7a0832bd17303480e72 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 28 Feb 2025 12:08:31 -0600 Subject: [PATCH 06/46] feat(developer): API for resolving line numbers for messages - change to use a different fast-xml-parser PR, https://github.com/NaturalIntelligence/fast-xml-parser/pull/729 Fixes: #10622 --- developer/src/common/web/utils/package.json | 2 +- .../src/common/web/utils/src/xml-utils.ts | 2 +- .../utils/test/kpj/kpj-file-writer.tests.ts | 1 - .../common/web/utils/test/xml-utils.tests.ts | 7 ++- developer/src/kmc-ldml/test/helpers/index.ts | 9 +++- developer/src/kmc-ldml/test/keys.tests.ts | 4 +- package-lock.json | 46 +++++++++++-------- 7 files changed, 42 insertions(+), 29 deletions(-) diff --git a/developer/src/common/web/utils/package.json b/developer/src/common/web/utils/package.json index 571123ec68..a56fdc386a 100644 --- a/developer/src/common/web/utils/package.json +++ b/developer/src/common/web/utils/package.json @@ -12,7 +12,7 @@ "@keymanapp/common-types": "*", "@sentry/node": "^7.57.0", "eventemitter3": "^5.0.0", - "fast-xml-parser": "github:srl295/fast-xml-parser#srl295/line-number-593", + "fast-xml-parser": "github:srl295/fast-xml-parser#line-number-593bis", "path-browserify": "^1.0.1", "restructure": "^3.0.1", "sax": ">=0.6.0", diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 9fbebdf397..6365134db7 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -8,7 +8,7 @@ import { XMLParser, XMLBuilder } from 'fast-xml-parser'; -export const START_INDEX = new XMLParser().getStartIndexSymbol(); +export const START_INDEX = XMLParser.getStartIndexSymbol(); export type KeymanXMLType = 'keyboard3' // LDML diff --git a/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts b/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts index 700d223406..899527edbb 100644 --- a/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts +++ b/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts @@ -26,7 +26,6 @@ describe('kpj-file-writer', function () { // The outputKpj may not contain all the fields from the inputKpj, only the // essential fields. Many of the fields in .kpj are deprecated, when they // relate to file content (e.g. parented files, file details) - assert.deepEqual(outputKpj.KeymanDeveloperProject.Options, { "BuildPath": "$PROJECTPATH\\build", "CompilerWarningsAsErrors": "True", diff --git a/developer/src/common/web/utils/test/xml-utils.tests.ts b/developer/src/common/web/utils/test/xml-utils.tests.ts index bdcd18817d..44b4d95f21 100644 --- a/developer/src/common/web/utils/test/xml-utils.tests.ts +++ b/developer/src/common/web/utils/test/xml-utils.tests.ts @@ -180,9 +180,8 @@ describe(`XML Reader line number test`, () => { assert.equal(actual.keyboard3[START_INDEX as any], 40); // index of element assert.equal(actual.keyboard3.info[START_INDEX as any], 136); // index of etc assert.equal(actual.keyboard3.transforms[START_INDEX as any], 186); - - assert.equal(KeymanXMLReader.offsetToLineColumn(actual.keyboard3[START_INDEX as any], xml), { line: 3, column: 0 }); - assert.equal(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.info[START_INDEX as any], xml), { line: 4, column: 2 }); - assert.equal(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.transforms[START_INDEX as any], xml), { line: 8, column: 2 }); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3[START_INDEX as any], xml), { line: 3, column: 0 }); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.info[START_INDEX as any], xml), { line: 4, column: 2 }); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.transforms[START_INDEX as any], xml), { line: 8, column: 2 }); }); }); diff --git a/developer/src/kmc-ldml/test/helpers/index.ts b/developer/src/kmc-ldml/test/helpers/index.ts index 7b67ee4e4c..5599f695f6 100644 --- a/developer/src/kmc-ldml/test/helpers/index.ts +++ b/developer/src/kmc-ldml/test/helpers/index.ts @@ -8,7 +8,7 @@ import * as path from 'path'; import { fileURLToPath } from 'url'; import { SectionCompiler, SectionCompilerNew } from '../../src/compiler/section-compiler.js'; import { util, KMXPlus, LdmlKeyboardTypes } from '@keymanapp/common-types'; -import { CompilerEvent, compilerEventFormat, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboard, } from "@keymanapp/developer-utils"; +import { CompilerEvent, compilerEventFormat, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboard, START_INDEX, } from "@keymanapp/developer-utils"; import { LdmlKeyboardCompiler } from '../../src/main.js'; // make sure main.js compiles import { assert } from 'chai'; import { KMXPlusMetadataCompiler } from '../../src/compiler/metadata-compiler.js'; @@ -327,3 +327,10 @@ const dontEscape = /[a-zA-Z0-9\.${}\[\]-]/; export function hex_str(s?: string) : string { return [...s].map(ch => dontEscape.test(ch) ? ch : util.escapeRegexChar(ch)).join(''); } + +/** return an object simulating an XML object with a column number */ +export function withColumn(c: number) : any { + return { + [START_INDEX as any]: c + }; +} diff --git a/developer/src/kmc-ldml/test/keys.tests.ts b/developer/src/kmc-ldml/test/keys.tests.ts index cfb37e6baf..1b74763646 100644 --- a/developer/src/kmc-ldml/test/keys.tests.ts +++ b/developer/src/kmc-ldml/test/keys.tests.ts @@ -1,7 +1,7 @@ import 'mocha'; import { assert } from 'chai'; import { KeysCompiler } from '../src/compiler/keys.js'; -import { assertCodePoints, compilerTestCallbacks, loadSectionFixture, testCompilationCases } from './helpers/index.js'; +import { assertCodePoints, compilerTestCallbacks, loadSectionFixture, testCompilationCases, withColumn } from './helpers/index.js'; import { KMXPlus, Constants, LdmlKeyboardTypes } from '@keymanapp/common-types'; import { LdmlCompilerMessages } from '../src/compiler/ldml-compiler-messages.js'; import { constants } from '@keymanapp/ldml-keyboard-constants'; @@ -442,7 +442,7 @@ describe('keys.kmap', function () { assert.isNull(keys); assert.equal(compilerTestCallbacks.messages.length, 1); - assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_HardwareLayerHasTooManyRows()); + assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_HardwareLayerHasTooManyRows(withColumn(276))); }); it('should reject layouts with too many hardware keys', async function() { diff --git a/package-lock.json b/package-lock.json index 0f62509ac5..c44fd00061 100644 --- a/package-lock.json +++ b/package-lock.json @@ -342,7 +342,7 @@ "@keymanapp/common-types": "*", "@sentry/node": "^7.57.0", "eventemitter3": "^5.0.0", - "fast-xml-parser": "github:srl295/fast-xml-parser#srl295/line-number-593", + "fast-xml-parser": "github:srl295/fast-xml-parser#line-number-593bis", "path-browserify": "^1.0.1", "restructure": "^3.0.1", "sax": ">=0.6.0", @@ -372,21 +372,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "developer/src/common/web/utils/node_modules/fast-xml-parser": { - "version": "4.1.3", - "resolved": "git+ssh://git@github.com/srl295/fast-xml-parser.git#d7601c31f39fe69e6612af79f9d778dee5dfcb42", - "license": "MIT", - "dependencies": { - "strnum": "^1.0.5" - }, - "bin": { - "fxparser": "src/cli/cli.js" - }, - "funding": { - "type": "paypal", - "url": "https://paypal.me/naturalintelligence" - } - }, "developer/src/common/web/utils/node_modules/js-yaml": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", @@ -9031,6 +9016,23 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "node_modules/fast-xml-parser": { + "version": "5.0.8", + "resolved": "git+ssh://git@github.com/srl295/fast-xml-parser.git#72f20480e8591aca819fac684d46f8d678fcb7f3", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "strnum": "^2.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fastq": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", @@ -13888,9 +13890,15 @@ "link": true }, "node_modules/strnum": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", - "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==", + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.0.5.tgz", + "integrity": "sha512-YAT3K/sgpCUxhxNMrrdhtod3jckkpYwH6JAuwmUdXZsmzH1wUyzTMrrK2wYCEEqlKwrWDd35NeuUkbBy/1iK+Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], "license": "MIT" }, "node_modules/supports-color": { From 4d9ea2db2d1a1768225a074ad58d91da77941ebc Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 28 Feb 2025 14:24:12 -0600 Subject: [PATCH 07/46] feat(developer): work around line number symbols - add KeymanXMLReader.removeSymbols() and call it from tests Fixes: #10622 --- developer/src/common/web/utils/src/xml-utils.ts | 11 ++++++++++- .../web/utils/test/kpj/kpj-file-writer.tests.ts | 5 +++-- .../web/utils/test/kps/kps-file-reader.tests.ts | 5 +++-- .../src/common/web/utils/test/kvks/kvks-file.tests.ts | 5 +++-- .../src/common/web/utils/test/xml-utils.tests.ts | 2 +- developer/src/kmc-ldml/test/helpers/compareXml.ts | 2 +- 6 files changed, 21 insertions(+), 9 deletions(-) diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 6365134db7..98af170790 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -156,13 +156,22 @@ export class KeymanXMLReader { * @param from source for symbols * @returns the onto object */ - private static copySymbols(onto: T, from: any): T { + public static copySymbols(onto: T, from: any): T { const o = onto as any; for (const sym of Object.getOwnPropertySymbols(from)) { o[sym] = from[sym]; } return onto; } + + /** use Object.entries to remove all symbols */ + public static removeSymbols(from: T): T { + if (Array.isArray(from)) { + return from.map(o => KeymanXMLReader.removeSymbols(o)) as T; + } + if (typeof from !== "object") return from; + return Object.fromEntries(Object.entries(from).map(([k, v]) => ([k, KeymanXMLReader.removeSymbols(v)]))) as T; + } /** move `{ $abc: 4 }` into `{ $: { abc: 4 } }` */ private static fixupDollarAttributes(data: any) : any { diff --git a/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts b/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts index 899527edbb..5fc223b240 100644 --- a/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts +++ b/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts @@ -6,6 +6,7 @@ import { KPJFileReader } from "../../src/types/kpj/kpj-file-reader.js"; import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; import { KPJFileWriter } from '../../src/types/kpj/kpj-file-writer.js'; import { KeymanDeveloperProjectOptions } from '../../src/types/kpj/keyman-developer-project.js'; +import { KeymanXMLReader } from '../../src/xml-utils.js'; const callbacks = new TestCompilerCallbacks(); @@ -21,7 +22,7 @@ describe('kpj-file-writer', function () { const writer = new KPJFileWriter(); const output = writer.write(project); - const outputKpj = reader.read(new TextEncoder().encode(output)); + const outputKpj = KeymanXMLReader.removeSymbols(reader.read(new TextEncoder().encode(output))); // The outputKpj may not contain all the fields from the inputKpj, only the // essential fields. Many of the fields in .kpj are deprecated, when they @@ -60,7 +61,7 @@ describe('kpj-file-writer', function () { const writer = new KPJFileWriter(); const output = writer.write(project); - const outputKpj = reader.read(new TextEncoder().encode(output)); + const outputKpj = KeymanXMLReader.removeSymbols(reader.read(new TextEncoder().encode(output))); // The outputKpj may not contain all the fields from the inputKpj, only the // essential fields. Many of the fields in .kpj are deprecated, when they diff --git a/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts b/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts index 1b809f39d0..94a1628bd9 100644 --- a/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts +++ b/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts @@ -15,6 +15,7 @@ import { makePathToFixture } from '../helpers/index.js'; import { KpsFileReader } from "../../src/types/kps/kps-file-reader.js"; import { KpsFileWriter } from '../../src/types/kps/kps-file-writer.js'; import { CommonTypesMessages } from '../../src/common-messages.js'; +import { KeymanXMLReader } from '../../src/xml-utils.js'; const callbacks = new TestCompilerCallbacks(); @@ -61,13 +62,13 @@ describe('kps-file-reader', function () { it('kps-file-reader should round-trip with kps-file-writer', function() { const input = fs.readFileSync(makePathToFixture('kps', 'khmer_angkor.kps')); const reader = new KpsFileReader(callbacks); - const kps = reader.read(input); + const kps = KeymanXMLReader.removeSymbols(reader.read(input)); const writer = new KpsFileWriter(); const output = writer.write(kps); // Round Trip - const kps2 = reader.read(new TextEncoder().encode(output)); + const kps2 = KeymanXMLReader.removeSymbols(reader.read(new TextEncoder().encode(output))); assert.deepEqual(kps2, kps); }); diff --git a/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts b/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts index 4c009e9d5c..96be2283b8 100644 --- a/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts +++ b/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts @@ -5,6 +5,7 @@ import KvksFileReader from "../../src/types/kvks/kvks-file-reader.js"; import KvksFileWriter from "../../src/types/kvks/kvks-file-writer.js"; import { verify_khmer_angkor, verify_balochi_inpage } from './kvk-utils.tests.js'; import { assert } from 'chai'; +import { KeymanXMLReader } from '../../src/xml-utils.js'; describe('kvks-file-reader', function() { it('should read a valid file', function() { @@ -48,7 +49,7 @@ describe('kvks-file-writer', function() { const input = fs.readFileSync(path); const reader = new KvksFileReader(); - const kvksExpected = reader.read(input); + const kvksExpected = KeymanXMLReader.removeSymbols(reader.read(input)); const invalidVkeys: string[] = []; const vk = reader.transform(kvksExpected, invalidVkeys); assert.isEmpty(invalidVkeys); @@ -59,6 +60,6 @@ describe('kvks-file-writer', function() { // We compare the (re)loaded data, because there may be // minor, irrelevant formatting differences in the emitted xml const kvks = reader.read(Buffer.from(output, 'utf8')); - assert.deepEqual(kvks, kvksExpected); + assert.deepEqual(KeymanXMLReader.removeSymbols(kvks), kvksExpected); }); }); diff --git a/developer/src/common/web/utils/test/xml-utils.tests.ts b/developer/src/common/web/utils/test/xml-utils.tests.ts index 44b4d95f21..9039760913 100644 --- a/developer/src/common/web/utils/test/xml-utils.tests.ts +++ b/developer/src/common/web/utils/test/xml-utils.tests.ts @@ -119,7 +119,7 @@ describe(`XML Reader Test ${GEN_XML_FIXTURES && '(update mode!)' || ''}`, () => writeJson(jsonPath, actual); } else { assert.ok(expect, `Could not read ${jsonPath} - run with env GEN_XML_FIXTURES=1 to update.`); - assert.deepEqual(actual, expect, `Mismatch of ${xmlPath} vs ${jsonPath}`); + assert.deepEqual(KeymanXMLReader.removeSymbols(actual), expect, `Mismatch of ${xmlPath} vs ${jsonPath}`); } }); } diff --git a/developer/src/kmc-ldml/test/helpers/compareXml.ts b/developer/src/kmc-ldml/test/helpers/compareXml.ts index f245da51a9..1503d4c606 100644 --- a/developer/src/kmc-ldml/test/helpers/compareXml.ts +++ b/developer/src/kmc-ldml/test/helpers/compareXml.ts @@ -21,5 +21,5 @@ export function compareXml(actual : string, expect: string, mutator?: (input: an const actualParsed = mutator(reader.parse(actualStr)); const expectParsed = mutator(reader.parse(expectStr)); - assert.deepEqual(actualParsed, expectParsed); + assert.deepEqual(KeymanXMLReader.removeSymbols(actualParsed), KeymanXMLReader.removeSymbols(expectParsed)); } From 3e73a3f0819ffa5a38cf4075f0c6bdab2b573137 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 3 Mar 2025 12:36:42 -0600 Subject: [PATCH 08/46] feat(developer): add CompilerEvent.offset - per review, keep this as a separate field. Fixes: #10622 --- .../common/web/utils/src/compiler-interfaces.ts | 8 ++++++-- .../src/compiler/ldml-compiler-messages.ts | 14 +++++++------- .../src/kmc/src/util/NodeCompilerCallbacks.ts | 2 +- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/developer/src/common/web/utils/src/compiler-interfaces.ts b/developer/src/common/web/utils/src/compiler-interfaces.ts index 390eea2f7e..dc24a772e9 100644 --- a/developer/src/common/web/utils/src/compiler-interfaces.ts +++ b/developer/src/common/web/utils/src/compiler-interfaces.ts @@ -7,11 +7,15 @@ export interface CompilerEvent { filename?: string; /** line where a message applies */ line?: number; - /** + /** * column where a message applies. - * where line is unset, column is a file offset */ column?: number; + /** + * offset where a message applies. + * If set, encompasses line and column. + */ + offset?: number; code: number; message: string; /** diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 6f8cc5fa3d..27f587b04f 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -10,7 +10,7 @@ const SevError = CompilerErrorSeverity.Error | CompilerErrorNamespace.LdmlKeyboa const SevErrorTransform = SevError | 0xF00; // like m() but takes an XML object for line numbers -const mx = (x: any, code: number, message: string, detail?: string): CompilerEvent => LdmlCompilerMessages.col(m(code, message, detail), x); +const mx = (x: any, code: number, message: string, detail?: string): CompilerEvent => LdmlCompilerMessages.offset(m(code, message, detail), x); /** * @internal @@ -269,14 +269,14 @@ export class LdmlCompilerMessages { ); /** - * Get a column number from o and set e's column field + * Get an offset from o and set e's offset field * @param event a compiler event, such as from functions in this class * @param x any object parsed from XML or with the START_INDEX symbol copied over * @returns modified event object */ - static col(event: CompilerEvent, x?: any): CompilerEvent { + static offset(event: CompilerEvent, x?: any): CompilerEvent { if(x) { - event.column = (x as any)[START_INDEX as any]; + event.offset = (x as any)[START_INDEX as any]; } return event; } @@ -287,9 +287,9 @@ export class LdmlCompilerMessages { * @param xml XML source * @returns the modified event */ - static resolveLineNumber(event: CompilerEvent, xml: string) : CompilerEvent { - if (event.column && !event.line && xml) { - const loc = KeymanXMLReader.offsetToLineColumn(event.column, xml); + static resolveLineNumber(event: CompilerEvent, xml: string) : CompilerEvent { + if (event.offset && !event.line && xml) { + const loc = KeymanXMLReader.offsetToLineColumn(event.offset, xml); event.line = loc.line; event.column = loc.column; } diff --git a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts index 7c184c1417..8186f29d70 100644 --- a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts +++ b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts @@ -161,7 +161,7 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { this.messageFiletext = ''; } - if (event.column && !event.line && event.filename) { + if (event.offset && !event.line && event.filename) { try { let text = ''; if (this.messageFilename == event.filename) { From d7cb7b63a3db987229e42a3b3fcbf78ff906eb1e Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 3 Mar 2025 13:14:41 -0600 Subject: [PATCH 09/46] feat(developer): add an EventResolver - it's a hook to the hook to allow the LDML compiler to resolve error message line numbers - TODO still not extra performant, but maybe the right separation of concerns Fixes: #10622 --- .../web/test-helpers/TestCompilerCallbacks.ts | 13 +++++++- .../web/utils/src/compiler-callbacks.ts | 28 +++++++++++++++- developer/src/common/web/utils/src/index.ts | 2 ++ .../src/kmc-ldml/src/compiler/compiler.ts | 4 +++ .../kmc-ldml/src/compiler/eventresolver.ts | 21 ++++++++++++ .../src/kmc/src/util/NodeCompilerCallbacks.ts | 32 +++++++------------ 6 files changed, 78 insertions(+), 22 deletions(-) create mode 100644 developer/src/kmc-ldml/src/compiler/eventresolver.ts diff --git a/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts b/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts index fad1dd646f..4771b16417 100644 --- a/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts +++ b/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts @@ -2,7 +2,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { CompilerEvent, CompilerCallbacks, CompilerPathCallbacks, CompilerFileSystemCallbacks, CompilerError, CompilerNetAsyncCallbacks, DefaultCompilerFileSystemAsyncCallbacks, - CompilerFileSystemAsyncCallbacks, + CompilerFileSystemAsyncCallbacks, EventResolver, NullEventResolver, CompilerErrorSeverity} from '@keymanapp/developer-utils'; import { fileURLToPath } from 'url'; @@ -111,7 +111,18 @@ export class TestCompilerCallbacks implements CompilerCallbacks { return filename; } + private eventResolver = new NullEventResolver(); + + setEventResolver(eventResolver: EventResolver): void { + this.eventResolver = eventResolver; + } + reportMessage(event: CompilerEvent): void { + this.eventResolver.resolve(event); + this.handleReportMessage(event); + } + + handleReportMessage(event: CompilerEvent): void { // console.log(event.message); this.messages.push(event); } diff --git a/developer/src/common/web/utils/src/compiler-callbacks.ts b/developer/src/common/web/utils/src/compiler-callbacks.ts index a7163175e8..bebfe5fd55 100644 --- a/developer/src/common/web/utils/src/compiler-callbacks.ts +++ b/developer/src/common/web/utils/src/compiler-callbacks.ts @@ -7,6 +7,21 @@ import { CompilerEvent, CompilerCallbackOptions, CompilerErrorSeverity, CompilerError, CompilerMessageOverrideMap, CompilerErrorMask } from "./compiler-interfaces.js"; +/** + * The EventResolver implementation is able to expand error messages, + * particularly to change a byte offset into a line number. + */ +export interface EventResolver { + resolve(event: CompilerEvent): void; +} + +/** A do-nothing EventResolver. Implementations can use this as their default EventResolver. */ +export class NullEventResolver implements EventResolver { + resolve(event: CompilerEvent): void { + // do nothing, this is the null event resolver. + } +} + /** * Abstract interface for callbacks, to abstract out file i/o */ @@ -34,12 +49,20 @@ export interface CompilerCallbacks { get net(): CompilerNetAsyncCallbacks; /** - * Resolves a file path relative to the baseFilename + * Resolves a file path relative to the baseFilename. * @param baseFilename * @param filename */ resolveFilename(baseFilename: string, filename: string): string; + /** change the EventResolver to use. If called more than once, the last resolver replaces a prior one. */ + setEventResolver(eventResolver: EventResolver): void; + + /** + * Report a message from the compiler back to the caller. + * It is expected that eventResolver.resolve() will be called on the message before processing. + * @param event + */ reportMessage(event: CompilerEvent): void; debug(msg: string): void; @@ -115,6 +138,9 @@ export class CompilerFileCallbacks implements CompilerCallbacks { constructor(private filename: string, private options: CompilerCallbackOptions, private parent: CompilerCallbacks) { } + setEventResolver(eventResolver: EventResolver): void { + throw new Error("Method not implemented."); + } /** * Returns `true` if any message in the `messages` array is a Fatal or Error diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index 1a56a735e8..812d503e96 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -42,6 +42,8 @@ export { CompilerPathCallbacks, CompilerFileSystemCallbacksFolderEntry as FileSystemFolderEntry, DefaultCompilerFileSystemAsyncCallbacks, + EventResolver, + NullEventResolver, } from './compiler-callbacks.js'; export { defaultCompilerOptions, CompilerBaseOptions, CompilerOptions, CompilerEvent, CompilerErrorNamespace, diff --git a/developer/src/kmc-ldml/src/compiler/compiler.ts b/developer/src/kmc-ldml/src/compiler/compiler.ts index e8b38184e9..e0b86dc712 100644 --- a/developer/src/kmc-ldml/src/compiler/compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/compiler.ts @@ -30,6 +30,7 @@ import { KmnCompiler } from '@keymanapp/kmc-kmn'; import { KMXPlusMetadataCompiler } from './metadata-compiler.js'; import { LdmlKeyboardVisualKeyboardCompiler } from './visual-keyboard-compiler.js'; import { LinterKeycaps } from './linter-keycaps.js'; +import { LdmlEventResolver } from './eventresolver.js'; export const SECTION_COMPILERS = [ // These are in dependency order. @@ -94,6 +95,7 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { // uset parser private usetparser?: LdmlKeyboardTypes.UnicodeSetParser = undefined; + private static eventResolver: LdmlEventResolver = new LdmlEventResolver(); /** * Initialize the compiler, including loading the WASM host for uset parsing. @@ -106,6 +108,7 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { async init(callbacks: CompilerCallbacks, options: LdmlCompilerOptions): Promise { this.options = { ...options }; this.callbacks = callbacks; + this.callbacks.setEventResolver(LdmlKeyboardCompiler.eventResolver); return true; } @@ -245,6 +248,7 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidFile({ errorText: 'Unable to read XML file' })); return null; } + LdmlKeyboardCompiler.eventResolver.addFile(filename, new TextDecoder().decode(data)); // TODO: double decode // parse (load) the string into an object tree const source = reader.load(data); if (!source) { diff --git a/developer/src/kmc-ldml/src/compiler/eventresolver.ts b/developer/src/kmc-ldml/src/compiler/eventresolver.ts new file mode 100644 index 0000000000..491e4a3423 --- /dev/null +++ b/developer/src/kmc-ldml/src/compiler/eventresolver.ts @@ -0,0 +1,21 @@ +import { CompilerEvent, EventResolver } from "@keymanapp/developer-utils"; +import { LdmlCompilerMessages } from "./ldml-compiler-messages.js"; + +export class LdmlEventResolver implements EventResolver { + + contentsCache: Map = new Map(); + + /** add or update a file */ + addFile(filename: string, xml: any) { + if (typeof xml !== 'string') { + xml = xml.toString(); + } + this.contentsCache.set(filename, xml); + } + + resolve(event: CompilerEvent) { + if (event.offset && !event.line && event.filename) { + LdmlCompilerMessages.resolveLineNumber(event, this.contentsCache.get(event.filename)); + } + } +} diff --git a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts index 8186f29d70..5f4d0e9018 100644 --- a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts +++ b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { platform } from 'os'; -import { CompilerCallbacks, CompilerEvent, +import { CompilerCallbacks, CompilerEvent, EventResolver, NullEventResolver, CompilerPathCallbacks, CompilerFileSystemCallbacks, compilerLogLevelToSeverity, CompilerErrorSeverity, CompilerError, @@ -15,7 +15,6 @@ import chalk from 'chalk'; import supportsColor from 'supports-color'; import { KeymanSentry } from '@keymanapp/developer-utils'; import { fileURLToPath } from 'url'; -import { LdmlCompilerMessages } from '@keymanapp/kmc-ldml'; const color = chalk.default; const severityColors: {[value in CompilerErrorSeverity]: chalk.Chalk} = { @@ -149,7 +148,18 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { return fileURLToPath(url); } + private eventResolver = new NullEventResolver(); + + setEventResolver(eventResolver: EventResolver): void { + this.eventResolver = eventResolver; + } + reportMessage(event: CompilerEvent): void { + this.eventResolver.resolve(event); + this.handleReportMessage(event); + } + + handleReportMessage(event: CompilerEvent): void { if(!event.filename) { event.filename = this.messageFilename; } @@ -161,24 +171,6 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { this.messageFiletext = ''; } - if (event.offset && !event.line && event.filename) { - try { - let text = ''; - if (this.messageFilename == event.filename) { - text = this.messageFiletext; - } - if (text == '') { - text = this.fs.readFileSync(event.filename, "utf-8"); - if (this.messageFilename == event.filename) { - this.messageFiletext = text; - } - } - LdmlCompilerMessages.resolveLineNumber(event, text); - } catch(e) { - console.error(e); // resolving line numbers - } - } - const disable = CompilerFileCallbacks.applyMessageOverridesToEvent(event, this.options.messageOverrides); this.messages.push({...event}); From 1febf9a0de36c2e979642291f91904d337c3c17c Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 3 Mar 2025 13:18:02 -0600 Subject: [PATCH 10/46] Update developer/src/common/web/utils/src/xml-utils.ts Co-authored-by: Marc Durdin --- developer/src/common/web/utils/src/xml-utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 98af170790..8c084231fe 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -8,7 +8,7 @@ import { XMLParser, XMLBuilder } from 'fast-xml-parser'; -export const START_INDEX = XMLParser.getStartIndexSymbol(); +export const XML_START_INDEX_SYMBOL = XMLParser.getStartIndexSymbol(); export type KeymanXMLType = 'keyboard3' // LDML From b015221814f83463908d88a5209376dc14916ff5 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 3 Mar 2025 17:08:40 -0600 Subject: [PATCH 11/46] feat(developer): line number symbols - change special symbol to XML_START_INDEX_SYMBOL - use an array of line lengths rather than an array of strings for the file info - add an EventResolver interface to the callbacks, such that it has an opportunity to resolve events Fixes: #10622 --- .../web/utils/src/compiler-callbacks.ts | 14 ++++++++++--- developer/src/common/web/utils/src/index.ts | 2 +- .../src/common/web/utils/src/xml-utils.ts | 19 ++++++++++------- .../common/web/utils/test/xml-utils.tests.ts | 14 ++++++------- .../kmc-ldml/src/compiler/eventresolver.ts | 16 +++++++------- .../src/compiler/ldml-compiler-messages.ts | 21 +++---------------- developer/src/kmc-ldml/test/helpers/index.ts | 6 +++--- 7 files changed, 44 insertions(+), 48 deletions(-) diff --git a/developer/src/common/web/utils/src/compiler-callbacks.ts b/developer/src/common/web/utils/src/compiler-callbacks.ts index bebfe5fd55..bce9d9f395 100644 --- a/developer/src/common/web/utils/src/compiler-callbacks.ts +++ b/developer/src/common/web/utils/src/compiler-callbacks.ts @@ -138,9 +138,6 @@ export class CompilerFileCallbacks implements CompilerCallbacks { constructor(private filename: string, private options: CompilerCallbackOptions, private parent: CompilerCallbacks) { } - setEventResolver(eventResolver: EventResolver): void { - throw new Error("Method not implemented."); - } /** * Returns `true` if any message in the `messages` array is a Fatal or Error @@ -228,7 +225,18 @@ export class CompilerFileCallbacks implements CompilerCallbacks { return this.parent.resolveFilename(baseFilename, filename); } + private eventResolver = new NullEventResolver(); + + setEventResolver(eventResolver: EventResolver): void { + this.eventResolver = eventResolver; + } + reportMessage(event: CompilerEvent): void { + this.eventResolver.resolve(event); + this.handleReportMessage(event); + } + + handleReportMessage(event: CompilerEvent): void { const disable = CompilerFileCallbacks.applyMessageOverridesToEvent(event, this.options.messageOverrides); this.messages.push(event); if (!disable) { diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index 812d503e96..c703e48aa2 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -67,7 +67,7 @@ export { UrlSubpathCompilerCallback } from './utils/UrlSubpathCompilerCallback.j export { CommonTypesMessages } from './common-messages.js'; export * as SourceFilenamePatterns from './source-filename-patterns.js'; -export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, START_INDEX } from './xml-utils.js'; +export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, XML_START_INDEX_SYMBOL } from './xml-utils.js'; export * as GitHubUrls from './github-urls.js'; export * as CloudUrls from './cloud-urls.js'; diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 8c084231fe..de78fa5b54 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -134,18 +134,21 @@ export interface LineColumn { export class KeymanXMLReader { public constructor(public type: KeymanXMLType) { } - - public static offsetToLineColumn(offset: number, text: string) : LineColumn { - const lines: string[] = text.split("\n"); + /** preprocess text to turn it into arrays of line lengths */ + public static textToLines(text: string) : number[] { + return text.replaceAll("\r\n", "\n").split("\n") + .map(l => l.length + 1); // line length (counting the trailing newline) + } + public static offsetToLineColumn(offset: number, lines: number[]) : LineColumn { for (let line = 1; line < lines.length + 1; line++) { // 1-based - const linestr = lines[line - 1]; - if (linestr.length < offset) { - offset = offset - (linestr.length + 1); // count newline at end + if (lines[line-1] < offset) { + offset = offset - (lines[line-1]); // count newline at end continue; } return { line, column: offset }; } - return { line: lines.length + 1 }; + // default: line 0, error + return { line: 0 } } /** @@ -172,7 +175,7 @@ export class KeymanXMLReader { if (typeof from !== "object") return from; return Object.fromEntries(Object.entries(from).map(([k, v]) => ([k, KeymanXMLReader.removeSymbols(v)]))) as T; } - + /** move `{ $abc: 4 }` into `{ $: { abc: 4 } }` */ private static fixupDollarAttributes(data: any) : any { if (typeof data === 'object') { diff --git a/developer/src/common/web/utils/test/xml-utils.tests.ts b/developer/src/common/web/utils/test/xml-utils.tests.ts index 9039760913..f487573724 100644 --- a/developer/src/common/web/utils/test/xml-utils.tests.ts +++ b/developer/src/common/web/utils/test/xml-utils.tests.ts @@ -12,7 +12,7 @@ import { env } from 'node:process'; import { readFileSync, writeFileSync } from 'node:fs'; -import { KeymanXMLType, KeymanXMLReader, KeymanXMLWriter, START_INDEX } from '../src/xml-utils.js'; +import { KeymanXMLType, KeymanXMLReader, KeymanXMLWriter, XML_START_INDEX_SYMBOL } from '../src/xml-utils.js'; import { makePathToFixture } from './helpers/index.js'; // if true, attempt to WRITE the fixtures @@ -177,11 +177,11 @@ describe(`XML Reader line number test`, () => { assert.ok(actual, `Parser failed on ${xmlPath}`); // now, assert char offset - assert.equal(actual.keyboard3[START_INDEX as any], 40); // index of element - assert.equal(actual.keyboard3.info[START_INDEX as any], 136); // index of etc - assert.equal(actual.keyboard3.transforms[START_INDEX as any], 186); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3[START_INDEX as any], xml), { line: 3, column: 0 }); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.info[START_INDEX as any], xml), { line: 4, column: 2 }); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.transforms[START_INDEX as any], xml), { line: 8, column: 2 }); + assert.equal(actual.keyboard3[XML_START_INDEX_SYMBOL as any], 40); // index of element + assert.equal(actual.keyboard3.info[XML_START_INDEX_SYMBOL as any], 136); // index of etc + assert.equal(actual.keyboard3.transforms[XML_START_INDEX_SYMBOL as any], 186); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3[XML_START_INDEX_SYMBOL as any], xml), { line: 3, column: 0 }); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.info[XML_START_INDEX_SYMBOL as any], xml), { line: 4, column: 2 }); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.transforms[XML_START_INDEX_SYMBOL as any], xml), { line: 8, column: 2 }); }); }); diff --git a/developer/src/kmc-ldml/src/compiler/eventresolver.ts b/developer/src/kmc-ldml/src/compiler/eventresolver.ts index 491e4a3423..7e1dd87fc3 100644 --- a/developer/src/kmc-ldml/src/compiler/eventresolver.ts +++ b/developer/src/kmc-ldml/src/compiler/eventresolver.ts @@ -1,21 +1,21 @@ -import { CompilerEvent, EventResolver } from "@keymanapp/developer-utils"; -import { LdmlCompilerMessages } from "./ldml-compiler-messages.js"; +import { CompilerEvent, EventResolver, KeymanXMLReader } from "@keymanapp/developer-utils"; export class LdmlEventResolver implements EventResolver { - contentsCache: Map = new Map(); + contentsCache: Map = new Map(); /** add or update a file */ addFile(filename: string, xml: any) { - if (typeof xml !== 'string') { - xml = xml.toString(); - } - this.contentsCache.set(filename, xml); + this.contentsCache.set(filename, KeymanXMLReader.textToLines(xml)); } resolve(event: CompilerEvent) { if (event.offset && !event.line && event.filename) { - LdmlCompilerMessages.resolveLineNumber(event, this.contentsCache.get(event.filename)); + const lines = this.contentsCache.get(event.filename); + const loc = KeymanXMLReader.offsetToLineColumn(event.offset, lines); + event.line = loc.line; + event.column = loc.column; } + return event; } } diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 27f587b04f..1298d0f0cd 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -1,5 +1,5 @@ import { util } from "@keymanapp/common-types"; -import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, START_INDEX, CompilerEvent, KeymanXMLReader } from '@keymanapp/developer-utils'; +import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_START_INDEX_SYMBOL, CompilerEvent } from '@keymanapp/developer-utils'; // const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.LdmlKeyboardCompiler; @@ -271,27 +271,12 @@ export class LdmlCompilerMessages { /** * Get an offset from o and set e's offset field * @param event a compiler event, such as from functions in this class - * @param x any object parsed from XML or with the START_INDEX symbol copied over + * @param x any object parsed from XML or with the XML_START_INDEX_SYMBOL symbol copied over * @returns modified event object */ static offset(event: CompilerEvent, x?: any): CompilerEvent { if(x) { - event.offset = (x as any)[START_INDEX as any]; - } - return event; - } - - /** - * Given an event with a column number but no line, resolve it into line:column - * @param event event to modify - * @param xml XML source - * @returns the modified event - */ - static resolveLineNumber(event: CompilerEvent, xml: string) : CompilerEvent { - if (event.offset && !event.line && xml) { - const loc = KeymanXMLReader.offsetToLineColumn(event.offset, xml); - event.line = loc.line; - event.column = loc.column; + event.offset = (x as any)[XML_START_INDEX_SYMBOL as any]; } return event; } diff --git a/developer/src/kmc-ldml/test/helpers/index.ts b/developer/src/kmc-ldml/test/helpers/index.ts index 5599f695f6..497f8ef919 100644 --- a/developer/src/kmc-ldml/test/helpers/index.ts +++ b/developer/src/kmc-ldml/test/helpers/index.ts @@ -8,7 +8,7 @@ import * as path from 'path'; import { fileURLToPath } from 'url'; import { SectionCompiler, SectionCompilerNew } from '../../src/compiler/section-compiler.js'; import { util, KMXPlus, LdmlKeyboardTypes } from '@keymanapp/common-types'; -import { CompilerEvent, compilerEventFormat, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboard, START_INDEX, } from "@keymanapp/developer-utils"; +import { CompilerEvent, compilerEventFormat, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboard, XML_START_INDEX_SYMBOL, } from "@keymanapp/developer-utils"; import { LdmlKeyboardCompiler } from '../../src/main.js'; // make sure main.js compiles import { assert } from 'chai'; import { KMXPlusMetadataCompiler } from '../../src/compiler/metadata-compiler.js'; @@ -290,7 +290,7 @@ export function testCompilationCases(compiler: SectionCompilerNew, cases : Compi // no warnings, so expect zero messages assert.sameDeepMembers(callbacks.messages, [], 'expected zero messages but got ' + callbacks.messages); } - + if (expectFailure) { assert.isNull(section, 'expected compilation result failure (null)'); } else { @@ -331,6 +331,6 @@ export function hex_str(s?: string) : string { /** return an object simulating an XML object with a column number */ export function withColumn(c: number) : any { return { - [START_INDEX as any]: c + [XML_START_INDEX_SYMBOL as any]: c }; } From 92e16b90a033da5e3d9133e9521fc9b6df5088db Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 3 Mar 2025 19:12:34 -0600 Subject: [PATCH 12/46] feat(developer): line number symbols - move file read bottleneck out of compiler.ts into the LDML XML reader - move the eventresolver into the LDML XML reader - add XML_FILENAME_SYMBOL - for tagging when an object is from a different file than the default (such as with an import) Fixes: #10622 --- .../web/utils/src/compiler-callbacks.ts | 1 + developer/src/common/web/utils/src/index.ts | 2 +- .../src/types/ldml-keyboard}/eventresolver.ts | 2 +- .../ldml-keyboard/ldml-keyboard-xml-reader.ts | 40 ++++++++++++++----- .../src/common/web/utils/src/xml-utils.ts | 3 ++ .../src/kmc-ldml/src/compiler/compiler.ts | 6 +-- developer/src/kmc-ldml/src/compiler/keys.ts | 4 +- .../src/compiler/ldml-compiler-messages.ts | 7 ++-- 8 files changed, 44 insertions(+), 21 deletions(-) rename developer/src/{kmc-ldml/src/compiler => common/web/utils/src/types/ldml-keyboard}/eventresolver.ts (86%) diff --git a/developer/src/common/web/utils/src/compiler-callbacks.ts b/developer/src/common/web/utils/src/compiler-callbacks.ts index bce9d9f395..f8b42afb7a 100644 --- a/developer/src/common/web/utils/src/compiler-callbacks.ts +++ b/developer/src/common/web/utils/src/compiler-callbacks.ts @@ -229,6 +229,7 @@ export class CompilerFileCallbacks implements CompilerCallbacks { setEventResolver(eventResolver: EventResolver): void { this.eventResolver = eventResolver; + this.parent.setEventResolver(eventResolver); } reportMessage(event: CompilerEvent): void { diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index c703e48aa2..b722a11b21 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -67,7 +67,7 @@ export { UrlSubpathCompilerCallback } from './utils/UrlSubpathCompilerCallback.j export { CommonTypesMessages } from './common-messages.js'; export * as SourceFilenamePatterns from './source-filename-patterns.js'; -export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, XML_START_INDEX_SYMBOL } from './xml-utils.js'; +export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, XML_START_INDEX_SYMBOL, XML_FILENAME_SYMBOL } from './xml-utils.js'; export * as GitHubUrls from './github-urls.js'; export * as CloudUrls from './cloud-urls.js'; diff --git a/developer/src/kmc-ldml/src/compiler/eventresolver.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts similarity index 86% rename from developer/src/kmc-ldml/src/compiler/eventresolver.ts rename to developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts index 7e1dd87fc3..3299169642 100644 --- a/developer/src/kmc-ldml/src/compiler/eventresolver.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts @@ -1,4 +1,4 @@ -import { CompilerEvent, EventResolver, KeymanXMLReader } from "@keymanapp/developer-utils"; +import { CompilerEvent, EventResolver, KeymanXMLReader } from "../../index.js"; export class LdmlEventResolver implements EventResolver { diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index 0a5abf6355..ca32fe4905 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -11,6 +11,8 @@ import { constants } from '@keymanapp/ldml-keyboard-constants'; import { LDMLKeyboardTestDataXMLSourceFile, LKTTest, LKTTests } from './ldml-keyboard-testdata-xml.js'; import { KeymanXMLReader } from '@keymanapp/developer-utils'; import boxXmlArray = util.boxXmlArray; +import { LdmlEventResolver } from './eventresolver.js'; +import { XML_FILENAME_SYMBOL } from 'src/xml-utils.js'; interface NameAndProps { '$'?: any; // content @@ -26,27 +28,39 @@ export class LDMLKeyboardXMLSourceFileReaderOptions { }; export class LDMLKeyboardXMLSourceFileReader { - constructor(private options: LDMLKeyboardXMLSourceFileReaderOptions, private callbacks : CompilerCallbacks) { + static eventResolver: LdmlEventResolver = new LdmlEventResolver(); + constructor(private options: LDMLKeyboardXMLSourceFileReaderOptions, private callbacks : CompilerCallbacks) { + callbacks.setEventResolver(LDMLKeyboardXMLSourceFileReader.eventResolver); } static get defaultImportsURL(): [string,string] { return ['../import/', import.meta.url]; } - readImportFile(version: string, subpath: string): Uint8Array { - const importPath = this.callbacks.resolveFilename(this.options.cldrImportsPath, `${version}/${subpath}`); - return this.callbacks.loadFile(importPath); + /** bottleneck for reading keyboard XML files */ + readFile(path: string): Uint8Array { + const data = this.callbacks.loadFile(path); + if (data) { + LDMLKeyboardXMLSourceFileReader.eventResolver.addFile(path, new TextDecoder().decode(data)); + } + return data; } - readLocalImportFile(path: string): Uint8Array { + /** @returns [data, filename] */ + readImportFile(version: string, subpath: string): [Uint8Array, string] { + const importPath = this.callbacks.resolveFilename(this.options.cldrImportsPath, `${version}/${subpath}`); + return [this.readFile(importPath), importPath]; + } + + readLocalImportFile(path: string): [Uint8Array, string] { // try each of the local imports paths for (const localPath of this.options.localImportsPaths) { const importPath = this.callbacks.path.join(localPath, path); if(this.callbacks.fs.existsSync(importPath)) { - return this.callbacks.loadFile(importPath); + return [this.readFile(importPath), importPath]; } } - return null; // was not able to load from any of the paths + return [null, null]; // was not able to load from any of the paths } /** @@ -223,6 +237,7 @@ export class LDMLKeyboardXMLSourceFileReader { return false; } let importData: Uint8Array; + let importPath: string; if (base === constants.cldr_import_base) { // CLDR import @@ -235,10 +250,10 @@ export class LDMLKeyboardXMLSourceFileReader { /** There's no data or DTD change in 45, 46, 46.1, 47 so map them all to 46 at present. */ paths[0] = constants.cldr_version_latest; } - importData = this.readImportFile(paths[0], paths[1]); + [importData, importPath] = this.readImportFile(paths[0], paths[1]); } else { // local import - importData = this.readLocalImportFile(path); + [importData, importPath] = this.readLocalImportFile(path); } if (!importData || !importData.length) { this.callbacks.reportMessage(CommonTypesMessages.Error_ImportReadFail({base, path, subtag})); @@ -246,6 +261,7 @@ export class LDMLKeyboardXMLSourceFileReader { } const importXml: any = this.loadUnboxed(importData); // TODO-LDML: have to load as any because it is an arbitrary part const importRootNode = importXml[subtag]; // e.g. + LDMLKeyboardXMLSourceFileReader.eventResolver.addFile(importPath, new TextDecoder().decode(importData)); // TODO: double decode // importXml will have one property: the root element. if (!importRootNode) { @@ -263,7 +279,11 @@ export class LDMLKeyboardXMLSourceFileReader { return false; } // Mark all children as an import - subsubval.forEach(o => o[ImportStatus.import] = basePath); + subsubval.forEach(o => { + o[ImportStatus.import] = basePath; + o[XML_FILENAME_SYMBOL] = importPath; // mark overriding import path + }); + if (implied) { // mark all children as an implied import subsubval.forEach(o => o[ImportStatus.impliedImport] = basePath); diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index de78fa5b54..ee33b40563 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -8,7 +8,10 @@ import { XMLParser, XMLBuilder } from 'fast-xml-parser'; +/** Symbol giving the start offset, in chars, of the node */ export const XML_START_INDEX_SYMBOL = XMLParser.getStartIndexSymbol(); +/** Symbol giving an override which file a node came from */ +export const XML_FILENAME_SYMBOL = Symbol("XML Filename"); export type KeymanXMLType = 'keyboard3' // LDML diff --git a/developer/src/kmc-ldml/src/compiler/compiler.ts b/developer/src/kmc-ldml/src/compiler/compiler.ts index e0b86dc712..5488afb188 100644 --- a/developer/src/kmc-ldml/src/compiler/compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/compiler.ts @@ -30,7 +30,6 @@ import { KmnCompiler } from '@keymanapp/kmc-kmn'; import { KMXPlusMetadataCompiler } from './metadata-compiler.js'; import { LdmlKeyboardVisualKeyboardCompiler } from './visual-keyboard-compiler.js'; import { LinterKeycaps } from './linter-keycaps.js'; -import { LdmlEventResolver } from './eventresolver.js'; export const SECTION_COMPILERS = [ // These are in dependency order. @@ -95,7 +94,6 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { // uset parser private usetparser?: LdmlKeyboardTypes.UnicodeSetParser = undefined; - private static eventResolver: LdmlEventResolver = new LdmlEventResolver(); /** * Initialize the compiler, including loading the WASM host for uset parsing. @@ -108,7 +106,6 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { async init(callbacks: CompilerCallbacks, options: LdmlCompilerOptions): Promise { this.options = { ...options }; this.callbacks = callbacks; - this.callbacks.setEventResolver(LdmlKeyboardCompiler.eventResolver); return true; } @@ -243,12 +240,11 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { public load(filename: string): LDMLKeyboardXMLSourceFile | null { const reader = new LDMLKeyboardXMLSourceFileReader(this.options.readerOptions, this.callbacks); // load the file from disk into a string - const data = this.callbacks.loadFile(filename); + const data = reader.readFile(filename); if (!data) { this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidFile({ errorText: 'Unable to read XML file' })); return null; } - LdmlKeyboardCompiler.eventResolver.addFile(filename, new TextDecoder().decode(data)); // TODO: double decode // parse (load) the string into an object tree const source = reader.load(data); if (!source) { diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index 35a931dd94..5906646587 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -485,7 +485,9 @@ export class KeysCompiler extends SectionCompiler { } if (!keydef.output && !keydef.gap && !keydef.layerId) { this.callbacks.reportMessage( - LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch({ keyId: key }) + LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch({ keyId: key }, + keydef + ) ); valid = false; continue; diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 1298d0f0cd..1fcb82193b 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -1,5 +1,5 @@ import { util } from "@keymanapp/common-types"; -import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_START_INDEX_SYMBOL, CompilerEvent } from '@keymanapp/developer-utils'; +import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_START_INDEX_SYMBOL, XML_FILENAME_SYMBOL, CompilerEvent } from '@keymanapp/developer-utils'; // const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.LdmlKeyboardCompiler; @@ -91,8 +91,8 @@ export class LdmlCompilerMessages { m(this.ERROR_DisplayIsRepeated, `display ${LdmlCompilerMessages.outputOrKeyId(o)} has more than one display entry.`); static ERROR_KeyMissingToGapOrSwitch = SevError | 0x0011; - static Error_KeyMissingToGapOrSwitch = (o:{keyId: string}) => - m(this.ERROR_KeyMissingToGapOrSwitch, `key id='${def(o.keyId)}' must have either output=, gap=, or layerId=.`); + static Error_KeyMissingToGapOrSwitch = (o:{keyId: string}, x: any) => mx(x, + this.ERROR_KeyMissingToGapOrSwitch, `key id='${def(o.keyId)}' must have either output=, gap=, or layerId=.`); static ERROR_ExcessHardware = SevError | 0x0012; static Error_ExcessHardware = (o:{formId: string}) => m(this.ERROR_ExcessHardware, @@ -277,6 +277,7 @@ export class LdmlCompilerMessages { static offset(event: CompilerEvent, x?: any): CompilerEvent { if(x) { event.offset = (x as any)[XML_START_INDEX_SYMBOL as any]; + event.filename = event.filename || (x as any)[XML_FILENAME_SYMBOL as any]; } return event; } From 0ba005a55b21b728a92ad11171c788025155eb99 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Tue, 4 Mar 2025 09:34:47 -0600 Subject: [PATCH 13/46] feat(developer): line number symbols - improve and simplify lookup algorithm - add test for override filename on imports Fixes: #10622 --- .../src/common/web/utils/src/xml-utils.ts | 22 ++++++++++++------- .../kmx/ldml-keyboard-xml-reader.tests.ts | 13 ++++++++--- .../common/web/utils/test/xml-utils.tests.ts | 7 +++--- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index ee33b40563..7d7dc10011 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -137,18 +137,24 @@ export interface LineColumn { export class KeymanXMLReader { public constructor(public type: KeymanXMLType) { } - /** preprocess text to turn it into arrays of line lengths */ + /** + * preprocess text to turn it into arrays of line lengths. + * This is effectively a 1-based line length, since line 0 has length of + * 0. + */ public static textToLines(text: string) : number[] { - return text.replaceAll("\r\n", "\n").split("\n") - .map(l => l.length + 1); // line length (counting the trailing newline) + return [ + 0, // "line 0" is empty + ...text.replaceAll("\r\n", "\n").split("\n") + .map(l => l.length + 1) // line length (counting the trailing newline) + ]; } public static offsetToLineColumn(offset: number, lines: number[]) : LineColumn { - for (let line = 1; line < lines.length + 1; line++) { // 1-based - if (lines[line-1] < offset) { - offset = offset - (lines[line-1]); // count newline at end - continue; + for (let line = 1; line < lines.length; line++) { // 1-based (assume the first row is 0) + if (lines[line] > offset) { + return { line, column: offset }; } - return { line, column: offset }; + offset = offset - (lines[line]); // count newline at end } // default: line 0, error return { line: 0 } diff --git a/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts b/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts index fcb3628144..56a70e3ea0 100644 --- a/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts +++ b/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts @@ -8,6 +8,7 @@ import { Constants } from '@keymanapp/common-types'; import CLDRScanToVkey = Constants.CLDRScanToVkey; import CLDRScanToKeyMap = Constants.CLDRScanToKeyMap; import USVirtualKeyCodes = Constants.USVirtualKeyCodes; +import { XML_FILENAME_SYMBOL } from '../../src/xml-utils.js'; function pluckKeysFromKeybag(keys: LKKey[], ids: string[]) { return keys.filter(({id}) => ids.indexOf(id) !== -1); @@ -140,10 +141,16 @@ describe('ldml keyboard xml reader tests', function () { { id: 'interrobang', output: '‽' }, { id: 'snail', output: '@' }, ]); + const snailKey = source?.keyboard3?.keys.key.find(({ id }) => id === 'snail'); // all of the keys are implied imports here - assert.isFalse(ImportStatus.isImpliedImport(source?.keyboard3?.keys.key.find(({id}) => id === 'snail'))); - assert.isTrue(ImportStatus.isImport(source?.keyboard3?.keys.key.find(({id}) => id === 'snail'))); - assert.isTrue(ImportStatus.isLocalImport(source?.keyboard3?.keys.key.find(({id}) => id === 'snail'))); + assert.isFalse(ImportStatus.isImpliedImport(snailKey)); + assert.isTrue(ImportStatus.isImport(snailKey)); + assert.isTrue(ImportStatus.isLocalImport(snailKey)); + // get the actual filename of where the import was located + const snailFilename = (snailKey as any)[XML_FILENAME_SYMBOL as any]; + assert.ok(snailFilename); + assert.ok(/\/keys-Zyyy-morepunctuation.xml$/.test(snailFilename) + , `snail key filename is ${snailFilename}`); }, }, { diff --git a/developer/src/common/web/utils/test/xml-utils.tests.ts b/developer/src/common/web/utils/test/xml-utils.tests.ts index f487573724..5045bcfcb3 100644 --- a/developer/src/common/web/utils/test/xml-utils.tests.ts +++ b/developer/src/common/web/utils/test/xml-utils.tests.ts @@ -174,14 +174,15 @@ describe(`XML Reader line number test`, () => { // now, parse. subsitute endings for Win const actual = reader.parse(xml); + const lines = KeymanXMLReader.textToLines(xml); assert.ok(actual, `Parser failed on ${xmlPath}`); // now, assert char offset assert.equal(actual.keyboard3[XML_START_INDEX_SYMBOL as any], 40); // index of element assert.equal(actual.keyboard3.info[XML_START_INDEX_SYMBOL as any], 136); // index of etc assert.equal(actual.keyboard3.transforms[XML_START_INDEX_SYMBOL as any], 186); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3[XML_START_INDEX_SYMBOL as any], xml), { line: 3, column: 0 }); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.info[XML_START_INDEX_SYMBOL as any], xml), { line: 4, column: 2 }); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.transforms[XML_START_INDEX_SYMBOL as any], xml), { line: 8, column: 2 }); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3[XML_START_INDEX_SYMBOL as any], lines), { line: 3, column: 0 }); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.info[XML_START_INDEX_SYMBOL as any], lines), { line: 4, column: 2 }); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.transforms[XML_START_INDEX_SYMBOL as any], lines), { line: 8, column: 2 }); }); }); From 0c290bd37ad8b7454783f01d6a9b3043251d3af2 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 31 Mar 2025 16:11:19 -0500 Subject: [PATCH 14/46] feat(developer): update for line numbers - update to metadata branch of fast-xml-parser - restructure metadata handling - use type safety for fast-xml-parser options Fixes: #10622 --- developer/src/common/web/utils/src/index.ts | 4 +- .../src/common/web/utils/src/symbol-utils.ts | 33 ++++++++ .../ldml-keyboard/ldml-keyboard-xml-reader.ts | 4 +- .../src/common/web/utils/src/xml-utils.ts | 84 ++++++++++--------- .../kmx/ldml-keyboard-xml-reader.tests.ts | 6 +- .../utils/test/kpj/kpj-file-writer.tests.ts | 6 +- .../utils/test/kps/kps-file-reader.tests.ts | 6 +- .../web/utils/test/kvks/kvks-file.tests.ts | 6 +- .../common/web/utils/test/xml-utils.tests.ts | 28 +++++-- .../src/compiler/ldml-compiler-messages.ts | 9 +- .../src/kmc-ldml/test/helpers/compareXml.ts | 4 +- developer/src/kmc-ldml/test/helpers/index.ts | 13 +-- developer/src/kmc-ldml/test/keys.tests.ts | 2 +- package-lock.json | 4 +- 14 files changed, 132 insertions(+), 77 deletions(-) create mode 100644 developer/src/common/web/utils/src/symbol-utils.ts diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index b722a11b21..95ef39ffcb 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -67,8 +67,8 @@ export { UrlSubpathCompilerCallback } from './utils/UrlSubpathCompilerCallback.j export { CommonTypesMessages } from './common-messages.js'; export * as SourceFilenamePatterns from './source-filename-patterns.js'; -export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, XML_START_INDEX_SYMBOL, XML_FILENAME_SYMBOL } from './xml-utils.js'; - +export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, KeymanXMLMetadata, XML_FILENAME_SYMBOL } from './xml-utils.js'; +export { SymbolUtils } from './symbol-utils.js'; export * as GitHubUrls from './github-urls.js'; export * as CloudUrls from './cloud-urls.js'; diff --git a/developer/src/common/web/utils/src/symbol-utils.ts b/developer/src/common/web/utils/src/symbol-utils.ts new file mode 100644 index 0000000000..971d02cc16 --- /dev/null +++ b/developer/src/common/web/utils/src/symbol-utils.ts @@ -0,0 +1,33 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by srl on 2025-03-27 + * + * Utilities for manipulating Symbol properties + */ + +export class SymbolUtils { + /** + * Copy symbols from 'from' onto 'onto' + * @param onto object to copy onto + * @param from source for symbols + * @returns the onto object + */ + public static copySymbols(onto: T, from: any): T { + const o = onto as any; + for (const sym of Object.getOwnPropertySymbols(from)) { + o[sym] = from[sym]; + } + return onto; + } + + /** use Object.entries to remove all symbols */ + public static removeSymbols(from: T): T { + if (Array.isArray(from)) { + return from.map(o => SymbolUtils.removeSymbols(o)) as T; + } + if (typeof from !== "object") return from; + return Object.fromEntries(Object.entries(from).map(([k, v]) => ([k, SymbolUtils.removeSymbols(v)]))) as T; + } + +} diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index ca32fe4905..84e82c62ea 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -12,7 +12,7 @@ import { LDMLKeyboardTestDataXMLSourceFile, LKTTest, LKTTests } from './ldml-key import { KeymanXMLReader } from '@keymanapp/developer-utils'; import boxXmlArray = util.boxXmlArray; import { LdmlEventResolver } from './eventresolver.js'; -import { XML_FILENAME_SYMBOL } from 'src/xml-utils.js'; +import { XML_FILENAME_SYMBOL } from '../../xml-utils.js'; interface NameAndProps { '$'?: any; // content @@ -281,7 +281,7 @@ export class LDMLKeyboardXMLSourceFileReader { // Mark all children as an import subsubval.forEach(o => { o[ImportStatus.import] = basePath; - o[XML_FILENAME_SYMBOL] = importPath; // mark overriding import path + KeymanXMLReader.setMetaData(o, {[XML_FILENAME_SYMBOL]: importPath}); // mark overriding import path }); if (implied) { diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 7d7dc10011..eadeb8659d 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -6,10 +6,11 @@ * Abstraction for XML reading and writing */ -import { XMLParser, XMLBuilder } from 'fast-xml-parser'; +import { XMLParser, XMLBuilder, XMLMetaData, X2jOptions, XmlBuilderOptions } from 'fast-xml-parser'; +import { SymbolUtils } from "./symbol-utils.js"; /** Symbol giving the start offset, in chars, of the node */ -export const XML_START_INDEX_SYMBOL = XMLParser.getStartIndexSymbol(); +const XML_META_DATA_SYMBOL = XMLParser.getMetaDataSymbol(); /** Symbol giving an override which file a node came from */ export const XML_FILENAME_SYMBOL = Symbol("XML Filename"); @@ -22,12 +23,12 @@ export type KeymanXMLType = ; /** Bag of options, maximally one for each KeymanXMLType */ -type KeymanXMLOptionsBag = { - [key in KeymanXMLType]?: any +type KeymanXMLParserOptionsBag = { + [key in KeymanXMLType]?: X2jOptions }; /** map of options for the XML parser */ -const PARSER_OPTIONS: KeymanXMLOptionsBag = { +const PARSER_OPTIONS: KeymanXMLParserOptionsBag = { 'keyboard3': { ignoreAttributes: false, // We'd like attributes, please attributeNamePrefix: '@__', // We'll use this to convert attributes to strings and subobjects to arrays, when empty. @@ -40,6 +41,7 @@ const PARSER_OPTIONS: KeymanXMLOptionsBag = { // if we do need elements in the future, we'd check the preserve-space attribute here. return tagValue?.trim(); }, + captureMetaData: true, }, 'keyboardTest3': { ignorePiTags: true, @@ -87,7 +89,7 @@ const PARSER_OPTIONS: KeymanXMLOptionsBag = { eNotation: null, }, trimValues: false, // preserve spaces, but: - tagValueProcessor: (tagName: string, tagValue: string, jPath: string, hasAttributes: string, isLeafNode: boolean) : string | undefined => { + tagValueProcessor: (tagName: string, tagValue: string, jPath: string, hasAttributes: boolean, isLeafNode: boolean) : string | undefined => { if (!isLeafNode) { return tagValue?.trim(); // trimmed value } else { @@ -97,7 +99,11 @@ const PARSER_OPTIONS: KeymanXMLOptionsBag = { }, }; -const GENERATOR_OPTIONS: KeymanXMLOptionsBag = { +type KeymanXMLGeneratorOptionsBag = { + [key in KeymanXMLType]?: XmlBuilderOptions +}; + +const GENERATOR_OPTIONS: KeymanXMLGeneratorOptionsBag = { kvks: { attributeNamePrefix: '$', ignoreAttributes: false, @@ -133,10 +139,37 @@ export interface LineColumn { column?: number; } +export interface KeymanXMLMetadata extends XMLMetaData { + /** override of name of XML file */ + [XML_FILENAME_SYMBOL]?: string; +} + /** wrapper for XML parsing support */ export class KeymanXMLReader { public constructor(public type: KeymanXMLType) { } + + /** Get metadata on a node if not already set */ + static getMetaData(o: any) : KeymanXMLMetadata { + if(!o) return o; + const metadata : KeymanXMLMetadata = o[XML_META_DATA_SYMBOL as any]; + return metadata; + } + + /** Set metadata if not already set */ + public static setMetaData(o: any, metadata: KeymanXMLMetadata) : KeymanXMLMetadata { + let m : KeymanXMLMetadata = KeymanXMLReader.getMetaData(o); + if (!m) { + m = {}; + } + // copy non-symbols + m = {...metadata, ...m}; + // copy symbols + SymbolUtils.copySymbols(m, metadata); + o[XML_META_DATA_SYMBOL as any] = m; + return m; + } + /** * preprocess text to turn it into arrays of line lengths. * This is effectively a 1-based line length, since line 0 has length of @@ -160,31 +193,6 @@ export class KeymanXMLReader { return { line: 0 } } - /** - * Copy symbols from 'from' onto 'onto' - * This is used to propagate special symbols - * and XML - * @param onto object to copy onto - * @param from source for symbols - * @returns the onto object - */ - public static copySymbols(onto: T, from: any): T { - const o = onto as any; - for (const sym of Object.getOwnPropertySymbols(from)) { - o[sym] = from[sym]; - } - return onto; - } - - /** use Object.entries to remove all symbols */ - public static removeSymbols(from: T): T { - if (Array.isArray(from)) { - return from.map(o => KeymanXMLReader.removeSymbols(o)) as T; - } - if (typeof from !== "object") return from; - return Object.fromEntries(Object.entries(from).map(([k, v]) => ([k, KeymanXMLReader.removeSymbols(v)]))) as T; - } - /** move `{ $abc: 4 }` into `{ $: { abc: 4 } }` */ private static fixupDollarAttributes(data: any) : any { if (typeof data === 'object') { @@ -203,9 +211,9 @@ export class KeymanXMLReader { } }); if (attrs.length) { - e.push(['$', this.copySymbols(Object.fromEntries(attrs), data)]); + e.push(['$', SymbolUtils.copySymbols(Object.fromEntries(attrs), data)]); } - return this.copySymbols(Object.fromEntries(e), data); + return SymbolUtils.copySymbols(Object.fromEntries(e), data); } else { return data; } @@ -241,7 +249,7 @@ export class KeymanXMLReader { } } }); - return this.copySymbols(Object.fromEntries(e), data); + return SymbolUtils.copySymbols(Object.fromEntries(e), data); } else { return data; } @@ -321,15 +329,11 @@ export class KeymanXMLReader { } public parser() { - let options = PARSER_OPTIONS[this.type]; + const options = PARSER_OPTIONS[this.type]; if (!options) { /* c8 ignore next 1 */ throw Error(`Internal error: unhandled XML type ${this.type}`); } - options = Object.assign({}, options); // TODO: xml2js likes to mutate the options here. Shallow clone the object. - if (options.emptyTag) { - options.emptyTag = {}; // TODO: xml2js likes to mutate the options here. Reset it. - } return new XMLParser(options); } } diff --git a/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts b/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts index 56a70e3ea0..bfd076a34d 100644 --- a/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts +++ b/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts @@ -8,7 +8,7 @@ import { Constants } from '@keymanapp/common-types'; import CLDRScanToVkey = Constants.CLDRScanToVkey; import CLDRScanToKeyMap = Constants.CLDRScanToKeyMap; import USVirtualKeyCodes = Constants.USVirtualKeyCodes; -import { XML_FILENAME_SYMBOL } from '../../src/xml-utils.js'; +import { KeymanXMLReader, XML_FILENAME_SYMBOL } from '../../src/xml-utils.js'; function pluckKeysFromKeybag(keys: LKKey[], ids: string[]) { return keys.filter(({id}) => ids.indexOf(id) !== -1); @@ -147,7 +147,9 @@ describe('ldml keyboard xml reader tests', function () { assert.isTrue(ImportStatus.isImport(snailKey)); assert.isTrue(ImportStatus.isLocalImport(snailKey)); // get the actual filename of where the import was located - const snailFilename = (snailKey as any)[XML_FILENAME_SYMBOL as any]; + const metadata = KeymanXMLReader.getMetaData(snailKey); + assert.ok(metadata); + const snailFilename = (metadata)[XML_FILENAME_SYMBOL]; assert.ok(snailFilename); assert.ok(/\/keys-Zyyy-morepunctuation.xml$/.test(snailFilename) , `snail key filename is ${snailFilename}`); diff --git a/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts b/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts index 5fc223b240..425d76a6ef 100644 --- a/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts +++ b/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts @@ -6,7 +6,7 @@ import { KPJFileReader } from "../../src/types/kpj/kpj-file-reader.js"; import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; import { KPJFileWriter } from '../../src/types/kpj/kpj-file-writer.js'; import { KeymanDeveloperProjectOptions } from '../../src/types/kpj/keyman-developer-project.js'; -import { KeymanXMLReader } from '../../src/xml-utils.js'; +import { SymbolUtils } from '../../src/symbol-utils.js'; const callbacks = new TestCompilerCallbacks(); @@ -22,7 +22,7 @@ describe('kpj-file-writer', function () { const writer = new KPJFileWriter(); const output = writer.write(project); - const outputKpj = KeymanXMLReader.removeSymbols(reader.read(new TextEncoder().encode(output))); + const outputKpj = SymbolUtils.removeSymbols(reader.read(new TextEncoder().encode(output))); // The outputKpj may not contain all the fields from the inputKpj, only the // essential fields. Many of the fields in .kpj are deprecated, when they @@ -61,7 +61,7 @@ describe('kpj-file-writer', function () { const writer = new KPJFileWriter(); const output = writer.write(project); - const outputKpj = KeymanXMLReader.removeSymbols(reader.read(new TextEncoder().encode(output))); + const outputKpj = SymbolUtils.removeSymbols(reader.read(new TextEncoder().encode(output))); // The outputKpj may not contain all the fields from the inputKpj, only the // essential fields. Many of the fields in .kpj are deprecated, when they diff --git a/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts b/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts index 94a1628bd9..9b7955819d 100644 --- a/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts +++ b/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts @@ -15,7 +15,7 @@ import { makePathToFixture } from '../helpers/index.js'; import { KpsFileReader } from "../../src/types/kps/kps-file-reader.js"; import { KpsFileWriter } from '../../src/types/kps/kps-file-writer.js'; import { CommonTypesMessages } from '../../src/common-messages.js'; -import { KeymanXMLReader } from '../../src/xml-utils.js'; +import { SymbolUtils } from '../../src/symbol-utils.js'; const callbacks = new TestCompilerCallbacks(); @@ -62,13 +62,13 @@ describe('kps-file-reader', function () { it('kps-file-reader should round-trip with kps-file-writer', function() { const input = fs.readFileSync(makePathToFixture('kps', 'khmer_angkor.kps')); const reader = new KpsFileReader(callbacks); - const kps = KeymanXMLReader.removeSymbols(reader.read(input)); + const kps = SymbolUtils.removeSymbols(reader.read(input)); const writer = new KpsFileWriter(); const output = writer.write(kps); // Round Trip - const kps2 = KeymanXMLReader.removeSymbols(reader.read(new TextEncoder().encode(output))); + const kps2 = SymbolUtils.removeSymbols(reader.read(new TextEncoder().encode(output))); assert.deepEqual(kps2, kps); }); diff --git a/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts b/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts index 96be2283b8..d264dc86aa 100644 --- a/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts +++ b/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts @@ -5,7 +5,7 @@ import KvksFileReader from "../../src/types/kvks/kvks-file-reader.js"; import KvksFileWriter from "../../src/types/kvks/kvks-file-writer.js"; import { verify_khmer_angkor, verify_balochi_inpage } from './kvk-utils.tests.js'; import { assert } from 'chai'; -import { KeymanXMLReader } from '../../src/xml-utils.js'; +import { SymbolUtils } from '../../src/symbol-utils.js'; describe('kvks-file-reader', function() { it('should read a valid file', function() { @@ -49,7 +49,7 @@ describe('kvks-file-writer', function() { const input = fs.readFileSync(path); const reader = new KvksFileReader(); - const kvksExpected = KeymanXMLReader.removeSymbols(reader.read(input)); + const kvksExpected = SymbolUtils.removeSymbols(reader.read(input)); const invalidVkeys: string[] = []; const vk = reader.transform(kvksExpected, invalidVkeys); assert.isEmpty(invalidVkeys); @@ -60,6 +60,6 @@ describe('kvks-file-writer', function() { // We compare the (re)loaded data, because there may be // minor, irrelevant formatting differences in the emitted xml const kvks = reader.read(Buffer.from(output, 'utf8')); - assert.deepEqual(KeymanXMLReader.removeSymbols(kvks), kvksExpected); + assert.deepEqual(SymbolUtils.removeSymbols(kvks), kvksExpected); }); }); diff --git a/developer/src/common/web/utils/test/xml-utils.tests.ts b/developer/src/common/web/utils/test/xml-utils.tests.ts index 5045bcfcb3..8728eec207 100644 --- a/developer/src/common/web/utils/test/xml-utils.tests.ts +++ b/developer/src/common/web/utils/test/xml-utils.tests.ts @@ -12,8 +12,9 @@ import { env } from 'node:process'; import { readFileSync, writeFileSync } from 'node:fs'; -import { KeymanXMLType, KeymanXMLReader, KeymanXMLWriter, XML_START_INDEX_SYMBOL } from '../src/xml-utils.js'; +import { KeymanXMLType, KeymanXMLReader, KeymanXMLWriter } from '../src/xml-utils.js'; import { makePathToFixture } from './helpers/index.js'; +import { SymbolUtils } from '../src/symbol-utils.js'; // if true, attempt to WRITE the fixtures const { GEN_XML_FIXTURES } = env; @@ -119,7 +120,7 @@ describe(`XML Reader Test ${GEN_XML_FIXTURES && '(update mode!)' || ''}`, () => writeJson(jsonPath, actual); } else { assert.ok(expect, `Could not read ${jsonPath} - run with env GEN_XML_FIXTURES=1 to update.`); - assert.deepEqual(KeymanXMLReader.removeSymbols(actual), expect, `Mismatch of ${xmlPath} vs ${jsonPath}`); + assert.deepEqual(SymbolUtils.removeSymbols(actual), expect, `Mismatch of ${xmlPath} vs ${jsonPath}`); } }); } @@ -178,11 +179,22 @@ describe(`XML Reader line number test`, () => { assert.ok(actual, `Parser failed on ${xmlPath}`); // now, assert char offset - assert.equal(actual.keyboard3[XML_START_INDEX_SYMBOL as any], 40); // index of element - assert.equal(actual.keyboard3.info[XML_START_INDEX_SYMBOL as any], 136); // index of etc - assert.equal(actual.keyboard3.transforms[XML_START_INDEX_SYMBOL as any], 186); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3[XML_START_INDEX_SYMBOL as any], lines), { line: 3, column: 0 }); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.info[XML_START_INDEX_SYMBOL as any], lines), { line: 4, column: 2 }); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.transforms[XML_START_INDEX_SYMBOL as any], lines), { line: 8, column: 2 }); + const getMetaData = KeymanXMLReader.getMetaData; + assert.ok(getMetaData(actual.keyboard3)); + assert.equal( + getMetaData(actual.keyboard3)?.startIndex, 40); // index of element + assert.equal( + getMetaData(actual.keyboard3.info)?.startIndex, 136); // index of etc + assert.equal( + getMetaData(actual.keyboard3.transforms)?.startIndex, 186); + assert.deepEqual( + KeymanXMLReader.offsetToLineColumn( + getMetaData(actual.keyboard3).startIndex, lines), { line: 3, column: 0 }); + assert.deepEqual( + KeymanXMLReader.offsetToLineColumn( + getMetaData(actual.keyboard3.info).startIndex, lines), { line: 4, column: 2 }); + assert.deepEqual( + KeymanXMLReader.offsetToLineColumn( + getMetaData(actual.keyboard3.transforms).startIndex, lines), { line: 8, column: 2 }); }); }); diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 1fcb82193b..d363fa03db 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -1,5 +1,5 @@ import { util } from "@keymanapp/common-types"; -import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_START_INDEX_SYMBOL, XML_FILENAME_SYMBOL, CompilerEvent } from '@keymanapp/developer-utils'; +import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_FILENAME_SYMBOL, CompilerEvent, KeymanXMLReader } from '@keymanapp/developer-utils'; // const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.LdmlKeyboardCompiler; @@ -271,13 +271,14 @@ export class LdmlCompilerMessages { /** * Get an offset from o and set e's offset field * @param event a compiler event, such as from functions in this class - * @param x any object parsed from XML or with the XML_START_INDEX_SYMBOL symbol copied over + * @param x any object parsed from XML or with the XML_META_DATA_SYMBOL symbol copied over * @returns modified event object */ static offset(event: CompilerEvent, x?: any): CompilerEvent { if(x) { - event.offset = (x as any)[XML_START_INDEX_SYMBOL as any]; - event.filename = event.filename || (x as any)[XML_FILENAME_SYMBOL as any]; + const metadata = KeymanXMLReader.getMetaData(x) || {}; + event.offset = metadata?.startIndex; + event.filename = event.filename || metadata[XML_FILENAME_SYMBOL]; } return event; } diff --git a/developer/src/kmc-ldml/test/helpers/compareXml.ts b/developer/src/kmc-ldml/test/helpers/compareXml.ts index 1503d4c606..057986bdb2 100644 --- a/developer/src/kmc-ldml/test/helpers/compareXml.ts +++ b/developer/src/kmc-ldml/test/helpers/compareXml.ts @@ -1,6 +1,6 @@ import {assert} from 'chai'; import {readFileSync} from 'node:fs'; -import { KeymanXMLReader } from "@keymanapp/developer-utils"; +import { KeymanXMLReader, SymbolUtils } from "@keymanapp/developer-utils"; /** * @@ -21,5 +21,5 @@ export function compareXml(actual : string, expect: string, mutator?: (input: an const actualParsed = mutator(reader.parse(actualStr)); const expectParsed = mutator(reader.parse(expectStr)); - assert.deepEqual(KeymanXMLReader.removeSymbols(actualParsed), KeymanXMLReader.removeSymbols(expectParsed)); + assert.deepEqual(SymbolUtils.removeSymbols(actualParsed), SymbolUtils.removeSymbols(expectParsed)); } diff --git a/developer/src/kmc-ldml/test/helpers/index.ts b/developer/src/kmc-ldml/test/helpers/index.ts index 497f8ef919..bb10825315 100644 --- a/developer/src/kmc-ldml/test/helpers/index.ts +++ b/developer/src/kmc-ldml/test/helpers/index.ts @@ -8,7 +8,7 @@ import * as path from 'path'; import { fileURLToPath } from 'url'; import { SectionCompiler, SectionCompilerNew } from '../../src/compiler/section-compiler.js'; import { util, KMXPlus, LdmlKeyboardTypes } from '@keymanapp/common-types'; -import { CompilerEvent, compilerEventFormat, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboard, XML_START_INDEX_SYMBOL, } from "@keymanapp/developer-utils"; +import { CompilerEvent, compilerEventFormat, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboard, KeymanXMLMetadata, KeymanXMLReader } from "@keymanapp/developer-utils"; import { LdmlKeyboardCompiler } from '../../src/main.js'; // make sure main.js compiles import { assert } from 'chai'; import { KMXPlusMetadataCompiler } from '../../src/compiler/metadata-compiler.js'; @@ -329,8 +329,11 @@ export function hex_str(s?: string) : string { } /** return an object simulating an XML object with a column number */ -export function withColumn(c: number) : any { - return { - [XML_START_INDEX_SYMBOL as any]: c - }; +export function withColumn(c: number) : KeymanXMLMetadata { + // set metadata on an empty object + const o = {}; + KeymanXMLReader.setMetaData(o, { + startIndex: c + }); + return o; } diff --git a/developer/src/kmc-ldml/test/keys.tests.ts b/developer/src/kmc-ldml/test/keys.tests.ts index 1b74763646..3d87caaa7a 100644 --- a/developer/src/kmc-ldml/test/keys.tests.ts +++ b/developer/src/kmc-ldml/test/keys.tests.ts @@ -464,7 +464,7 @@ describe('keys.kmap', function () { let keys = await loadSectionFixture(KeysCompiler, 'sections/keys/invalid-key-missing-attrs.xml', compilerTestCallbacks, keysDependencies) as Keys; assert.isNull(keys); assert.equal(compilerTestCallbacks.messages.length, 1); - assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch({keyId: 'Q'})); + assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch({keyId: 'Q'}, withColumn(188))); }); it('should accept layouts with gap/switch keys', async function() { let keys = await loadSectionFixture(KeysCompiler, 'sections/keys/gap-switch.xml', compilerTestCallbacks, keysDependencies) as Keys; diff --git a/package-lock.json b/package-lock.json index 192f383713..10a5e5d9d6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9094,8 +9094,8 @@ "dev": true }, "node_modules/fast-xml-parser": { - "version": "5.0.8", - "resolved": "git+ssh://git@github.com/srl295/fast-xml-parser.git#72f20480e8591aca819fac684d46f8d678fcb7f3", + "version": "5.0.9", + "resolved": "git+ssh://git@github.com/srl295/fast-xml-parser.git#985271d7c731a6777456c362849d106073feaddd", "funding": [ { "type": "github", From 1f2b8463f5c368bd5fd79584b4249e00a78041ad Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 3 Apr 2025 12:15:20 -0500 Subject: [PATCH 15/46] chore(developer): deps: update to fast-xml-parser@5.1.0 (Metadata branch) Fixes: #10622 --- developer/src/common/web/utils/package.json | 2 +- package-lock.json | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/developer/src/common/web/utils/package.json b/developer/src/common/web/utils/package.json index a56fdc386a..4bceb5d6e4 100644 --- a/developer/src/common/web/utils/package.json +++ b/developer/src/common/web/utils/package.json @@ -12,7 +12,7 @@ "@keymanapp/common-types": "*", "@sentry/node": "^7.57.0", "eventemitter3": "^5.0.0", - "fast-xml-parser": "github:srl295/fast-xml-parser#line-number-593bis", + "fast-xml-parser": "github:NaturalIntelligence/fast-xml-parser#metadata", "path-browserify": "^1.0.1", "restructure": "^3.0.1", "sax": ">=0.6.0", diff --git a/package-lock.json b/package-lock.json index 10a5e5d9d6..05ed59e500 100644 --- a/package-lock.json +++ b/package-lock.json @@ -342,7 +342,7 @@ "@keymanapp/common-types": "*", "@sentry/node": "^7.57.0", "eventemitter3": "^5.0.0", - "fast-xml-parser": "github:srl295/fast-xml-parser#line-number-593bis", + "fast-xml-parser": "github:NaturalIntelligence/fast-xml-parser#metadata", "path-browserify": "^1.0.1", "restructure": "^3.0.1", "sax": ">=0.6.0", @@ -9094,15 +9094,14 @@ "dev": true }, "node_modules/fast-xml-parser": { - "version": "5.0.9", - "resolved": "git+ssh://git@github.com/srl295/fast-xml-parser.git#985271d7c731a6777456c362849d106073feaddd", + "version": "5.1.0", + "resolved": "git+ssh://git@github.com/NaturalIntelligence/fast-xml-parser.git#ab8d40a3077f109f34d02d05653146b8621bd255", "funding": [ { "type": "github", "url": "https://github.com/sponsors/NaturalIntelligence" } ], - "license": "MIT", "dependencies": { "strnum": "^2.0.5" }, From 8d438283c61afb6f26a5a70d106ed41ec91d3a8c Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 3 Apr 2025 16:40:53 -0500 Subject: [PATCH 16/46] feat(developer): update for line numbers - split out LineFinder to separate class Fixes: #10622 --- developer/src/common/web/utils/src/index.ts | 1 + .../src/common/web/utils/src/line-utils.ts | 61 +++++++++++++++++++ .../src/types/ldml-keyboard/eventresolver.ts | 20 ++++-- .../src/common/web/utils/src/xml-utils.ts | 28 --------- .../common/web/utils/test/line-utils.tests.ts | 28 +++++++++ .../common/web/utils/test/xml-utils.tests.ts | 10 +-- 6 files changed, 110 insertions(+), 38 deletions(-) create mode 100644 developer/src/common/web/utils/src/line-utils.ts create mode 100644 developer/src/common/web/utils/test/line-utils.tests.ts diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index 95ef39ffcb..dc51720b25 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -69,6 +69,7 @@ export { CommonTypesMessages } from './common-messages.js'; export * as SourceFilenamePatterns from './source-filename-patterns.js'; export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, KeymanXMLMetadata, XML_FILENAME_SYMBOL } from './xml-utils.js'; export { SymbolUtils } from './symbol-utils.js'; +export { LineFinder, LineColumn } from './line-utils.js'; export * as GitHubUrls from './github-urls.js'; export * as CloudUrls from './cloud-urls.js'; diff --git a/developer/src/common/web/utils/src/line-utils.ts b/developer/src/common/web/utils/src/line-utils.ts new file mode 100644 index 0000000000..5c9a4dec31 --- /dev/null +++ b/developer/src/common/web/utils/src/line-utils.ts @@ -0,0 +1,61 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by srl on 2025-04-03 + * + * Abstraction for line number processing + */ + +/** line number with an optional column number */ +export interface LineColumn { + line: number; + column?: number; +} + +export type LineList = number[]; + +/** a LineFinder is able to convert from an offset to a line/column */ +export class LineFinder { + private list: LineList; + constructor(public text: string) { + this.list = LineFinder.textToLines(text); + } + + /** + * Convert an offset into line:column + * @param offset input offset into the text + * @returns line:column information + */ + public findOffset(offset: number): LineColumn { + return LineFinder.offsetToLineColumn(offset, this.list); + } + + /** + * preprocess text to turn it into arrays of line lengths. + * This is effectively a 1-based line length, since line 0 has length of + * 0. + */ + public static textToLines(text: string): LineList { + return [ + 0, // "line 0" is empty + ...text.replaceAll("\r\n", "\n").split("\n") + .map(l => l.length + 1) // line length (counting the trailing newline) + ]; + } + + /** + * convert a line number array to a line/column + */ + public static offsetToLineColumn(offset: number, list: LineList): LineColumn { + for (let line = 1; line < list.length; line++) { // 1-based (assume the first row is 0) + if (list[line] > offset) { + return { line, column: offset }; + } + offset = offset - (list[line]); // count newline at end + } + // default: line 0, error + return { line: 0 } + } +} + + diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts index 3299169642..d155cf27d2 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts @@ -1,18 +1,26 @@ -import { CompilerEvent, EventResolver, KeymanXMLReader } from "../../index.js"; +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by srl on 2025-03-03 + * + * Implementation of a hook on LDML events + */ +import { CompilerEvent, EventResolver, LineFinder } from "../../index.js"; export class LdmlEventResolver implements EventResolver { - contentsCache: Map = new Map(); + contentsCache: Map = new Map(); - /** add or update a file */ + /** add or update a source file */ addFile(filename: string, xml: any) { - this.contentsCache.set(filename, KeymanXMLReader.textToLines(xml)); + this.contentsCache.set(filename, new LineFinder(xml)); } + /** resolve a CompilerEvent by expanding the line numbers */ resolve(event: CompilerEvent) { if (event.offset && !event.line && event.filename) { - const lines = this.contentsCache.get(event.filename); - const loc = KeymanXMLReader.offsetToLineColumn(event.offset, lines); + const lf = this.contentsCache.get(event.filename); + const loc = lf.findOffset(event.offset); event.line = loc.line; event.column = loc.column; } diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 435fbb4c20..9ec8872975 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -115,11 +115,6 @@ const GENERATOR_OPTIONS: KeymanXMLGeneratorOptionsBag = { }, }; -export interface LineColumn { - line: number; - column?: number; -} - export interface KeymanXMLMetadata extends XMLMetaData { /** override of name of XML file */ [XML_FILENAME_SYMBOL]?: string; @@ -151,29 +146,6 @@ export class KeymanXMLReader { return m; } - /** - * preprocess text to turn it into arrays of line lengths. - * This is effectively a 1-based line length, since line 0 has length of - * 0. - */ - public static textToLines(text: string) : number[] { - return [ - 0, // "line 0" is empty - ...text.replaceAll("\r\n", "\n").split("\n") - .map(l => l.length + 1) // line length (counting the trailing newline) - ]; - } - public static offsetToLineColumn(offset: number, lines: number[]) : LineColumn { - for (let line = 1; line < lines.length; line++) { // 1-based (assume the first row is 0) - if (lines[line] > offset) { - return { line, column: offset }; - } - offset = offset - (lines[line]); // count newline at end - } - // default: line 0, error - return { line: 0 } - } - /** move `{ $abc: 4 }` into `{ $: { abc: 4 } }` */ private static fixupDollarAttributes(data: any) : any { if (typeof data === 'object') { diff --git a/developer/src/common/web/utils/test/line-utils.tests.ts b/developer/src/common/web/utils/test/line-utils.tests.ts new file mode 100644 index 0000000000..9c7b5da01c --- /dev/null +++ b/developer/src/common/web/utils/test/line-utils.tests.ts @@ -0,0 +1,28 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by srl on 2025-04-03 + * + * Test for LineFinder + */ + +import { assert } from 'chai'; +import { readFileSync } from 'node:fs'; +import 'mocha'; +import { LineFinder } from '../src/line-utils.js'; +import { makePathToFixture } from './helpers/index.js'; + +describe(`LineFinder test`, () => { + const path = 'tran_fail-empty.xml'; + + it(`Should be able to process ${path}`, () => { + const xmlPath = makePathToFixture('xml', `${path}`); + const data = readFileSync(xmlPath, 'utf-8'); + assert.ok(data); + const lf = new LineFinder(data); + assert.ok(lf); + assert.deepEqual(lf.findOffset(40), {line: 3, column: 0}); + assert.deepEqual(lf.findOffset(136), {line: 4, column: 2}); + assert.deepEqual(lf.findOffset(186), {line: 8, column: 2}); + }); +}); diff --git a/developer/src/common/web/utils/test/xml-utils.tests.ts b/developer/src/common/web/utils/test/xml-utils.tests.ts index 8728eec207..4b3df736b6 100644 --- a/developer/src/common/web/utils/test/xml-utils.tests.ts +++ b/developer/src/common/web/utils/test/xml-utils.tests.ts @@ -13,6 +13,7 @@ import { readFileSync, writeFileSync } from 'node:fs'; import { KeymanXMLType, KeymanXMLReader, KeymanXMLWriter } from '../src/xml-utils.js'; +import { LineFinder } from '../src/line-utils.js'; import { makePathToFixture } from './helpers/index.js'; import { SymbolUtils } from '../src/symbol-utils.js'; @@ -175,7 +176,7 @@ describe(`XML Reader line number test`, () => { // now, parse. subsitute endings for Win const actual = reader.parse(xml); - const lines = KeymanXMLReader.textToLines(xml); + const lines = LineFinder.textToLines(xml); assert.ok(actual, `Parser failed on ${xmlPath}`); // now, assert char offset @@ -188,13 +189,14 @@ describe(`XML Reader line number test`, () => { assert.equal( getMetaData(actual.keyboard3.transforms)?.startIndex, 186); assert.deepEqual( - KeymanXMLReader.offsetToLineColumn( + LineFinder.offsetToLineColumn( getMetaData(actual.keyboard3).startIndex, lines), { line: 3, column: 0 }); assert.deepEqual( - KeymanXMLReader.offsetToLineColumn( + LineFinder.offsetToLineColumn( getMetaData(actual.keyboard3.info).startIndex, lines), { line: 4, column: 2 }); assert.deepEqual( - KeymanXMLReader.offsetToLineColumn( + LineFinder.offsetToLineColumn( getMetaData(actual.keyboard3.transforms).startIndex, lines), { line: 8, column: 2 }); }); }); + From 11f77ed87647ff4e1e32c90388d664a939168092 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 3 Apr 2025 16:47:04 -0500 Subject: [PATCH 17/46] feat(developer): update for line numbers - fix ts issue on reader Fixes: #10622 --- .../utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index 84e82c62ea..2b5b0fc746 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -281,7 +281,7 @@ export class LDMLKeyboardXMLSourceFileReader { // Mark all children as an import subsubval.forEach(o => { o[ImportStatus.import] = basePath; - KeymanXMLReader.setMetaData(o, {[XML_FILENAME_SYMBOL]: importPath}); // mark overriding import path + KeymanXMLReader.setMetaData(o, {[XML_FILENAME_SYMBOL as any]: importPath}); // mark overriding import path }); if (implied) { From 53fd228d0889da1fb1f3aabf791af3fae9bb9d2b Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 3 Apr 2025 17:24:57 -0500 Subject: [PATCH 18/46] feat(developer): update for line numbers - fix windows path specific issue in test Fixes: #10622 --- .../web/utils/test/ldml/ldml-keyboard-xml-reader.tests.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/src/common/web/utils/test/ldml/ldml-keyboard-xml-reader.tests.ts b/developer/src/common/web/utils/test/ldml/ldml-keyboard-xml-reader.tests.ts index 6da13e0645..af0005cb6d 100644 --- a/developer/src/common/web/utils/test/ldml/ldml-keyboard-xml-reader.tests.ts +++ b/developer/src/common/web/utils/test/ldml/ldml-keyboard-xml-reader.tests.ts @@ -151,7 +151,7 @@ describe('ldml keyboard xml reader tests', function () { assert.ok(metadata); const snailFilename = (metadata)[XML_FILENAME_SYMBOL]; assert.ok(snailFilename); - assert.ok(/\/keys-Zyyy-morepunctuation.xml$/.test(snailFilename) + assert.ok(/keys-Zyyy-morepunctuation.xml$/.test(snailFilename) , `snail key filename is ${snailFilename}`); }, }, From e003612ea05b5bdcb048868b7900250088e82ca7 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 10 Apr 2025 17:22:51 -0500 Subject: [PATCH 19/46] feat(developer): update for line numbers - remove resolver complexity in callback subclasses - call LDMLKeyboardXMLSourceFileReader.eventResolver.resolve() statically from ldml-compiler-messages Fixes: #10622 --- .../web/test-helpers/TestCompilerCallbacks.ts | 13 +------------ .../common/web/utils/src/compiler-callbacks.ts | 16 ---------------- .../ldml-keyboard/ldml-keyboard-xml-reader.ts | 1 - .../src/compiler/ldml-compiler-messages.ts | 9 +++++++-- .../src/kmc/src/util/NodeCompilerCallbacks.ts | 13 +------------ 5 files changed, 9 insertions(+), 43 deletions(-) diff --git a/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts b/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts index cd1c816aaa..893e3363ed 100644 --- a/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts +++ b/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts @@ -2,7 +2,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { CompilerEvent, CompilerCallbacks, CompilerPathCallbacks, CompilerFileSystemCallbacks, CompilerError, CompilerNetAsyncCallbacks, DefaultCompilerFileSystemAsyncCallbacks, - CompilerFileSystemAsyncCallbacks, EventResolver, NullEventResolver, + CompilerFileSystemAsyncCallbacks, CompilerErrorSeverity} from '@keymanapp/developer-utils'; import { fileURLToPath } from 'url'; @@ -111,18 +111,7 @@ export class TestCompilerCallbacks implements CompilerCallbacks { return filename; } - private eventResolver = new NullEventResolver(); - - setEventResolver(eventResolver: EventResolver): void { - this.eventResolver = eventResolver; - } - reportMessage(event: CompilerEvent): void { - this.eventResolver.resolve(event); - this.handleReportMessage(event); - } - - handleReportMessage(event: CompilerEvent): void { // console.log(event.message); this.messages.push(event); } diff --git a/developer/src/common/web/utils/src/compiler-callbacks.ts b/developer/src/common/web/utils/src/compiler-callbacks.ts index f8b42afb7a..aabf1794e5 100644 --- a/developer/src/common/web/utils/src/compiler-callbacks.ts +++ b/developer/src/common/web/utils/src/compiler-callbacks.ts @@ -55,12 +55,8 @@ export interface CompilerCallbacks { */ resolveFilename(baseFilename: string, filename: string): string; - /** change the EventResolver to use. If called more than once, the last resolver replaces a prior one. */ - setEventResolver(eventResolver: EventResolver): void; - /** * Report a message from the compiler back to the caller. - * It is expected that eventResolver.resolve() will be called on the message before processing. * @param event */ reportMessage(event: CompilerEvent): void; @@ -225,19 +221,7 @@ export class CompilerFileCallbacks implements CompilerCallbacks { return this.parent.resolveFilename(baseFilename, filename); } - private eventResolver = new NullEventResolver(); - - setEventResolver(eventResolver: EventResolver): void { - this.eventResolver = eventResolver; - this.parent.setEventResolver(eventResolver); - } - reportMessage(event: CompilerEvent): void { - this.eventResolver.resolve(event); - this.handleReportMessage(event); - } - - handleReportMessage(event: CompilerEvent): void { const disable = CompilerFileCallbacks.applyMessageOverridesToEvent(event, this.options.messageOverrides); this.messages.push(event); if (!disable) { diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index 36dab45202..2d099849cd 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -30,7 +30,6 @@ export class LDMLKeyboardXMLSourceFileReaderOptions { export class LDMLKeyboardXMLSourceFileReader { static eventResolver: LdmlEventResolver = new LdmlEventResolver(); constructor(private options: LDMLKeyboardXMLSourceFileReaderOptions, private callbacks : CompilerCallbacks) { - callbacks.setEventResolver(LDMLKeyboardXMLSourceFileReader.eventResolver); } static get defaultImportsURL(): [string,string] { diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 3a26c9a5fe..6deeedfceb 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -1,5 +1,5 @@ import { util } from "@keymanapp/common-types"; -import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_FILENAME_SYMBOL, CompilerEvent, KeymanXMLReader } from '@keymanapp/developer-utils'; +import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_FILENAME_SYMBOL, CompilerEvent, KeymanXMLReader, LDMLKeyboardXMLSourceFileReader } from '@keymanapp/developer-utils'; // const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.LdmlKeyboardCompiler; @@ -10,7 +10,12 @@ const SevError = CompilerErrorSeverity.Error | CompilerErrorNamespace.LdmlKeyboa const SevErrorTransform = SevError | 0xF00; // like m() but takes an XML object for line numbers -const mx = (x: any, code: number, message: string, detail?: string): CompilerEvent => LdmlCompilerMessages.offset(m(code, message, detail), x); +const mx = (x: any, code: number, message: string, detail?: string): CompilerEvent => { + let evt = m(code, message, detail); // raw message + evt = LdmlCompilerMessages.offset(evt, x); // with offset + evt = LDMLKeyboardXMLSourceFileReader.eventResolver.resolve(evt); // offset expanded to line number + return evt; +}; /** * @internal diff --git a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts index d42e3b6545..2ec3854c3e 100644 --- a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts +++ b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { platform } from 'os'; -import { CompilerCallbacks, CompilerEvent, EventResolver, NullEventResolver, +import { CompilerCallbacks, CompilerEvent, CompilerPathCallbacks, CompilerFileSystemCallbacks, compilerLogLevelToSeverity, CompilerErrorSeverity, CompilerError, @@ -148,18 +148,7 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { return fileURLToPath(url); } - private eventResolver = new NullEventResolver(); - - setEventResolver(eventResolver: EventResolver): void { - this.eventResolver = eventResolver; - } - reportMessage(event: CompilerEvent): void { - this.eventResolver.resolve(event); - this.handleReportMessage(event); - } - - handleReportMessage(event: CompilerEvent): void { if(!event.filename) { event.filename = this.messageFilename; } From 3109e9e8abf487bcc12874406d37847b624f5dbd Mon Sep 17 00:00:00 2001 From: Shawn Schantz Date: Mon, 5 May 2025 11:53:49 +0700 Subject: [PATCH 20/46] chore(mac): remove unused String extension functions --- .../KME/Categories/NSString+XString.h | 8 - .../KME/Categories/NSString+XString.m | 166 ------------------ 2 files changed, 174 deletions(-) diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/Categories/NSString+XString.h b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/Categories/NSString+XString.h index 3647ea2c3f..93268d5917 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/Categories/NSString+XString.h +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/Categories/NSString+XString.h @@ -11,9 +11,6 @@ @interface NSString (XString) -- (NSUInteger)xLength; -- (NSUInteger)xLengthIncludingIf; -- (NSUInteger)deadKeyCount; - (NSString *)lastNChars:(NSUInteger)n; - (NSString *)codeString; - (BOOL)isValidCode; @@ -23,11 +20,6 @@ @interface NSMutableString (XString) -- (void)appendDeadkey:(NSUInteger)index; -- (void)appendNullChar; -- (NSUInteger)deleteLastDeadkeys; -- (NSUInteger)deleteLastNullChars; -- (NSUInteger)removeAllNullChars; - (void)deleteLastNChars:(NSUInteger)n; @end diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/Categories/NSString+XString.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/Categories/NSString+XString.m index f7d5c8c882..7b280a9946 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/Categories/NSString+XString.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/Categories/NSString+XString.m @@ -10,121 +10,6 @@ @implementation NSString (XString) -// For calculating offset including system stores -- (NSUInteger)xLengthIncludingIf { - NSUInteger cx = 0; - for (int i = 0; i < self.length;) { - unichar c = [self characterAtIndex:i]; - if (c == UC_SENTINEL) { - c = [self characterAtIndex:i+1]; - switch (c) { - case CODE_ANY: - case CODE_NOTANY: - case CODE_CONTEXTEX: - case CODE_DEADKEY: - case CODE_NULLCHAR: - cx++; - i+=3; - break; - case CODE_INDEX: - cx++; - i+=4; - break; - case CODE_IFOPT: - case CODE_IFSYSTEMSTORE: - cx++; - i+=5; - break; - case CODE_NUL: - default: - i+=2; - break; - } - } - else { - cx++; - i++; - } - } - - return cx; -} - -- (NSUInteger)xLength { - NSUInteger cx = 0; - for (int i = 0; i < self.length;) { - unichar c = [self characterAtIndex:i]; - if (c == UC_SENTINEL) { - c = [self characterAtIndex:i+1]; - switch (c) { - case CODE_ANY: - case CODE_NOTANY: - case CODE_CONTEXTEX: - case CODE_DEADKEY: - case CODE_NULLCHAR: - cx++; - i+=3; - break; - case CODE_INDEX: - cx++; - i+=4; - break; - case CODE_IFOPT: - case CODE_IFSYSTEMSTORE: - i+=5; - break; - case CODE_NUL: - default: - i+=2; - break; - } - } - else { - cx++; - i++; - } - } - - return cx; -} - -- (NSUInteger)deadKeyCount { - NSUInteger dk = 0; - for (int i = 0; i < self.length;) { - unichar c = [self characterAtIndex:i]; - if (c == UC_SENTINEL) { - c = [self characterAtIndex:i+1]; - switch (c) { - case CODE_DEADKEY: - dk++; - i+=3; - break; - case CODE_NULLCHAR: - case CODE_ANY: - case CODE_NOTANY: - case CODE_CONTEXTEX: - i+=3; - break; - case CODE_INDEX: - i+=4; - break; - case CODE_IFOPT: - case CODE_IFSYSTEMSTORE: - i+=5; - break; - case CODE_NUL: - default: - i+=2; - break; - } - } - else { - i++; - } - } - - return dk; -} - (NSString *)lastNChars:(NSUInteger)n { if (n == 0) @@ -395,57 +280,6 @@ @implementation NSMutableString (XString) -- (void)appendDeadkey:(NSUInteger)index { - NSString *dk = [NSString stringWithFormat:@"%C%C%C", UC_SENTINEL, CODE_DEADKEY, index]; - [self appendString:dk]; -} - -- (void)appendNullChar { - [self appendString:[[self class] nullChar]]; -} - -- (NSUInteger)deleteLastDeadkeys { - NSUInteger dkc = 0; - if (self.length >= 3) { - unichar c1 = [self characterAtIndex:self.length - 3]; - unichar c2 = [self characterAtIndex:self.length - 2]; - while (c1 == UC_SENTINEL && c2 == CODE_DEADKEY) { - NSRange range = NSMakeRange(self.length - 3, 3); - [self deleteCharactersInRange:range]; - dkc++; - if (self.length < 3) - break; - c1 = [self characterAtIndex:self.length - 3]; - c2 = [self characterAtIndex:self.length - 2]; - } - } - - return dkc; -} - -- (NSUInteger)deleteLastNullChars { - NSUInteger dsc = 0; - if (self.length >= 3) { - unichar c1 = [self characterAtIndex:self.length - 3]; - unichar c2 = [self characterAtIndex:self.length - 2]; - while (c1 == UC_SENTINEL && c2 == CODE_NULLCHAR) { - NSRange range = NSMakeRange(self.length - 3, 3); - [self deleteCharactersInRange:range]; - dsc++; - if (self.length < 3) - break; - c1 = [self characterAtIndex:self.length - 3]; - c2 = [self characterAtIndex:self.length - 2]; - } - } - - return dsc; -} - -- (NSUInteger)removeAllNullChars { - return [self replaceOccurrencesOfString:[[self class] nullChar] withString:@"" options:NSLiteralSearch range:NSMakeRange(0, self.length)]; -} - - (void)deleteLastNChars:(NSUInteger)n { if (n > 0) { if (self.length >= 3) { From a401d7eec34d1321249cea060243257ce6b6518b Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 5 May 2025 09:17:36 -0500 Subject: [PATCH 21/46] feat(developer): update for line numbers - make line finder cache lazy loading - rename LDMLResolver to LineFinderResolver Fixes: #10622 --- .../src/common/web/utils/src/line-utils.ts | 36 +++++++++++++++++-- .../src/types/ldml-keyboard/eventresolver.ts | 15 ++++---- .../ldml-keyboard/ldml-keyboard-xml-reader.ts | 4 +-- 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/developer/src/common/web/utils/src/line-utils.ts b/developer/src/common/web/utils/src/line-utils.ts index 5c9a4dec31..e496f9b6ac 100644 --- a/developer/src/common/web/utils/src/line-utils.ts +++ b/developer/src/common/web/utils/src/line-utils.ts @@ -18,7 +18,13 @@ export type LineList = number[]; export class LineFinder { private list: LineList; constructor(public text: string) { - this.list = LineFinder.textToLines(text); + } + + public getLineList(): LineList { + if (!this.list) { + this.list = LineFinder.textToLines(this.text); + } + return this.list; } /** @@ -27,7 +33,7 @@ export class LineFinder { * @returns line:column information */ public findOffset(offset: number): LineColumn { - return LineFinder.offsetToLineColumn(offset, this.list); + return LineFinder.offsetToLineColumn(offset, this.getLineList()); } /** @@ -58,4 +64,30 @@ export class LineFinder { } } +/** + * Interface for a class which can receive file contents, + * keyed by the filename + */ +export interface FileConsumer { + /** + * @param filename name of the file + * @param contents string contents of the file + */ + addFile(filename: string, contents: string): void; +} + +/** Cache of LineFinder elements, organized by filename */ +export class LineFinderCache implements FileConsumer { + contentsCache: Map = new Map(); + + /** add or update a source file */ + addFile(filename: string, contents: string): void { + this.contentsCache.set(filename, new LineFinder(contents)); + } + + getByFilename(filename: string) : LineFinder | undefined { + const lf = this.contentsCache.get(filename); + return lf; + } +} diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts index d155cf27d2..b97e8908fd 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts @@ -6,20 +6,19 @@ * Implementation of a hook on LDML events */ -import { CompilerEvent, EventResolver, LineFinder } from "../../index.js"; -export class LdmlEventResolver implements EventResolver { +import { LineFinderCache, FileConsumer } from "../../line-utils.js"; +import { CompilerEvent, EventResolver } from "../../index.js"; +export class LineFinderEventResolver implements EventResolver, FileConsumer { + private lfcache = new LineFinderCache(); - contentsCache: Map = new Map(); - - /** add or update a source file */ - addFile(filename: string, xml: any) { - this.contentsCache.set(filename, new LineFinder(xml)); + addFile(filename: string, contents: string): void { + this.lfcache.addFile(filename, contents); } /** resolve a CompilerEvent by expanding the line numbers */ resolve(event: CompilerEvent) { if (event.offset && !event.line && event.filename) { - const lf = this.contentsCache.get(event.filename); + const lf = this.lfcache.getByFilename(event.filename); const loc = lf.findOffset(event.offset); event.line = loc.line; event.column = loc.column; diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index 2d099849cd..9e1d1e2e06 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -11,7 +11,7 @@ import { constants } from '@keymanapp/ldml-keyboard-constants'; import { LDMLKeyboardTestDataXMLSourceFile, LKTTest, LKTTests } from './ldml-keyboard-testdata-xml.js'; import { KeymanXMLReader } from '@keymanapp/developer-utils'; import boxXmlArray = util.boxXmlArray; -import { LdmlEventResolver } from './eventresolver.js'; +import { LineFinderEventResolver } from './eventresolver.js'; import { XML_FILENAME_SYMBOL } from '../../xml-utils.js'; interface NameAndProps { @@ -28,7 +28,7 @@ export class LDMLKeyboardXMLSourceFileReaderOptions { }; export class LDMLKeyboardXMLSourceFileReader { - static eventResolver: LdmlEventResolver = new LdmlEventResolver(); + static eventResolver: LineFinderEventResolver = new LineFinderEventResolver(); constructor(private options: LDMLKeyboardXMLSourceFileReaderOptions, private callbacks : CompilerCallbacks) { } From e827827bb40835529c9249b45485fc8dd85f2817 Mon Sep 17 00:00:00 2001 From: Shawn Schantz Date: Tue, 6 May 2025 17:06:29 +0700 Subject: [PATCH 22/46] chore(mac): removed references to groups and load only system stores --- mac/Keyman4Mac/Keyman4Mac/AppDelegate.m | 5 + .../KMConfigurationWindowController.m | 2 +- .../KME/Categories/NSString+XString.h | 9 - .../KME/Categories/NSString+XString.m | 306 ------------------ .../KeymanEngine4Mac/KME/KMBinaryFileFormat.h | 2 + .../KeymanEngine4Mac/KME/KMCompGroup.m | 4 +- .../KeymanEngine4Mac/KME/KMCompKey.m | 4 +- .../KeymanEngine4Mac/KME/KMEngine.m | 5 +- .../KeymanEngine4Mac/KME/KMXFile.h | 6 +- .../KeymanEngine4Mac/KME/KMXFile.m | 60 ++-- .../KeymanEngine4MacTests/KMXFileTests.m | 2 + 11 files changed, 56 insertions(+), 349 deletions(-) diff --git a/mac/Keyman4Mac/Keyman4Mac/AppDelegate.m b/mac/Keyman4Mac/Keyman4Mac/AppDelegate.m index f6bf6c81fe..fb31d3d7b3 100644 --- a/mac/Keyman4Mac/Keyman4Mac/AppDelegate.m +++ b/mac/Keyman4Mac/Keyman4Mac/AppDelegate.m @@ -196,6 +196,8 @@ CGEventRef eventTapFunction(CGEventTapProxy proxy, CGEventType type, CGEventRef [self clearContext]; NSString *kmxPath = [self.kmxList objectAtIndex:self.kbComboBox.indexOfSelectedItem-1]; KMXFile *kmx = [[KMXFile alloc] initWithFilePath:kmxPath]; + + /* if (![kmx isValid]) { NSAlert *alert = [[NSAlert alloc] init]; [alert addButtonWithTitle:@"OK"]; @@ -204,6 +206,7 @@ CGEventRef eventTapFunction(CGEventTapProxy proxy, CGEventType type, CGEventRef [alert setAlertStyle:NSAlertStyleWarning]; [alert runModal]; } + */ NSString *packagePath = [kmxPath stringByDeletingLastPathComponent]; NSDictionary *infoDict = [KMXFile keyboardInfoFromKmxFile:kmxPath]; @@ -218,11 +221,13 @@ CGEventRef eventTapFunction(CGEventTapProxy proxy, CGEventType type, CGEventRef [self.kbData setObject:kmx forKey:kKMXFileKey]; self.imgView.image = kmx.bitmap; + /* int index = 0; for (NSObject *kmStore in kmx.store) { NSLog(@"%d: %@", index, kmStore); index++; } + */ } } diff --git a/mac/Keyman4MacIM/Keyman4MacIM/KMConfiguration/KMConfigurationWindowController.m b/mac/Keyman4MacIM/Keyman4MacIM/KMConfiguration/KMConfigurationWindowController.m index 2b06db4cad..8c12e64580 100644 --- a/mac/Keyman4MacIM/Keyman4MacIM/KMConfiguration/KMConfigurationWindowController.m +++ b/mac/Keyman4MacIM/Keyman4MacIM/KMConfiguration/KMConfigurationWindowController.m @@ -337,7 +337,7 @@ NSArray *filenames = [pboard propertyListForType:NSFilenamesPboardType]; NSMutableArray *kmpFiles = nil; for (NSString *filename in filenames) { - if ([[filename lastNChars:4] isEqualTo:@".kmp"]) { + if ([filename hasSuffix:@".kmp"]) { if (kmpFiles == nil) kmpFiles = [NSMutableArray arrayWithObject:filename]; else diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/Categories/NSString+XString.h b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/Categories/NSString+XString.h index 93268d5917..4367a4aa27 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/Categories/NSString+XString.h +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/Categories/NSString+XString.h @@ -11,15 +11,6 @@ @interface NSString (XString) -- (NSString *)lastNChars:(NSUInteger)n; -- (NSString *)codeString; -- (BOOL)isValidCode; + (NSString *)nullChar; @end - -@interface NSMutableString (XString) - -- (void)deleteLastNChars:(NSUInteger)n; - -@end diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/Categories/NSString+XString.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/Categories/NSString+XString.m index 7b280a9946..5a3db3b1d0 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/Categories/NSString+XString.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/Categories/NSString+XString.m @@ -10,314 +10,8 @@ @implementation NSString (XString) - -- (NSString *)lastNChars:(NSUInteger)n { - if (n == 0) - return @""; - - if (self.length >= 3) { - NSInteger len = 0; - NSInteger chars = 0; - for (NSInteger i = self.length - 1; i >= 0;) { - NSInteger index = i - 2; - if (index >= 0) { - unichar c1 = [self characterAtIndex:index]; - unichar c2 = [self characterAtIndex:index+1]; - if (c1 == UC_SENTINEL && (c2 == CODE_DEADKEY || c2 == CODE_NULLCHAR)) { - i -= 3; - len += 3; - } - else { - i--; - len++; - } - } - else { - i--; - len++; - } - - chars++; - if (chars == n) - break; - } - - return [NSString stringWithString:[self substringFromIndex:self.length - len]]; - } - else { - if (n >= self.length) - return [NSString stringWithString:self]; - else - return [NSString stringWithString:[self substringFromIndex:self.length - n]]; - } -} - -- (NSString *)codeString { - if (self == nil) - return nil; - - NSMutableString *mStr = [NSMutableString stringWithString:@""]; - for (int i=0; i < self.length;) { - unichar c = [self characterAtIndex:i]; - if (c == UC_SENTINEL) { - if ((i+1) < self.length) - c = [self characterAtIndex:i+1]; - else { - [mStr appendString:@"sentinel(INVALID)"]; - return mStr; - } - - switch (c) { - case CODE_ANY: { - if ((i+2) < self.length) { - DWORD index = [self characterAtIndex:i+2]; - [mStr appendString:[NSString stringWithFormat:@"any(%d)", index]]; - } - else { - [mStr appendString:@"any(INVALID)"]; - } - - i+=3; - break; - } - case CODE_NOTANY: { - if ((i+2) < self.length) { - DWORD index = [self characterAtIndex:i+2]; - [mStr appendString:[NSString stringWithFormat:@"notany(%d)", index]]; - } - else { - [mStr appendString:@"notany(INVALID)"]; - } - - i+=3; - break; - } - case CODE_INDEX: { - if ((i+3) < self.length) { - DWORD x1 = [self characterAtIndex:i+2]; - DWORD x2 = [self characterAtIndex:i+3]; - [mStr appendString:[NSString stringWithFormat:@"index(%d,%d)", x1, x2]]; - } - else { - [mStr appendString:@"index(INVALID)"]; - } - - i+=4; - break; - } - case CODE_NUL: - [mStr appendString:@"nul()"]; - i+=2; - break; - case CODE_DEADKEY: { - if ((i+2) < self.length) { - c = [self characterAtIndex:i+2]; - [mStr appendString:[NSString stringWithFormat:@"dk(%d)", c]]; - } - else { - [mStr appendString:@"dk(INVALID)"]; - } - - i+=3; - break; - } - case CODE_NULLCHAR: { - if ((i+2) < self.length) { - c = [self characterAtIndex:i+2]; - [mStr appendString:[NSString stringWithFormat:@"nc(%d)", c]]; - } - else { - [mStr appendString:@"nc(INVALID)"]; - } - - i+=3; - break; - } - case CODE_CONTEXT: { - [mStr appendString:[NSString stringWithFormat:@"ctx()"]]; - i+=2; - break; - } - case CODE_CONTEXTEX: { - if ((i+2) < self.length) { - c = [self characterAtIndex:i+2]; - [mStr appendString:[NSString stringWithFormat:@"ctxex(%d)", c]]; - } - else { - [mStr appendString:@"ctxex(INVALID)"]; - } - - i+=3; - break; - } - case CODE_USE: { - if ((i+2) < self.length) { - c = [self characterAtIndex:i+2]; - [mStr appendString:[NSString stringWithFormat:@"use(%d)", c]]; - } - else { - [mStr appendString:@"use(INVALID)"]; - } - - i+=3; - break; - } - case CODE_BEEP: { - [mStr appendString:[NSString stringWithFormat:@"beep()"]]; - i+=2; - break; - } - case CODE_RETURN: { - [mStr appendString:[NSString stringWithFormat:@"return()"]]; - i+=2; - break; - } - case CODE_IFOPT: { - if ((i+4) < self.length) { - DWORD x1 = [self characterAtIndex:i+2]; - DWORD x2 = [self characterAtIndex:i+3]; - DWORD x3 = [self characterAtIndex:i+4]; - [mStr appendString:[NSString stringWithFormat:@"ifopt(%d %d %d)", x1, x2, x3]]; - } - else { - [mStr appendString:@"ifopt(INVALID)"]; - } - - i+=5; - break; - } - case CODE_IFSYSTEMSTORE: { - if ((i+4) < self.length) { - DWORD x1 = [self characterAtIndex:i+2]; - DWORD x2 = [self characterAtIndex:i+3]; - DWORD x3 = [self characterAtIndex:i+4]; - [mStr appendString:[NSString stringWithFormat:@"ifsys(%d %d %d)", x1, x2, x3]]; - } - else { - [mStr appendString:@"ifsys(INVALID)"]; - } - - i+=5; - break; - } - default: - [mStr appendString:@"other()"]; - i+=2; - break; - } - } - else { - [mStr appendString:[NSString stringWithFormat:@"U+%x", c]]; - i++; - } - } - - return mStr; -} - -- (BOOL)isValidCode { - if (self == nil) - return NO; - - for (int i=0; i < self.length;) { - unichar c = [self characterAtIndex:i]; - if (c == UC_SENTINEL) { - if ((i+1) < self.length) - c = [self characterAtIndex:i+1]; - else - return NO; - - switch (c) { - case CODE_ANY: - case CODE_NOTANY: - case CODE_DEADKEY: - case CODE_NULLCHAR: - case CODE_CONTEXTEX: - case CODE_USE: { - if ((i+2) >= self.length) - return NO; - - i+=3; - break; - } - case CODE_INDEX: { - if ((i+3) >= self.length) - return NO; - - i+=4; - break; - } - case CODE_IFOPT: - case CODE_IFSYSTEMSTORE: { - if ((i+4) >= self.length) - return NO; - - i+=5; - break; - } - case CODE_NUL: - case CODE_CONTEXT: - case CODE_BEEP: - case CODE_RETURN: - default: - i+=2; - break; - } - } - else { - i++; - } - } - - return YES; -} - + (NSString *)nullChar { return [NSString stringWithFormat:@"%C%C%d", UC_SENTINEL, CODE_NULLCHAR, 1]; } @end - -@implementation NSMutableString (XString) - -- (void)deleteLastNChars:(NSUInteger)n { - if (n > 0) { - if (self.length >= 3) { - NSInteger len = 0; - NSInteger chars = 0; - for (NSInteger i = self.length - 1; i >= 0;) { - NSInteger index = i - 2; - if (index >= 0) { - unichar c1 = [self characterAtIndex:index]; - unichar c2 = [self characterAtIndex:index+1]; - if (c1 == UC_SENTINEL && (c2 == CODE_DEADKEY || c2 == CODE_NULLCHAR)) { - i -= 3; - len += 3; - } - else { - i--; - len++; - } - } - else { - i--; - len++; - } - - chars++; - if (chars == n) - break; - } - - [self deleteCharactersInRange:NSMakeRange(self.length - len, len)]; - } - else { - if (self.length >= n) - [self deleteCharactersInRange:NSMakeRange(self.length - n, n)]; - else - [self deleteCharactersInRange:NSMakeRange(0, self.length)]; - } - } -} - -@end diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMBinaryFileFormat.h b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMBinaryFileFormat.h index 894f0f4b9c..6e306861f6 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMBinaryFileFormat.h +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMBinaryFileFormat.h @@ -49,6 +49,7 @@ struct COMP_KEYBOARD { #define VERSION_MIN VERSION_50 #define VERSION_MAX VERSION_170 +/* struct COMP_GROUP { DWORD dpName; // string (only debug) DWORD dpKeyArray; // [LPCOMP_KEY] address of first item in key array @@ -57,6 +58,7 @@ struct COMP_GROUP { DWORD cxKeyArray; // in array entries BOOL fUsingKeys; // group(xx) [using keys] <-- specified or not }; +*/ struct COMP_STORE { DWORD dwSystemID; diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompGroup.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompGroup.m index 1aaa376084..4da8d3bc96 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompGroup.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompGroup.m @@ -24,9 +24,9 @@ } - (NSString *)description { - NSString *format = @"<%@:%p, Name:%@ Match:%@ NoMatch:%@ fUsingKeys:%d Keys:%@>"; + NSString *format = @"<%@:%p, Name:%@ fUsingKeys:%d Keys:%@>"; NSString *str = [NSString stringWithFormat:format, - [self className], self, _name, [_match codeString], [_noMatch codeString], _fUsingKeys, _keys]; + [self className], self, _name, _fUsingKeys, _keys]; return str; } diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompKey.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompKey.m index f8508dbf4c..8d77dcf072 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompKey.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompKey.m @@ -22,9 +22,9 @@ } - (NSString *)description { - NSString *format = @"<%@:%p K:0x%X L:0x%X F:0x%X O:%@ C:%@>"; + NSString *format = @"<%@:%p K:0x%X L:0x%X F:0x%X>"; NSString *str = [NSString stringWithFormat:format, - self.className, self, self.key, self.line, self.shiftFlags, [self.output codeString], [self.context codeString]]; + self.className, self, self.key, self.line, self.shiftFlags]; return str; } diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m index 080c7023c3..2bd63d5131 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m @@ -8,9 +8,8 @@ #import "KMEngine.h" #import "KMBinaryFileFormat.h" -#import "KMCompStore.h" -#import "KMCompGroup.h" -#import "KMCompKey.h" +//#import "KMCompStore.h" +//#import "KMCompKey.h" #import "NSString+XString.h" #import "WindowsVKCodes.h" #import "MacVKCodes.h" diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMXFile.h b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMXFile.h index 8eec8bb4e3..76b9d8287b 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMXFile.h +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMXFile.h @@ -28,16 +28,16 @@ extern NSString *const kKMVisualKeyboardKey; @property (assign, nonatomic, readonly) BOOL isMnemonic; @property (assign, nonatomic, readonly) DWORD version; @property (strong, nonatomic, readonly) NSArray *store; +/* @property (strong, nonatomic, readonly) NSArray *storeSaved; @property (strong, nonatomic, readonly) NSArray *group; @property (strong, nonatomic, readonly) NSArray *startGroup; -@property (assign, nonatomic, readonly) DWORD flags; -@property (assign, nonatomic, readonly) DWORD hotKey; + */ @property (strong, nonatomic, readonly) NSImage *bitmap; @property (strong, nonatomic, readonly) NSString *filePath; - (id)initWithFilePath:(NSString *)path; -- (BOOL)isValid; +//- (BOOL)isValid; /*! * Provides an info dictionary from a KMX file which includes keyboard name, version, copyright information and keyboard icon. diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMXFile.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMXFile.m index 2ce2fbfa89..812cd60a5d 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMXFile.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMXFile.m @@ -8,8 +8,8 @@ #import "KMXFile.h" #import "KMCompStore.h" -#import "KMCompGroup.h" -#import "KMCompKey.h" +//#import "KMCompGroup.h" +//#import "KMCompKey.h" #import "NSString+XString.h" #import "KMELogs.h" @@ -22,6 +22,7 @@ NSString *const kKMVisualKeyboardKey = @"KMVisualKeyboardKey"; @implementation KMXFile - (id)initWithFilePath:(NSString *)path { + os_log_info([KMELogs configLog], "initWithFilePath, path: %{public}@", path); self = [super init]; if (self) { if (path == nil) { @@ -56,43 +57,53 @@ NSString *const kKMVisualKeyboardKey = @"KMVisualKeyboardKey"; _keyboardID = cmp_kb.KeyboardID; _isRegistered = (cmp_kb.IsRegistered == 0?NO:YES); _version = cmp_kb.version; + +/* NSMutableArray *mStartGroup = [[NSMutableArray alloc] initWithCapacity:2]; [mStartGroup addObject:[NSNumber numberWithInt:cmp_kb.StartGroup[0]]]; [mStartGroup addObject:[NSNumber numberWithInt:cmp_kb.StartGroup[1]]]; _startGroup = [[NSArray alloc] initWithArray:mStartGroup]; - _flags = cmp_kb.dwFlags; - _hotKey = cmp_kb.dwHotKey; + */ struct COMP_STORE cmp_str[cmp_kb.cxStoreArray]; [file seekToFileOffset:cmp_kb.dpStoreArray]; size = sizeof(cmp_str); dataBuffer = [file readDataOfLength:size]; [dataBuffer getBytes:cmp_str length:size]; NSMutableArray *mStore = [[NSMutableArray alloc] initWithCapacity:cmp_kb.cxStoreArray]; + os_log_info([KMELogs configLog], "COMP_STORE, number of entries: %u", cmp_kb.cxStoreArray); for (int i = 0; i < cmp_kb.cxStoreArray; i++) { KMCompStore *kmStore = [[KMCompStore alloc] init]; kmStore.dwSystemID = cmp_str[i].dwSystemID; - kmStore.name = [KMXFile UTF16StringWithPointer:cmp_str[i].dpName inFile:file]; - if (kmStore.dwSystemID == TSS_VERSION) - kmStore.string = [[KMXFile UTF16StringWithPointer:cmp_str[i].dpString inFile:file] stringByReplacingOccurrencesOfString:@"\n" withString:@""]; - else { - kmStore.string = [KMXFile UTF16StringWithPointer:cmp_str[i].dpString inFile:file]; - if (kmStore.dwSystemID == TSS_MNEMONIC && [kmStore.string isEqualToString:@"1"]) - _isMnemonic = YES; + /* Keyman for Mac is not processing keys, so no need to save normal stores. + If dwSystemID is not zero, then this is a system store and may contain + useful metadata */ + + if (kmStore.dwSystemID != 0) { + kmStore.name = [KMXFile UTF16StringWithPointer:cmp_str[i].dpName inFile:file]; + + if (kmStore.dwSystemID == TSS_VERSION) + kmStore.string = [[KMXFile UTF16StringWithPointer:cmp_str[i].dpString inFile:file] stringByReplacingOccurrencesOfString:@"\n" withString:@""]; + else { + kmStore.string = [KMXFile UTF16StringWithPointer:cmp_str[i].dpString inFile:file]; + if (kmStore.dwSystemID == TSS_MNEMONIC && [kmStore.string isEqualToString:@"1"]) + _isMnemonic = YES; + } + + if (kmStore.dwSystemID == TSS_VERSION) + kmStore.string = [[KMXFile UTF16StringWithPointer:cmp_str[i].dpString inFile:file] stringByReplacingOccurrencesOfString:@"\n" withString:@""]; + else + kmStore.string = [KMXFile UTF16StringWithPointer:cmp_str[i].dpString inFile:file]; + + [mStore addObject:kmStore]; + os_log_info([KMELogs configLog], "loaded store, systemId: %u name: %{public}@ string: %{public}@", kmStore.dwSystemID, kmStore.name, kmStore.string); } - - if (kmStore.dwSystemID == TSS_VERSION) - kmStore.string = [[KMXFile UTF16StringWithPointer:cmp_str[i].dpString inFile:file] stringByReplacingOccurrencesOfString:@"\n" withString:@""]; - else - kmStore.string = [KMXFile UTF16StringWithPointer:cmp_str[i].dpString inFile:file]; - - [mStore addObject:kmStore]; } - _store = [[NSArray alloc] initWithArray:mStore]; - _storeSaved = [[NSArray alloc] initWithArray:mStore copyItems:YES]; - + //_storeSaved = [[NSArray alloc] initWithArray:mStore copyItems:YES]; + + /* struct COMP_GROUP cmp_grp[cmp_kb.cxGroupArray]; [file seekToFileOffset:cmp_kb.dpGroupArray]; size = sizeof(cmp_grp); @@ -126,6 +137,7 @@ NSString *const kKMVisualKeyboardKey = @"KMVisualKeyboardKey"; [mGroup addObject:kmGrp]; } _group = [[NSArray alloc] initWithArray:mGroup]; + */ [file seekToFileOffset:cmp_kb.dpBitmapOffset]; NSData *bitmapData = [file readDataOfLength:cmp_kb.dwBitmapSize]; @@ -139,12 +151,13 @@ NSString *const kKMVisualKeyboardKey = @"KMVisualKeyboardKey"; } - (NSString *)description { - NSString *format = @"<%@: %p, File version: 0x%X Flags: 0x%X Store: %@ Group: %@>"; + NSString *format = @"<%@: %p, File version: 0x%X>"; NSString *str = [NSString stringWithFormat:format, - self.className, self, self.fileVersion, self.flags, self.store, self.group]; + self.className, self, self.fileVersion]; return str; } +/* - (BOOL)isValid { for (KMCompGroup *gp in self.group) { if ((gp.match && ![gp.match isValidCode]) || (gp.noMatch && ![gp.noMatch isValidCode])) @@ -158,6 +171,7 @@ NSString *const kKMVisualKeyboardKey = @"KMVisualKeyboardKey"; return YES; } +*/ + (NSDictionary *)keyboardInfoFromKmxFile:(NSString *)path { NSFileHandle *file = [NSFileHandle fileHandleForReadingAtPath:path]; diff --git a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMXFileTests.m b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMXFileTests.m index 6b13479d87..1a2ac73e4b 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMXFileTests.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMXFileTests.m @@ -36,10 +36,12 @@ XCTAssert(kmxFile == nil, @"Expected nil file"); } +/* - (void)testinitWithFilePath_Valid_ReturnsInitializedKmxFile { KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileTestMacEngine]; XCTAssert(kmxFile != nil, @"Expected non-nil file"); XCTAssert([kmxFile isValid], @"Expected valid KMX file."); } + */ @end From 623d5af5ab517cd72c03c864c504d84854e2b410 Mon Sep 17 00:00:00 2001 From: Shawn Schantz Date: Wed, 7 May 2025 12:07:59 +0700 Subject: [PATCH 23/46] chore(mac): remove group code, stop reading stores at init --- .../KeymanEngine4Mac/KME/KMBinaryFileFormat.h | 27 +--- .../KeymanEngine4Mac/KME/KMEngine.m | 2 - .../KeymanEngine4Mac/KME/KMXFile.h | 7 - .../KeymanEngine4Mac/KME/KMXFile.m | 151 ++++++------------ 4 files changed, 58 insertions(+), 129 deletions(-) diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMBinaryFileFormat.h b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMBinaryFileFormat.h index 6e306861f6..211a40d9d3 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMBinaryFileFormat.h +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMBinaryFileFormat.h @@ -6,12 +6,18 @@ // Copyright (c) 2017 SIL International. All rights reserved. // +/** + * Note that many of the constants defined here are also defined in the common header`kmx_file.h` + * Keyman for Mac only reads the kmx file to pass the contents to Keyman Core and for meta data. + * If the meta data can instead be returned by Keyman Core in the future, then the duplicate definitions + * found here can be removed. + */ + #ifndef KMBinaryFileFormat_h #define KMBinaryFileFormat_h typedef uint16_t WORD; typedef uint32_t DWORD; -//typedef wchar_t* PWSTR; struct COMP_KEYBOARD { DWORD dwIdentifier; // 0000 Keyman compiled keyboard id @@ -49,17 +55,6 @@ struct COMP_KEYBOARD { #define VERSION_MIN VERSION_50 #define VERSION_MAX VERSION_170 -/* -struct COMP_GROUP { - DWORD dpName; // string (only debug) - DWORD dpKeyArray; // [LPCOMP_KEY] address of first item in key array - DWORD dpMatch; // extended string - DWORD dpNoMatch; // extended string - DWORD cxKeyArray; // in array entries - BOOL fUsingKeys; // group(xx) [using keys] <-- specified or not -}; -*/ - struct COMP_STORE { DWORD dwSystemID; DWORD dpName; // string (only debug) @@ -150,14 +145,6 @@ struct COMP_STORE { #define K_MODIFIERFLAG 0x007F #define K_CAPITALMASK (CAPITALFLAG|NOTCAPITALFLAG) -struct COMP_KEY { - WORD Key; // Windows VK code or character value (VIRTUALCHARKEY, ISVIRTUALKEY) - DWORD Line; - DWORD ShiftFlags; - DWORD dpOutput; // extended string - DWORD dpContext; // extended string -}; - #define UC_SENTINEL 0xFFFF #define UC_SENTINEL_EXTENDEDEND 0x10 diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m index 2bd63d5131..4fca35955b 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m @@ -8,8 +8,6 @@ #import "KMEngine.h" #import "KMBinaryFileFormat.h" -//#import "KMCompStore.h" -//#import "KMCompKey.h" #import "NSString+XString.h" #import "WindowsVKCodes.h" #import "MacVKCodes.h" diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMXFile.h b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMXFile.h index 76b9d8287b..9ef03a0891 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMXFile.h +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMXFile.h @@ -27,17 +27,10 @@ extern NSString *const kKMVisualKeyboardKey; @property (assign, nonatomic, readonly) BOOL isRegistered; @property (assign, nonatomic, readonly) BOOL isMnemonic; @property (assign, nonatomic, readonly) DWORD version; -@property (strong, nonatomic, readonly) NSArray *store; -/* -@property (strong, nonatomic, readonly) NSArray *storeSaved; -@property (strong, nonatomic, readonly) NSArray *group; -@property (strong, nonatomic, readonly) NSArray *startGroup; - */ @property (strong, nonatomic, readonly) NSImage *bitmap; @property (strong, nonatomic, readonly) NSString *filePath; - (id)initWithFilePath:(NSString *)path; -//- (BOOL)isValid; /*! * Provides an info dictionary from a KMX file which includes keyboard name, version, copyright information and keyboard icon. diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMXFile.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMXFile.m index 812cd60a5d..63eaf93359 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMXFile.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMXFile.m @@ -8,8 +8,6 @@ #import "KMXFile.h" #import "KMCompStore.h" -//#import "KMCompGroup.h" -//#import "KMCompKey.h" #import "NSString+XString.h" #import "KMELogs.h" @@ -30,7 +28,6 @@ NSString *const kKMVisualKeyboardKey = @"KMVisualKeyboardKey"; return nil; } - NSFileHandle *file = [NSFileHandle fileHandleForReadingAtPath:path]; if (file == nil) { os_log_error([KMELogs configLog], "Failed to open kmx file"); @@ -57,88 +54,10 @@ NSString *const kKMVisualKeyboardKey = @"KMVisualKeyboardKey"; _keyboardID = cmp_kb.KeyboardID; _isRegistered = (cmp_kb.IsRegistered == 0?NO:YES); _version = cmp_kb.version; - -/* - NSMutableArray *mStartGroup = [[NSMutableArray alloc] initWithCapacity:2]; - [mStartGroup addObject:[NSNumber numberWithInt:cmp_kb.StartGroup[0]]]; - [mStartGroup addObject:[NSNumber numberWithInt:cmp_kb.StartGroup[1]]]; - _startGroup = [[NSArray alloc] initWithArray:mStartGroup]; - - */ - struct COMP_STORE cmp_str[cmp_kb.cxStoreArray]; - [file seekToFileOffset:cmp_kb.dpStoreArray]; - size = sizeof(cmp_str); - dataBuffer = [file readDataOfLength:size]; - [dataBuffer getBytes:cmp_str length:size]; - NSMutableArray *mStore = [[NSMutableArray alloc] initWithCapacity:cmp_kb.cxStoreArray]; - os_log_info([KMELogs configLog], "COMP_STORE, number of entries: %u", cmp_kb.cxStoreArray); - for (int i = 0; i < cmp_kb.cxStoreArray; i++) { - KMCompStore *kmStore = [[KMCompStore alloc] init]; - kmStore.dwSystemID = cmp_str[i].dwSystemID; - - /* Keyman for Mac is not processing keys, so no need to save normal stores. - If dwSystemID is not zero, then this is a system store and may contain - useful metadata */ - - if (kmStore.dwSystemID != 0) { - kmStore.name = [KMXFile UTF16StringWithPointer:cmp_str[i].dpName inFile:file]; - - if (kmStore.dwSystemID == TSS_VERSION) - kmStore.string = [[KMXFile UTF16StringWithPointer:cmp_str[i].dpString inFile:file] stringByReplacingOccurrencesOfString:@"\n" withString:@""]; - else { - kmStore.string = [KMXFile UTF16StringWithPointer:cmp_str[i].dpString inFile:file]; - if (kmStore.dwSystemID == TSS_MNEMONIC && [kmStore.string isEqualToString:@"1"]) - _isMnemonic = YES; - } - - if (kmStore.dwSystemID == TSS_VERSION) - kmStore.string = [[KMXFile UTF16StringWithPointer:cmp_str[i].dpString inFile:file] stringByReplacingOccurrencesOfString:@"\n" withString:@""]; - else - kmStore.string = [KMXFile UTF16StringWithPointer:cmp_str[i].dpString inFile:file]; - - [mStore addObject:kmStore]; - os_log_info([KMELogs configLog], "loaded store, systemId: %u name: %{public}@ string: %{public}@", kmStore.dwSystemID, kmStore.name, kmStore.string); - } - } - _store = [[NSArray alloc] initWithArray:mStore]; - //_storeSaved = [[NSArray alloc] initWithArray:mStore copyItems:YES]; - /* - struct COMP_GROUP cmp_grp[cmp_kb.cxGroupArray]; - [file seekToFileOffset:cmp_kb.dpGroupArray]; - size = sizeof(cmp_grp); - dataBuffer = [file readDataOfLength:size]; - [dataBuffer getBytes:cmp_grp length:size]; - - NSMutableArray *mGroup = [[NSMutableArray alloc] initWithCapacity:cmp_kb.cxGroupArray]; - for (int i = 0; i < cmp_kb.cxGroupArray; i++) { - KMCompGroup *kmGrp = [[KMCompGroup alloc] init]; - kmGrp.name = [KMXFile UTF16StringWithPointer:cmp_grp[i].dpName inFile:file]; - kmGrp.match = [KMXFile UTF16StringWithPointer:cmp_grp[i].dpMatch inFile:file]; - kmGrp.noMatch = [KMXFile UTF16StringWithPointer:cmp_grp[i].dpNoMatch inFile:file]; - kmGrp.fUsingKeys = cmp_grp[i].fUsingKeys; - - struct COMP_KEY cmp_keys[cmp_grp[i].cxKeyArray]; - [file seekToFileOffset:cmp_grp[i].dpKeyArray]; - size = sizeof(cmp_keys); - dataBuffer = [file readDataOfLength:size]; - [dataBuffer getBytes:cmp_keys length:size]; - - for (int j = 0; j < cmp_grp[i].cxKeyArray; j++) { - KMCompKey *kmKey = [[KMCompKey alloc] init]; - kmKey.key = cmp_keys[j].Key; - kmKey.line = cmp_keys[j].Line; - kmKey.shiftFlags = cmp_keys[j].ShiftFlags; - kmKey.output = [KMXFile UTF16StringWithPointer:cmp_keys[j].dpOutput inFile:file]; - kmKey.context = [KMXFile UTF16StringWithPointer:cmp_keys[j].dpContext inFile:file]; - [kmGrp.keys addObject:kmKey]; - } - - [mGroup addObject:kmGrp]; - } - _group = [[NSArray alloc] initWithArray:mGroup]; - */ - + // stores not used at init time + //NSArray *systemStores = [self readSystemStores:cmp_kb file: file]; + [file seekToFileOffset:cmp_kb.dpBitmapOffset]; NSData *bitmapData = [file readDataOfLength:cmp_kb.dwBitmapSize]; NSBitmapImageRep *imageRep = [NSBitmapImageRep imageRepWithData:bitmapData]; @@ -150,6 +69,53 @@ NSString *const kKMVisualKeyboardKey = @"KMVisualKeyboardKey"; return self; } +/** + * Read the system stores from the kmx file into an array. + * Skips over all the normal stores as Keyman for Mac does not process keys but only needs the + * meta data contained in system stores. + */ +- (NSArray *) readSystemStores:(struct COMP_KEYBOARD)cmp_kb file: (NSFileHandle *) file { + struct COMP_STORE cmp_str[cmp_kb.cxStoreArray]; + [file seekToFileOffset:cmp_kb.dpStoreArray]; + size_t size = sizeof(cmp_str); + NSData *dataBuffer = [file readDataOfLength:size]; + [dataBuffer getBytes:cmp_str length:size]; + + NSMutableArray *systemStoreArray = [[NSMutableArray alloc] initWithCapacity:cmp_kb.cxStoreArray]; + + os_log_info([KMELogs configLog], "COMP_STORE, number of entries: %u", cmp_kb.cxStoreArray); + for (int i = 0; i < cmp_kb.cxStoreArray; i++) { + KMCompStore *kmStore = [[KMCompStore alloc] init]; + kmStore.dwSystemID = cmp_str[i].dwSystemID; + + /* Keyman for Mac is not processing keys, so no need to save normal stores. + If dwSystemID is not zero, then this is a system store and may contain + useful metadata */ + + if (kmStore.dwSystemID != 0) { + kmStore.name = [KMXFile UTF16StringWithPointer:cmp_str[i].dpName inFile:file]; + + if (kmStore.dwSystemID == TSS_VERSION) + kmStore.string = [[KMXFile UTF16StringWithPointer:cmp_str[i].dpString inFile:file] stringByReplacingOccurrencesOfString:@"\n" withString:@""]; + else { + kmStore.string = [KMXFile UTF16StringWithPointer:cmp_str[i].dpString inFile:file]; + if (kmStore.dwSystemID == TSS_MNEMONIC && [kmStore.string isEqualToString:@"1"]) + _isMnemonic = YES; + } + + if (kmStore.dwSystemID == TSS_VERSION) + kmStore.string = [[KMXFile UTF16StringWithPointer:cmp_str[i].dpString inFile:file] stringByReplacingOccurrencesOfString:@"\n" withString:@""]; + else + kmStore.string = [KMXFile UTF16StringWithPointer:cmp_str[i].dpString inFile:file]; + + [systemStoreArray addObject:kmStore]; + os_log_info([KMELogs configLog], "loaded store, systemId: %u name: %{public}@ string: %{public}@", kmStore.dwSystemID, kmStore.name, kmStore.string); + } + } + + return systemStoreArray; +} + - (NSString *)description { NSString *format = @"<%@: %p, File version: 0x%X>"; NSString *str = [NSString stringWithFormat:format, @@ -157,23 +123,8 @@ NSString *const kKMVisualKeyboardKey = @"KMVisualKeyboardKey"; return str; } -/* -- (BOOL)isValid { - for (KMCompGroup *gp in self.group) { - if ((gp.match && ![gp.match isValidCode]) || (gp.noMatch && ![gp.noMatch isValidCode])) - return NO; - - for (KMCompKey *kmKey in gp.keys) { - if (![kmKey.context isValidCode] || ![kmKey.output isValidCode]) - return NO; - } - } - - return YES; -} -*/ - + (NSDictionary *)keyboardInfoFromKmxFile:(NSString *)path { + os_log_info([KMELogs configLog], "keyboardInfoFromKmxFile, path: %{public}@", path); NSFileHandle *file = [NSFileHandle fileHandleForReadingAtPath:path]; if (file == nil) { From cd59c298a8fc13ca63039af68e076c68a43d536b Mon Sep 17 00:00:00 2001 From: Shawn Schantz Date: Wed, 7 May 2025 12:36:17 +0700 Subject: [PATCH 24/46] chore(mac): remove NSString extension --- .../Keyman4MacIM/KMInputMethodAppDelegate.m | 15 ++++++--------- mac/Keyman4MacIM/KeymanTests/TestAppDelegate.m | 7 ------- .../KeymanEngine4Mac.xcodeproj/project.pbxproj | 6 ------ .../KeymanEngine4Mac/KME/KMCompGroup.m | 1 - .../KeymanEngine4Mac/KME/KMCompKey.m | 1 - .../KeymanEngine4Mac/KME/KMEngine.m | 1 - .../KeymanEngine4Mac/KME/KMXFile.m | 1 - .../KeymanEngine4Mac/KeymanEngine4Mac.h | 1 - 8 files changed, 6 insertions(+), 27 deletions(-) diff --git a/mac/Keyman4MacIM/Keyman4MacIM/KMInputMethodAppDelegate.m b/mac/Keyman4MacIM/Keyman4MacIM/KMInputMethodAppDelegate.m index 168c0400fa..9bac3e9c89 100644 --- a/mac/Keyman4MacIM/Keyman4MacIM/KMInputMethodAppDelegate.m +++ b/mac/Keyman4MacIM/Keyman4MacIM/KMInputMethodAppDelegate.m @@ -740,11 +740,8 @@ CGEventRef eventTapFunction(CGEventTapProxy proxy, CGEventType type, CGEventRef return _contextBuffer; } -- (void)setContextBuffer:(NSMutableString *)contextBuffer { - _contextBuffer = [contextBuffer mutableCopy]; - if (_contextBuffer.length) - [_contextBuffer replaceOccurrencesOfString:@"\0" withString:[NSString nullChar] options:0 range:NSMakeRange(0, 1)]; - [self.kme setCoreContextIfNeeded:self.contextBuffer]; +- (void)clearContextBuffer { + [self.kme clearCoreContext]; } - (void)awakeFromNib { @@ -880,8 +877,8 @@ CGEventRef eventTapFunction(CGEventTapProxy proxy, CGEventType type, CGEventRef NSString *keyboardName = [self.activeKeyboards objectAtIndex:0]; [self setSelectedKeyboard:keyboardName inMenuItem:menuItem]; [self setSelectedKeyboard:keyboardName]; - [self setContextBuffer:nil]; - + [self clearContextBuffer]; + [KMSentryHelper addInfoBreadCrumb:@"startup" message:[NSString stringWithFormat:@"setDefaultSelectedKeyboard: %@", keyboardName]]; } @@ -893,7 +890,7 @@ CGEventRef eventTapFunction(CGEventTapProxy proxy, CGEventType type, CGEventRef [self setKvk:nil]; [self setKeyboardName:nil]; [self setKeyboardIcon:nil]; - [self setContextBuffer:nil]; + [self clearContextBuffer]; [self setSelectedKeyboard:@""]; } @@ -932,7 +929,7 @@ CGEventRef eventTapFunction(CGEventTapProxy proxy, CGEventType type, CGEventRef os_log_info([KMLogs keyboardLog], "Selected keyboard from menu: %{public}@", keyboardName); [self setKeyboardName:keyboardName]; [self setKeyboardIcon:[kmxInfo objectForKey:kKMKeyboardIconKey]]; - [self setContextBuffer:nil]; + [self clearContextBuffer]; [self setSelectedKeyboard:path]; [self applyPersistedOptions]; } diff --git a/mac/Keyman4MacIM/KeymanTests/TestAppDelegate.m b/mac/Keyman4MacIM/KeymanTests/TestAppDelegate.m index c2078c95ea..5733aa8a6a 100644 --- a/mac/Keyman4MacIM/KeymanTests/TestAppDelegate.m +++ b/mac/Keyman4MacIM/KeymanTests/TestAppDelegate.m @@ -35,13 +35,6 @@ return _contextBuffer; } -- (void)setContextBuffer:(NSMutableString *)contextBuffer { - _contextBuffer = [contextBuffer mutableCopy]; - if (_contextBuffer.length) - [_contextBuffer replaceOccurrencesOfString:@"\0" withString:[NSString nullChar] options:0 range:NSMakeRange(0, 1)]; - [self.kme setCoreContextIfNeeded:self.contextBuffer]; -} - -(NSEvent *)keyStrokeEventForCharacter:(NSString *)character keyCode:(unsigned short) keyCode { return [NSEvent keyEventWithType:NSEventTypeKeyDown location:NSZeroPoint modifierFlags:0 timestamp:NSTimeIntervalSince1970 windowNumber:0 context:nil characters:character charactersIgnoringModifiers:character isARepeat:NO keyCode:keyCode]; } diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac.xcodeproj/project.pbxproj b/mac/KeymanEngine4Mac/KeymanEngine4Mac.xcodeproj/project.pbxproj index f87288a953..e705118e5b 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac.xcodeproj/project.pbxproj +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac.xcodeproj/project.pbxproj @@ -52,12 +52,9 @@ 984C2ADE1A79C98D0023F89D /* KeymanEngine4Mac.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 984C2AD21A79C98D0023F89D /* KeymanEngine4Mac.framework */; }; 984C2AE51A79C98D0023F89D /* KMXFileTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 984C2AE41A79C98D0023F89D /* KMXFileTests.m */; }; 984C2B061A79CA4F0023F89D /* NSString+XString.h in Headers */ = {isa = PBXBuildFile; fileRef = 984C2AF21A79CA4F0023F89D /* NSString+XString.h */; settings = {ATTRIBUTES = (Public, ); }; }; - 984C2B071A79CA4F0023F89D /* NSString+XString.m in Sources */ = {isa = PBXBuildFile; fileRef = 984C2AF31A79CA4F0023F89D /* NSString+XString.m */; }; 984C2B091A79CA4F0023F89D /* KMBinaryFileFormat.h in Headers */ = {isa = PBXBuildFile; fileRef = 984C2AF51A79CA4F0023F89D /* KMBinaryFileFormat.h */; settings = {ATTRIBUTES = (Public, ); }; }; 984C2B0C1A79CA4F0023F89D /* KMCompGroup.h in Headers */ = {isa = PBXBuildFile; fileRef = 984C2AF81A79CA4F0023F89D /* KMCompGroup.h */; }; - 984C2B0D1A79CA4F0023F89D /* KMCompGroup.m in Sources */ = {isa = PBXBuildFile; fileRef = 984C2AF91A79CA4F0023F89D /* KMCompGroup.m */; }; 984C2B0E1A79CA4F0023F89D /* KMCompKey.h in Headers */ = {isa = PBXBuildFile; fileRef = 984C2AFA1A79CA4F0023F89D /* KMCompKey.h */; }; - 984C2B0F1A79CA4F0023F89D /* KMCompKey.m in Sources */ = {isa = PBXBuildFile; fileRef = 984C2AFB1A79CA4F0023F89D /* KMCompKey.m */; }; 984C2B101A79CA4F0023F89D /* KMCompStore.h in Headers */ = {isa = PBXBuildFile; fileRef = 984C2AFC1A79CA4F0023F89D /* KMCompStore.h */; }; 984C2B111A79CA4F0023F89D /* KMCompStore.m in Sources */ = {isa = PBXBuildFile; fileRef = 984C2AFD1A79CA4F0023F89D /* KMCompStore.m */; }; 984C2B121A79CA4F0023F89D /* KMEngine.h in Headers */ = {isa = PBXBuildFile; fileRef = 984C2AFE1A79CA4F0023F89D /* KMEngine.h */; settings = {ATTRIBUTES = (Public, ); }; }; @@ -525,7 +522,6 @@ 29B4A0D92BFAE8A400682049 /* KMELogs.m in Sources */, 981880781BC1EA5800A1FBA5 /* OSKView.m in Sources */, 2993C32E29B1D92E00FB5E95 /* CoreWrapper.m in Sources */, - 984C2B0F1A79CA4F0023F89D /* KMCompKey.m in Sources */, 984C2B131A79CA4F0023F89D /* KMEngine.m in Sources */, 984C2B171A79CA4F0023F89D /* KVKFile.m in Sources */, 29A1971F2AF091E600512A37 /* CoreKeyOutput.m in Sources */, @@ -533,12 +529,10 @@ 981880761BC1EA5800A1FBA5 /* OSKKey.m in Sources */, 29BE6F042C9D700B00635B1A /* CoreKey.m in Sources */, 2993C32F29B1D92E00FB5E95 /* CoreHelper.m in Sources */, - 984C2B0D1A79CA4F0023F89D /* KMCompGroup.m in Sources */, 984C2B111A79CA4F0023F89D /* KMCompStore.m in Sources */, 2993C32C29B1D92E00FB5E95 /* CoreAction.m in Sources */, 984C2B151A79CA4F0023F89D /* KMXFile.m in Sources */, 980053361B37C9B50088DDDD /* NFont.m in Sources */, - 984C2B071A79CA4F0023F89D /* NSString+XString.m in Sources */, 29E50EB02CA121BA00B59959 /* CoreKeyboardInfo.m in Sources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompGroup.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompGroup.m index 4da8d3bc96..802eed40d6 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompGroup.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompGroup.m @@ -7,7 +7,6 @@ // #import "KMCompGroup.h" -#import "NSString+XString.h" @implementation KMCompGroup diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompKey.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompKey.m index 8d77dcf072..0e2b43afa6 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompKey.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompKey.m @@ -7,7 +7,6 @@ // #import "KMCompKey.h" -#import "NSString+XString.h" @implementation KMCompKey diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m index 4fca35955b..fd59c48b62 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMEngine.m @@ -8,7 +8,6 @@ #import "KMEngine.h" #import "KMBinaryFileFormat.h" -#import "NSString+XString.h" #import "WindowsVKCodes.h" #import "MacVKCodes.h" #import "CoreWrapper.h" diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMXFile.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMXFile.m index 63eaf93359..27a2848426 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMXFile.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMXFile.m @@ -8,7 +8,6 @@ #import "KMXFile.h" #import "KMCompStore.h" -#import "NSString+XString.h" #import "KMELogs.h" NSString *const kKMKeyboardNameKey = @"KMKeyboardNameKey"; diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KeymanEngine4Mac.h b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KeymanEngine4Mac.h index 6addfc4b47..197f256851 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KeymanEngine4Mac.h +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KeymanEngine4Mac.h @@ -24,7 +24,6 @@ FOUNDATION_EXPORT const unsigned char KeymanEngine4MacVersionString[]; #import #import #import -#import #import #import #import From db5a49370bc9d3a951cc6db3f24402f238d4b632 Mon Sep 17 00:00:00 2001 From: Shawn Schantz Date: Wed, 7 May 2025 12:51:51 +0700 Subject: [PATCH 25/46] chore(mac): remove files for KMCompGroup, KMCompKey, NSString extension --- .../project.pbxproj | 18 ----------- .../KME/Categories/NSString+XString.h | 16 ---------- .../KME/Categories/NSString+XString.m | 17 ---------- .../KeymanEngine4Mac/KME/KMCompGroup.h | 26 --------------- .../KeymanEngine4Mac/KME/KMCompGroup.m | 32 ------------------- .../KeymanEngine4Mac/KME/KMCompKey.h | 25 --------------- .../KeymanEngine4Mac/KME/KMCompKey.m | 30 ----------------- 7 files changed, 164 deletions(-) delete mode 100644 mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/Categories/NSString+XString.h delete mode 100644 mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/Categories/NSString+XString.m delete mode 100644 mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompGroup.h delete mode 100644 mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompGroup.m delete mode 100644 mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompKey.h delete mode 100644 mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompKey.m diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac.xcodeproj/project.pbxproj b/mac/KeymanEngine4Mac/KeymanEngine4Mac.xcodeproj/project.pbxproj index e705118e5b..6ea81dec08 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac.xcodeproj/project.pbxproj +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac.xcodeproj/project.pbxproj @@ -51,10 +51,7 @@ 984C2AD81A79C98D0023F89D /* KeymanEngine4Mac.h in Headers */ = {isa = PBXBuildFile; fileRef = 984C2AD71A79C98D0023F89D /* KeymanEngine4Mac.h */; settings = {ATTRIBUTES = (Public, ); }; }; 984C2ADE1A79C98D0023F89D /* KeymanEngine4Mac.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 984C2AD21A79C98D0023F89D /* KeymanEngine4Mac.framework */; }; 984C2AE51A79C98D0023F89D /* KMXFileTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 984C2AE41A79C98D0023F89D /* KMXFileTests.m */; }; - 984C2B061A79CA4F0023F89D /* NSString+XString.h in Headers */ = {isa = PBXBuildFile; fileRef = 984C2AF21A79CA4F0023F89D /* NSString+XString.h */; settings = {ATTRIBUTES = (Public, ); }; }; 984C2B091A79CA4F0023F89D /* KMBinaryFileFormat.h in Headers */ = {isa = PBXBuildFile; fileRef = 984C2AF51A79CA4F0023F89D /* KMBinaryFileFormat.h */; settings = {ATTRIBUTES = (Public, ); }; }; - 984C2B0C1A79CA4F0023F89D /* KMCompGroup.h in Headers */ = {isa = PBXBuildFile; fileRef = 984C2AF81A79CA4F0023F89D /* KMCompGroup.h */; }; - 984C2B0E1A79CA4F0023F89D /* KMCompKey.h in Headers */ = {isa = PBXBuildFile; fileRef = 984C2AFA1A79CA4F0023F89D /* KMCompKey.h */; }; 984C2B101A79CA4F0023F89D /* KMCompStore.h in Headers */ = {isa = PBXBuildFile; fileRef = 984C2AFC1A79CA4F0023F89D /* KMCompStore.h */; }; 984C2B111A79CA4F0023F89D /* KMCompStore.m in Sources */ = {isa = PBXBuildFile; fileRef = 984C2AFD1A79CA4F0023F89D /* KMCompStore.m */; }; 984C2B121A79CA4F0023F89D /* KMEngine.h in Headers */ = {isa = PBXBuildFile; fileRef = 984C2AFE1A79CA4F0023F89D /* KMEngine.h */; settings = {ATTRIBUTES = (Public, ); }; }; @@ -131,13 +128,7 @@ 984C2ADD1A79C98D0023F89D /* KeymanEngine4MacTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = KeymanEngine4MacTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; 984C2AE31A79C98D0023F89D /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; 984C2AE41A79C98D0023F89D /* KMXFileTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = KMXFileTests.m; sourceTree = ""; }; - 984C2AF21A79CA4F0023F89D /* NSString+XString.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "NSString+XString.h"; sourceTree = ""; }; - 984C2AF31A79CA4F0023F89D /* NSString+XString.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "NSString+XString.m"; sourceTree = ""; }; 984C2AF51A79CA4F0023F89D /* KMBinaryFileFormat.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = KMBinaryFileFormat.h; sourceTree = ""; }; - 984C2AF81A79CA4F0023F89D /* KMCompGroup.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = KMCompGroup.h; sourceTree = ""; }; - 984C2AF91A79CA4F0023F89D /* KMCompGroup.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = KMCompGroup.m; sourceTree = ""; }; - 984C2AFA1A79CA4F0023F89D /* KMCompKey.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = KMCompKey.h; sourceTree = ""; }; - 984C2AFB1A79CA4F0023F89D /* KMCompKey.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = KMCompKey.m; sourceTree = ""; }; 984C2AFC1A79CA4F0023F89D /* KMCompStore.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = KMCompStore.h; sourceTree = ""; }; 984C2AFD1A79CA4F0023F89D /* KMCompStore.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = KMCompStore.m; sourceTree = ""; }; 984C2AFE1A79CA4F0023F89D /* KMEngine.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = KMEngine.h; sourceTree = ""; }; @@ -304,10 +295,6 @@ 981880641BC1EA5800A1FBA5 /* OnScreenKeyboard */, 984C2AEF1A79CA4F0023F89D /* Categories */, 984C2AF51A79CA4F0023F89D /* KMBinaryFileFormat.h */, - 984C2AF81A79CA4F0023F89D /* KMCompGroup.h */, - 984C2AF91A79CA4F0023F89D /* KMCompGroup.m */, - 984C2AFA1A79CA4F0023F89D /* KMCompKey.h */, - 984C2AFB1A79CA4F0023F89D /* KMCompKey.m */, 984C2AFC1A79CA4F0023F89D /* KMCompStore.h */, 984C2AFD1A79CA4F0023F89D /* KMCompStore.m */, 984C2AFE1A79CA4F0023F89D /* KMEngine.h */, @@ -329,8 +316,6 @@ 984C2AEF1A79CA4F0023F89D /* Categories */ = { isa = PBXGroup; children = ( - 984C2AF21A79CA4F0023F89D /* NSString+XString.h */, - 984C2AF31A79CA4F0023F89D /* NSString+XString.m */, ); path = Categories; sourceTree = ""; @@ -348,7 +333,6 @@ 984C2B141A79CA4F0023F89D /* KMXFile.h in Headers */, 981880771BC1EA5800A1FBA5 /* OSKView.h in Headers */, 984C2B161A79CA4F0023F89D /* KVKFile.h in Headers */, - 984C2B061A79CA4F0023F89D /* NSString+XString.h in Headers */, 984C2AD81A79C98D0023F89D /* KeymanEngine4Mac.h in Headers */, 2993C33429B1D92E00FB5E95 /* CoreAction.h in Headers */, 980053351B37C9B50088DDDD /* NFont.h in Headers */, @@ -357,9 +341,7 @@ 29CBB2C62CA137A50097556B /* CoreKey.h in Headers */, 984C2B191A79CB020023F89D /* WindowsVKCodes.h in Headers */, 29E50EB12CA121BA00B59959 /* CoreKeyboardInfo.h in Headers */, - 984C2B0C1A79CA4F0023F89D /* KMCompGroup.h in Headers */, 984C2B101A79CA4F0023F89D /* KMCompStore.h in Headers */, - 984C2B0E1A79CA4F0023F89D /* KMCompKey.h in Headers */, 980053371B37C9B50088DDDD /* NKey.h in Headers */, 9840F5ED1B43B02500410ABA /* MacVKCodes.h in Headers */, 981880711BC1EA5800A1FBA5 /* KeyLabelCell.h in Headers */, diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/Categories/NSString+XString.h b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/Categories/NSString+XString.h deleted file mode 100644 index 4367a4aa27..0000000000 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/Categories/NSString+XString.h +++ /dev/null @@ -1,16 +0,0 @@ -// -// NSString+XString.h -// KeymanEngine4Mac -// -// Created by Serkan Kurt on 8/01/2015. -// Copyright (c) 2017 SIL International. All rights reserved. -// - -#import -#import "KMBinaryFileFormat.h" - -@interface NSString (XString) - -+ (NSString *)nullChar; - -@end diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/Categories/NSString+XString.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/Categories/NSString+XString.m deleted file mode 100644 index 5a3db3b1d0..0000000000 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/Categories/NSString+XString.m +++ /dev/null @@ -1,17 +0,0 @@ -// -// NSString+XString.m -// KeymanEngine4Mac -// -// Created by Serkan Kurt on 8/01/2015. -// Copyright (c) 2017 SIL International. All rights reserved. -// - -#import "NSString+XString.h" - -@implementation NSString (XString) - -+ (NSString *)nullChar { - return [NSString stringWithFormat:@"%C%C%d", UC_SENTINEL, CODE_NULLCHAR, 1]; -} - -@end diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompGroup.h b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompGroup.h deleted file mode 100644 index 2bc3fc0725..0000000000 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompGroup.h +++ /dev/null @@ -1,26 +0,0 @@ -// -// KMCompGroup.h -// KeymanEngine4Mac -// -// Created by Serkan Kurt on 28/11/2014. -// Copyright (c) 2017 SIL International. All rights reserved. -// - -#ifndef KMCompGroup_h -#define KMCompGroup_h - -#import -#import -#import "KMBinaryFileFormat.h" - -@interface KMCompGroup : NSObject - -@property (strong, nonatomic) NSString *name; -@property (strong, nonatomic) NSMutableArray *keys; -@property (strong, nonatomic) NSString *match; -@property (strong, nonatomic) NSString *noMatch; -@property (assign, nonatomic) BOOL fUsingKeys; - -@end - -#endif /* KMCompGroup_h */ diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompGroup.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompGroup.m deleted file mode 100644 index 802eed40d6..0000000000 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompGroup.m +++ /dev/null @@ -1,32 +0,0 @@ -// -// KMCompGroup.m -// KeymanEngine4Mac -// -// Created by Serkan Kurt on 28/11/2014. -// Copyright (c) 2017 SIL International. All rights reserved. -// - -#import "KMCompGroup.h" - -@implementation KMCompGroup - -- (id)init { - self = [super init]; - if (self) { - _name = @""; - _keys = [[NSMutableArray alloc] initWithCapacity:0]; - _match = @""; - _noMatch = @""; - } - - return self; -} - -- (NSString *)description { - NSString *format = @"<%@:%p, Name:%@ fUsingKeys:%d Keys:%@>"; - NSString *str = [NSString stringWithFormat:format, - [self className], self, _name, _fUsingKeys, _keys]; - return str; -} - -@end diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompKey.h b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompKey.h deleted file mode 100644 index 20dcf001b3..0000000000 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompKey.h +++ /dev/null @@ -1,25 +0,0 @@ -// -// KMCompKey.h -// KeymanEngine4Mac -// -// Created by Serkan Kurt on 28/11/2014. -// Copyright (c) 2017 SIL International. All rights reserved. -// - -#ifndef KMCompKey_h -#define KMCompKey_h - -#import -#import "KMBinaryFileFormat.h" - -@interface KMCompKey : NSObject - -@property (assign, nonatomic) WORD key; -@property (assign, nonatomic) DWORD line; -@property (assign, nonatomic) DWORD shiftFlags; -@property (strong, nonatomic) NSString *output; -@property (strong, nonatomic) NSString *context; - -@end - -#endif /* KMCompKey_h */ diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompKey.m b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompKey.m deleted file mode 100644 index 0e2b43afa6..0000000000 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMCompKey.m +++ /dev/null @@ -1,30 +0,0 @@ -// -// KMCompKey.m -// KeymanEngine4Mac -// -// Created by Serkan Kurt on 28/11/2014. -// Copyright (c) 2017 SIL International. All rights reserved. -// - -#import "KMCompKey.h" - -@implementation KMCompKey - -- (id)init { - self = [super init]; - if (self) { - _output = @""; - _context = @""; - } - - return self; -} - -- (NSString *)description { - NSString *format = @"<%@:%p K:0x%X L:0x%X F:0x%X>"; - NSString *str = [NSString stringWithFormat:format, - self.className, self, self.key, self.line, self.shiftFlags]; - return str; -} - -@end From e619e9f89a38a1715a5f151ffe7cbaaac507f953 Mon Sep 17 00:00:00 2001 From: Shawn Schantz Date: Wed, 7 May 2025 16:08:14 +0700 Subject: [PATCH 26/46] chore(mac): removed commented-out code Fixes: #11104 --- mac/Keyman4Mac/Keyman4Mac/AppDelegate.m | 19 ------------------- .../KeymanEngine4MacTests/KMXFileTests.m | 8 -------- 2 files changed, 27 deletions(-) diff --git a/mac/Keyman4Mac/Keyman4Mac/AppDelegate.m b/mac/Keyman4Mac/Keyman4Mac/AppDelegate.m index fb31d3d7b3..b873a34290 100644 --- a/mac/Keyman4Mac/Keyman4Mac/AppDelegate.m +++ b/mac/Keyman4Mac/Keyman4Mac/AppDelegate.m @@ -197,17 +197,6 @@ CGEventRef eventTapFunction(CGEventTapProxy proxy, CGEventType type, CGEventRef NSString *kmxPath = [self.kmxList objectAtIndex:self.kbComboBox.indexOfSelectedItem-1]; KMXFile *kmx = [[KMXFile alloc] initWithFilePath:kmxPath]; - /* - if (![kmx isValid]) { - NSAlert *alert = [[NSAlert alloc] init]; - [alert addButtonWithTitle:@"OK"]; - [alert setMessageText:@"Invalid KMX file"]; - [alert setInformativeText:@"This KMX file contains some invalid code!"]; - [alert setAlertStyle:NSAlertStyleWarning]; - [alert runModal]; - } - */ - NSString *packagePath = [kmxPath stringByDeletingLastPathComponent]; NSDictionary *infoDict = [KMXFile keyboardInfoFromKmxFile:kmxPath]; NSString *kvkFilename = [infoDict objectForKey:kKMVisualKeyboardKey]; @@ -220,14 +209,6 @@ CGEventRef eventTapFunction(CGEventTapProxy proxy, CGEventType type, CGEventRef [self.kbData setObject:kmx forKey:kKMXFileKey]; self.imgView.image = kmx.bitmap; - - /* - int index = 0; - for (NSObject *kmStore in kmx.store) { - NSLog(@"%d: %@", index, kmStore); - index++; - } - */ } } diff --git a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMXFileTests.m b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMXFileTests.m index 1a2ac73e4b..6d6a696252 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMXFileTests.m +++ b/mac/KeymanEngine4Mac/KeymanEngine4MacTests/KMXFileTests.m @@ -36,12 +36,4 @@ XCTAssert(kmxFile == nil, @"Expected nil file"); } -/* -- (void)testinitWithFilePath_Valid_ReturnsInitializedKmxFile { - KMXFile *kmxFile = [KeymanEngineTestsStaticHelperMethods getKmxFileTestMacEngine]; - XCTAssert(kmxFile != nil, @"Expected non-nil file"); - XCTAssert([kmxFile isValid], @"Expected valid KMX file."); -} - */ - @end From c8cd5b13b372350503e0879d7c1e28b884374903 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Tue, 6 May 2025 09:09:39 +0700 Subject: [PATCH 27/46] maint(android): Add setting JAVA_HOME_21 env --- resources/build/build-utils.sh | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/resources/build/build-utils.sh b/resources/build/build-utils.sh index 2cbbcbfba9..d857cad5b6 100755 --- a/resources/build/build-utils.sh +++ b/resources/build/build-utils.sh @@ -328,11 +328,29 @@ set_keyman_standard_build_path() { PATH="$KEYMAN_ROOT/node_modules/.bin:$PATH" } -# For CI compatbility of building Keyman for Android 16.0 with OpenJDK 8, -# this overrides JAVA_HOME for the builder script to use OpenJDK 11. +# For CI compatibility of building Keyman for Android 18.0 with OpenJDK 11, +# this overrides JAVA_HOME for the builder script to use OpenJDK 21. set_java_home() { + set_java_home_21 +} + +set_java_home_21() { + if [[ ! -z ${JAVA_HOME_21+x} ]]; then + builder_echo "Setting JAVA_HOME to JAVA_HOME_21 (${JAVA_HOME_21})" + export JAVA_HOME="${JAVA_HOME_21}" + fi +} + +set_java_home_17() { + if [[ ! -z ${JAVA_HOME_17+x} ]]; then + builder_echo "Setting JAVA_HOME to JAVA_HOME_17 (${JAVA_HOME_17})" + export JAVA_HOME="${JAVA_HOME_17}" + fi +} + +set_java_home_11() { if [[ ! -z ${JAVA_HOME_11+x} ]]; then - builder_echo "Setting JAVA_HOME to JAVA_HOME_11 ($JAVA_HOME_11)" + builder_echo "Setting JAVA_HOME to JAVA_HOME_11 (${JAVA_HOME_11})" export JAVA_HOME="${JAVA_HOME_11}" fi } From 5f0cd83d055d48186d5a4215b22ff14fc815bdb3 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Tue, 6 May 2025 09:12:57 +0700 Subject: [PATCH 28/46] maint(android): Update Gradle to use Java 21 --- android/KMAPro/build.gradle | 4 +- android/KMAPro/build.sh | 2 +- android/KMAPro/gradle.properties | 3 + .../gradle/wrapper/gradle-wrapper.properties | 2 +- android/KMAPro/kMAPro/build.gradle | 11 +++- .../com/tavultesoft/kmapro/MainActivity.java | 66 +++++++++---------- android/KMEA/app/build.gradle | 35 ++++++---- .../keyman/engine/cloud/CloudDownloadMgr.java | 3 + android/KMEA/build.gradle | 4 +- android/KMEA/build.sh | 2 +- .../gradle/wrapper/gradle-wrapper.properties | 3 +- 11 files changed, 81 insertions(+), 54 deletions(-) diff --git a/android/KMAPro/build.gradle b/android/KMAPro/build.gradle index 5614fcae6a..c644d44fca 100644 --- a/android/KMAPro/build.gradle +++ b/android/KMAPro/build.gradle @@ -8,10 +8,10 @@ buildscript { } } dependencies { - classpath 'com.android.tools.build:gradle:7.4.2' + classpath 'com.android.tools.build:gradle:8.9.2' // sentry-android-gradle-plugin 2.1.5+ requires AGP 7.0 classpath 'io.sentry:sentry-android-gradle-plugin:4.6.0' - classpath 'name.remal:gradle-plugins:1.5.0' + classpath 'name.remal:gradle-plugins:1.9.2' // From jcenter() which was deprecated August 2024 // https://jfrog.com/blog/into-the-sunset-bintray-jcenter-gocenter-and-chartcenter/ diff --git a/android/KMAPro/build.sh b/android/KMAPro/build.sh index bfe81bba5b..4f94236cfa 100755 --- a/android/KMAPro/build.sh +++ b/android/KMAPro/build.sh @@ -37,7 +37,7 @@ builder_parse "$@" if builder_is_debug_build; then builder_heading "### Debug config ####" CONFIG="debug" - BUILD_FLAGS="assembleDebug -x lint -x test" + BUILD_FLAGS="assembleDebug -x lintDebug -x testDebug" TEST_FLAGS="-x assembleDebug lintDebug testDebug" fi diff --git a/android/KMAPro/gradle.properties b/android/KMAPro/gradle.properties index 93aaf1410c..f56c02ac58 100644 --- a/android/KMAPro/gradle.properties +++ b/android/KMAPro/gradle.properties @@ -1,4 +1,7 @@ android.enableJetifier=true android.useAndroidX=true +# AGP 8.0 generates R classes for resources defined in the current module only. +android.nonTransitiveRClass=false + org.gradle.jvmargs=-Xmx1024m diff --git a/android/KMAPro/gradle/wrapper/gradle-wrapper.properties b/android/KMAPro/gradle/wrapper/gradle-wrapper.properties index 3994438e22..d6e308a637 100644 --- a/android/KMAPro/gradle/wrapper/gradle-wrapper.properties +++ b/android/KMAPro/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.4-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/android/KMAPro/kMAPro/build.gradle b/android/KMAPro/kMAPro/build.gradle index 768d98d933..774a452190 100644 --- a/android/KMAPro/kMAPro/build.gradle +++ b/android/KMAPro/kMAPro/build.gradle @@ -20,6 +20,15 @@ android { noCompress "kmp" } + buildFeatures { + buildConfig = true + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_21 + targetCompatibility = JavaVersion.VERSION_21 + } + defaultConfig { applicationId "com.tavultesoft.kmapro" minSdkVersion 21 @@ -156,7 +165,7 @@ repositories { dependencies { implementation fileTree(dir: 'libs', include: ['*.jar']) - implementation 'androidx.appcompat:appcompat:1.6.1' + implementation 'androidx.appcompat:appcompat:1.7.0' implementation 'com.google.android.material:material:1.12.0' implementation 'com.stepstone.stepper:material-stepper:4.3.1' api(name: 'keyman-engine', ext: 'aar') diff --git a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/MainActivity.java b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/MainActivity.java index 07ce5c4cbb..ee6f6705a3 100644 --- a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/MainActivity.java +++ b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/MainActivity.java @@ -440,41 +440,41 @@ public class MainActivity extends BaseActivity implements OnKeyboardEventListene @Override public boolean onOptionsItemSelected(MenuItem item) { - switch (item.getItemId()) { - case R.id.action_info: - showInfo(); - return true; - case R.id.action_share: - showShareDialog(); - return true; + // Android Gradle 8.0 no longer declares resources final, so can't use switch statement here + if (item.getItemId() == R.id.action_info) { + showInfo(); + return true; + } else if (item.getItemId() == R.id.action_share) { + showShareDialog(); + return true; /* Disable Web Browser to investigate Google sign-in - case R.id.action_web: + } else if ((item.getItemId() == R.id.action_web) { showWebBrowser(); return true;*/ - case R.id.action_text_size: - showTextSizeDialog(); - return true; - case R.id.action_clear_text: - showClearTextDialog(); - return true; - case R.id.action_get_started: - showGetStarted(); - return true; - case R.id.action_settings: - showSettings(); - return true; - case R.id.action_update_keyboards: - KMManager.getUpdateTool().executeOpenUpdates(); - // Dismiss icon - updateUpdateCountIndicator(0); - final MenuItem _keyboardupdate = menu.findItem(R.id.action_update_keyboards); - if (_keyboardupdate != null && _keyboardupdate.isVisible()) { - _keyboardupdate.setVisible(false); - } + } else if (item.getItemId() == R.id.action_text_size) { + showTextSizeDialog(); + return true; + } else if (item.getItemId() == R.id.action_clear_text){ + showClearTextDialog(); + return true; + } else if (item.getItemId() == R.id.action_get_started) { + showGetStarted(); + return true; + } else if (item.getItemId() == R.id.action_settings) { + showSettings(); + return true; + } else if (item.getItemId() == R.id.action_update_keyboards) { + KMManager.getUpdateTool().executeOpenUpdates(); + // Dismiss icon + updateUpdateCountIndicator(0); + final MenuItem _keyboardupdate = menu.findItem(R.id.action_update_keyboards); + if (_keyboardupdate != null && _keyboardupdate.isVisible()) { + _keyboardupdate.setVisible(false); + } - return true; - default: - return super.onOptionsItemSelected(item); + return true; + } else { + return super.onOptionsItemSelected(item); } } @@ -632,7 +632,7 @@ public class MainActivity extends BaseActivity implements OnKeyboardEventListene Intent i = new Intent(this, InfoActivity.class); i.addFlags(Intent.FLAG_ACTIVITY_CLEAR_WHEN_TASK_RESET); startActivity(i); - overridePendingTransition(android.R.anim.fade_in, R.anim.hold); + overridePendingTransition(android.R.anim.fade_in, com.keyman.engine.R.anim.hold); } private void showWebBrowser() { @@ -643,7 +643,7 @@ public class MainActivity extends BaseActivity implements OnKeyboardEventListene Intent i = new Intent(this, WebBrowserActivity.class); i.addFlags(Intent.FLAG_ACTIVITY_CLEAR_WHEN_TASK_RESET); startActivity(i); - overridePendingTransition(android.R.anim.fade_in, R.anim.hold); + overridePendingTransition(android.R.anim.fade_in, com.keyman.engine.R.anim.hold); } private void showShareDialog() { diff --git a/android/KMEA/app/build.gradle b/android/KMEA/app/build.gradle index c80a4fd66e..cf527ca01c 100644 --- a/android/KMEA/app/build.gradle +++ b/android/KMEA/app/build.gradle @@ -1,4 +1,5 @@ plugins { + id 'base' id 'com.android.library' id 'name.remal.default-plugins' } @@ -6,10 +7,25 @@ plugins { ext.rootPath = '../../' apply from: "$rootPath/version.gradle" +base { + // Specify library filename keyman-engine-$CONFIG.aar + archivesName = "keyman-engine" +} + android { compileSdk 34 namespace "com.keyman.engine" + buildFeatures { + // needed for custom BuildConfigField values in defaultConfig + buildConfig = true + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_21 + targetCompatibility = JavaVersion.VERSION_21 + } + defaultConfig { minSdkVersion 21 targetSdkVersion 34 @@ -17,9 +33,6 @@ android { // VERSION_CODE and VERSION_NAME from version.gradle but Gradle removes them for libraries buildConfigField "String", "KEYMAN_ENGINE_VERSION_NAME", "\""+VERSION_NAME+"\"" buildConfigField "String", "VERSION_ENVIRONMENT", "\""+VERSION_ENVIRONMENT+"\"" - - // Sepcify library filename keyman-engine-$CONFIG.aar - setProperty("archivesBaseName", "keyman-engine") } buildTypes { @@ -36,10 +49,6 @@ android { } testOptions { - unitTests { - // Allows use of a simulated Android API for tests. (Thanks, roboelectric!) - includeAndroidResources = true - } unitTests.all { testLogging { // May also add "passed", "skipped". @@ -55,17 +64,19 @@ android { dependencies { // 1.6.0-rc01 needed to resolve https://issuetracker.google.com/issues/238425626 - implementation 'androidx.appcompat:appcompat:1.6.1' + implementation 'androidx.appcompat:appcompat:1.7.0' // material:1.7.0 will need Gradle plugin 7.1.0+ implementation 'com.google.android.material:material:1.12.0' - implementation 'commons-io:commons-io:2.13.0' + implementation 'commons-io:commons-io:2.16.1' implementation 'io.sentry:sentry-android:7.8.0' implementation 'androidx.preference:preference:1.2.1' // Robolectric - testImplementation 'androidx.test:core:1.5.0' - testImplementation 'androidx.test.ext:junit:1.1.5' - testImplementation 'org.robolectric:robolectric:4.10.3' + testImplementation 'androidx.test.ext:junit:1.2.1' + testImplementation 'org.robolectric:robolectric:4.14.1' + testImplementation 'androidx.test:core:1.6.1' + testImplementation 'androidx.test.ext:junit:1.2.1' + testImplementation 'org.robolectric:robolectric:4.14.1' // Generate QR Codes implementation ('com.github.kenglxn.QRGen:android:3.0.1') { diff --git a/android/KMEA/app/src/main/java/com/keyman/engine/cloud/CloudDownloadMgr.java b/android/KMEA/app/src/main/java/com/keyman/engine/cloud/CloudDownloadMgr.java index bb8ee190b6..ce32f2beeb 100644 --- a/android/KMEA/app/src/main/java/com/keyman/engine/cloud/CloudDownloadMgr.java +++ b/android/KMEA/app/src/main/java/com/keyman/engine/cloud/CloudDownloadMgr.java @@ -1,5 +1,6 @@ package com.keyman.engine.cloud; +import android.annotation.SuppressLint; import android.app.DownloadManager; import android.content.BroadcastReceiver; import android.content.Context; @@ -60,6 +61,7 @@ public class CloudDownloadMgr{ * Append downloadreceiver to the main context. * @param aContext the context */ + @SuppressLint("UnspecifiedRegisterReceiverFlag") public synchronized void initialize(Context aContext) { if(isInitialized) @@ -73,6 +75,7 @@ public class CloudDownloadMgr{ aContext.registerReceiver(completeListener, new IntentFilter(DownloadManager.ACTION_DOWNLOAD_COMPLETE), Context.RECEIVER_EXPORTED); } else { + // Needs @SuppressLint UnspecifiedRegisterReceiverFlag aContext.registerReceiver(completeListener, new IntentFilter(DownloadManager.ACTION_DOWNLOAD_COMPLETE)); } } catch (IllegalArgumentException e) { diff --git a/android/KMEA/build.gradle b/android/KMEA/build.gradle index 9aa2d0dcc7..4ca999ea60 100644 --- a/android/KMEA/build.gradle +++ b/android/KMEA/build.gradle @@ -5,10 +5,10 @@ buildscript { mavenCentral() } dependencies { - classpath 'com.android.tools.build:gradle:7.4.2' + classpath 'com.android.tools.build:gradle:8.9.2' // io.sentry:sentry-android-gradle-plugin not available for library project classpath 'io.sentry:sentry-android:7.8.0' - classpath 'name.remal:gradle-plugins:1.5.0' + classpath 'name.remal:gradle-plugins:1.9.2' } } diff --git a/android/KMEA/build.sh b/android/KMEA/build.sh index c8c79c13de..dcd4c9510a 100755 --- a/android/KMEA/build.sh +++ b/android/KMEA/build.sh @@ -37,7 +37,7 @@ builder_parse "$@" if builder_is_debug_build; then builder_heading "### Debug config ####" CONFIG="debug" - BUILD_FLAGS="assembleDebug -x lint -x test" + BUILD_FLAGS="assembleDebug -x lintDebug -x testDebug" TEST_FLAGS="-x assembleDebug lintDebug testDebug" JUNIT_RESULTS="##teamcity[importData type='junit' path='keyman\android\KMEA\app\build\test-results\testDebugUnitTest\']" fi diff --git a/android/KMEA/gradle/wrapper/gradle-wrapper.properties b/android/KMEA/gradle/wrapper/gradle-wrapper.properties index 3994438e22..8330c6a158 100644 --- a/android/KMEA/gradle/wrapper/gradle-wrapper.properties +++ b/android/KMEA/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,6 @@ +#Wed Apr 30 11:17:37 ICT 2025 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.4-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From 80710a8303ead93cb8a2b40e19b644b4ecf5ab63 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Thu, 8 May 2025 15:36:52 +0700 Subject: [PATCH 29/46] Apply suggestions from code review Co-authored-by: Joshua Horton --- android/KMEA/app/build.gradle | 2 -- 1 file changed, 2 deletions(-) diff --git a/android/KMEA/app/build.gradle b/android/KMEA/app/build.gradle index cf527ca01c..e4e98f74f3 100644 --- a/android/KMEA/app/build.gradle +++ b/android/KMEA/app/build.gradle @@ -75,8 +75,6 @@ dependencies { testImplementation 'androidx.test.ext:junit:1.2.1' testImplementation 'org.robolectric:robolectric:4.14.1' testImplementation 'androidx.test:core:1.6.1' - testImplementation 'androidx.test.ext:junit:1.2.1' - testImplementation 'org.robolectric:robolectric:4.14.1' // Generate QR Codes implementation ('com.github.kenglxn.QRGen:android:3.0.1') { From ad0ea8678b9337f3823eaa04d992347184dc8b53 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 8 May 2025 17:29:30 -0500 Subject: [PATCH 30/46] feat(developer): update for line numbers - added new DelegatingCompilerCallbacks class - added ResolvingCompilerCallbacks which does event resolution - added the XML file name metadata to *every object* after XML parse. Fixes: #10622 --- .../web/utils/src/compiler-callbacks.ts | 106 +++++++++++------- developer/src/common/web/utils/src/index.ts | 6 +- .../src/common/web/utils/src/line-utils.ts | 19 ++++ .../src/types/ldml-keyboard/eventresolver.ts | 28 ----- .../ldml-keyboard/ldml-keyboard-xml-reader.ts | 10 +- .../src/common/web/utils/src/xml-utils.ts | 17 +++ .../src/kmc-ldml/src/compiler/compiler.ts | 16 ++- .../src/compiler/ldml-compiler-messages.ts | 3 +- .../src/commands/buildClasses/BuildProject.ts | 2 +- 9 files changed, 124 insertions(+), 83 deletions(-) delete mode 100644 developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts diff --git a/developer/src/common/web/utils/src/compiler-callbacks.ts b/developer/src/common/web/utils/src/compiler-callbacks.ts index aabf1794e5..78187b9a58 100644 --- a/developer/src/common/web/utils/src/compiler-callbacks.ts +++ b/developer/src/common/web/utils/src/compiler-callbacks.ts @@ -125,14 +125,64 @@ export interface CompilerFileSystemAsyncCallbacks { resolveFilename(baseFilename: string, filename: string): string; } +/** a CompilerCallbacks implementation that delegates to a parent */ +export class DelegatingCompilerCallbacks implements CompilerCallbacks { + constructor(protected options: CompilerCallbackOptions, protected parent: CompilerCallbacks) { + } + + loadFile(filename: string): Uint8Array { + return this.parent.loadFile(filename); + } + + fileSize(filename: string): number { + return this.parent.fileSize(filename); + } + + isDirectory(filename: string): boolean { + return this.parent.isDirectory(filename); + } + + get path(): CompilerPathCallbacks { + return this.parent.path; + } + + get fs(): CompilerFileSystemCallbacks { + return this.parent.fs; + } + + get net(): CompilerNetAsyncCallbacks { + return this.parent.net; + } + + get fsAsync(): CompilerFileSystemAsyncCallbacks { + return this.parent.fsAsync; + } + + resolveFilename(baseFilename: string, filename: string): string { + return this.parent.resolveFilename(baseFilename, filename); + } + + debug(msg: string): void { + return this.parent.debug(msg); + } + + fileURLToPath(url: string | URL): string { + return this.parent.fileURLToPath(url); + } + + reportMessage(event: CompilerEvent) { + this.parent.reportMessage(event); + } +} /** * Wrapper class for CompilerCallbacks for a given input file */ -export class CompilerFileCallbacks implements CompilerCallbacks { +export class CompilerFileCallbacks extends DelegatingCompilerCallbacks { messages: CompilerEvent[] = []; - constructor(private filename: string, private options: CompilerCallbackOptions, private parent: CompilerCallbacks) { + constructor(private filename: string, options: CompilerCallbackOptions, parent: CompilerCallbacks) { + super(options, parent); } /** @@ -189,38 +239,6 @@ export class CompilerFileCallbacks implements CompilerCallbacks { this.messages = []; } - loadFile(filename: string): Uint8Array { - return this.parent.loadFile(filename); - } - - fileSize(filename: string): number { - return this.parent.fileSize(filename); - } - - isDirectory(filename: string): boolean { - return this.parent.isDirectory(filename); - } - - get path(): CompilerPathCallbacks { - return this.parent.path; - } - - get fs(): CompilerFileSystemCallbacks { - return this.parent.fs; - } - - get net(): CompilerNetAsyncCallbacks { - return this.parent.net; - } - - get fsAsync(): CompilerFileSystemAsyncCallbacks { - return this.parent.fsAsync; - } - - resolveFilename(baseFilename: string, filename: string): string { - return this.parent.resolveFilename(baseFilename, filename); - } - reportMessage(event: CompilerEvent): void { const disable = CompilerFileCallbacks.applyMessageOverridesToEvent(event, this.options.messageOverrides); this.messages.push(event); @@ -228,14 +246,6 @@ export class CompilerFileCallbacks implements CompilerCallbacks { this.parent.reportMessage({ filename: this.filename, ...event }); } } - - debug(msg: string): void { - return this.parent.debug(msg); - } - - fileURLToPath(url: string | URL): string { - return this.parent.fileURLToPath(url); - } } export class DefaultCompilerFileSystemAsyncCallbacks implements CompilerFileSystemAsyncCallbacks { @@ -261,3 +271,15 @@ export class DefaultCompilerFileSystemAsyncCallbacks implements CompilerFileSyst return this.owner.resolveFilename(baseFilename, filename); } } + +/** a CompilerCallbacks that applies the EventResolver to any reported message */ +export class ResolvingCompilerCallbacks extends DelegatingCompilerCallbacks { + constructor(private eventResolver: EventResolver, options: CompilerCallbackOptions, parent: CompilerCallbacks) { + super(options, parent); + } + + reportMessage(event: CompilerEvent) { + this.eventResolver.resolve(event); + this.parent.reportMessage(event); + } +} diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index 822dbffed1..2224463408 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -42,6 +42,8 @@ export { DefaultCompilerFileSystemAsyncCallbacks, EventResolver, NullEventResolver, + DelegatingCompilerCallbacks, + ResolvingCompilerCallbacks, } from './compiler-callbacks.js'; export { defaultCompilerOptions, CompilerBaseOptions, CompilerOptions, CompilerEvent, CompilerErrorNamespace, @@ -67,7 +69,7 @@ export { DeveloperUtilsMessages } from './developer-utils-messages.js'; export * as SourceFilenamePatterns from './source-filename-patterns.js'; export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, KeymanXMLMetadata, XML_FILENAME_SYMBOL } from './xml-utils.js'; export { SymbolUtils } from './symbol-utils.js'; -export { LineFinder, LineColumn } from './line-utils.js'; +export { LineFinder, LineColumn, LineFinderEventResolver } from './line-utils.js'; export * as GitHubUrls from './github-urls.js'; export * as CloudUrls from './cloud-urls.js'; @@ -75,4 +77,4 @@ export { getFontFamily, getFontFamilySync } from './font-family.js'; export * as ValidIds from './valid-ids.js'; -export * as ProjectLoader from './project-loader.js'; \ No newline at end of file +export * as ProjectLoader from './project-loader.js'; diff --git a/developer/src/common/web/utils/src/line-utils.ts b/developer/src/common/web/utils/src/line-utils.ts index e496f9b6ac..95f5744c55 100644 --- a/developer/src/common/web/utils/src/line-utils.ts +++ b/developer/src/common/web/utils/src/line-utils.ts @@ -6,6 +6,8 @@ * Abstraction for line number processing */ +import { CompilerEvent, EventResolver } from "./index.js"; + /** line number with an optional column number */ export interface LineColumn { line: number; @@ -90,4 +92,21 @@ export class LineFinderCache implements FileConsumer { return lf; } } +export class LineFinderEventResolver implements EventResolver, FileConsumer { + private lfcache = new LineFinderCache(); + addFile(filename: string, contents: string): void { + this.lfcache.addFile(filename, contents); + } + + /** resolve a CompilerEvent by expanding the line numbers */ + resolve(event: CompilerEvent) { + if (event.offset && !event.line && event.filename) { + const lf = this.lfcache.getByFilename(event.filename); + const loc = lf.findOffset(event.offset); + event.line = loc.line; + event.column = loc.column; + } + return event; + } +} diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts deleted file mode 100644 index b97e8908fd..0000000000 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Keyman is copyright (C) SIL Global. MIT License. - * - * Created by srl on 2025-03-03 - * - * Implementation of a hook on LDML events - */ - -import { LineFinderCache, FileConsumer } from "../../line-utils.js"; -import { CompilerEvent, EventResolver } from "../../index.js"; -export class LineFinderEventResolver implements EventResolver, FileConsumer { - private lfcache = new LineFinderCache(); - - addFile(filename: string, contents: string): void { - this.lfcache.addFile(filename, contents); - } - - /** resolve a CompilerEvent by expanding the line numbers */ - resolve(event: CompilerEvent) { - if (event.offset && !event.line && event.filename) { - const lf = this.lfcache.getByFilename(event.filename); - const loc = lf.findOffset(event.offset); - event.line = loc.line; - event.column = loc.column; - } - return event; - } -} diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index 9e1d1e2e06..f581d661cb 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -11,7 +11,7 @@ import { constants } from '@keymanapp/ldml-keyboard-constants'; import { LDMLKeyboardTestDataXMLSourceFile, LKTTest, LKTTests } from './ldml-keyboard-testdata-xml.js'; import { KeymanXMLReader } from '@keymanapp/developer-utils'; import boxXmlArray = util.boxXmlArray; -import { LineFinderEventResolver } from './eventresolver.js'; +import { LineFinderEventResolver } from '../../index.js'; import { XML_FILENAME_SYMBOL } from '../../xml-utils.js'; interface NameAndProps { @@ -28,7 +28,9 @@ export class LDMLKeyboardXMLSourceFileReaderOptions { }; export class LDMLKeyboardXMLSourceFileReader { - static eventResolver: LineFinderEventResolver = new LineFinderEventResolver(); + /** for resolving messages involving line numbers */ + eventResolver: LineFinderEventResolver = new LineFinderEventResolver(); + constructor(private options: LDMLKeyboardXMLSourceFileReaderOptions, private callbacks : CompilerCallbacks) { } @@ -40,7 +42,7 @@ export class LDMLKeyboardXMLSourceFileReader { readFile(path: string): Uint8Array { const data = this.callbacks.loadFile(path); if (data) { - LDMLKeyboardXMLSourceFileReader.eventResolver.addFile(path, new TextDecoder().decode(data)); + this.eventResolver.addFile(path, new TextDecoder().decode(data)); } return data; } @@ -260,7 +262,7 @@ export class LDMLKeyboardXMLSourceFileReader { } const importXml: any = this.loadUnboxed(importData); // TODO-LDML: have to load as any because it is an arbitrary part const importRootNode = importXml[subtag]; // e.g. - LDMLKeyboardXMLSourceFileReader.eventResolver.addFile(importPath, new TextDecoder().decode(importData)); // TODO: double decode + this.eventResolver.addFile(importPath, new TextDecoder().decode(importData)); // TODO: double decode // importXml will have one property: the root element. if (!importRootNode) { diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 9ec8872975..f83ca75dc7 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -146,6 +146,23 @@ export class KeymanXMLReader { return m; } + /** set metadata on this and children with the default filename - if not already set */ + public static setDefaultFilename(data: any, filename: string) { + if (!data || !filename) return; + if (typeof data === 'object') { + const m = KeymanXMLReader.getMetaData(data) || {}; + if (!m[XML_FILENAME_SYMBOL]) { + (m as any)[XML_FILENAME_SYMBOL] = filename; + KeymanXMLReader.setMetaData(data, m); + } + if (Array.isArray(data)) { + data.forEach(e => KeymanXMLReader.setDefaultFilename(e, filename)); + } else for(const k of Object.keys(data)) { + KeymanXMLReader.setDefaultFilename(data[k], filename); + } + } + } + /** move `{ $abc: 4 }` into `{ $: { abc: 4 } }` */ private static fixupDollarAttributes(data: any) : any { if (typeof data === 'object') { diff --git a/developer/src/kmc-ldml/src/compiler/compiler.ts b/developer/src/kmc-ldml/src/compiler/compiler.ts index 1163bb77d8..4564230e04 100644 --- a/developer/src/kmc-ldml/src/compiler/compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/compiler.ts @@ -8,7 +8,9 @@ import { CompilerCallbacks, KeymanCompiler, KeymanCompilerResult, KeymanCompilerArtifacts, defaultCompilerOptions, LDMLKeyboardXMLSourceFileReader, LDMLKeyboard, LDMLKeyboardTestDataXMLSourceFile, KMXBuilder, - KeymanCompilerArtifactOptional + KeymanCompilerArtifactOptional, + ResolvingCompilerCallbacks, + KeymanXMLReader, } from "@keymanapp/developer-utils"; import { LdmlCompilerOptions } from './ldml-compiler-options.js'; import { LdmlCompilerMessages } from './ldml-compiler-messages.js'; @@ -94,6 +96,7 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { // uset parser private usetparser?: LdmlKeyboardTypes.UnicodeSetParser = undefined; + private reader?: LDMLKeyboardXMLSourceFileReader; /** * Initialize the compiler, including loading the WASM host for uset parsing. @@ -105,7 +108,9 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { */ async init(callbacks: CompilerCallbacks, options: LdmlCompilerOptions): Promise { this.options = { ...options }; - this.callbacks = callbacks; + this.reader = new LDMLKeyboardXMLSourceFileReader(this.options.readerOptions, callbacks); + // wrap the callbacks so that the eventresolver is called + this.callbacks = new ResolvingCompilerCallbacks(this.reader.eventResolver, this.options, callbacks); return true; } @@ -238,7 +243,7 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { * @returns the source file, or null if invalid */ public load(filename: string): LDMLKeyboardXMLSourceFile | null { - const reader = new LDMLKeyboardXMLSourceFileReader(this.options.readerOptions, this.callbacks); + const reader = this.reader; // load the file from disk into a string const data = reader.readFile(filename); if (!data) { @@ -261,6 +266,9 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { return null; } + // record the default filename - for error reporting. + KeymanXMLReader.setDefaultFilename(source, filename); + return source; } @@ -272,7 +280,7 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { * @returns the source file, or null if invalid */ public loadTestData(filename: string): LDMLKeyboardTestDataXMLSourceFile | null { - const reader = new LDMLKeyboardXMLSourceFileReader(this.options.readerOptions, this.callbacks); + const reader = this.reader; const data = this.callbacks.loadFile(filename); if (!data) { this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidFile({ errorText: 'Unable to read XML file' })); diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 6deeedfceb..56111be865 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -1,5 +1,5 @@ import { util } from "@keymanapp/common-types"; -import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_FILENAME_SYMBOL, CompilerEvent, KeymanXMLReader, LDMLKeyboardXMLSourceFileReader } from '@keymanapp/developer-utils'; +import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_FILENAME_SYMBOL, CompilerEvent, KeymanXMLReader } from '@keymanapp/developer-utils'; // const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.LdmlKeyboardCompiler; @@ -13,7 +13,6 @@ const SevErrorTransform = SevError | 0xF00; const mx = (x: any, code: number, message: string, detail?: string): CompilerEvent => { let evt = m(code, message, detail); // raw message evt = LdmlCompilerMessages.offset(evt, x); // with offset - evt = LDMLKeyboardXMLSourceFileReader.eventResolver.resolve(evt); // offset expanded to line number return evt; }; diff --git a/developer/src/kmc/src/commands/buildClasses/BuildProject.ts b/developer/src/kmc/src/commands/buildClasses/BuildProject.ts index 82e1347aa5..434325e5cf 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildProject.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildProject.ts @@ -132,4 +132,4 @@ class ProjectBuilder { return result; } -} \ No newline at end of file +} From 307288dc3b043a78c5859e50b80259b5f6a4262c Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 8 May 2025 17:43:19 -0500 Subject: [PATCH 31/46] chore(developer): deps: fast-xml-parser@5.2.2 My line number changes were merged into the main branch Fixes: #10622 --- developer/src/common/web/utils/package.json | 2 +- package-lock.json | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/developer/src/common/web/utils/package.json b/developer/src/common/web/utils/package.json index d0fff064d7..20f71e6e9a 100644 --- a/developer/src/common/web/utils/package.json +++ b/developer/src/common/web/utils/package.json @@ -11,7 +11,7 @@ "dependencies": { "@keymanapp/common-types": "*", "eventemitter3": "^5.0.0", - "fast-xml-parser": "github:NaturalIntelligence/fast-xml-parser#metadata", + "fast-xml-parser": "^5.2.2", "path-browserify": "^1.0.1", "restructure": "^3.0.1", "sax": ">=0.6.0", diff --git a/package-lock.json b/package-lock.json index 5f407f851f..4c811b578e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -341,7 +341,7 @@ "dependencies": { "@keymanapp/common-types": "*", "eventemitter3": "^5.0.0", - "fast-xml-parser": "github:NaturalIntelligence/fast-xml-parser#metadata", + "fast-xml-parser": "^5.2.2", "path-browserify": "^1.0.1", "restructure": "^3.0.1", "sax": ">=0.6.0", @@ -9124,16 +9124,18 @@ "dev": true }, "node_modules/fast-xml-parser": { - "version": "5.1.0", - "resolved": "git+ssh://git@github.com/NaturalIntelligence/fast-xml-parser.git#ab8d40a3077f109f34d02d05653146b8621bd255", + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.2.2.tgz", + "integrity": "sha512-ZaCmslH75Jkfowo/x44Uq8KT5SutC5BFxHmY61nmTXPccw11PVuIXKUqC2hembMkJ3nPwTkQESXiUlsKutCbMg==", "funding": [ { "type": "github", "url": "https://github.com/sponsors/NaturalIntelligence" } ], + "license": "MIT", "dependencies": { - "strnum": "^2.0.5" + "strnum": "^2.1.0" }, "bin": { "fxparser": "src/cli/cli.js" @@ -13991,9 +13993,9 @@ "link": true }, "node_modules/strnum": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.0.5.tgz", - "integrity": "sha512-YAT3K/sgpCUxhxNMrrdhtod3jckkpYwH6JAuwmUdXZsmzH1wUyzTMrrK2wYCEEqlKwrWDd35NeuUkbBy/1iK+Q==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.1.0.tgz", + "integrity": "sha512-w0S//9BqZZGw0L0Y8uLSelFGnDJgTyyNQLmSlPnVz43zPAiqu3w4t8J8sDqqANOGeZIZ/9jWuPguYcEnsoHv4A==", "funding": [ { "type": "github", From 428178ce8c8bcda8c0c0daf4a23921160b23869d Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Fri, 9 May 2025 09:31:17 +0700 Subject: [PATCH 32/46] maint(android): Update Java and Gradle for Sample projects --- android/Samples/KMSample1/app/build.gradle | 2 +- android/Samples/KMSample1/build.gradle | 2 +- android/Samples/KMSample1/gradle.properties | 4 ++++ .../KMSample1/gradle/wrapper/gradle-wrapper.properties | 2 +- android/Samples/KMSample2/app/build.gradle | 2 +- android/Samples/KMSample2/build.gradle | 2 +- android/Samples/KMSample2/gradle.properties | 4 ++++ .../KMSample2/gradle/wrapper/gradle-wrapper.properties | 2 +- 8 files changed, 14 insertions(+), 6 deletions(-) diff --git a/android/Samples/KMSample1/app/build.gradle b/android/Samples/KMSample1/app/build.gradle index 51490c5ab2..cc92dee298 100644 --- a/android/Samples/KMSample1/app/build.gradle +++ b/android/Samples/KMSample1/app/build.gradle @@ -36,7 +36,7 @@ repositories { dependencies { implementation fileTree(dir: 'libs', include: ['*.jar']) - implementation 'androidx.appcompat:appcompat:1.6.1' + implementation 'androidx.appcompat:appcompat:1.7.0' implementation 'com.google.android.material:material:1.12.0' api(name: 'keyman-engine', ext: 'aar') implementation 'androidx.preference:preference:1.2.1' diff --git a/android/Samples/KMSample1/build.gradle b/android/Samples/KMSample1/build.gradle index 15824022e2..18d4c46cd5 100644 --- a/android/Samples/KMSample1/build.gradle +++ b/android/Samples/KMSample1/build.gradle @@ -7,7 +7,7 @@ buildscript { mavenCentral() } dependencies { - classpath 'com.android.tools.build:gradle:7.4.2' + classpath 'com.android.tools.build:gradle:8.9.2' // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files diff --git a/android/Samples/KMSample1/gradle.properties b/android/Samples/KMSample1/gradle.properties index 80f4cae5a8..f486cc756a 100644 --- a/android/Samples/KMSample1/gradle.properties +++ b/android/Samples/KMSample1/gradle.properties @@ -18,4 +18,8 @@ # org.gradle.parallel=true android.enableJetifier=true android.useAndroidX=true + +# AGP 8.0 generates R classes for resources defined in the current module only. +android.nonTransitiveRClass=false + org.gradle.jvmargs=-Xmx1024m diff --git a/android/Samples/KMSample1/gradle/wrapper/gradle-wrapper.properties b/android/Samples/KMSample1/gradle/wrapper/gradle-wrapper.properties index fb79de47c0..4fba274552 100644 --- a/android/Samples/KMSample1/gradle/wrapper/gradle-wrapper.properties +++ b/android/Samples/KMSample1/gradle/wrapper/gradle-wrapper.properties @@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.4-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-all.zip diff --git a/android/Samples/KMSample2/app/build.gradle b/android/Samples/KMSample2/app/build.gradle index 71ab1dab03..cd2f4d6225 100644 --- a/android/Samples/KMSample2/app/build.gradle +++ b/android/Samples/KMSample2/app/build.gradle @@ -35,7 +35,7 @@ repositories { dependencies { implementation fileTree(dir: 'libs', include: ['*.jar']) - implementation 'androidx.appcompat:appcompat:1.6.1' + implementation 'androidx.appcompat:appcompat:1.7.0' implementation 'com.google.android.material:material:1.12.0' api (name:'keyman-engine', ext:'aar') implementation 'androidx.preference:preference:1.2.1' diff --git a/android/Samples/KMSample2/build.gradle b/android/Samples/KMSample2/build.gradle index 15824022e2..18d4c46cd5 100644 --- a/android/Samples/KMSample2/build.gradle +++ b/android/Samples/KMSample2/build.gradle @@ -7,7 +7,7 @@ buildscript { mavenCentral() } dependencies { - classpath 'com.android.tools.build:gradle:7.4.2' + classpath 'com.android.tools.build:gradle:8.9.2' // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files diff --git a/android/Samples/KMSample2/gradle.properties b/android/Samples/KMSample2/gradle.properties index 80f4cae5a8..f486cc756a 100644 --- a/android/Samples/KMSample2/gradle.properties +++ b/android/Samples/KMSample2/gradle.properties @@ -18,4 +18,8 @@ # org.gradle.parallel=true android.enableJetifier=true android.useAndroidX=true + +# AGP 8.0 generates R classes for resources defined in the current module only. +android.nonTransitiveRClass=false + org.gradle.jvmargs=-Xmx1024m diff --git a/android/Samples/KMSample2/gradle/wrapper/gradle-wrapper.properties b/android/Samples/KMSample2/gradle/wrapper/gradle-wrapper.properties index 4b0560a7a2..0c0a7aec7d 100644 --- a/android/Samples/KMSample2/gradle/wrapper/gradle-wrapper.properties +++ b/android/Samples/KMSample2/gradle/wrapper/gradle-wrapper.properties @@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.4-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-all.zip From 3990ab45db184b6dbb3191a01cfb9400c504e31b Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Fri, 9 May 2025 09:33:00 +0700 Subject: [PATCH 33/46] maint(android): Update Java and Gradle for KeyboardHarness --- android/Tests/KeyboardHarness/app/build.gradle | 6 +++++- android/Tests/KeyboardHarness/build.gradle | 2 +- android/Tests/KeyboardHarness/build.sh | 2 +- android/Tests/KeyboardHarness/gradle.properties | 3 +++ .../gradle/wrapper/gradle-wrapper.properties | 2 +- .../Tests/keycode/gradle/wrapper/gradle-wrapper.properties | 2 +- 6 files changed, 12 insertions(+), 5 deletions(-) diff --git a/android/Tests/KeyboardHarness/app/build.gradle b/android/Tests/KeyboardHarness/app/build.gradle index 5462622cfa..cd172dd14e 100644 --- a/android/Tests/KeyboardHarness/app/build.gradle +++ b/android/Tests/KeyboardHarness/app/build.gradle @@ -14,6 +14,10 @@ android { noCompress "kmp" } + buildFeatures { + buildConfig = true + } + defaultConfig { applicationId "com.keyman.android.tests.keyboardHarness" minSdkVersion 21 @@ -42,7 +46,7 @@ repositories { dependencies { implementation fileTree(dir: 'libs', include: ['*.jar']) - implementation 'androidx.appcompat:appcompat:1.6.1' + implementation 'androidx.appcompat:appcompat:1.7.0' implementation 'com.google.android.material:material:1.12.0' api (name:'keyman-engine', ext:'aar') implementation 'androidx.preference:preference:1.2.1' diff --git a/android/Tests/KeyboardHarness/build.gradle b/android/Tests/KeyboardHarness/build.gradle index ce0b936509..8fdf2aa422 100644 --- a/android/Tests/KeyboardHarness/build.gradle +++ b/android/Tests/KeyboardHarness/build.gradle @@ -6,7 +6,7 @@ buildscript { mavenCentral() } dependencies { - classpath 'com.android.tools.build:gradle:7.4.0' + classpath 'com.android.tools.build:gradle:8.9.2' // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files diff --git a/android/Tests/KeyboardHarness/build.sh b/android/Tests/KeyboardHarness/build.sh index 65671872f8..c23f8ca0f5 100755 --- a/android/Tests/KeyboardHarness/build.sh +++ b/android/Tests/KeyboardHarness/build.sh @@ -33,7 +33,7 @@ ARTIFACT="app-release-unsigned.apk" if builder_is_debug_build; then builder_heading "### Debug config ####" CONFIG="debug" - BUILD_FLAGS="assembleDebug -x lint -x test" + BUILD_FLAGS="assembleDebug -x lintDebug -x testDebug" TEST_FLAGS="-x assembleDebug lintDebug testDebug" ARTIFACT="app-$CONFIG.apk" fi diff --git a/android/Tests/KeyboardHarness/gradle.properties b/android/Tests/KeyboardHarness/gradle.properties index 80f4cae5a8..8bb139036f 100644 --- a/android/Tests/KeyboardHarness/gradle.properties +++ b/android/Tests/KeyboardHarness/gradle.properties @@ -19,3 +19,6 @@ android.enableJetifier=true android.useAndroidX=true org.gradle.jvmargs=-Xmx1024m + +# AGP 8.0 generates R classes for resources defined in the current module only. +android.nonTransitiveRClass=false diff --git a/android/Tests/KeyboardHarness/gradle/wrapper/gradle-wrapper.properties b/android/Tests/KeyboardHarness/gradle/wrapper/gradle-wrapper.properties index da62d22563..3b14c8260c 100644 --- a/android/Tests/KeyboardHarness/gradle/wrapper/gradle-wrapper.properties +++ b/android/Tests/KeyboardHarness/gradle/wrapper/gradle-wrapper.properties @@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.4-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-all.zip diff --git a/android/Tests/keycode/gradle/wrapper/gradle-wrapper.properties b/android/Tests/keycode/gradle/wrapper/gradle-wrapper.properties index 763217c50f..8357d84852 100644 --- a/android/Tests/keycode/gradle/wrapper/gradle-wrapper.properties +++ b/android/Tests/keycode/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.4-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From 5d5609c343c5cee508f4575344163cd13d320e7a Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Fri, 9 May 2025 09:43:22 +0700 Subject: [PATCH 34/46] maint(android): Update Java and Gradle for FirstVoices --- oem/firstvoices/android/app/build.gradle | 14 ++++++-------- oem/firstvoices/android/build.gradle | 4 ++-- oem/firstvoices/android/build.sh | 2 +- oem/firstvoices/android/gradle.properties | 4 ++++ .../gradle/wrapper/gradle-wrapper.properties | 2 +- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/oem/firstvoices/android/app/build.gradle b/oem/firstvoices/android/app/build.gradle index 95bbcf2fa2..334ae2892d 100644 --- a/oem/firstvoices/android/app/build.gradle +++ b/oem/firstvoices/android/app/build.gradle @@ -11,6 +11,10 @@ android { compileSdk 34 namespace="com.firstvoices.keyboards" + buildFeatures { + buildConfig = true + } + defaultConfig { applicationId "com.firstvoices.keyboards" minSdkVersion 21 @@ -66,12 +70,6 @@ android { } } -java { - toolchain { - languageVersion.set(JavaLanguageVersion.of(11)) - } -} - // how to configure the sentry android gradle plugin task publishSentry { doLast { @@ -124,10 +122,10 @@ repositories { dependencies { implementation fileTree(dir: 'libs', include: ['*.jar']) - implementation 'androidx.appcompat:appcompat:1.6.1' + implementation 'androidx.appcompat:appcompat:1.7.0' implementation 'com.google.android.material:material:1.12.0' api(name: 'keyman-engine', ext: 'aar') - implementation 'io.sentry:sentry-android:6.9.2' + implementation 'io.sentry:sentry-android:7.8.0' implementation 'androidx.preference:preference:1.2.1' } diff --git a/oem/firstvoices/android/build.gradle b/oem/firstvoices/android/build.gradle index 4e544b7663..b4bbee2968 100644 --- a/oem/firstvoices/android/build.gradle +++ b/oem/firstvoices/android/build.gradle @@ -6,8 +6,8 @@ buildscript { } dependencies { - classpath 'com.android.tools.build:gradle:7.4.2' - classpath 'io.sentry:sentry-android-gradle-plugin:2.1.2' + classpath 'com.android.tools.build:gradle:8.9.2' + classpath 'io.sentry:sentry-android-gradle-plugin:4.6.0' // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files } diff --git a/oem/firstvoices/android/build.sh b/oem/firstvoices/android/build.sh index f7cb441f00..919c81572c 100755 --- a/oem/firstvoices/android/build.sh +++ b/oem/firstvoices/android/build.sh @@ -34,7 +34,7 @@ builder_parse "$@" if builder_is_debug_build; then builder_heading "### Debug config ####" CONFIG="debug" - BUILD_FLAGS="assembleDebug -x lint -x test" + BUILD_FLAGS="assembleDebug -x lintDebug -x testDebug" TEST_FLAGS="-x assembleDebug lintDebug testDebug" fi diff --git a/oem/firstvoices/android/gradle.properties b/oem/firstvoices/android/gradle.properties index c8e906dff4..6831edcea1 100644 --- a/oem/firstvoices/android/gradle.properties +++ b/oem/firstvoices/android/gradle.properties @@ -19,4 +19,8 @@ android.enableJetifier=true android.useAndroidX=true + +# AGP 8.0 generates R classes for resources defined in the current module only. +android.nonTransitiveRClass=false + org.gradle.jvmargs=-Xmx1024m diff --git a/oem/firstvoices/android/gradle/wrapper/gradle-wrapper.properties b/oem/firstvoices/android/gradle/wrapper/gradle-wrapper.properties index 3994438e22..d6e308a637 100644 --- a/oem/firstvoices/android/gradle/wrapper/gradle-wrapper.properties +++ b/oem/firstvoices/android/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.4-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From e061d3e755b59a6681f61727aa520699f63231f9 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Fri, 9 May 2025 09:43:54 +0700 Subject: [PATCH 35/46] docs(android): Update minimum versions docs --- android/README.md | 6 +++++- docs/minimum-versions.md | 4 ++-- resources/build/minimum-versions.inc.sh | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/android/README.md b/android/README.md index 6abb94161d..db1eb592b0 100644 --- a/android/README.md +++ b/android/README.md @@ -141,6 +141,10 @@ android { aaptOptions { noCompress "kmp" } + + buildFeatures { + buildConfig = true + } ``` 5. After the `android {}` object, include the following: ````gradle @@ -154,7 +158,7 @@ repositories { dependencies { implementation fileTree(dir: 'libs', include: ['*.jar']) - implementation 'androidx.appcompat:appcompat:1.6.1' + implementation 'androidx.appcompat:appcompat:1.7.0' implementation 'com.google.android.material:material:1.12.0' api (name:'keyman-engine', ext:'aar') implementation 'androidx.preference:preference:1.2.1' diff --git a/docs/minimum-versions.md b/docs/minimum-versions.md index 0bd31eebac..b01ca18b51 100644 --- a/docs/minimum-versions.md +++ b/docs/minimum-versions.md @@ -67,10 +67,10 @@ https://help.keyman.com/developer/engine/android/latest-version/ | KEYMAN_MIN_VERSION_NPM | 10.5.1 | | KEYMAN_MIN_VERSION_VISUAL_STUDIO | 2019 | | KEYMAN_VERSION_CLDR | 46 | -| KEYMAN_VERSION_GRADLE | 7.6.4 | +| KEYMAN_VERSION_GRADLE | 8.12 | | KEYMAN_VERSION_ICU | 73.1 | | KEYMAN_VERSION_ISO639_3 | 2024-05-22 | -| KEYMAN_VERSION_JAVA | 11 | +| KEYMAN_VERSION_JAVA | 21 | | KEYMAN_VERSION_LANGTAGS | 2025-02-18 | | KEYMAN_VERSION_LANGUAGE_SUBTAG_REGISTRY | 2025-03-10 | | KEYMAN_VERSION_UNICODE | 16.0.0 | diff --git a/resources/build/minimum-versions.inc.sh b/resources/build/minimum-versions.inc.sh index eadd2442cf..eed047e39d 100644 --- a/resources/build/minimum-versions.inc.sh +++ b/resources/build/minimum-versions.inc.sh @@ -31,13 +31,13 @@ KEYMAN_MIN_VERSION_EMSCRIPTEN=3.1.58 # Use KEYMAN_USE_EMSDK to automati KEYMAN_MIN_VERSION_VISUAL_STUDIO=2019 KEYMAN_MIN_VERSION_MESON=1.0.0 -KEYMAN_VERSION_GRADLE=7.6.4 # See /android/KMEA/gradle/wrapper/gradle-wrapper.properties +KEYMAN_VERSION_GRADLE=8.12 # See /android/KMEA/gradle/wrapper/gradle-wrapper.properties # Dependencies with fixed versions KEYMAN_VERSION_ICU=73.1 # See /core/subprojects/icu-minimal.wrap # Language and runtime versions -KEYMAN_VERSION_JAVA=11 # We're using Java/OpenJDK 11 +KEYMAN_VERSION_JAVA=21 # We're using Java/OpenJDK 21 KEYMAN_MIN_VERSION_CPP=17 # C++17 KEYMAN_MIN_VERSION_ANDROID_SDK=21 From 47407a3164556b790e6ba247cf2547f1124414ef Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Fri, 9 May 2025 18:05:13 +0200 Subject: [PATCH 36/46] maint(common): remove obsolete code from docker-images `run.sh` PR #13861 missed the fact that docker-images' `run.sh` script also contains code that got re-added during a faulty merge. This change cleans this up. --- resources/docker-images/run.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/resources/docker-images/run.sh b/resources/docker-images/run.sh index c71047e22f..05124768e9 100755 --- a/resources/docker-images/run.sh +++ b/resources/docker-images/run.sh @@ -67,14 +67,6 @@ run_web() { "${builder_extra_params[@]}" } -if [[ -z "${DISTRO_VERSION:-}" ]]; then - image_version=default - build_dir=default -else - image_version="${DISTRO:-}-${DISTRO_VERSION}-java${KEYMAN_VERSION_JAVA}-node$(_print_expected_node_version)-emsdk${KEYMAN_MIN_VERSION_EMSCRIPTEN}" - build_dir="${DISTRO:-}-${DISTRO_VERSION}" -fi - mkdir -p "${KEYMAN_ROOT}/core/build/docker-core/${build_dir}" builder_run_action android run_android From 8e2eb498e3b0b9a7030fb7aa1cfcd23550affd20 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 9 May 2025 11:41:23 -0500 Subject: [PATCH 37/46] feat(developer): update for line numbers - reduce whitespace churn Fixes: #10622 --- developer/src/common/web/test-helpers/TestCompilerCallbacks.ts | 2 +- developer/src/common/web/utils/src/compiler-callbacks.ts | 2 +- developer/src/common/web/utils/src/index.ts | 2 +- developer/src/common/web/utils/src/line-utils.ts | 2 +- .../utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts | 2 +- developer/src/kmc/src/commands/buildClasses/BuildProject.ts | 2 +- developer/src/kmc/src/util/NodeCompilerCallbacks.ts | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts b/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts index 893e3363ed..04e5734f18 100644 --- a/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts +++ b/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts @@ -2,7 +2,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { CompilerEvent, CompilerCallbacks, CompilerPathCallbacks, CompilerFileSystemCallbacks, CompilerError, CompilerNetAsyncCallbacks, DefaultCompilerFileSystemAsyncCallbacks, - CompilerFileSystemAsyncCallbacks, + CompilerFileSystemAsyncCallbacks, CompilerErrorSeverity} from '@keymanapp/developer-utils'; import { fileURLToPath } from 'url'; diff --git a/developer/src/common/web/utils/src/compiler-callbacks.ts b/developer/src/common/web/utils/src/compiler-callbacks.ts index 78187b9a58..dda2c1485b 100644 --- a/developer/src/common/web/utils/src/compiler-callbacks.ts +++ b/developer/src/common/web/utils/src/compiler-callbacks.ts @@ -49,7 +49,7 @@ export interface CompilerCallbacks { get net(): CompilerNetAsyncCallbacks; /** - * Resolves a file path relative to the baseFilename. + * Resolves a file path relative to the baseFilename * @param baseFilename * @param filename */ diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index 2224463408..fd442356a7 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -77,4 +77,4 @@ export { getFontFamily, getFontFamilySync } from './font-family.js'; export * as ValidIds from './valid-ids.js'; -export * as ProjectLoader from './project-loader.js'; +export * as ProjectLoader from './project-loader.js'; \ No newline at end of file diff --git a/developer/src/common/web/utils/src/line-utils.ts b/developer/src/common/web/utils/src/line-utils.ts index 95f5744c55..2d2282db41 100644 --- a/developer/src/common/web/utils/src/line-utils.ts +++ b/developer/src/common/web/utils/src/line-utils.ts @@ -38,7 +38,7 @@ export class LineFinder { return LineFinder.offsetToLineColumn(offset, this.getLineList()); } - /** + /** * preprocess text to turn it into arrays of line lengths. * This is effectively a 1-based line length, since line 0 has length of * 0. diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index f581d661cb..8569e617b9 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -31,7 +31,7 @@ export class LDMLKeyboardXMLSourceFileReader { /** for resolving messages involving line numbers */ eventResolver: LineFinderEventResolver = new LineFinderEventResolver(); - constructor(private options: LDMLKeyboardXMLSourceFileReaderOptions, private callbacks : CompilerCallbacks) { + constructor(private options: LDMLKeyboardXMLSourceFileReaderOptions, private callbacks : CompilerCallbacks) { } static get defaultImportsURL(): [string,string] { diff --git a/developer/src/kmc/src/commands/buildClasses/BuildProject.ts b/developer/src/kmc/src/commands/buildClasses/BuildProject.ts index 434325e5cf..82e1347aa5 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildProject.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildProject.ts @@ -132,4 +132,4 @@ class ProjectBuilder { return result; } -} +} \ No newline at end of file diff --git a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts index 2ec3854c3e..a92314aa5f 100644 --- a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts +++ b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { platform } from 'os'; -import { CompilerCallbacks, CompilerEvent, +import { CompilerCallbacks, CompilerEvent, CompilerPathCallbacks, CompilerFileSystemCallbacks, compilerLogLevelToSeverity, CompilerErrorSeverity, CompilerError, From 61943923100c67beef99cae33099b429b2860732 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 9 May 2025 13:07:51 -0500 Subject: [PATCH 38/46] feat(developer): update for line numbers - reduce whitespace churn - other review comments Fixes: #10622 Co-authored-by: Marc Durdin --- developer/src/common/web/utils/src/index.ts | 4 +- .../src/common/web/utils/src/line-utils.ts | 124 ++++++++++-------- .../src/common/web/utils/src/symbol-utils.ts | 38 +++--- .../ldml-keyboard/ldml-keyboard-xml-reader.ts | 11 +- .../common/web/utils/test/line-utils.tests.ts | 22 ++-- .../src/kmc-ldml/src/compiler/compiler.ts | 2 +- developer/src/kmc-ldml/src/compiler/keys.ts | 4 +- .../src/compiler/ldml-compiler-messages.ts | 32 ++++- developer/src/kmc-ldml/test/keys.tests.ts | 4 +- 9 files changed, 140 insertions(+), 101 deletions(-) diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index fd442356a7..937cd98642 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -69,7 +69,7 @@ export { DeveloperUtilsMessages } from './developer-utils-messages.js'; export * as SourceFilenamePatterns from './source-filename-patterns.js'; export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, KeymanXMLMetadata, XML_FILENAME_SYMBOL } from './xml-utils.js'; export { SymbolUtils } from './symbol-utils.js'; -export { LineFinder, LineColumn, LineFinderEventResolver } from './line-utils.js'; +export * as LineUtils from './line-utils.js'; export * as GitHubUrls from './github-urls.js'; export * as CloudUrls from './cloud-urls.js'; @@ -77,4 +77,4 @@ export { getFontFamily, getFontFamilySync } from './font-family.js'; export * as ValidIds from './valid-ids.js'; -export * as ProjectLoader from './project-loader.js'; \ No newline at end of file +export * as ProjectLoader from './project-loader.js'; diff --git a/developer/src/common/web/utils/src/line-utils.ts b/developer/src/common/web/utils/src/line-utils.ts index 2d2282db41..2e1cced292 100644 --- a/developer/src/common/web/utils/src/line-utils.ts +++ b/developer/src/common/web/utils/src/line-utils.ts @@ -10,60 +10,66 @@ import { CompilerEvent, EventResolver } from "./index.js"; /** line number with an optional column number */ export interface LineColumn { - line: number; - column?: number; + line: number; + column?: number; } export type LineList = number[]; /** a LineFinder is able to convert from an offset to a line/column */ export class LineFinder { - private list: LineList; - constructor(public text: string) { - } + private list: LineList; + constructor(public text: string) { + } - public getLineList(): LineList { - if (!this.list) { - this.list = LineFinder.textToLines(this.text); - } - return this.list; + public getLineList(): LineList { + if (!this.list) { + this.list = LineFinder.textToLines(this.text); } + return this.list; + } - /** - * Convert an offset into line:column - * @param offset input offset into the text - * @returns line:column information - */ - public findOffset(offset: number): LineColumn { - return LineFinder.offsetToLineColumn(offset, this.getLineList()); - } + /** + * Convert an offset into line:column + * @param offset input offset into the text + * @returns line:column information + */ + public findOffset(offset: number): LineColumn { + return LineFinder.offsetToLineColumn(offset, this.getLineList()); + } - /** - * preprocess text to turn it into arrays of line lengths. - * This is effectively a 1-based line length, since line 0 has length of - * 0. - */ - public static textToLines(text: string): LineList { - return [ - 0, // "line 0" is empty - ...text.replaceAll("\r\n", "\n").split("\n") - .map(l => l.length + 1) // line length (counting the trailing newline) - ]; - } + /** + * Preprocess text to turn it into arrays of line lengths. + * This is effectively a 1-based line length, since line 0 has length of + * 0. + * Note that the fast-xml-parser converts \r\n to \n before processing, + * and this function does as well. So, any offsets passed in will need to + * have that taken into consideration. + */ + public static textToLines(text: string): LineList { + return [ + 0, // "line 0" is empty + ...text.replaceAll("\r\n", "\n").split("\n") + .map(l => l.length + 1) // line length (counting the trailing newline) + ]; + } - /** - * convert a line number array to a line/column - */ - public static offsetToLineColumn(offset: number, list: LineList): LineColumn { - for (let line = 1; line < list.length; line++) { // 1-based (assume the first row is 0) - if (list[line] > offset) { - return { line, column: offset }; - } - offset = offset - (list[line]); // count newline at end - } - // default: line 0, error - return { line: 0 } + /** + * convert a line number array to a line/column. + * Note that the fast-xml-parser converts \r\n to \n before processing, + * and this function does as well. So, any offsets passed in will need to + * have that taken into consideration. + */ + public static offsetToLineColumn(offset: number, list: LineList): LineColumn { + for (let line = 1; line < list.length; line++) { // 1-based (assume the first row is 0) + if (list[line] > offset) { + return { line, column: offset }; + } + offset = offset - (list[line]); // count newline at end } + // default: line 0, error + return { line: 0 } + } } /** @@ -71,27 +77,33 @@ export class LineFinder { * keyed by the filename */ export interface FileConsumer { - /** - * @param filename name of the file - * @param contents string contents of the file - */ - addFile(filename: string, contents: string): void; + /** + * @param filename name of the file + * @param contents string contents of the file + */ + addFile(filename: string, contents: string): void; } /** Cache of LineFinder elements, organized by filename */ export class LineFinderCache implements FileConsumer { - contentsCache: Map = new Map(); + contentsCache: Map = new Map(); - /** add or update a source file */ - addFile(filename: string, contents: string): void { - this.contentsCache.set(filename, new LineFinder(contents)); - } + /** add or update a source file */ + addFile(filename: string, contents: string): void { + this.contentsCache.set(filename, new LineFinder(contents)); + } - getByFilename(filename: string) : LineFinder | undefined { - const lf = this.contentsCache.get(filename); - return lf; - } + getByFilename(filename: string): LineFinder | undefined { + const lf = this.contentsCache.get(filename); + return lf; + } } + +/** + * This EventResolver expands events that have a filename and offset set, + * by looking up the event.offset in the LineFinder list. + * The expanded event has the event.line and event.column fields set. + */ export class LineFinderEventResolver implements EventResolver, FileConsumer { private lfcache = new LineFinderCache(); diff --git a/developer/src/common/web/utils/src/symbol-utils.ts b/developer/src/common/web/utils/src/symbol-utils.ts index 971d02cc16..92cf07dd02 100644 --- a/developer/src/common/web/utils/src/symbol-utils.ts +++ b/developer/src/common/web/utils/src/symbol-utils.ts @@ -7,27 +7,27 @@ */ export class SymbolUtils { - /** - * Copy symbols from 'from' onto 'onto' - * @param onto object to copy onto - * @param from source for symbols - * @returns the onto object - */ - public static copySymbols(onto: T, from: any): T { - const o = onto as any; - for (const sym of Object.getOwnPropertySymbols(from)) { - o[sym] = from[sym]; - } - return onto; + /** + * Copy symbols shallowly from 'from' onto 'onto' + * @param onto object to copy onto + * @param from source for symbols + * @returns the onto object + */ + public static copySymbols(onto: T, from: any): T { + const o = onto as any; + for (const sym of Object.getOwnPropertySymbols(from)) { + o[sym] = from[sym]; } + return onto; + } - /** use Object.entries to remove all symbols */ - public static removeSymbols(from: T): T { - if (Array.isArray(from)) { - return from.map(o => SymbolUtils.removeSymbols(o)) as T; - } - if (typeof from !== "object") return from; - return Object.fromEntries(Object.entries(from).map(([k, v]) => ([k, SymbolUtils.removeSymbols(v)]))) as T; + /** use Object.entries to remove all symbols, recursively. */ + public static removeSymbols(from: T): T { + if (Array.isArray(from)) { + return from.map(o => SymbolUtils.removeSymbols(o)) as T; } + if (typeof from !== "object") return from; + return Object.fromEntries(Object.entries(from).map(([k, v]) => ([k, SymbolUtils.removeSymbols(v)]))) as T; + } } diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index 8569e617b9..6d06bd2546 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -9,9 +9,9 @@ import { CompilerCallbacks } from "../../compiler-callbacks.js"; import { LDMLKeyboardXMLSourceFile, LKImport, ImportStatus } from './ldml-keyboard-xml.js'; import { constants } from '@keymanapp/ldml-keyboard-constants'; import { LDMLKeyboardTestDataXMLSourceFile, LKTTest, LKTTests } from './ldml-keyboard-testdata-xml.js'; -import { KeymanXMLReader } from '@keymanapp/developer-utils'; +import { CompilerEvent, EventResolver, KeymanXMLReader } from '@keymanapp/developer-utils'; import boxXmlArray = util.boxXmlArray; -import { LineFinderEventResolver } from '../../index.js'; +import { LineFinderEventResolver } from '../../line-utils.js'; import { XML_FILENAME_SYMBOL } from '../../xml-utils.js'; interface NameAndProps { @@ -27,12 +27,15 @@ export class LDMLKeyboardXMLSourceFileReaderOptions { localImportsPaths: string[]; }; -export class LDMLKeyboardXMLSourceFileReader { +export class LDMLKeyboardXMLSourceFileReader implements EventResolver { /** for resolving messages involving line numbers */ - eventResolver: LineFinderEventResolver = new LineFinderEventResolver(); + private eventResolver: LineFinderEventResolver = new LineFinderEventResolver(); constructor(private options: LDMLKeyboardXMLSourceFileReaderOptions, private callbacks : CompilerCallbacks) { } + resolve(event: CompilerEvent): void { + this.eventResolver.resolve(event); + } static get defaultImportsURL(): [string,string] { return ['../import/', import.meta.url]; diff --git a/developer/src/common/web/utils/test/line-utils.tests.ts b/developer/src/common/web/utils/test/line-utils.tests.ts index 9c7b5da01c..ffbdba5b0e 100644 --- a/developer/src/common/web/utils/test/line-utils.tests.ts +++ b/developer/src/common/web/utils/test/line-utils.tests.ts @@ -13,16 +13,16 @@ import { LineFinder } from '../src/line-utils.js'; import { makePathToFixture } from './helpers/index.js'; describe(`LineFinder test`, () => { - const path = 'tran_fail-empty.xml'; + const path = 'tran_fail-empty.xml'; - it(`Should be able to process ${path}`, () => { - const xmlPath = makePathToFixture('xml', `${path}`); - const data = readFileSync(xmlPath, 'utf-8'); - assert.ok(data); - const lf = new LineFinder(data); - assert.ok(lf); - assert.deepEqual(lf.findOffset(40), {line: 3, column: 0}); - assert.deepEqual(lf.findOffset(136), {line: 4, column: 2}); - assert.deepEqual(lf.findOffset(186), {line: 8, column: 2}); - }); + it(`Should be able to process ${path}`, () => { + const xmlPath = makePathToFixture('xml', `${path}`); + const data = readFileSync(xmlPath, 'utf-8'); + assert.ok(data); + const lf = new LineFinder(data); + assert.ok(lf); + assert.deepEqual(lf.findOffset(40), { line: 3, column: 0 }); + assert.deepEqual(lf.findOffset(136), { line: 4, column: 2 }); + assert.deepEqual(lf.findOffset(186), { line: 8, column: 2 }); + }); }); diff --git a/developer/src/kmc-ldml/src/compiler/compiler.ts b/developer/src/kmc-ldml/src/compiler/compiler.ts index 4564230e04..b69e984728 100644 --- a/developer/src/kmc-ldml/src/compiler/compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/compiler.ts @@ -110,7 +110,7 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { this.options = { ...options }; this.reader = new LDMLKeyboardXMLSourceFileReader(this.options.readerOptions, callbacks); // wrap the callbacks so that the eventresolver is called - this.callbacks = new ResolvingCompilerCallbacks(this.reader.eventResolver, this.options, callbacks); + this.callbacks = new ResolvingCompilerCallbacks(this.reader, this.options, callbacks); return true; } diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index 6776057756..3b0d228e34 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -485,8 +485,8 @@ export class KeysCompiler extends SectionCompiler { } if (!keydef.output && !keydef.gap && !keydef.layerId) { this.callbacks.reportMessage( - LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch({ keyId: key }, - keydef + LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch(keydef, + { keyId: key }, ) ); valid = false; diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 56111be865..fba40669fb 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -9,13 +9,32 @@ const SevError = CompilerErrorSeverity.Error | CompilerErrorNamespace.LdmlKeyboa // sub-numberspace for transform errors const SevErrorTransform = SevError | 0xF00; -// like m() but takes an XML object for line numbers -const mx = (x: any, code: number, message: string, detail?: string): CompilerEvent => { +/** + * Any object with metadata, for line number errs. + * Could be for example an LKKeys or KMXPlus.KeysKeys object. + * Defined as 'any' here to reduce noise on the client side. + * @see {@link KeymanXMLReader.getMetaData()} + */ +type ObjectWithMetadata = any; + +/** + * Convenience function for constructing CompilerEvents with line numbers + * @param x Object to be used as a source for line number information + * @param code Unique numeric value of the event + * @param message A short description of the error presented to the user + * @param detail Detailed Markdown-formatted description of the error + * including references to documentation, remediation options. + * @see CompilerMessageSpec + * @returns + */ +function CompilerMessageObjectSpec(x: ObjectWithMetadata, code: number, message: string, detail?: string): CompilerEvent { let evt = m(code, message, detail); // raw message evt = LdmlCompilerMessages.offset(evt, x); // with offset return evt; }; +const mx = CompilerMessageObjectSpec; + /** * @internal */ @@ -28,7 +47,8 @@ export class LdmlCompilerMessages { static ERROR_HardwareLayerHasTooManyRows = SevError | 0x0003; static Error_HardwareLayerHasTooManyRows = (x: any) => mx(x, - this.ERROR_HardwareLayerHasTooManyRows, `'hardware' layer has too many rows`); + this.ERROR_HardwareLayerHasTooManyRows, `'hardware' layer has too many rows`, + ); static ERROR_RowOnHardwareLayerHasTooManyKeys = SevError | 0x0004; static Error_RowOnHardwareLayerHasTooManyKeys = (o:{row: number, hardware: string, modifiers: string}) => m(this.ERROR_RowOnHardwareLayerHasTooManyKeys, `Row #${def(o.row)} on 'hardware' ${def(o.hardware)} layer for modifier ${o.modifiers || 'none'} has too many keys`); @@ -105,8 +125,10 @@ export class LdmlCompilerMessages { m(this.ERROR_DisplayIsRepeated, `display ${LdmlCompilerMessages.outputOrKeyId(o)} has more than one display entry.`); static ERROR_KeyMissingToGapOrSwitch = SevError | 0x0011; - static Error_KeyMissingToGapOrSwitch = (o:{keyId: string}, x: any) => mx(x, - this.ERROR_KeyMissingToGapOrSwitch, `key id='${def(o.keyId)}' must have either output=, gap=, or layerId=.`); + static Error_KeyMissingToGapOrSwitch = (x: any, o:{keyId: string}) => mx(x, + this.ERROR_KeyMissingToGapOrSwitch, + `key id='${def(o.keyId)}' must have either output=, gap=, or layerId=.`, + ); static ERROR_ExcessHardware = SevError | 0x0012; static Error_ExcessHardware = (o:{formId: string}) => m(this.ERROR_ExcessHardware, diff --git a/developer/src/kmc-ldml/test/keys.tests.ts b/developer/src/kmc-ldml/test/keys.tests.ts index 73036be877..cd90def1cd 100644 --- a/developer/src/kmc-ldml/test/keys.tests.ts +++ b/developer/src/kmc-ldml/test/keys.tests.ts @@ -464,7 +464,9 @@ describe('keys.kmap', function () { const keys = await loadSectionFixture(KeysCompiler, 'sections/keys/invalid-key-missing-attrs.xml', compilerTestCallbacks, keysDependencies) as Keys; assert.isNull(keys); assert.equal(compilerTestCallbacks.messages.length, 1); - assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch({keyId: 'Q'}, withColumn(188))); + assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch(withColumn(188), + {keyId: 'Q'} + )); }); it('should accept layouts with gap/switch keys', async function() { const keys = await loadSectionFixture(KeysCompiler, 'sections/keys/gap-switch.xml', compilerTestCallbacks, keysDependencies) as Keys; From 8ec8fb189096559f21039ea0a6ba520e6aa1a865 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 9 May 2025 13:09:53 -0500 Subject: [PATCH 39/46] feat(developer): update for line numbers - annotate why we're calling SymbolUtils.removeSymbols() Fixes: #10622 --- .../src/common/web/utils/test/kpj/kpj-file-writer.tests.ts | 2 ++ .../src/common/web/utils/test/kps/kps-file-reader.tests.ts | 2 ++ developer/src/common/web/utils/test/kvks/kvks-file.tests.ts | 2 ++ 3 files changed, 6 insertions(+) diff --git a/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts b/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts index 425d76a6ef..d1a7cdddb1 100644 --- a/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts +++ b/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts @@ -22,6 +22,7 @@ describe('kpj-file-writer', function () { const writer = new KPJFileWriter(); const output = writer.write(project); + // Remove XML metadata symbols to reduce clutter for testing purposes const outputKpj = SymbolUtils.removeSymbols(reader.read(new TextEncoder().encode(output))); // The outputKpj may not contain all the fields from the inputKpj, only the @@ -61,6 +62,7 @@ describe('kpj-file-writer', function () { const writer = new KPJFileWriter(); const output = writer.write(project); + // Remove XML metadata symbols to reduce clutter for testing purposes const outputKpj = SymbolUtils.removeSymbols(reader.read(new TextEncoder().encode(output))); // The outputKpj may not contain all the fields from the inputKpj, only the diff --git a/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts b/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts index 78e69c6ef6..117a48a281 100644 --- a/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts +++ b/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts @@ -60,12 +60,14 @@ describe('kps-file-reader', function () { it('kps-file-reader should round-trip with kps-file-writer', function() { const input = fs.readFileSync(makePathToFixture('kps', 'khmer_angkor.kps')); const reader = new KpsFileReader(callbacks); + // Remove XML metadata symbols to reduce clutter for testing purposes const kps = SymbolUtils.removeSymbols(reader.read(input)); const writer = new KpsFileWriter(); const output = writer.write(kps); // Round Trip + // Remove XML metadata symbols to reduce clutter for testing purposes const kps2 = SymbolUtils.removeSymbols(reader.read(new TextEncoder().encode(output))); assert.deepEqual(kps2, kps); }); diff --git a/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts b/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts index 9f7cca504e..7e60119d28 100644 --- a/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts +++ b/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts @@ -64,6 +64,7 @@ describe('kvks-file-writer', function() { const input = fs.readFileSync(path); const reader = new KvksFileReader(); + // Remove XML metadata symbols to reduce clutter for testing purposes const kvksExpected = SymbolUtils.removeSymbols(reader.read(input)); const invalidVkeys: string[] = []; const vk = reader.transform(kvksExpected, invalidVkeys); @@ -75,6 +76,7 @@ describe('kvks-file-writer', function() { // We compare the (re)loaded data, because there may be // minor, irrelevant formatting differences in the emitted xml const kvks = reader.read(Buffer.from(output, 'utf8')); + // Remove XML metadata symbols to reduce clutter for testing purposes assert.deepEqual(SymbolUtils.removeSymbols(kvks), kvksExpected); }); }); From b23275d7e17d3b52f27de8e5aa17b9a8e034cec4 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 9 May 2025 13:32:15 -0500 Subject: [PATCH 40/46] feat(developer): fix imports in ldml-keyboard-xml-reader.ts Fixes: #10622 --- .../src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index 6d06bd2546..0a31bc051b 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -5,14 +5,14 @@ */ import { SchemaValidators, util } from '@keymanapp/common-types'; import { DeveloperUtilsMessages } from '../../developer-utils-messages.js'; -import { CompilerCallbacks } from "../../compiler-callbacks.js"; +import { CompilerCallbacks, EventResolver } from "../../compiler-callbacks.js"; +import { CompilerEvent } from "../../compiler-interfaces.js"; import { LDMLKeyboardXMLSourceFile, LKImport, ImportStatus } from './ldml-keyboard-xml.js'; import { constants } from '@keymanapp/ldml-keyboard-constants'; import { LDMLKeyboardTestDataXMLSourceFile, LKTTest, LKTTests } from './ldml-keyboard-testdata-xml.js'; -import { CompilerEvent, EventResolver, KeymanXMLReader } from '@keymanapp/developer-utils'; import boxXmlArray = util.boxXmlArray; import { LineFinderEventResolver } from '../../line-utils.js'; -import { XML_FILENAME_SYMBOL } from '../../xml-utils.js'; +import { XML_FILENAME_SYMBOL, KeymanXMLReader } from '../../xml-utils.js'; interface NameAndProps { '$'?: any; // content From 42d9c68e2b31daf02745937443df7a3cb8040e93 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 9 May 2025 16:43:59 -0500 Subject: [PATCH 41/46] feat(developer): fix Error() function - swap x, o because the 'api' call getMessageDetail() is going to call with {} Fixes: #10622 --- developer/src/kmc-ldml/src/compiler/keys.ts | 3 ++- .../kmc-ldml/src/compiler/ldml-compiler-messages.ts | 13 ++++++++----- developer/src/kmc-ldml/test/keys.tests.ts | 5 +++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index 3b0d228e34..7b79a3be83 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -485,8 +485,9 @@ export class KeysCompiler extends SectionCompiler { } if (!keydef.output && !keydef.gap && !keydef.layerId) { this.callbacks.reportMessage( - LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch(keydef, + LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch( { keyId: key }, + keydef, ) ); valid = false; diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index fba40669fb..28c5b52e1f 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -19,15 +19,15 @@ type ObjectWithMetadata = any; /** * Convenience function for constructing CompilerEvents with line numbers - * @param x Object to be used as a source for line number information * @param code Unique numeric value of the event * @param message A short description of the error presented to the user + * @param x Object to be used as a source for line number information * @param detail Detailed Markdown-formatted description of the error * including references to documentation, remediation options. * @see CompilerMessageSpec * @returns */ -function CompilerMessageObjectSpec(x: ObjectWithMetadata, code: number, message: string, detail?: string): CompilerEvent { +function CompilerMessageObjectSpec(code: number, message: string, x: ObjectWithMetadata, detail?: string): CompilerEvent { let evt = m(code, message, detail); // raw message evt = LdmlCompilerMessages.offset(evt, x); // with offset return evt; @@ -46,8 +46,10 @@ export class LdmlCompilerMessages { static Error_InvalidLocale = (o:{tag: string}) => m(this.ERROR_InvalidLocale, `Invalid BCP 47 locale form '${def(o.tag)}'`); static ERROR_HardwareLayerHasTooManyRows = SevError | 0x0003; - static Error_HardwareLayerHasTooManyRows = (x: any) => mx(x, - this.ERROR_HardwareLayerHasTooManyRows, `'hardware' layer has too many rows`, + static Error_HardwareLayerHasTooManyRows = (x: any) => mx( + this.ERROR_HardwareLayerHasTooManyRows, + `'hardware' layer has too many rows`, + x, ); static ERROR_RowOnHardwareLayerHasTooManyKeys = SevError | 0x0004; @@ -125,9 +127,10 @@ export class LdmlCompilerMessages { m(this.ERROR_DisplayIsRepeated, `display ${LdmlCompilerMessages.outputOrKeyId(o)} has more than one display entry.`); static ERROR_KeyMissingToGapOrSwitch = SevError | 0x0011; - static Error_KeyMissingToGapOrSwitch = (x: any, o:{keyId: string}) => mx(x, + static Error_KeyMissingToGapOrSwitch = (o:{keyId: string}, x: ObjectWithMetadata) => mx( this.ERROR_KeyMissingToGapOrSwitch, `key id='${def(o.keyId)}' must have either output=, gap=, or layerId=.`, + x, ); static ERROR_ExcessHardware = SevError | 0x0012; diff --git a/developer/src/kmc-ldml/test/keys.tests.ts b/developer/src/kmc-ldml/test/keys.tests.ts index cd90def1cd..3db4388a35 100644 --- a/developer/src/kmc-ldml/test/keys.tests.ts +++ b/developer/src/kmc-ldml/test/keys.tests.ts @@ -464,8 +464,9 @@ describe('keys.kmap', function () { const keys = await loadSectionFixture(KeysCompiler, 'sections/keys/invalid-key-missing-attrs.xml', compilerTestCallbacks, keysDependencies) as Keys; assert.isNull(keys); assert.equal(compilerTestCallbacks.messages.length, 1); - assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch(withColumn(188), - {keyId: 'Q'} + assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch( + {keyId: 'Q'}, + withColumn(188) )); }); it('should accept layouts with gap/switch keys', async function() { From c91e309c549602fc36f98640e721cc684d978264 Mon Sep 17 00:00:00 2001 From: Shawn Schantz Date: Mon, 12 May 2025 16:34:32 +0700 Subject: [PATCH 42/46] chore(mac): deleted another swath of unused constants --- .../KeymanEngine4Mac/KME/KMBinaryFileFormat.h | 64 ------------------- 1 file changed, 64 deletions(-) diff --git a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMBinaryFileFormat.h b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMBinaryFileFormat.h index 211a40d9d3..3951d5831d 100644 --- a/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMBinaryFileFormat.h +++ b/mac/KeymanEngine4Mac/KeymanEngine4Mac/KME/KMBinaryFileFormat.h @@ -122,70 +122,6 @@ struct COMP_STORE { #define TSS__MAX 43 -// ShiftFlags -#define LCTRLFLAG 0x0001 // Left Control flag -#define RCTRLFLAG 0x0002 // Right Control flag -#define LALTFLAG 0x0004 // Left Alt flag -#define RALTFLAG 0x0008 // Right Alt flag -#define K_SHIFTFLAG 0x0010 // Either shift flag -#define K_CTRLFLAG 0x0020 // Either ctrl flag -#define K_ALTFLAG 0x0040 // Either alt flag -//#define K_METAFLAG 0x0080 // Either Meta-key flag (tentative). Not usable in keyboard rules; -// Used internally (currently, only by KMW) to ensure Meta-key -// shortcuts safely bypass rules -// Meta key = Command key on macOS, Windows key on Windows -#define CAPITALFLAG 0x0100 // Caps lock on -#define NOTCAPITALFLAG 0x0200 // Caps lock NOT on -#define NUMLOCKFLAG 0x0400 // Num lock on -#define NOTNUMLOCKFLAG 0x0800 // Num lock NOT on -#define SCROLLFLAG 0x1000 // Scroll lock on -#define NOTSCROLLFLAG 0x2000 // Scroll lock NOT on -#define ISVIRTUALKEY 0x4000 // It is a Virtual Key Sequence -#define VIRTUALCHARKEY 0x8000 // Keyman 6.0: Virtual Key Cap Sequence -#define K_MODIFIERFLAG 0x007F -#define K_CAPITALMASK (CAPITALFLAG|NOTCAPITALFLAG) - -#define UC_SENTINEL 0xFFFF -#define UC_SENTINEL_EXTENDEDEND 0x10 - -#define CODE_ANY 0x01 -#define CODE_INDEX 0x02 -#define CODE_CONTEXT 0x03 -#define CODE_NUL 0x04 -#define CODE_USE 0x05 -#define CODE_RETURN 0x06 -#define CODE_BEEP 0x07 -#define CODE_DEADKEY 0x08 -// 0x09 = bkspace.-- we don't need to keep this separate though with UC_SENTINEL -#define CODE_EXTENDED 0x0A -//#define CODE_EXTENDEDEND 0x0B deprecated -#define CODE_SWITCH 0x0C -#define CODE_KEY 0x0D -#define CODE_CLEARCONTEXT 0x0E -#define CODE_CALL 0x0F -// UC_SENTINEL_EXTENDEDEND 0x10 -#define CODE_CONTEXTEX 0x11 -#define CODE_NOTANY 0x12 -#define CODE_KEYMAN70_LASTCODE 0x12 - -#define CODE_SETOPT 0x13 -#define CODE_IFOPT 0x14 -#define CODE_SAVEOPT 0x15 -#define CODE_RESETOPT 0x16 -#define CODE_KEYMAN80_LASTCODE 0x16 - -/* Keyman 9.0 codes */ -#define CODE_IFSYSTEMSTORE 0x17 -#define CODE_SETSYSTEMSTORE 0x18 -#define CODE_LASTCODE 0x18 - -/* Keyman4Mac codes */ -#define CODE_NULLCHAR 0x20 - - -#define NOT_EQUAL 0 -#define EQUAL 1 - /* KVK file format definitions */ // KVK header flags From 4645d1471a1eaf14cd93e7429f740e9cac34da63 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Mon, 12 May 2025 20:53:20 +0700 Subject: [PATCH 43/46] maint(android): Remove set_java_home_17 Unused per review comment --- resources/build/build-utils.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/resources/build/build-utils.sh b/resources/build/build-utils.sh index d857cad5b6..47feb3ee8c 100755 --- a/resources/build/build-utils.sh +++ b/resources/build/build-utils.sh @@ -341,13 +341,6 @@ set_java_home_21() { fi } -set_java_home_17() { - if [[ ! -z ${JAVA_HOME_17+x} ]]; then - builder_echo "Setting JAVA_HOME to JAVA_HOME_17 (${JAVA_HOME_17})" - export JAVA_HOME="${JAVA_HOME_17}" - fi -} - set_java_home_11() { if [[ ! -z ${JAVA_HOME_11+x} ]]; then builder_echo "Setting JAVA_HOME to JAVA_HOME_11 (${JAVA_HOME_11})" From 87369f6750a2d0e148fdae375334b67e31588da0 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Mon, 12 May 2025 14:02:07 -0400 Subject: [PATCH 44/46] auto: increment master version to 19.0.36 --- HISTORY.md | 8 ++++++++ VERSION.md | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 53b5c68dcd..bf2f6887cc 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,13 @@ # Keyman Version History +## 19.0.35 alpha 2025-05-12 + +* maint(common): add two more functions to detect if running on CI (#13869) +* maint(common): use unique names for Keyman version variables (#13854) +* maint(common): use new `builder_is_running_on_*` functions (#13875) +* maint(common): remove obsolete code from docker-images `run.sh` (#13879) +* chore(mac): remove obsolete key processing code (#13848) + ## 19.0.34 alpha 2025-05-08 * docs(common): update `builder_echo` documentation for TC blocks (#13855) diff --git a/VERSION.md b/VERSION.md index d8676f6b6a..e99a6f6be4 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -19.0.35 \ No newline at end of file +19.0.36 \ No newline at end of file From 2ad0c9cea753f53611ac120bc190b13b5ebc156e Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Tue, 13 May 2025 08:13:02 +0700 Subject: [PATCH 45/46] maint(android): Set Java language version 21 --- android/KMEA/app/build.gradle | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/android/KMEA/app/build.gradle b/android/KMEA/app/build.gradle index dd4251025b..8cc641ca60 100644 --- a/android/KMEA/app/build.gradle +++ b/android/KMEA/app/build.gradle @@ -62,6 +62,12 @@ android { } } +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(21)) + } +} + dependencies { // 1.6.0-rc01 needed to resolve https://issuetracker.google.com/issues/238425626 implementation 'androidx.appcompat:appcompat:1.7.0' From f4c5c557739355789d170c0fb811336106d958d8 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Tue, 13 May 2025 10:02:56 +0700 Subject: [PATCH 46/46] fix(android): Revert setting java language version --- android/KMEA/app/build.gradle | 6 ------ 1 file changed, 6 deletions(-) diff --git a/android/KMEA/app/build.gradle b/android/KMEA/app/build.gradle index 8cc641ca60..dd4251025b 100644 --- a/android/KMEA/app/build.gradle +++ b/android/KMEA/app/build.gradle @@ -62,12 +62,6 @@ android { } } -java { - toolchain { - languageVersion.set(JavaLanguageVersion.of(21)) - } -} - dependencies { // 1.6.0-rc01 needed to resolve https://issuetracker.google.com/issues/238425626 implementation 'androidx.appcompat:appcompat:1.7.0'