From ec68e74bde8418472b9641ee33e9a88b63845ad9 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 27 Feb 2025 16:24:21 -0600 Subject: [PATCH 001/431] feat(developer): add experimental char-offset data Depends on PR https://github.com/NaturalIntelligence/fast-xml-parser/pull/728 so uses srl repository "temporarily" Fixes: #10622 --- developer/src/common/web/utils/package.json | 2 +- .../src/common/web/utils/src/xml-utils.ts | 24 ++++++++++-- .../common/web/utils/test/xml-utils.tests.ts | 24 +++++++++++- package-lock.json | 39 ++++++++----------- 4 files changed, 61 insertions(+), 28 deletions(-) diff --git a/developer/src/common/web/utils/package.json b/developer/src/common/web/utils/package.json index 4f066313f6..571123ec68 100644 --- a/developer/src/common/web/utils/package.json +++ b/developer/src/common/web/utils/package.json @@ -12,7 +12,7 @@ "@keymanapp/common-types": "*", "@sentry/node": "^7.57.0", "eventemitter3": "^5.0.0", - "fast-xml-parser": "^4.5.0", + "fast-xml-parser": "github:srl295/fast-xml-parser#srl295/line-number-593", "path-browserify": "^1.0.1", "restructure": "^3.0.1", "sax": ">=0.6.0", diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 69a039aa00..626233dd17 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -8,6 +8,8 @@ import { XMLParser, XMLBuilder } from 'fast-xml-parser'; +export const START_INDEX = new XMLParser().getStartIndexSymbol(); + export type KeymanXMLType = 'keyboard3' // LDML | 'keyboardTest3' // LDML @@ -128,6 +130,22 @@ export class KeymanXMLReader { public constructor(public type: KeymanXMLType) { } + /** + * Copy symbols from 'from' onto 'onto' + * This is used to propagate special symbols + * and XML + * @param onto object to copy onto + * @param from source for symbols + * @returns the onto object + */ + public static copySymbols(onto: T, from: any): T { + const o = onto as any; + for (const sym of Object.getOwnPropertySymbols(from)) { + o[sym] = from[sym]; + } + return onto; + } + /** move `{ $abc: 4 }` into `{ $: { abc: 4 } }` */ private static fixupDollarAttributes(data: any) : any { if (typeof data === 'object') { @@ -146,9 +164,9 @@ export class KeymanXMLReader { } }); if (attrs.length) { - e.push(['$', Object.fromEntries(attrs)]); + e.push(['$', this.copySymbols(Object.fromEntries(attrs), data)]); } - return Object.fromEntries(e); + return this.copySymbols(Object.fromEntries(e), data); } else { return data; } @@ -184,7 +202,7 @@ export class KeymanXMLReader { } } }); - return Object.fromEntries(e); + return this.copySymbols(Object.fromEntries(e), data); } else { return data; } diff --git a/developer/src/common/web/utils/test/xml-utils.tests.ts b/developer/src/common/web/utils/test/xml-utils.tests.ts index 1cc9724b27..65ee826f57 100644 --- a/developer/src/common/web/utils/test/xml-utils.tests.ts +++ b/developer/src/common/web/utils/test/xml-utils.tests.ts @@ -12,7 +12,7 @@ import { env } from 'node:process'; import { readFileSync, writeFileSync } from 'node:fs'; -import { KeymanXMLType, KeymanXMLReader, KeymanXMLWriter } from '../src/xml-utils.js'; +import { KeymanXMLType, KeymanXMLReader, KeymanXMLWriter, START_INDEX } from '../src/xml-utils.js'; import { makePathToFixture } from './helpers/index.js'; // if true, attempt to WRITE the fixtures @@ -160,3 +160,25 @@ describe(`XML Writer Test ${GEN_XML_FIXTURES && '(update mode!)' || ''}`, () => }); } }); + +describe(`XML Reader line number test`, () => { + const path = 'tran_fail-empty.xml'; + const xmlPath = makePathToFixture('xml', `${path}`); + const type: KeymanXMLType = 'keyboard3'; + it(`Should report line numbers on parse of ${type} ${path}`, () => { + const xml = readData(xmlPath); + assert.ok(xml, `Could not read ${xmlPath}`); + + const reader = new KeymanXMLReader(type); + assert.ok(reader); + + // now, parse. subsitute endings for Win + const actual = reader.parse(xml.replace(/\r\n/g, '\n')); + assert.ok(actual, `Parser failed on ${xmlPath}`); + + // now, assert char offset + assert.equal(actual.keyboard3[START_INDEX as any], 40); // index of element + assert.equal(actual.keyboard3.info[START_INDEX as any], 136); // index of etc + assert.equal(actual.keyboard3.transforms[START_INDEX as any], 186); + }); +}); diff --git a/package-lock.json b/package-lock.json index 4e340f8af4..0f62509ac5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -342,7 +342,7 @@ "@keymanapp/common-types": "*", "@sentry/node": "^7.57.0", "eventemitter3": "^5.0.0", - "fast-xml-parser": "^4.5.0", + "fast-xml-parser": "github:srl295/fast-xml-parser#srl295/line-number-593", "path-browserify": "^1.0.1", "restructure": "^3.0.1", "sax": ">=0.6.0", @@ -372,6 +372,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "developer/src/common/web/utils/node_modules/fast-xml-parser": { + "version": "4.1.3", + "resolved": "git+ssh://git@github.com/srl295/fast-xml-parser.git#d7601c31f39fe69e6612af79f9d778dee5dfcb42", + "license": "MIT", + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + }, + "funding": { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + }, "developer/src/common/web/utils/node_modules/js-yaml": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", @@ -9016,28 +9031,6 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, - "node_modules/fast-xml-parser": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.5.0.tgz", - "integrity": "sha512-/PlTQCI96+fZMAOLMZK4CWG1ItCbfZ/0jx7UIJFChPNrx7tcEgerUgWbeieCM9MfHInUDyK8DWYZ+YrywDJuTg==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - }, - { - "type": "paypal", - "url": "https://paypal.me/naturalintelligence" - } - ], - "license": "MIT", - "dependencies": { - "strnum": "^1.0.5" - }, - "bin": { - "fxparser": "src/cli/cli.js" - } - }, "node_modules/fastq": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", From b60c64c3d3f1b4ec072090215be95c5d0d7e8464 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 27 Feb 2025 16:42:41 -0600 Subject: [PATCH 002/431] feat(developer): test for line number extension to xml-utils Fixes: #10622 --- .../src/common/web/utils/src/xml-utils.ts | 30 +++++++++++++++---- .../common/web/utils/test/xml-utils.tests.ts | 10 +++++-- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 626233dd17..9fbebdf397 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -125,11 +125,29 @@ const GENERATOR_OPTIONS: KeymanXMLOptionsBag = { }, }; +export interface LineColumn { + line: number; + column?: number; +} + /** wrapper for XML parsing support */ export class KeymanXMLReader { public constructor(public type: KeymanXMLType) { } + public static offsetToLineColumn(offset: number, text: string) : LineColumn { + const lines: string[] = text.split("\n"); + for (let line = 1; line < lines.length + 1; line++) { // 1-based + const linestr = lines[line - 1]; + if (linestr.length < offset) { + offset = offset - (linestr.length + 1); // count newline at end + continue; + } + return { line, column: offset }; + } + return { line: lines.length + 1 }; + } + /** * Copy symbols from 'from' onto 'onto' * This is used to propagate special symbols @@ -138,13 +156,13 @@ export class KeymanXMLReader { * @param from source for symbols * @returns the onto object */ - public static copySymbols(onto: T, from: any): T { - const o = onto as any; - for (const sym of Object.getOwnPropertySymbols(from)) { - o[sym] = from[sym]; - } - return onto; + private static copySymbols(onto: T, from: any): T { + const o = onto as any; + for (const sym of Object.getOwnPropertySymbols(from)) { + o[sym] = from[sym]; } + return onto; + } /** move `{ $abc: 4 }` into `{ $: { abc: 4 } }` */ private static fixupDollarAttributes(data: any) : any { diff --git a/developer/src/common/web/utils/test/xml-utils.tests.ts b/developer/src/common/web/utils/test/xml-utils.tests.ts index 65ee826f57..bdcd18817d 100644 --- a/developer/src/common/web/utils/test/xml-utils.tests.ts +++ b/developer/src/common/web/utils/test/xml-utils.tests.ts @@ -166,19 +166,23 @@ describe(`XML Reader line number test`, () => { const xmlPath = makePathToFixture('xml', `${path}`); const type: KeymanXMLType = 'keyboard3'; it(`Should report line numbers on parse of ${type} ${path}`, () => { - const xml = readData(xmlPath); + let xml = readData(xmlPath); assert.ok(xml, `Could not read ${xmlPath}`); - + xml = xml.replace(/\r\n/g, '\n'); const reader = new KeymanXMLReader(type); assert.ok(reader); // now, parse. subsitute endings for Win - const actual = reader.parse(xml.replace(/\r\n/g, '\n')); + const actual = reader.parse(xml); assert.ok(actual, `Parser failed on ${xmlPath}`); // now, assert char offset assert.equal(actual.keyboard3[START_INDEX as any], 40); // index of element assert.equal(actual.keyboard3.info[START_INDEX as any], 136); // index of etc assert.equal(actual.keyboard3.transforms[START_INDEX as any], 186); + + assert.equal(KeymanXMLReader.offsetToLineColumn(actual.keyboard3[START_INDEX as any], xml), { line: 3, column: 0 }); + assert.equal(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.info[START_INDEX as any], xml), { line: 4, column: 2 }); + assert.equal(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.transforms[START_INDEX as any], xml), { line: 8, column: 2 }); }); }); From c4ca359d182d28cf900b15a9c0c13fb63e54bb04 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 28 Feb 2025 09:27:25 -0600 Subject: [PATCH 003/431] feat(developer): API for resolving line numbers for messages - add 'column' to CompilerEvent - Add a LdmlCompilerMessages.col() function to set the column number - add a LdmlCompilerMessages.resolveLineNumber() to map offsets to line number Fixes: #10622 --- .../web/utils/src/compiler-interfaces.ts | 6 +++++ developer/src/common/web/utils/src/index.ts | 2 +- developer/src/kmc-ldml/src/compiler/keys.ts | 5 +++- .../src/compiler/ldml-compiler-messages.ts | 26 ++++++++++++++++++- .../src/kmc/src/util/NodeCompilerCallbacks.ts | 11 ++++++++ 5 files changed, 47 insertions(+), 3 deletions(-) diff --git a/developer/src/common/web/utils/src/compiler-interfaces.ts b/developer/src/common/web/utils/src/compiler-interfaces.ts index 869e0165da..390eea2f7e 100644 --- a/developer/src/common/web/utils/src/compiler-interfaces.ts +++ b/developer/src/common/web/utils/src/compiler-interfaces.ts @@ -5,7 +5,13 @@ import { CompilerCallbacks } from "./compiler-callbacks.js"; */ export interface CompilerEvent { filename?: string; + /** line where a message applies */ line?: number; + /** + * column where a message applies. + * where line is unset, column is a file offset + */ + column?: number; code: number; message: string; /** diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index f3e23e79f2..1a56a735e8 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -65,7 +65,7 @@ export { UrlSubpathCompilerCallback } from './utils/UrlSubpathCompilerCallback.j export { CommonTypesMessages } from './common-messages.js'; export * as SourceFilenamePatterns from './source-filename-patterns.js'; -export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader } from './xml-utils.js'; +export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, START_INDEX } from './xml-utils.js'; export * as GitHubUrls from './github-urls.js'; export * as CloudUrls from './cloud-urls.js'; diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index d86d7e2bf5..372bbea891 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -446,7 +446,10 @@ export class KeysCompiler extends SectionCompiler { if (layer.row.length > keymap.length) { this.callbacks.reportMessage( - LdmlCompilerMessages.Error_HardwareLayerHasTooManyRows() + LdmlCompilerMessages.col( + LdmlCompilerMessages.Error_HardwareLayerHasTooManyRows(), + layer + ) ); valid = false; } diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 1a84b10937..b6293d0158 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -1,5 +1,5 @@ import { util } from "@keymanapp/common-types"; -import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def } from '@keymanapp/developer-utils'; +import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, START_INDEX, CompilerEvent, KeymanXMLReader } from '@keymanapp/developer-utils'; // const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.LdmlKeyboardCompiler; @@ -264,5 +264,29 @@ export class LdmlCompilerMessages { `Invalid transform to="${def(o.to)}": "${def(o.message)}"`, ); + /** + * Get a column number from o and set e's column field + * @param event a compiler event, such as from functions in this class + * @param o any object parsed from XML or with the START_INDEX symbol copied over + * @returns modified event object + */ + static col(event: CompilerEvent, o: any): CompilerEvent { + event.column = (o as any)[START_INDEX as any]; + return event; + } + /** + * Given an event with a column number but no line, resolve it into line:column + * @param event event to modify + * @param xml XML source + * @returns the modified event + */ + static resolveLineNumber(event: CompilerEvent, xml: string) : CompilerEvent { + if (event.column && !event.line && xml) { + const loc = KeymanXMLReader.offsetToLineColumn(event.column, xml); + event.line = loc.line; + event.column = loc.column; + } + return event; + } } diff --git a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts index f3cdf1e7b2..5b340633fa 100644 --- a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts +++ b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts @@ -15,6 +15,7 @@ import chalk from 'chalk'; import supportsColor from 'supports-color'; import { KeymanSentry } from '@keymanapp/developer-utils'; import { fileURLToPath } from 'url'; +import { LdmlCompilerMessages } from '@keymanapp/kmc-ldml'; const color = chalk.default; const severityColors: {[value in CompilerErrorSeverity]: chalk.Chalk} = { @@ -150,6 +151,16 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { event.filename = this.messageFilename; } + if (event.column && !event.line && event.filename) { + try { + // TODO: rereads XML every single time. + // Should cache unless the filename changes (this.messageFilename) + LdmlCompilerMessages.resolveLineNumber(event, this.fs.readFileSync(event.filename, "utf-8")); + } catch(e) { + console.error(e); // resolving line numbers + } + } + if(this.messageFilename != event.filename) { // Reset max message limit when a new file is being processed this.messageFilename = event.filename; From d38025b4a19418e80931059ea7767895842dba83 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 28 Feb 2025 10:32:05 -0600 Subject: [PATCH 004/431] feat(developer): API for resolving line numbers for messages - Only read the xml file once Fixes: #10622 --- .../src/kmc/src/util/NodeCompilerCallbacks.ts | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts index 5b340633fa..7c184c1417 100644 --- a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts +++ b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts @@ -46,6 +46,8 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { messages: CompilerEvent[] = []; messageCount = 0; messageFilename: string = ''; + /** cache of the contentes of the text */ + messageFiletext: string = ''; maxLogMessages = MaxMessagesDefault; constructor(private options: CompilerCallbackOptions) { @@ -56,6 +58,7 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { this.messages = []; this.messageCount = 0; this.messageFilename = ''; + this.messageFiletext = ''; } /** @@ -151,20 +154,29 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { event.filename = this.messageFilename; } - if (event.column && !event.line && event.filename) { - try { - // TODO: rereads XML every single time. - // Should cache unless the filename changes (this.messageFilename) - LdmlCompilerMessages.resolveLineNumber(event, this.fs.readFileSync(event.filename, "utf-8")); - } catch(e) { - console.error(e); // resolving line numbers - } - } - if(this.messageFilename != event.filename) { // Reset max message limit when a new file is being processed this.messageFilename = event.filename; this.messageCount = 0; + this.messageFiletext = ''; + } + + if (event.column && !event.line && event.filename) { + try { + let text = ''; + if (this.messageFilename == event.filename) { + text = this.messageFiletext; + } + if (text == '') { + text = this.fs.readFileSync(event.filename, "utf-8"); + if (this.messageFilename == event.filename) { + this.messageFiletext = text; + } + } + LdmlCompilerMessages.resolveLineNumber(event, text); + } catch(e) { + console.error(e); // resolving line numbers + } } const disable = CompilerFileCallbacks.applyMessageOverridesToEvent(event, this.options.messageOverrides); From 650042fb6e7ce831d7e14bd0c74d5fd886a88588 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 28 Feb 2025 10:53:21 -0600 Subject: [PATCH 005/431] feat(developer): API for resolving line numbers for messages - simplify the error call site again Fixes: #10622 --- developer/src/kmc-ldml/src/compiler/keys.ts | 5 +---- .../src/compiler/ldml-compiler-messages.ts | 14 ++++++++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index 372bbea891..35a931dd94 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -446,10 +446,7 @@ export class KeysCompiler extends SectionCompiler { if (layer.row.length > keymap.length) { this.callbacks.reportMessage( - LdmlCompilerMessages.col( - LdmlCompilerMessages.Error_HardwareLayerHasTooManyRows(), - layer - ) + LdmlCompilerMessages.Error_HardwareLayerHasTooManyRows(layer) ); valid = false; } diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index b6293d0158..6f8cc5fa3d 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -9,6 +9,9 @@ const SevError = CompilerErrorSeverity.Error | CompilerErrorNamespace.LdmlKeyboa // sub-numberspace for transform errors const SevErrorTransform = SevError | 0xF00; +// like m() but takes an XML object for line numbers +const mx = (x: any, code: number, message: string, detail?: string): CompilerEvent => LdmlCompilerMessages.col(m(code, message, detail), x); + /** * @internal */ @@ -20,7 +23,8 @@ export class LdmlCompilerMessages { static Error_InvalidLocale = (o:{tag: string}) => m(this.ERROR_InvalidLocale, `Invalid BCP 47 locale form '${def(o.tag)}'`); static ERROR_HardwareLayerHasTooManyRows = SevError | 0x0003; - static Error_HardwareLayerHasTooManyRows = () => m(this.ERROR_HardwareLayerHasTooManyRows, `'hardware' layer has too many rows`); + static Error_HardwareLayerHasTooManyRows = (x: any) => mx(x, + this.ERROR_HardwareLayerHasTooManyRows, `'hardware' layer has too many rows`); static ERROR_RowOnHardwareLayerHasTooManyKeys = SevError | 0x0004; static Error_RowOnHardwareLayerHasTooManyKeys = (o:{row: number, hardware: string, modifiers: string}) => m(this.ERROR_RowOnHardwareLayerHasTooManyKeys, `Row #${def(o.row)} on 'hardware' ${def(o.hardware)} layer for modifier ${o.modifiers || 'none'} has too many keys`); @@ -267,11 +271,13 @@ export class LdmlCompilerMessages { /** * Get a column number from o and set e's column field * @param event a compiler event, such as from functions in this class - * @param o any object parsed from XML or with the START_INDEX symbol copied over + * @param x any object parsed from XML or with the START_INDEX symbol copied over * @returns modified event object */ - static col(event: CompilerEvent, o: any): CompilerEvent { - event.column = (o as any)[START_INDEX as any]; + static col(event: CompilerEvent, x?: any): CompilerEvent { + if(x) { + event.column = (x as any)[START_INDEX as any]; + } return event; } From 819d9bf5620a65ae9ef9d7a0832bd17303480e72 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 28 Feb 2025 12:08:31 -0600 Subject: [PATCH 006/431] feat(developer): API for resolving line numbers for messages - change to use a different fast-xml-parser PR, https://github.com/NaturalIntelligence/fast-xml-parser/pull/729 Fixes: #10622 --- developer/src/common/web/utils/package.json | 2 +- .../src/common/web/utils/src/xml-utils.ts | 2 +- .../utils/test/kpj/kpj-file-writer.tests.ts | 1 - .../common/web/utils/test/xml-utils.tests.ts | 7 ++- developer/src/kmc-ldml/test/helpers/index.ts | 9 +++- developer/src/kmc-ldml/test/keys.tests.ts | 4 +- package-lock.json | 46 +++++++++++-------- 7 files changed, 42 insertions(+), 29 deletions(-) diff --git a/developer/src/common/web/utils/package.json b/developer/src/common/web/utils/package.json index 571123ec68..a56fdc386a 100644 --- a/developer/src/common/web/utils/package.json +++ b/developer/src/common/web/utils/package.json @@ -12,7 +12,7 @@ "@keymanapp/common-types": "*", "@sentry/node": "^7.57.0", "eventemitter3": "^5.0.0", - "fast-xml-parser": "github:srl295/fast-xml-parser#srl295/line-number-593", + "fast-xml-parser": "github:srl295/fast-xml-parser#line-number-593bis", "path-browserify": "^1.0.1", "restructure": "^3.0.1", "sax": ">=0.6.0", diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 9fbebdf397..6365134db7 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -8,7 +8,7 @@ import { XMLParser, XMLBuilder } from 'fast-xml-parser'; -export const START_INDEX = new XMLParser().getStartIndexSymbol(); +export const START_INDEX = XMLParser.getStartIndexSymbol(); export type KeymanXMLType = 'keyboard3' // LDML diff --git a/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts b/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts index 700d223406..899527edbb 100644 --- a/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts +++ b/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts @@ -26,7 +26,6 @@ describe('kpj-file-writer', function () { // The outputKpj may not contain all the fields from the inputKpj, only the // essential fields. Many of the fields in .kpj are deprecated, when they // relate to file content (e.g. parented files, file details) - assert.deepEqual(outputKpj.KeymanDeveloperProject.Options, { "BuildPath": "$PROJECTPATH\\build", "CompilerWarningsAsErrors": "True", diff --git a/developer/src/common/web/utils/test/xml-utils.tests.ts b/developer/src/common/web/utils/test/xml-utils.tests.ts index bdcd18817d..44b4d95f21 100644 --- a/developer/src/common/web/utils/test/xml-utils.tests.ts +++ b/developer/src/common/web/utils/test/xml-utils.tests.ts @@ -180,9 +180,8 @@ describe(`XML Reader line number test`, () => { assert.equal(actual.keyboard3[START_INDEX as any], 40); // index of element assert.equal(actual.keyboard3.info[START_INDEX as any], 136); // index of etc assert.equal(actual.keyboard3.transforms[START_INDEX as any], 186); - - assert.equal(KeymanXMLReader.offsetToLineColumn(actual.keyboard3[START_INDEX as any], xml), { line: 3, column: 0 }); - assert.equal(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.info[START_INDEX as any], xml), { line: 4, column: 2 }); - assert.equal(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.transforms[START_INDEX as any], xml), { line: 8, column: 2 }); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3[START_INDEX as any], xml), { line: 3, column: 0 }); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.info[START_INDEX as any], xml), { line: 4, column: 2 }); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.transforms[START_INDEX as any], xml), { line: 8, column: 2 }); }); }); diff --git a/developer/src/kmc-ldml/test/helpers/index.ts b/developer/src/kmc-ldml/test/helpers/index.ts index 7b67ee4e4c..5599f695f6 100644 --- a/developer/src/kmc-ldml/test/helpers/index.ts +++ b/developer/src/kmc-ldml/test/helpers/index.ts @@ -8,7 +8,7 @@ import * as path from 'path'; import { fileURLToPath } from 'url'; import { SectionCompiler, SectionCompilerNew } from '../../src/compiler/section-compiler.js'; import { util, KMXPlus, LdmlKeyboardTypes } from '@keymanapp/common-types'; -import { CompilerEvent, compilerEventFormat, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboard, } from "@keymanapp/developer-utils"; +import { CompilerEvent, compilerEventFormat, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboard, START_INDEX, } from "@keymanapp/developer-utils"; import { LdmlKeyboardCompiler } from '../../src/main.js'; // make sure main.js compiles import { assert } from 'chai'; import { KMXPlusMetadataCompiler } from '../../src/compiler/metadata-compiler.js'; @@ -327,3 +327,10 @@ const dontEscape = /[a-zA-Z0-9\.${}\[\]-]/; export function hex_str(s?: string) : string { return [...s].map(ch => dontEscape.test(ch) ? ch : util.escapeRegexChar(ch)).join(''); } + +/** return an object simulating an XML object with a column number */ +export function withColumn(c: number) : any { + return { + [START_INDEX as any]: c + }; +} diff --git a/developer/src/kmc-ldml/test/keys.tests.ts b/developer/src/kmc-ldml/test/keys.tests.ts index cfb37e6baf..1b74763646 100644 --- a/developer/src/kmc-ldml/test/keys.tests.ts +++ b/developer/src/kmc-ldml/test/keys.tests.ts @@ -1,7 +1,7 @@ import 'mocha'; import { assert } from 'chai'; import { KeysCompiler } from '../src/compiler/keys.js'; -import { assertCodePoints, compilerTestCallbacks, loadSectionFixture, testCompilationCases } from './helpers/index.js'; +import { assertCodePoints, compilerTestCallbacks, loadSectionFixture, testCompilationCases, withColumn } from './helpers/index.js'; import { KMXPlus, Constants, LdmlKeyboardTypes } from '@keymanapp/common-types'; import { LdmlCompilerMessages } from '../src/compiler/ldml-compiler-messages.js'; import { constants } from '@keymanapp/ldml-keyboard-constants'; @@ -442,7 +442,7 @@ describe('keys.kmap', function () { assert.isNull(keys); assert.equal(compilerTestCallbacks.messages.length, 1); - assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_HardwareLayerHasTooManyRows()); + assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_HardwareLayerHasTooManyRows(withColumn(276))); }); it('should reject layouts with too many hardware keys', async function() { diff --git a/package-lock.json b/package-lock.json index 0f62509ac5..c44fd00061 100644 --- a/package-lock.json +++ b/package-lock.json @@ -342,7 +342,7 @@ "@keymanapp/common-types": "*", "@sentry/node": "^7.57.0", "eventemitter3": "^5.0.0", - "fast-xml-parser": "github:srl295/fast-xml-parser#srl295/line-number-593", + "fast-xml-parser": "github:srl295/fast-xml-parser#line-number-593bis", "path-browserify": "^1.0.1", "restructure": "^3.0.1", "sax": ">=0.6.0", @@ -372,21 +372,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "developer/src/common/web/utils/node_modules/fast-xml-parser": { - "version": "4.1.3", - "resolved": "git+ssh://git@github.com/srl295/fast-xml-parser.git#d7601c31f39fe69e6612af79f9d778dee5dfcb42", - "license": "MIT", - "dependencies": { - "strnum": "^1.0.5" - }, - "bin": { - "fxparser": "src/cli/cli.js" - }, - "funding": { - "type": "paypal", - "url": "https://paypal.me/naturalintelligence" - } - }, "developer/src/common/web/utils/node_modules/js-yaml": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", @@ -9031,6 +9016,23 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "node_modules/fast-xml-parser": { + "version": "5.0.8", + "resolved": "git+ssh://git@github.com/srl295/fast-xml-parser.git#72f20480e8591aca819fac684d46f8d678fcb7f3", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "strnum": "^2.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fastq": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", @@ -13888,9 +13890,15 @@ "link": true }, "node_modules/strnum": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", - "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==", + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.0.5.tgz", + "integrity": "sha512-YAT3K/sgpCUxhxNMrrdhtod3jckkpYwH6JAuwmUdXZsmzH1wUyzTMrrK2wYCEEqlKwrWDd35NeuUkbBy/1iK+Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], "license": "MIT" }, "node_modules/supports-color": { From 4d9ea2db2d1a1768225a074ad58d91da77941ebc Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 28 Feb 2025 14:24:12 -0600 Subject: [PATCH 007/431] feat(developer): work around line number symbols - add KeymanXMLReader.removeSymbols() and call it from tests Fixes: #10622 --- developer/src/common/web/utils/src/xml-utils.ts | 11 ++++++++++- .../web/utils/test/kpj/kpj-file-writer.tests.ts | 5 +++-- .../web/utils/test/kps/kps-file-reader.tests.ts | 5 +++-- .../src/common/web/utils/test/kvks/kvks-file.tests.ts | 5 +++-- .../src/common/web/utils/test/xml-utils.tests.ts | 2 +- developer/src/kmc-ldml/test/helpers/compareXml.ts | 2 +- 6 files changed, 21 insertions(+), 9 deletions(-) diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 6365134db7..98af170790 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -156,13 +156,22 @@ export class KeymanXMLReader { * @param from source for symbols * @returns the onto object */ - private static copySymbols(onto: T, from: any): T { + public static copySymbols(onto: T, from: any): T { const o = onto as any; for (const sym of Object.getOwnPropertySymbols(from)) { o[sym] = from[sym]; } return onto; } + + /** use Object.entries to remove all symbols */ + public static removeSymbols(from: T): T { + if (Array.isArray(from)) { + return from.map(o => KeymanXMLReader.removeSymbols(o)) as T; + } + if (typeof from !== "object") return from; + return Object.fromEntries(Object.entries(from).map(([k, v]) => ([k, KeymanXMLReader.removeSymbols(v)]))) as T; + } /** move `{ $abc: 4 }` into `{ $: { abc: 4 } }` */ private static fixupDollarAttributes(data: any) : any { diff --git a/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts b/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts index 899527edbb..5fc223b240 100644 --- a/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts +++ b/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts @@ -6,6 +6,7 @@ import { KPJFileReader } from "../../src/types/kpj/kpj-file-reader.js"; import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; import { KPJFileWriter } from '../../src/types/kpj/kpj-file-writer.js'; import { KeymanDeveloperProjectOptions } from '../../src/types/kpj/keyman-developer-project.js'; +import { KeymanXMLReader } from '../../src/xml-utils.js'; const callbacks = new TestCompilerCallbacks(); @@ -21,7 +22,7 @@ describe('kpj-file-writer', function () { const writer = new KPJFileWriter(); const output = writer.write(project); - const outputKpj = reader.read(new TextEncoder().encode(output)); + const outputKpj = KeymanXMLReader.removeSymbols(reader.read(new TextEncoder().encode(output))); // The outputKpj may not contain all the fields from the inputKpj, only the // essential fields. Many of the fields in .kpj are deprecated, when they @@ -60,7 +61,7 @@ describe('kpj-file-writer', function () { const writer = new KPJFileWriter(); const output = writer.write(project); - const outputKpj = reader.read(new TextEncoder().encode(output)); + const outputKpj = KeymanXMLReader.removeSymbols(reader.read(new TextEncoder().encode(output))); // The outputKpj may not contain all the fields from the inputKpj, only the // essential fields. Many of the fields in .kpj are deprecated, when they diff --git a/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts b/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts index 1b809f39d0..94a1628bd9 100644 --- a/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts +++ b/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts @@ -15,6 +15,7 @@ import { makePathToFixture } from '../helpers/index.js'; import { KpsFileReader } from "../../src/types/kps/kps-file-reader.js"; import { KpsFileWriter } from '../../src/types/kps/kps-file-writer.js'; import { CommonTypesMessages } from '../../src/common-messages.js'; +import { KeymanXMLReader } from '../../src/xml-utils.js'; const callbacks = new TestCompilerCallbacks(); @@ -61,13 +62,13 @@ describe('kps-file-reader', function () { it('kps-file-reader should round-trip with kps-file-writer', function() { const input = fs.readFileSync(makePathToFixture('kps', 'khmer_angkor.kps')); const reader = new KpsFileReader(callbacks); - const kps = reader.read(input); + const kps = KeymanXMLReader.removeSymbols(reader.read(input)); const writer = new KpsFileWriter(); const output = writer.write(kps); // Round Trip - const kps2 = reader.read(new TextEncoder().encode(output)); + const kps2 = KeymanXMLReader.removeSymbols(reader.read(new TextEncoder().encode(output))); assert.deepEqual(kps2, kps); }); diff --git a/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts b/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts index 4c009e9d5c..96be2283b8 100644 --- a/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts +++ b/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts @@ -5,6 +5,7 @@ import KvksFileReader from "../../src/types/kvks/kvks-file-reader.js"; import KvksFileWriter from "../../src/types/kvks/kvks-file-writer.js"; import { verify_khmer_angkor, verify_balochi_inpage } from './kvk-utils.tests.js'; import { assert } from 'chai'; +import { KeymanXMLReader } from '../../src/xml-utils.js'; describe('kvks-file-reader', function() { it('should read a valid file', function() { @@ -48,7 +49,7 @@ describe('kvks-file-writer', function() { const input = fs.readFileSync(path); const reader = new KvksFileReader(); - const kvksExpected = reader.read(input); + const kvksExpected = KeymanXMLReader.removeSymbols(reader.read(input)); const invalidVkeys: string[] = []; const vk = reader.transform(kvksExpected, invalidVkeys); assert.isEmpty(invalidVkeys); @@ -59,6 +60,6 @@ describe('kvks-file-writer', function() { // We compare the (re)loaded data, because there may be // minor, irrelevant formatting differences in the emitted xml const kvks = reader.read(Buffer.from(output, 'utf8')); - assert.deepEqual(kvks, kvksExpected); + assert.deepEqual(KeymanXMLReader.removeSymbols(kvks), kvksExpected); }); }); diff --git a/developer/src/common/web/utils/test/xml-utils.tests.ts b/developer/src/common/web/utils/test/xml-utils.tests.ts index 44b4d95f21..9039760913 100644 --- a/developer/src/common/web/utils/test/xml-utils.tests.ts +++ b/developer/src/common/web/utils/test/xml-utils.tests.ts @@ -119,7 +119,7 @@ describe(`XML Reader Test ${GEN_XML_FIXTURES && '(update mode!)' || ''}`, () => writeJson(jsonPath, actual); } else { assert.ok(expect, `Could not read ${jsonPath} - run with env GEN_XML_FIXTURES=1 to update.`); - assert.deepEqual(actual, expect, `Mismatch of ${xmlPath} vs ${jsonPath}`); + assert.deepEqual(KeymanXMLReader.removeSymbols(actual), expect, `Mismatch of ${xmlPath} vs ${jsonPath}`); } }); } diff --git a/developer/src/kmc-ldml/test/helpers/compareXml.ts b/developer/src/kmc-ldml/test/helpers/compareXml.ts index f245da51a9..1503d4c606 100644 --- a/developer/src/kmc-ldml/test/helpers/compareXml.ts +++ b/developer/src/kmc-ldml/test/helpers/compareXml.ts @@ -21,5 +21,5 @@ export function compareXml(actual : string, expect: string, mutator?: (input: an const actualParsed = mutator(reader.parse(actualStr)); const expectParsed = mutator(reader.parse(expectStr)); - assert.deepEqual(actualParsed, expectParsed); + assert.deepEqual(KeymanXMLReader.removeSymbols(actualParsed), KeymanXMLReader.removeSymbols(expectParsed)); } From 3e73a3f0819ffa5a38cf4075f0c6bdab2b573137 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 3 Mar 2025 12:36:42 -0600 Subject: [PATCH 008/431] feat(developer): add CompilerEvent.offset - per review, keep this as a separate field. Fixes: #10622 --- .../common/web/utils/src/compiler-interfaces.ts | 8 ++++++-- .../src/compiler/ldml-compiler-messages.ts | 14 +++++++------- .../src/kmc/src/util/NodeCompilerCallbacks.ts | 2 +- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/developer/src/common/web/utils/src/compiler-interfaces.ts b/developer/src/common/web/utils/src/compiler-interfaces.ts index 390eea2f7e..dc24a772e9 100644 --- a/developer/src/common/web/utils/src/compiler-interfaces.ts +++ b/developer/src/common/web/utils/src/compiler-interfaces.ts @@ -7,11 +7,15 @@ export interface CompilerEvent { filename?: string; /** line where a message applies */ line?: number; - /** + /** * column where a message applies. - * where line is unset, column is a file offset */ column?: number; + /** + * offset where a message applies. + * If set, encompasses line and column. + */ + offset?: number; code: number; message: string; /** diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 6f8cc5fa3d..27f587b04f 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -10,7 +10,7 @@ const SevError = CompilerErrorSeverity.Error | CompilerErrorNamespace.LdmlKeyboa const SevErrorTransform = SevError | 0xF00; // like m() but takes an XML object for line numbers -const mx = (x: any, code: number, message: string, detail?: string): CompilerEvent => LdmlCompilerMessages.col(m(code, message, detail), x); +const mx = (x: any, code: number, message: string, detail?: string): CompilerEvent => LdmlCompilerMessages.offset(m(code, message, detail), x); /** * @internal @@ -269,14 +269,14 @@ export class LdmlCompilerMessages { ); /** - * Get a column number from o and set e's column field + * Get an offset from o and set e's offset field * @param event a compiler event, such as from functions in this class * @param x any object parsed from XML or with the START_INDEX symbol copied over * @returns modified event object */ - static col(event: CompilerEvent, x?: any): CompilerEvent { + static offset(event: CompilerEvent, x?: any): CompilerEvent { if(x) { - event.column = (x as any)[START_INDEX as any]; + event.offset = (x as any)[START_INDEX as any]; } return event; } @@ -287,9 +287,9 @@ export class LdmlCompilerMessages { * @param xml XML source * @returns the modified event */ - static resolveLineNumber(event: CompilerEvent, xml: string) : CompilerEvent { - if (event.column && !event.line && xml) { - const loc = KeymanXMLReader.offsetToLineColumn(event.column, xml); + static resolveLineNumber(event: CompilerEvent, xml: string) : CompilerEvent { + if (event.offset && !event.line && xml) { + const loc = KeymanXMLReader.offsetToLineColumn(event.offset, xml); event.line = loc.line; event.column = loc.column; } diff --git a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts index 7c184c1417..8186f29d70 100644 --- a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts +++ b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts @@ -161,7 +161,7 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { this.messageFiletext = ''; } - if (event.column && !event.line && event.filename) { + if (event.offset && !event.line && event.filename) { try { let text = ''; if (this.messageFilename == event.filename) { From d7cb7b63a3db987229e42a3b3fcbf78ff906eb1e Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 3 Mar 2025 13:14:41 -0600 Subject: [PATCH 009/431] feat(developer): add an EventResolver - it's a hook to the hook to allow the LDML compiler to resolve error message line numbers - TODO still not extra performant, but maybe the right separation of concerns Fixes: #10622 --- .../web/test-helpers/TestCompilerCallbacks.ts | 13 +++++++- .../web/utils/src/compiler-callbacks.ts | 28 +++++++++++++++- developer/src/common/web/utils/src/index.ts | 2 ++ .../src/kmc-ldml/src/compiler/compiler.ts | 4 +++ .../kmc-ldml/src/compiler/eventresolver.ts | 21 ++++++++++++ .../src/kmc/src/util/NodeCompilerCallbacks.ts | 32 +++++++------------ 6 files changed, 78 insertions(+), 22 deletions(-) create mode 100644 developer/src/kmc-ldml/src/compiler/eventresolver.ts diff --git a/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts b/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts index fad1dd646f..4771b16417 100644 --- a/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts +++ b/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts @@ -2,7 +2,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { CompilerEvent, CompilerCallbacks, CompilerPathCallbacks, CompilerFileSystemCallbacks, CompilerError, CompilerNetAsyncCallbacks, DefaultCompilerFileSystemAsyncCallbacks, - CompilerFileSystemAsyncCallbacks, + CompilerFileSystemAsyncCallbacks, EventResolver, NullEventResolver, CompilerErrorSeverity} from '@keymanapp/developer-utils'; import { fileURLToPath } from 'url'; @@ -111,7 +111,18 @@ export class TestCompilerCallbacks implements CompilerCallbacks { return filename; } + private eventResolver = new NullEventResolver(); + + setEventResolver(eventResolver: EventResolver): void { + this.eventResolver = eventResolver; + } + reportMessage(event: CompilerEvent): void { + this.eventResolver.resolve(event); + this.handleReportMessage(event); + } + + handleReportMessage(event: CompilerEvent): void { // console.log(event.message); this.messages.push(event); } diff --git a/developer/src/common/web/utils/src/compiler-callbacks.ts b/developer/src/common/web/utils/src/compiler-callbacks.ts index a7163175e8..bebfe5fd55 100644 --- a/developer/src/common/web/utils/src/compiler-callbacks.ts +++ b/developer/src/common/web/utils/src/compiler-callbacks.ts @@ -7,6 +7,21 @@ import { CompilerEvent, CompilerCallbackOptions, CompilerErrorSeverity, CompilerError, CompilerMessageOverrideMap, CompilerErrorMask } from "./compiler-interfaces.js"; +/** + * The EventResolver implementation is able to expand error messages, + * particularly to change a byte offset into a line number. + */ +export interface EventResolver { + resolve(event: CompilerEvent): void; +} + +/** A do-nothing EventResolver. Implementations can use this as their default EventResolver. */ +export class NullEventResolver implements EventResolver { + resolve(event: CompilerEvent): void { + // do nothing, this is the null event resolver. + } +} + /** * Abstract interface for callbacks, to abstract out file i/o */ @@ -34,12 +49,20 @@ export interface CompilerCallbacks { get net(): CompilerNetAsyncCallbacks; /** - * Resolves a file path relative to the baseFilename + * Resolves a file path relative to the baseFilename. * @param baseFilename * @param filename */ resolveFilename(baseFilename: string, filename: string): string; + /** change the EventResolver to use. If called more than once, the last resolver replaces a prior one. */ + setEventResolver(eventResolver: EventResolver): void; + + /** + * Report a message from the compiler back to the caller. + * It is expected that eventResolver.resolve() will be called on the message before processing. + * @param event + */ reportMessage(event: CompilerEvent): void; debug(msg: string): void; @@ -115,6 +138,9 @@ export class CompilerFileCallbacks implements CompilerCallbacks { constructor(private filename: string, private options: CompilerCallbackOptions, private parent: CompilerCallbacks) { } + setEventResolver(eventResolver: EventResolver): void { + throw new Error("Method not implemented."); + } /** * Returns `true` if any message in the `messages` array is a Fatal or Error diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index 1a56a735e8..812d503e96 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -42,6 +42,8 @@ export { CompilerPathCallbacks, CompilerFileSystemCallbacksFolderEntry as FileSystemFolderEntry, DefaultCompilerFileSystemAsyncCallbacks, + EventResolver, + NullEventResolver, } from './compiler-callbacks.js'; export { defaultCompilerOptions, CompilerBaseOptions, CompilerOptions, CompilerEvent, CompilerErrorNamespace, diff --git a/developer/src/kmc-ldml/src/compiler/compiler.ts b/developer/src/kmc-ldml/src/compiler/compiler.ts index e8b38184e9..e0b86dc712 100644 --- a/developer/src/kmc-ldml/src/compiler/compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/compiler.ts @@ -30,6 +30,7 @@ import { KmnCompiler } from '@keymanapp/kmc-kmn'; import { KMXPlusMetadataCompiler } from './metadata-compiler.js'; import { LdmlKeyboardVisualKeyboardCompiler } from './visual-keyboard-compiler.js'; import { LinterKeycaps } from './linter-keycaps.js'; +import { LdmlEventResolver } from './eventresolver.js'; export const SECTION_COMPILERS = [ // These are in dependency order. @@ -94,6 +95,7 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { // uset parser private usetparser?: LdmlKeyboardTypes.UnicodeSetParser = undefined; + private static eventResolver: LdmlEventResolver = new LdmlEventResolver(); /** * Initialize the compiler, including loading the WASM host for uset parsing. @@ -106,6 +108,7 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { async init(callbacks: CompilerCallbacks, options: LdmlCompilerOptions): Promise { this.options = { ...options }; this.callbacks = callbacks; + this.callbacks.setEventResolver(LdmlKeyboardCompiler.eventResolver); return true; } @@ -245,6 +248,7 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidFile({ errorText: 'Unable to read XML file' })); return null; } + LdmlKeyboardCompiler.eventResolver.addFile(filename, new TextDecoder().decode(data)); // TODO: double decode // parse (load) the string into an object tree const source = reader.load(data); if (!source) { diff --git a/developer/src/kmc-ldml/src/compiler/eventresolver.ts b/developer/src/kmc-ldml/src/compiler/eventresolver.ts new file mode 100644 index 0000000000..491e4a3423 --- /dev/null +++ b/developer/src/kmc-ldml/src/compiler/eventresolver.ts @@ -0,0 +1,21 @@ +import { CompilerEvent, EventResolver } from "@keymanapp/developer-utils"; +import { LdmlCompilerMessages } from "./ldml-compiler-messages.js"; + +export class LdmlEventResolver implements EventResolver { + + contentsCache: Map = new Map(); + + /** add or update a file */ + addFile(filename: string, xml: any) { + if (typeof xml !== 'string') { + xml = xml.toString(); + } + this.contentsCache.set(filename, xml); + } + + resolve(event: CompilerEvent) { + if (event.offset && !event.line && event.filename) { + LdmlCompilerMessages.resolveLineNumber(event, this.contentsCache.get(event.filename)); + } + } +} diff --git a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts index 8186f29d70..5f4d0e9018 100644 --- a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts +++ b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { platform } from 'os'; -import { CompilerCallbacks, CompilerEvent, +import { CompilerCallbacks, CompilerEvent, EventResolver, NullEventResolver, CompilerPathCallbacks, CompilerFileSystemCallbacks, compilerLogLevelToSeverity, CompilerErrorSeverity, CompilerError, @@ -15,7 +15,6 @@ import chalk from 'chalk'; import supportsColor from 'supports-color'; import { KeymanSentry } from '@keymanapp/developer-utils'; import { fileURLToPath } from 'url'; -import { LdmlCompilerMessages } from '@keymanapp/kmc-ldml'; const color = chalk.default; const severityColors: {[value in CompilerErrorSeverity]: chalk.Chalk} = { @@ -149,7 +148,18 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { return fileURLToPath(url); } + private eventResolver = new NullEventResolver(); + + setEventResolver(eventResolver: EventResolver): void { + this.eventResolver = eventResolver; + } + reportMessage(event: CompilerEvent): void { + this.eventResolver.resolve(event); + this.handleReportMessage(event); + } + + handleReportMessage(event: CompilerEvent): void { if(!event.filename) { event.filename = this.messageFilename; } @@ -161,24 +171,6 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { this.messageFiletext = ''; } - if (event.offset && !event.line && event.filename) { - try { - let text = ''; - if (this.messageFilename == event.filename) { - text = this.messageFiletext; - } - if (text == '') { - text = this.fs.readFileSync(event.filename, "utf-8"); - if (this.messageFilename == event.filename) { - this.messageFiletext = text; - } - } - LdmlCompilerMessages.resolveLineNumber(event, text); - } catch(e) { - console.error(e); // resolving line numbers - } - } - const disable = CompilerFileCallbacks.applyMessageOverridesToEvent(event, this.options.messageOverrides); this.messages.push({...event}); From 1febf9a0de36c2e979642291f91904d337c3c17c Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 3 Mar 2025 13:18:02 -0600 Subject: [PATCH 010/431] Update developer/src/common/web/utils/src/xml-utils.ts Co-authored-by: Marc Durdin --- developer/src/common/web/utils/src/xml-utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 98af170790..8c084231fe 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -8,7 +8,7 @@ import { XMLParser, XMLBuilder } from 'fast-xml-parser'; -export const START_INDEX = XMLParser.getStartIndexSymbol(); +export const XML_START_INDEX_SYMBOL = XMLParser.getStartIndexSymbol(); export type KeymanXMLType = 'keyboard3' // LDML From b015221814f83463908d88a5209376dc14916ff5 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 3 Mar 2025 17:08:40 -0600 Subject: [PATCH 011/431] feat(developer): line number symbols - change special symbol to XML_START_INDEX_SYMBOL - use an array of line lengths rather than an array of strings for the file info - add an EventResolver interface to the callbacks, such that it has an opportunity to resolve events Fixes: #10622 --- .../web/utils/src/compiler-callbacks.ts | 14 ++++++++++--- developer/src/common/web/utils/src/index.ts | 2 +- .../src/common/web/utils/src/xml-utils.ts | 19 ++++++++++------- .../common/web/utils/test/xml-utils.tests.ts | 14 ++++++------- .../kmc-ldml/src/compiler/eventresolver.ts | 16 +++++++------- .../src/compiler/ldml-compiler-messages.ts | 21 +++---------------- developer/src/kmc-ldml/test/helpers/index.ts | 6 +++--- 7 files changed, 44 insertions(+), 48 deletions(-) diff --git a/developer/src/common/web/utils/src/compiler-callbacks.ts b/developer/src/common/web/utils/src/compiler-callbacks.ts index bebfe5fd55..bce9d9f395 100644 --- a/developer/src/common/web/utils/src/compiler-callbacks.ts +++ b/developer/src/common/web/utils/src/compiler-callbacks.ts @@ -138,9 +138,6 @@ export class CompilerFileCallbacks implements CompilerCallbacks { constructor(private filename: string, private options: CompilerCallbackOptions, private parent: CompilerCallbacks) { } - setEventResolver(eventResolver: EventResolver): void { - throw new Error("Method not implemented."); - } /** * Returns `true` if any message in the `messages` array is a Fatal or Error @@ -228,7 +225,18 @@ export class CompilerFileCallbacks implements CompilerCallbacks { return this.parent.resolveFilename(baseFilename, filename); } + private eventResolver = new NullEventResolver(); + + setEventResolver(eventResolver: EventResolver): void { + this.eventResolver = eventResolver; + } + reportMessage(event: CompilerEvent): void { + this.eventResolver.resolve(event); + this.handleReportMessage(event); + } + + handleReportMessage(event: CompilerEvent): void { const disable = CompilerFileCallbacks.applyMessageOverridesToEvent(event, this.options.messageOverrides); this.messages.push(event); if (!disable) { diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index 812d503e96..c703e48aa2 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -67,7 +67,7 @@ export { UrlSubpathCompilerCallback } from './utils/UrlSubpathCompilerCallback.j export { CommonTypesMessages } from './common-messages.js'; export * as SourceFilenamePatterns from './source-filename-patterns.js'; -export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, START_INDEX } from './xml-utils.js'; +export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, XML_START_INDEX_SYMBOL } from './xml-utils.js'; export * as GitHubUrls from './github-urls.js'; export * as CloudUrls from './cloud-urls.js'; diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 8c084231fe..de78fa5b54 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -134,18 +134,21 @@ export interface LineColumn { export class KeymanXMLReader { public constructor(public type: KeymanXMLType) { } - - public static offsetToLineColumn(offset: number, text: string) : LineColumn { - const lines: string[] = text.split("\n"); + /** preprocess text to turn it into arrays of line lengths */ + public static textToLines(text: string) : number[] { + return text.replaceAll("\r\n", "\n").split("\n") + .map(l => l.length + 1); // line length (counting the trailing newline) + } + public static offsetToLineColumn(offset: number, lines: number[]) : LineColumn { for (let line = 1; line < lines.length + 1; line++) { // 1-based - const linestr = lines[line - 1]; - if (linestr.length < offset) { - offset = offset - (linestr.length + 1); // count newline at end + if (lines[line-1] < offset) { + offset = offset - (lines[line-1]); // count newline at end continue; } return { line, column: offset }; } - return { line: lines.length + 1 }; + // default: line 0, error + return { line: 0 } } /** @@ -172,7 +175,7 @@ export class KeymanXMLReader { if (typeof from !== "object") return from; return Object.fromEntries(Object.entries(from).map(([k, v]) => ([k, KeymanXMLReader.removeSymbols(v)]))) as T; } - + /** move `{ $abc: 4 }` into `{ $: { abc: 4 } }` */ private static fixupDollarAttributes(data: any) : any { if (typeof data === 'object') { diff --git a/developer/src/common/web/utils/test/xml-utils.tests.ts b/developer/src/common/web/utils/test/xml-utils.tests.ts index 9039760913..f487573724 100644 --- a/developer/src/common/web/utils/test/xml-utils.tests.ts +++ b/developer/src/common/web/utils/test/xml-utils.tests.ts @@ -12,7 +12,7 @@ import { env } from 'node:process'; import { readFileSync, writeFileSync } from 'node:fs'; -import { KeymanXMLType, KeymanXMLReader, KeymanXMLWriter, START_INDEX } from '../src/xml-utils.js'; +import { KeymanXMLType, KeymanXMLReader, KeymanXMLWriter, XML_START_INDEX_SYMBOL } from '../src/xml-utils.js'; import { makePathToFixture } from './helpers/index.js'; // if true, attempt to WRITE the fixtures @@ -177,11 +177,11 @@ describe(`XML Reader line number test`, () => { assert.ok(actual, `Parser failed on ${xmlPath}`); // now, assert char offset - assert.equal(actual.keyboard3[START_INDEX as any], 40); // index of element - assert.equal(actual.keyboard3.info[START_INDEX as any], 136); // index of etc - assert.equal(actual.keyboard3.transforms[START_INDEX as any], 186); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3[START_INDEX as any], xml), { line: 3, column: 0 }); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.info[START_INDEX as any], xml), { line: 4, column: 2 }); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.transforms[START_INDEX as any], xml), { line: 8, column: 2 }); + assert.equal(actual.keyboard3[XML_START_INDEX_SYMBOL as any], 40); // index of element + assert.equal(actual.keyboard3.info[XML_START_INDEX_SYMBOL as any], 136); // index of etc + assert.equal(actual.keyboard3.transforms[XML_START_INDEX_SYMBOL as any], 186); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3[XML_START_INDEX_SYMBOL as any], xml), { line: 3, column: 0 }); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.info[XML_START_INDEX_SYMBOL as any], xml), { line: 4, column: 2 }); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.transforms[XML_START_INDEX_SYMBOL as any], xml), { line: 8, column: 2 }); }); }); diff --git a/developer/src/kmc-ldml/src/compiler/eventresolver.ts b/developer/src/kmc-ldml/src/compiler/eventresolver.ts index 491e4a3423..7e1dd87fc3 100644 --- a/developer/src/kmc-ldml/src/compiler/eventresolver.ts +++ b/developer/src/kmc-ldml/src/compiler/eventresolver.ts @@ -1,21 +1,21 @@ -import { CompilerEvent, EventResolver } from "@keymanapp/developer-utils"; -import { LdmlCompilerMessages } from "./ldml-compiler-messages.js"; +import { CompilerEvent, EventResolver, KeymanXMLReader } from "@keymanapp/developer-utils"; export class LdmlEventResolver implements EventResolver { - contentsCache: Map = new Map(); + contentsCache: Map = new Map(); /** add or update a file */ addFile(filename: string, xml: any) { - if (typeof xml !== 'string') { - xml = xml.toString(); - } - this.contentsCache.set(filename, xml); + this.contentsCache.set(filename, KeymanXMLReader.textToLines(xml)); } resolve(event: CompilerEvent) { if (event.offset && !event.line && event.filename) { - LdmlCompilerMessages.resolveLineNumber(event, this.contentsCache.get(event.filename)); + const lines = this.contentsCache.get(event.filename); + const loc = KeymanXMLReader.offsetToLineColumn(event.offset, lines); + event.line = loc.line; + event.column = loc.column; } + return event; } } diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 27f587b04f..1298d0f0cd 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -1,5 +1,5 @@ import { util } from "@keymanapp/common-types"; -import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, START_INDEX, CompilerEvent, KeymanXMLReader } from '@keymanapp/developer-utils'; +import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_START_INDEX_SYMBOL, CompilerEvent } from '@keymanapp/developer-utils'; // const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.LdmlKeyboardCompiler; @@ -271,27 +271,12 @@ export class LdmlCompilerMessages { /** * Get an offset from o and set e's offset field * @param event a compiler event, such as from functions in this class - * @param x any object parsed from XML or with the START_INDEX symbol copied over + * @param x any object parsed from XML or with the XML_START_INDEX_SYMBOL symbol copied over * @returns modified event object */ static offset(event: CompilerEvent, x?: any): CompilerEvent { if(x) { - event.offset = (x as any)[START_INDEX as any]; - } - return event; - } - - /** - * Given an event with a column number but no line, resolve it into line:column - * @param event event to modify - * @param xml XML source - * @returns the modified event - */ - static resolveLineNumber(event: CompilerEvent, xml: string) : CompilerEvent { - if (event.offset && !event.line && xml) { - const loc = KeymanXMLReader.offsetToLineColumn(event.offset, xml); - event.line = loc.line; - event.column = loc.column; + event.offset = (x as any)[XML_START_INDEX_SYMBOL as any]; } return event; } diff --git a/developer/src/kmc-ldml/test/helpers/index.ts b/developer/src/kmc-ldml/test/helpers/index.ts index 5599f695f6..497f8ef919 100644 --- a/developer/src/kmc-ldml/test/helpers/index.ts +++ b/developer/src/kmc-ldml/test/helpers/index.ts @@ -8,7 +8,7 @@ import * as path from 'path'; import { fileURLToPath } from 'url'; import { SectionCompiler, SectionCompilerNew } from '../../src/compiler/section-compiler.js'; import { util, KMXPlus, LdmlKeyboardTypes } from '@keymanapp/common-types'; -import { CompilerEvent, compilerEventFormat, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboard, START_INDEX, } from "@keymanapp/developer-utils"; +import { CompilerEvent, compilerEventFormat, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboard, XML_START_INDEX_SYMBOL, } from "@keymanapp/developer-utils"; import { LdmlKeyboardCompiler } from '../../src/main.js'; // make sure main.js compiles import { assert } from 'chai'; import { KMXPlusMetadataCompiler } from '../../src/compiler/metadata-compiler.js'; @@ -290,7 +290,7 @@ export function testCompilationCases(compiler: SectionCompilerNew, cases : Compi // no warnings, so expect zero messages assert.sameDeepMembers(callbacks.messages, [], 'expected zero messages but got ' + callbacks.messages); } - + if (expectFailure) { assert.isNull(section, 'expected compilation result failure (null)'); } else { @@ -331,6 +331,6 @@ export function hex_str(s?: string) : string { /** return an object simulating an XML object with a column number */ export function withColumn(c: number) : any { return { - [START_INDEX as any]: c + [XML_START_INDEX_SYMBOL as any]: c }; } From 92e16b90a033da5e3d9133e9521fc9b6df5088db Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 3 Mar 2025 19:12:34 -0600 Subject: [PATCH 012/431] feat(developer): line number symbols - move file read bottleneck out of compiler.ts into the LDML XML reader - move the eventresolver into the LDML XML reader - add XML_FILENAME_SYMBOL - for tagging when an object is from a different file than the default (such as with an import) Fixes: #10622 --- .../web/utils/src/compiler-callbacks.ts | 1 + developer/src/common/web/utils/src/index.ts | 2 +- .../src/types/ldml-keyboard}/eventresolver.ts | 2 +- .../ldml-keyboard/ldml-keyboard-xml-reader.ts | 40 ++++++++++++++----- .../src/common/web/utils/src/xml-utils.ts | 3 ++ .../src/kmc-ldml/src/compiler/compiler.ts | 6 +-- developer/src/kmc-ldml/src/compiler/keys.ts | 4 +- .../src/compiler/ldml-compiler-messages.ts | 7 ++-- 8 files changed, 44 insertions(+), 21 deletions(-) rename developer/src/{kmc-ldml/src/compiler => common/web/utils/src/types/ldml-keyboard}/eventresolver.ts (86%) diff --git a/developer/src/common/web/utils/src/compiler-callbacks.ts b/developer/src/common/web/utils/src/compiler-callbacks.ts index bce9d9f395..f8b42afb7a 100644 --- a/developer/src/common/web/utils/src/compiler-callbacks.ts +++ b/developer/src/common/web/utils/src/compiler-callbacks.ts @@ -229,6 +229,7 @@ export class CompilerFileCallbacks implements CompilerCallbacks { setEventResolver(eventResolver: EventResolver): void { this.eventResolver = eventResolver; + this.parent.setEventResolver(eventResolver); } reportMessage(event: CompilerEvent): void { diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index c703e48aa2..b722a11b21 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -67,7 +67,7 @@ export { UrlSubpathCompilerCallback } from './utils/UrlSubpathCompilerCallback.j export { CommonTypesMessages } from './common-messages.js'; export * as SourceFilenamePatterns from './source-filename-patterns.js'; -export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, XML_START_INDEX_SYMBOL } from './xml-utils.js'; +export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, XML_START_INDEX_SYMBOL, XML_FILENAME_SYMBOL } from './xml-utils.js'; export * as GitHubUrls from './github-urls.js'; export * as CloudUrls from './cloud-urls.js'; diff --git a/developer/src/kmc-ldml/src/compiler/eventresolver.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts similarity index 86% rename from developer/src/kmc-ldml/src/compiler/eventresolver.ts rename to developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts index 7e1dd87fc3..3299169642 100644 --- a/developer/src/kmc-ldml/src/compiler/eventresolver.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts @@ -1,4 +1,4 @@ -import { CompilerEvent, EventResolver, KeymanXMLReader } from "@keymanapp/developer-utils"; +import { CompilerEvent, EventResolver, KeymanXMLReader } from "../../index.js"; export class LdmlEventResolver implements EventResolver { diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index 0a5abf6355..ca32fe4905 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -11,6 +11,8 @@ import { constants } from '@keymanapp/ldml-keyboard-constants'; import { LDMLKeyboardTestDataXMLSourceFile, LKTTest, LKTTests } from './ldml-keyboard-testdata-xml.js'; import { KeymanXMLReader } from '@keymanapp/developer-utils'; import boxXmlArray = util.boxXmlArray; +import { LdmlEventResolver } from './eventresolver.js'; +import { XML_FILENAME_SYMBOL } from 'src/xml-utils.js'; interface NameAndProps { '$'?: any; // content @@ -26,27 +28,39 @@ export class LDMLKeyboardXMLSourceFileReaderOptions { }; export class LDMLKeyboardXMLSourceFileReader { - constructor(private options: LDMLKeyboardXMLSourceFileReaderOptions, private callbacks : CompilerCallbacks) { + static eventResolver: LdmlEventResolver = new LdmlEventResolver(); + constructor(private options: LDMLKeyboardXMLSourceFileReaderOptions, private callbacks : CompilerCallbacks) { + callbacks.setEventResolver(LDMLKeyboardXMLSourceFileReader.eventResolver); } static get defaultImportsURL(): [string,string] { return ['../import/', import.meta.url]; } - readImportFile(version: string, subpath: string): Uint8Array { - const importPath = this.callbacks.resolveFilename(this.options.cldrImportsPath, `${version}/${subpath}`); - return this.callbacks.loadFile(importPath); + /** bottleneck for reading keyboard XML files */ + readFile(path: string): Uint8Array { + const data = this.callbacks.loadFile(path); + if (data) { + LDMLKeyboardXMLSourceFileReader.eventResolver.addFile(path, new TextDecoder().decode(data)); + } + return data; } - readLocalImportFile(path: string): Uint8Array { + /** @returns [data, filename] */ + readImportFile(version: string, subpath: string): [Uint8Array, string] { + const importPath = this.callbacks.resolveFilename(this.options.cldrImportsPath, `${version}/${subpath}`); + return [this.readFile(importPath), importPath]; + } + + readLocalImportFile(path: string): [Uint8Array, string] { // try each of the local imports paths for (const localPath of this.options.localImportsPaths) { const importPath = this.callbacks.path.join(localPath, path); if(this.callbacks.fs.existsSync(importPath)) { - return this.callbacks.loadFile(importPath); + return [this.readFile(importPath), importPath]; } } - return null; // was not able to load from any of the paths + return [null, null]; // was not able to load from any of the paths } /** @@ -223,6 +237,7 @@ export class LDMLKeyboardXMLSourceFileReader { return false; } let importData: Uint8Array; + let importPath: string; if (base === constants.cldr_import_base) { // CLDR import @@ -235,10 +250,10 @@ export class LDMLKeyboardXMLSourceFileReader { /** There's no data or DTD change in 45, 46, 46.1, 47 so map them all to 46 at present. */ paths[0] = constants.cldr_version_latest; } - importData = this.readImportFile(paths[0], paths[1]); + [importData, importPath] = this.readImportFile(paths[0], paths[1]); } else { // local import - importData = this.readLocalImportFile(path); + [importData, importPath] = this.readLocalImportFile(path); } if (!importData || !importData.length) { this.callbacks.reportMessage(CommonTypesMessages.Error_ImportReadFail({base, path, subtag})); @@ -246,6 +261,7 @@ export class LDMLKeyboardXMLSourceFileReader { } const importXml: any = this.loadUnboxed(importData); // TODO-LDML: have to load as any because it is an arbitrary part const importRootNode = importXml[subtag]; // e.g. + LDMLKeyboardXMLSourceFileReader.eventResolver.addFile(importPath, new TextDecoder().decode(importData)); // TODO: double decode // importXml will have one property: the root element. if (!importRootNode) { @@ -263,7 +279,11 @@ export class LDMLKeyboardXMLSourceFileReader { return false; } // Mark all children as an import - subsubval.forEach(o => o[ImportStatus.import] = basePath); + subsubval.forEach(o => { + o[ImportStatus.import] = basePath; + o[XML_FILENAME_SYMBOL] = importPath; // mark overriding import path + }); + if (implied) { // mark all children as an implied import subsubval.forEach(o => o[ImportStatus.impliedImport] = basePath); diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index de78fa5b54..ee33b40563 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -8,7 +8,10 @@ import { XMLParser, XMLBuilder } from 'fast-xml-parser'; +/** Symbol giving the start offset, in chars, of the node */ export const XML_START_INDEX_SYMBOL = XMLParser.getStartIndexSymbol(); +/** Symbol giving an override which file a node came from */ +export const XML_FILENAME_SYMBOL = Symbol("XML Filename"); export type KeymanXMLType = 'keyboard3' // LDML diff --git a/developer/src/kmc-ldml/src/compiler/compiler.ts b/developer/src/kmc-ldml/src/compiler/compiler.ts index e0b86dc712..5488afb188 100644 --- a/developer/src/kmc-ldml/src/compiler/compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/compiler.ts @@ -30,7 +30,6 @@ import { KmnCompiler } from '@keymanapp/kmc-kmn'; import { KMXPlusMetadataCompiler } from './metadata-compiler.js'; import { LdmlKeyboardVisualKeyboardCompiler } from './visual-keyboard-compiler.js'; import { LinterKeycaps } from './linter-keycaps.js'; -import { LdmlEventResolver } from './eventresolver.js'; export const SECTION_COMPILERS = [ // These are in dependency order. @@ -95,7 +94,6 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { // uset parser private usetparser?: LdmlKeyboardTypes.UnicodeSetParser = undefined; - private static eventResolver: LdmlEventResolver = new LdmlEventResolver(); /** * Initialize the compiler, including loading the WASM host for uset parsing. @@ -108,7 +106,6 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { async init(callbacks: CompilerCallbacks, options: LdmlCompilerOptions): Promise { this.options = { ...options }; this.callbacks = callbacks; - this.callbacks.setEventResolver(LdmlKeyboardCompiler.eventResolver); return true; } @@ -243,12 +240,11 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { public load(filename: string): LDMLKeyboardXMLSourceFile | null { const reader = new LDMLKeyboardXMLSourceFileReader(this.options.readerOptions, this.callbacks); // load the file from disk into a string - const data = this.callbacks.loadFile(filename); + const data = reader.readFile(filename); if (!data) { this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidFile({ errorText: 'Unable to read XML file' })); return null; } - LdmlKeyboardCompiler.eventResolver.addFile(filename, new TextDecoder().decode(data)); // TODO: double decode // parse (load) the string into an object tree const source = reader.load(data); if (!source) { diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index 35a931dd94..5906646587 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -485,7 +485,9 @@ export class KeysCompiler extends SectionCompiler { } if (!keydef.output && !keydef.gap && !keydef.layerId) { this.callbacks.reportMessage( - LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch({ keyId: key }) + LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch({ keyId: key }, + keydef + ) ); valid = false; continue; diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 1298d0f0cd..1fcb82193b 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -1,5 +1,5 @@ import { util } from "@keymanapp/common-types"; -import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_START_INDEX_SYMBOL, CompilerEvent } from '@keymanapp/developer-utils'; +import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_START_INDEX_SYMBOL, XML_FILENAME_SYMBOL, CompilerEvent } from '@keymanapp/developer-utils'; // const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.LdmlKeyboardCompiler; @@ -91,8 +91,8 @@ export class LdmlCompilerMessages { m(this.ERROR_DisplayIsRepeated, `display ${LdmlCompilerMessages.outputOrKeyId(o)} has more than one display entry.`); static ERROR_KeyMissingToGapOrSwitch = SevError | 0x0011; - static Error_KeyMissingToGapOrSwitch = (o:{keyId: string}) => - m(this.ERROR_KeyMissingToGapOrSwitch, `key id='${def(o.keyId)}' must have either output=, gap=, or layerId=.`); + static Error_KeyMissingToGapOrSwitch = (o:{keyId: string}, x: any) => mx(x, + this.ERROR_KeyMissingToGapOrSwitch, `key id='${def(o.keyId)}' must have either output=, gap=, or layerId=.`); static ERROR_ExcessHardware = SevError | 0x0012; static Error_ExcessHardware = (o:{formId: string}) => m(this.ERROR_ExcessHardware, @@ -277,6 +277,7 @@ export class LdmlCompilerMessages { static offset(event: CompilerEvent, x?: any): CompilerEvent { if(x) { event.offset = (x as any)[XML_START_INDEX_SYMBOL as any]; + event.filename = event.filename || (x as any)[XML_FILENAME_SYMBOL as any]; } return event; } From 0ba005a55b21b728a92ad11171c788025155eb99 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Tue, 4 Mar 2025 09:34:47 -0600 Subject: [PATCH 013/431] feat(developer): line number symbols - improve and simplify lookup algorithm - add test for override filename on imports Fixes: #10622 --- .../src/common/web/utils/src/xml-utils.ts | 22 ++++++++++++------- .../kmx/ldml-keyboard-xml-reader.tests.ts | 13 ++++++++--- .../common/web/utils/test/xml-utils.tests.ts | 7 +++--- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index ee33b40563..7d7dc10011 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -137,18 +137,24 @@ export interface LineColumn { export class KeymanXMLReader { public constructor(public type: KeymanXMLType) { } - /** preprocess text to turn it into arrays of line lengths */ + /** + * preprocess text to turn it into arrays of line lengths. + * This is effectively a 1-based line length, since line 0 has length of + * 0. + */ public static textToLines(text: string) : number[] { - return text.replaceAll("\r\n", "\n").split("\n") - .map(l => l.length + 1); // line length (counting the trailing newline) + return [ + 0, // "line 0" is empty + ...text.replaceAll("\r\n", "\n").split("\n") + .map(l => l.length + 1) // line length (counting the trailing newline) + ]; } public static offsetToLineColumn(offset: number, lines: number[]) : LineColumn { - for (let line = 1; line < lines.length + 1; line++) { // 1-based - if (lines[line-1] < offset) { - offset = offset - (lines[line-1]); // count newline at end - continue; + for (let line = 1; line < lines.length; line++) { // 1-based (assume the first row is 0) + if (lines[line] > offset) { + return { line, column: offset }; } - return { line, column: offset }; + offset = offset - (lines[line]); // count newline at end } // default: line 0, error return { line: 0 } diff --git a/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts b/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts index fcb3628144..56a70e3ea0 100644 --- a/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts +++ b/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts @@ -8,6 +8,7 @@ import { Constants } from '@keymanapp/common-types'; import CLDRScanToVkey = Constants.CLDRScanToVkey; import CLDRScanToKeyMap = Constants.CLDRScanToKeyMap; import USVirtualKeyCodes = Constants.USVirtualKeyCodes; +import { XML_FILENAME_SYMBOL } from '../../src/xml-utils.js'; function pluckKeysFromKeybag(keys: LKKey[], ids: string[]) { return keys.filter(({id}) => ids.indexOf(id) !== -1); @@ -140,10 +141,16 @@ describe('ldml keyboard xml reader tests', function () { { id: 'interrobang', output: '‽' }, { id: 'snail', output: '@' }, ]); + const snailKey = source?.keyboard3?.keys.key.find(({ id }) => id === 'snail'); // all of the keys are implied imports here - assert.isFalse(ImportStatus.isImpliedImport(source?.keyboard3?.keys.key.find(({id}) => id === 'snail'))); - assert.isTrue(ImportStatus.isImport(source?.keyboard3?.keys.key.find(({id}) => id === 'snail'))); - assert.isTrue(ImportStatus.isLocalImport(source?.keyboard3?.keys.key.find(({id}) => id === 'snail'))); + assert.isFalse(ImportStatus.isImpliedImport(snailKey)); + assert.isTrue(ImportStatus.isImport(snailKey)); + assert.isTrue(ImportStatus.isLocalImport(snailKey)); + // get the actual filename of where the import was located + const snailFilename = (snailKey as any)[XML_FILENAME_SYMBOL as any]; + assert.ok(snailFilename); + assert.ok(/\/keys-Zyyy-morepunctuation.xml$/.test(snailFilename) + , `snail key filename is ${snailFilename}`); }, }, { diff --git a/developer/src/common/web/utils/test/xml-utils.tests.ts b/developer/src/common/web/utils/test/xml-utils.tests.ts index f487573724..5045bcfcb3 100644 --- a/developer/src/common/web/utils/test/xml-utils.tests.ts +++ b/developer/src/common/web/utils/test/xml-utils.tests.ts @@ -174,14 +174,15 @@ describe(`XML Reader line number test`, () => { // now, parse. subsitute endings for Win const actual = reader.parse(xml); + const lines = KeymanXMLReader.textToLines(xml); assert.ok(actual, `Parser failed on ${xmlPath}`); // now, assert char offset assert.equal(actual.keyboard3[XML_START_INDEX_SYMBOL as any], 40); // index of element assert.equal(actual.keyboard3.info[XML_START_INDEX_SYMBOL as any], 136); // index of etc assert.equal(actual.keyboard3.transforms[XML_START_INDEX_SYMBOL as any], 186); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3[XML_START_INDEX_SYMBOL as any], xml), { line: 3, column: 0 }); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.info[XML_START_INDEX_SYMBOL as any], xml), { line: 4, column: 2 }); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.transforms[XML_START_INDEX_SYMBOL as any], xml), { line: 8, column: 2 }); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3[XML_START_INDEX_SYMBOL as any], lines), { line: 3, column: 0 }); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.info[XML_START_INDEX_SYMBOL as any], lines), { line: 4, column: 2 }); + assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.transforms[XML_START_INDEX_SYMBOL as any], lines), { line: 8, column: 2 }); }); }); From 0c290bd37ad8b7454783f01d6a9b3043251d3af2 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 31 Mar 2025 16:11:19 -0500 Subject: [PATCH 014/431] feat(developer): update for line numbers - update to metadata branch of fast-xml-parser - restructure metadata handling - use type safety for fast-xml-parser options Fixes: #10622 --- developer/src/common/web/utils/src/index.ts | 4 +- .../src/common/web/utils/src/symbol-utils.ts | 33 ++++++++ .../ldml-keyboard/ldml-keyboard-xml-reader.ts | 4 +- .../src/common/web/utils/src/xml-utils.ts | 84 ++++++++++--------- .../kmx/ldml-keyboard-xml-reader.tests.ts | 6 +- .../utils/test/kpj/kpj-file-writer.tests.ts | 6 +- .../utils/test/kps/kps-file-reader.tests.ts | 6 +- .../web/utils/test/kvks/kvks-file.tests.ts | 6 +- .../common/web/utils/test/xml-utils.tests.ts | 28 +++++-- .../src/compiler/ldml-compiler-messages.ts | 9 +- .../src/kmc-ldml/test/helpers/compareXml.ts | 4 +- developer/src/kmc-ldml/test/helpers/index.ts | 13 +-- developer/src/kmc-ldml/test/keys.tests.ts | 2 +- package-lock.json | 4 +- 14 files changed, 132 insertions(+), 77 deletions(-) create mode 100644 developer/src/common/web/utils/src/symbol-utils.ts diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index b722a11b21..95ef39ffcb 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -67,8 +67,8 @@ export { UrlSubpathCompilerCallback } from './utils/UrlSubpathCompilerCallback.j export { CommonTypesMessages } from './common-messages.js'; export * as SourceFilenamePatterns from './source-filename-patterns.js'; -export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, XML_START_INDEX_SYMBOL, XML_FILENAME_SYMBOL } from './xml-utils.js'; - +export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, KeymanXMLMetadata, XML_FILENAME_SYMBOL } from './xml-utils.js'; +export { SymbolUtils } from './symbol-utils.js'; export * as GitHubUrls from './github-urls.js'; export * as CloudUrls from './cloud-urls.js'; diff --git a/developer/src/common/web/utils/src/symbol-utils.ts b/developer/src/common/web/utils/src/symbol-utils.ts new file mode 100644 index 0000000000..971d02cc16 --- /dev/null +++ b/developer/src/common/web/utils/src/symbol-utils.ts @@ -0,0 +1,33 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by srl on 2025-03-27 + * + * Utilities for manipulating Symbol properties + */ + +export class SymbolUtils { + /** + * Copy symbols from 'from' onto 'onto' + * @param onto object to copy onto + * @param from source for symbols + * @returns the onto object + */ + public static copySymbols(onto: T, from: any): T { + const o = onto as any; + for (const sym of Object.getOwnPropertySymbols(from)) { + o[sym] = from[sym]; + } + return onto; + } + + /** use Object.entries to remove all symbols */ + public static removeSymbols(from: T): T { + if (Array.isArray(from)) { + return from.map(o => SymbolUtils.removeSymbols(o)) as T; + } + if (typeof from !== "object") return from; + return Object.fromEntries(Object.entries(from).map(([k, v]) => ([k, SymbolUtils.removeSymbols(v)]))) as T; + } + +} diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index ca32fe4905..84e82c62ea 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -12,7 +12,7 @@ import { LDMLKeyboardTestDataXMLSourceFile, LKTTest, LKTTests } from './ldml-key import { KeymanXMLReader } from '@keymanapp/developer-utils'; import boxXmlArray = util.boxXmlArray; import { LdmlEventResolver } from './eventresolver.js'; -import { XML_FILENAME_SYMBOL } from 'src/xml-utils.js'; +import { XML_FILENAME_SYMBOL } from '../../xml-utils.js'; interface NameAndProps { '$'?: any; // content @@ -281,7 +281,7 @@ export class LDMLKeyboardXMLSourceFileReader { // Mark all children as an import subsubval.forEach(o => { o[ImportStatus.import] = basePath; - o[XML_FILENAME_SYMBOL] = importPath; // mark overriding import path + KeymanXMLReader.setMetaData(o, {[XML_FILENAME_SYMBOL]: importPath}); // mark overriding import path }); if (implied) { diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 7d7dc10011..eadeb8659d 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -6,10 +6,11 @@ * Abstraction for XML reading and writing */ -import { XMLParser, XMLBuilder } from 'fast-xml-parser'; +import { XMLParser, XMLBuilder, XMLMetaData, X2jOptions, XmlBuilderOptions } from 'fast-xml-parser'; +import { SymbolUtils } from "./symbol-utils.js"; /** Symbol giving the start offset, in chars, of the node */ -export const XML_START_INDEX_SYMBOL = XMLParser.getStartIndexSymbol(); +const XML_META_DATA_SYMBOL = XMLParser.getMetaDataSymbol(); /** Symbol giving an override which file a node came from */ export const XML_FILENAME_SYMBOL = Symbol("XML Filename"); @@ -22,12 +23,12 @@ export type KeymanXMLType = ; /** Bag of options, maximally one for each KeymanXMLType */ -type KeymanXMLOptionsBag = { - [key in KeymanXMLType]?: any +type KeymanXMLParserOptionsBag = { + [key in KeymanXMLType]?: X2jOptions }; /** map of options for the XML parser */ -const PARSER_OPTIONS: KeymanXMLOptionsBag = { +const PARSER_OPTIONS: KeymanXMLParserOptionsBag = { 'keyboard3': { ignoreAttributes: false, // We'd like attributes, please attributeNamePrefix: '@__', // We'll use this to convert attributes to strings and subobjects to arrays, when empty. @@ -40,6 +41,7 @@ const PARSER_OPTIONS: KeymanXMLOptionsBag = { // if we do need elements in the future, we'd check the preserve-space attribute here. return tagValue?.trim(); }, + captureMetaData: true, }, 'keyboardTest3': { ignorePiTags: true, @@ -87,7 +89,7 @@ const PARSER_OPTIONS: KeymanXMLOptionsBag = { eNotation: null, }, trimValues: false, // preserve spaces, but: - tagValueProcessor: (tagName: string, tagValue: string, jPath: string, hasAttributes: string, isLeafNode: boolean) : string | undefined => { + tagValueProcessor: (tagName: string, tagValue: string, jPath: string, hasAttributes: boolean, isLeafNode: boolean) : string | undefined => { if (!isLeafNode) { return tagValue?.trim(); // trimmed value } else { @@ -97,7 +99,11 @@ const PARSER_OPTIONS: KeymanXMLOptionsBag = { }, }; -const GENERATOR_OPTIONS: KeymanXMLOptionsBag = { +type KeymanXMLGeneratorOptionsBag = { + [key in KeymanXMLType]?: XmlBuilderOptions +}; + +const GENERATOR_OPTIONS: KeymanXMLGeneratorOptionsBag = { kvks: { attributeNamePrefix: '$', ignoreAttributes: false, @@ -133,10 +139,37 @@ export interface LineColumn { column?: number; } +export interface KeymanXMLMetadata extends XMLMetaData { + /** override of name of XML file */ + [XML_FILENAME_SYMBOL]?: string; +} + /** wrapper for XML parsing support */ export class KeymanXMLReader { public constructor(public type: KeymanXMLType) { } + + /** Get metadata on a node if not already set */ + static getMetaData(o: any) : KeymanXMLMetadata { + if(!o) return o; + const metadata : KeymanXMLMetadata = o[XML_META_DATA_SYMBOL as any]; + return metadata; + } + + /** Set metadata if not already set */ + public static setMetaData(o: any, metadata: KeymanXMLMetadata) : KeymanXMLMetadata { + let m : KeymanXMLMetadata = KeymanXMLReader.getMetaData(o); + if (!m) { + m = {}; + } + // copy non-symbols + m = {...metadata, ...m}; + // copy symbols + SymbolUtils.copySymbols(m, metadata); + o[XML_META_DATA_SYMBOL as any] = m; + return m; + } + /** * preprocess text to turn it into arrays of line lengths. * This is effectively a 1-based line length, since line 0 has length of @@ -160,31 +193,6 @@ export class KeymanXMLReader { return { line: 0 } } - /** - * Copy symbols from 'from' onto 'onto' - * This is used to propagate special symbols - * and XML - * @param onto object to copy onto - * @param from source for symbols - * @returns the onto object - */ - public static copySymbols(onto: T, from: any): T { - const o = onto as any; - for (const sym of Object.getOwnPropertySymbols(from)) { - o[sym] = from[sym]; - } - return onto; - } - - /** use Object.entries to remove all symbols */ - public static removeSymbols(from: T): T { - if (Array.isArray(from)) { - return from.map(o => KeymanXMLReader.removeSymbols(o)) as T; - } - if (typeof from !== "object") return from; - return Object.fromEntries(Object.entries(from).map(([k, v]) => ([k, KeymanXMLReader.removeSymbols(v)]))) as T; - } - /** move `{ $abc: 4 }` into `{ $: { abc: 4 } }` */ private static fixupDollarAttributes(data: any) : any { if (typeof data === 'object') { @@ -203,9 +211,9 @@ export class KeymanXMLReader { } }); if (attrs.length) { - e.push(['$', this.copySymbols(Object.fromEntries(attrs), data)]); + e.push(['$', SymbolUtils.copySymbols(Object.fromEntries(attrs), data)]); } - return this.copySymbols(Object.fromEntries(e), data); + return SymbolUtils.copySymbols(Object.fromEntries(e), data); } else { return data; } @@ -241,7 +249,7 @@ export class KeymanXMLReader { } } }); - return this.copySymbols(Object.fromEntries(e), data); + return SymbolUtils.copySymbols(Object.fromEntries(e), data); } else { return data; } @@ -321,15 +329,11 @@ export class KeymanXMLReader { } public parser() { - let options = PARSER_OPTIONS[this.type]; + const options = PARSER_OPTIONS[this.type]; if (!options) { /* c8 ignore next 1 */ throw Error(`Internal error: unhandled XML type ${this.type}`); } - options = Object.assign({}, options); // TODO: xml2js likes to mutate the options here. Shallow clone the object. - if (options.emptyTag) { - options.emptyTag = {}; // TODO: xml2js likes to mutate the options here. Reset it. - } return new XMLParser(options); } } diff --git a/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts b/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts index 56a70e3ea0..bfd076a34d 100644 --- a/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts +++ b/developer/src/common/web/utils/test/kmx/ldml-keyboard-xml-reader.tests.ts @@ -8,7 +8,7 @@ import { Constants } from '@keymanapp/common-types'; import CLDRScanToVkey = Constants.CLDRScanToVkey; import CLDRScanToKeyMap = Constants.CLDRScanToKeyMap; import USVirtualKeyCodes = Constants.USVirtualKeyCodes; -import { XML_FILENAME_SYMBOL } from '../../src/xml-utils.js'; +import { KeymanXMLReader, XML_FILENAME_SYMBOL } from '../../src/xml-utils.js'; function pluckKeysFromKeybag(keys: LKKey[], ids: string[]) { return keys.filter(({id}) => ids.indexOf(id) !== -1); @@ -147,7 +147,9 @@ describe('ldml keyboard xml reader tests', function () { assert.isTrue(ImportStatus.isImport(snailKey)); assert.isTrue(ImportStatus.isLocalImport(snailKey)); // get the actual filename of where the import was located - const snailFilename = (snailKey as any)[XML_FILENAME_SYMBOL as any]; + const metadata = KeymanXMLReader.getMetaData(snailKey); + assert.ok(metadata); + const snailFilename = (metadata)[XML_FILENAME_SYMBOL]; assert.ok(snailFilename); assert.ok(/\/keys-Zyyy-morepunctuation.xml$/.test(snailFilename) , `snail key filename is ${snailFilename}`); diff --git a/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts b/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts index 5fc223b240..425d76a6ef 100644 --- a/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts +++ b/developer/src/common/web/utils/test/kpj/kpj-file-writer.tests.ts @@ -6,7 +6,7 @@ import { KPJFileReader } from "../../src/types/kpj/kpj-file-reader.js"; import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers'; import { KPJFileWriter } from '../../src/types/kpj/kpj-file-writer.js'; import { KeymanDeveloperProjectOptions } from '../../src/types/kpj/keyman-developer-project.js'; -import { KeymanXMLReader } from '../../src/xml-utils.js'; +import { SymbolUtils } from '../../src/symbol-utils.js'; const callbacks = new TestCompilerCallbacks(); @@ -22,7 +22,7 @@ describe('kpj-file-writer', function () { const writer = new KPJFileWriter(); const output = writer.write(project); - const outputKpj = KeymanXMLReader.removeSymbols(reader.read(new TextEncoder().encode(output))); + const outputKpj = SymbolUtils.removeSymbols(reader.read(new TextEncoder().encode(output))); // The outputKpj may not contain all the fields from the inputKpj, only the // essential fields. Many of the fields in .kpj are deprecated, when they @@ -61,7 +61,7 @@ describe('kpj-file-writer', function () { const writer = new KPJFileWriter(); const output = writer.write(project); - const outputKpj = KeymanXMLReader.removeSymbols(reader.read(new TextEncoder().encode(output))); + const outputKpj = SymbolUtils.removeSymbols(reader.read(new TextEncoder().encode(output))); // The outputKpj may not contain all the fields from the inputKpj, only the // essential fields. Many of the fields in .kpj are deprecated, when they diff --git a/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts b/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts index 94a1628bd9..9b7955819d 100644 --- a/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts +++ b/developer/src/common/web/utils/test/kps/kps-file-reader.tests.ts @@ -15,7 +15,7 @@ import { makePathToFixture } from '../helpers/index.js'; import { KpsFileReader } from "../../src/types/kps/kps-file-reader.js"; import { KpsFileWriter } from '../../src/types/kps/kps-file-writer.js'; import { CommonTypesMessages } from '../../src/common-messages.js'; -import { KeymanXMLReader } from '../../src/xml-utils.js'; +import { SymbolUtils } from '../../src/symbol-utils.js'; const callbacks = new TestCompilerCallbacks(); @@ -62,13 +62,13 @@ describe('kps-file-reader', function () { it('kps-file-reader should round-trip with kps-file-writer', function() { const input = fs.readFileSync(makePathToFixture('kps', 'khmer_angkor.kps')); const reader = new KpsFileReader(callbacks); - const kps = KeymanXMLReader.removeSymbols(reader.read(input)); + const kps = SymbolUtils.removeSymbols(reader.read(input)); const writer = new KpsFileWriter(); const output = writer.write(kps); // Round Trip - const kps2 = KeymanXMLReader.removeSymbols(reader.read(new TextEncoder().encode(output))); + const kps2 = SymbolUtils.removeSymbols(reader.read(new TextEncoder().encode(output))); assert.deepEqual(kps2, kps); }); diff --git a/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts b/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts index 96be2283b8..d264dc86aa 100644 --- a/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts +++ b/developer/src/common/web/utils/test/kvks/kvks-file.tests.ts @@ -5,7 +5,7 @@ import KvksFileReader from "../../src/types/kvks/kvks-file-reader.js"; import KvksFileWriter from "../../src/types/kvks/kvks-file-writer.js"; import { verify_khmer_angkor, verify_balochi_inpage } from './kvk-utils.tests.js'; import { assert } from 'chai'; -import { KeymanXMLReader } from '../../src/xml-utils.js'; +import { SymbolUtils } from '../../src/symbol-utils.js'; describe('kvks-file-reader', function() { it('should read a valid file', function() { @@ -49,7 +49,7 @@ describe('kvks-file-writer', function() { const input = fs.readFileSync(path); const reader = new KvksFileReader(); - const kvksExpected = KeymanXMLReader.removeSymbols(reader.read(input)); + const kvksExpected = SymbolUtils.removeSymbols(reader.read(input)); const invalidVkeys: string[] = []; const vk = reader.transform(kvksExpected, invalidVkeys); assert.isEmpty(invalidVkeys); @@ -60,6 +60,6 @@ describe('kvks-file-writer', function() { // We compare the (re)loaded data, because there may be // minor, irrelevant formatting differences in the emitted xml const kvks = reader.read(Buffer.from(output, 'utf8')); - assert.deepEqual(KeymanXMLReader.removeSymbols(kvks), kvksExpected); + assert.deepEqual(SymbolUtils.removeSymbols(kvks), kvksExpected); }); }); diff --git a/developer/src/common/web/utils/test/xml-utils.tests.ts b/developer/src/common/web/utils/test/xml-utils.tests.ts index 5045bcfcb3..8728eec207 100644 --- a/developer/src/common/web/utils/test/xml-utils.tests.ts +++ b/developer/src/common/web/utils/test/xml-utils.tests.ts @@ -12,8 +12,9 @@ import { env } from 'node:process'; import { readFileSync, writeFileSync } from 'node:fs'; -import { KeymanXMLType, KeymanXMLReader, KeymanXMLWriter, XML_START_INDEX_SYMBOL } from '../src/xml-utils.js'; +import { KeymanXMLType, KeymanXMLReader, KeymanXMLWriter } from '../src/xml-utils.js'; import { makePathToFixture } from './helpers/index.js'; +import { SymbolUtils } from '../src/symbol-utils.js'; // if true, attempt to WRITE the fixtures const { GEN_XML_FIXTURES } = env; @@ -119,7 +120,7 @@ describe(`XML Reader Test ${GEN_XML_FIXTURES && '(update mode!)' || ''}`, () => writeJson(jsonPath, actual); } else { assert.ok(expect, `Could not read ${jsonPath} - run with env GEN_XML_FIXTURES=1 to update.`); - assert.deepEqual(KeymanXMLReader.removeSymbols(actual), expect, `Mismatch of ${xmlPath} vs ${jsonPath}`); + assert.deepEqual(SymbolUtils.removeSymbols(actual), expect, `Mismatch of ${xmlPath} vs ${jsonPath}`); } }); } @@ -178,11 +179,22 @@ describe(`XML Reader line number test`, () => { assert.ok(actual, `Parser failed on ${xmlPath}`); // now, assert char offset - assert.equal(actual.keyboard3[XML_START_INDEX_SYMBOL as any], 40); // index of element - assert.equal(actual.keyboard3.info[XML_START_INDEX_SYMBOL as any], 136); // index of etc - assert.equal(actual.keyboard3.transforms[XML_START_INDEX_SYMBOL as any], 186); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3[XML_START_INDEX_SYMBOL as any], lines), { line: 3, column: 0 }); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.info[XML_START_INDEX_SYMBOL as any], lines), { line: 4, column: 2 }); - assert.deepEqual(KeymanXMLReader.offsetToLineColumn(actual.keyboard3.transforms[XML_START_INDEX_SYMBOL as any], lines), { line: 8, column: 2 }); + const getMetaData = KeymanXMLReader.getMetaData; + assert.ok(getMetaData(actual.keyboard3)); + assert.equal( + getMetaData(actual.keyboard3)?.startIndex, 40); // index of element + assert.equal( + getMetaData(actual.keyboard3.info)?.startIndex, 136); // index of etc + assert.equal( + getMetaData(actual.keyboard3.transforms)?.startIndex, 186); + assert.deepEqual( + KeymanXMLReader.offsetToLineColumn( + getMetaData(actual.keyboard3).startIndex, lines), { line: 3, column: 0 }); + assert.deepEqual( + KeymanXMLReader.offsetToLineColumn( + getMetaData(actual.keyboard3.info).startIndex, lines), { line: 4, column: 2 }); + assert.deepEqual( + KeymanXMLReader.offsetToLineColumn( + getMetaData(actual.keyboard3.transforms).startIndex, lines), { line: 8, column: 2 }); }); }); diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 1fcb82193b..d363fa03db 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -1,5 +1,5 @@ import { util } from "@keymanapp/common-types"; -import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_START_INDEX_SYMBOL, XML_FILENAME_SYMBOL, CompilerEvent } from '@keymanapp/developer-utils'; +import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_FILENAME_SYMBOL, CompilerEvent, KeymanXMLReader } from '@keymanapp/developer-utils'; // const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.LdmlKeyboardCompiler; @@ -271,13 +271,14 @@ export class LdmlCompilerMessages { /** * Get an offset from o and set e's offset field * @param event a compiler event, such as from functions in this class - * @param x any object parsed from XML or with the XML_START_INDEX_SYMBOL symbol copied over + * @param x any object parsed from XML or with the XML_META_DATA_SYMBOL symbol copied over * @returns modified event object */ static offset(event: CompilerEvent, x?: any): CompilerEvent { if(x) { - event.offset = (x as any)[XML_START_INDEX_SYMBOL as any]; - event.filename = event.filename || (x as any)[XML_FILENAME_SYMBOL as any]; + const metadata = KeymanXMLReader.getMetaData(x) || {}; + event.offset = metadata?.startIndex; + event.filename = event.filename || metadata[XML_FILENAME_SYMBOL]; } return event; } diff --git a/developer/src/kmc-ldml/test/helpers/compareXml.ts b/developer/src/kmc-ldml/test/helpers/compareXml.ts index 1503d4c606..057986bdb2 100644 --- a/developer/src/kmc-ldml/test/helpers/compareXml.ts +++ b/developer/src/kmc-ldml/test/helpers/compareXml.ts @@ -1,6 +1,6 @@ import {assert} from 'chai'; import {readFileSync} from 'node:fs'; -import { KeymanXMLReader } from "@keymanapp/developer-utils"; +import { KeymanXMLReader, SymbolUtils } from "@keymanapp/developer-utils"; /** * @@ -21,5 +21,5 @@ export function compareXml(actual : string, expect: string, mutator?: (input: an const actualParsed = mutator(reader.parse(actualStr)); const expectParsed = mutator(reader.parse(expectStr)); - assert.deepEqual(KeymanXMLReader.removeSymbols(actualParsed), KeymanXMLReader.removeSymbols(expectParsed)); + assert.deepEqual(SymbolUtils.removeSymbols(actualParsed), SymbolUtils.removeSymbols(expectParsed)); } diff --git a/developer/src/kmc-ldml/test/helpers/index.ts b/developer/src/kmc-ldml/test/helpers/index.ts index 497f8ef919..bb10825315 100644 --- a/developer/src/kmc-ldml/test/helpers/index.ts +++ b/developer/src/kmc-ldml/test/helpers/index.ts @@ -8,7 +8,7 @@ import * as path from 'path'; import { fileURLToPath } from 'url'; import { SectionCompiler, SectionCompilerNew } from '../../src/compiler/section-compiler.js'; import { util, KMXPlus, LdmlKeyboardTypes } from '@keymanapp/common-types'; -import { CompilerEvent, compilerEventFormat, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboard, XML_START_INDEX_SYMBOL, } from "@keymanapp/developer-utils"; +import { CompilerEvent, compilerEventFormat, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboard, KeymanXMLMetadata, KeymanXMLReader } from "@keymanapp/developer-utils"; import { LdmlKeyboardCompiler } from '../../src/main.js'; // make sure main.js compiles import { assert } from 'chai'; import { KMXPlusMetadataCompiler } from '../../src/compiler/metadata-compiler.js'; @@ -329,8 +329,11 @@ export function hex_str(s?: string) : string { } /** return an object simulating an XML object with a column number */ -export function withColumn(c: number) : any { - return { - [XML_START_INDEX_SYMBOL as any]: c - }; +export function withColumn(c: number) : KeymanXMLMetadata { + // set metadata on an empty object + const o = {}; + KeymanXMLReader.setMetaData(o, { + startIndex: c + }); + return o; } diff --git a/developer/src/kmc-ldml/test/keys.tests.ts b/developer/src/kmc-ldml/test/keys.tests.ts index 1b74763646..3d87caaa7a 100644 --- a/developer/src/kmc-ldml/test/keys.tests.ts +++ b/developer/src/kmc-ldml/test/keys.tests.ts @@ -464,7 +464,7 @@ describe('keys.kmap', function () { let keys = await loadSectionFixture(KeysCompiler, 'sections/keys/invalid-key-missing-attrs.xml', compilerTestCallbacks, keysDependencies) as Keys; assert.isNull(keys); assert.equal(compilerTestCallbacks.messages.length, 1); - assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch({keyId: 'Q'})); + assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch({keyId: 'Q'}, withColumn(188))); }); it('should accept layouts with gap/switch keys', async function() { let keys = await loadSectionFixture(KeysCompiler, 'sections/keys/gap-switch.xml', compilerTestCallbacks, keysDependencies) as Keys; diff --git a/package-lock.json b/package-lock.json index 192f383713..10a5e5d9d6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9094,8 +9094,8 @@ "dev": true }, "node_modules/fast-xml-parser": { - "version": "5.0.8", - "resolved": "git+ssh://git@github.com/srl295/fast-xml-parser.git#72f20480e8591aca819fac684d46f8d678fcb7f3", + "version": "5.0.9", + "resolved": "git+ssh://git@github.com/srl295/fast-xml-parser.git#985271d7c731a6777456c362849d106073feaddd", "funding": [ { "type": "github", From 1f2b8463f5c368bd5fd79584b4249e00a78041ad Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 3 Apr 2025 12:15:20 -0500 Subject: [PATCH 015/431] chore(developer): deps: update to fast-xml-parser@5.1.0 (Metadata branch) Fixes: #10622 --- developer/src/common/web/utils/package.json | 2 +- package-lock.json | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/developer/src/common/web/utils/package.json b/developer/src/common/web/utils/package.json index a56fdc386a..4bceb5d6e4 100644 --- a/developer/src/common/web/utils/package.json +++ b/developer/src/common/web/utils/package.json @@ -12,7 +12,7 @@ "@keymanapp/common-types": "*", "@sentry/node": "^7.57.0", "eventemitter3": "^5.0.0", - "fast-xml-parser": "github:srl295/fast-xml-parser#line-number-593bis", + "fast-xml-parser": "github:NaturalIntelligence/fast-xml-parser#metadata", "path-browserify": "^1.0.1", "restructure": "^3.0.1", "sax": ">=0.6.0", diff --git a/package-lock.json b/package-lock.json index 10a5e5d9d6..05ed59e500 100644 --- a/package-lock.json +++ b/package-lock.json @@ -342,7 +342,7 @@ "@keymanapp/common-types": "*", "@sentry/node": "^7.57.0", "eventemitter3": "^5.0.0", - "fast-xml-parser": "github:srl295/fast-xml-parser#line-number-593bis", + "fast-xml-parser": "github:NaturalIntelligence/fast-xml-parser#metadata", "path-browserify": "^1.0.1", "restructure": "^3.0.1", "sax": ">=0.6.0", @@ -9094,15 +9094,14 @@ "dev": true }, "node_modules/fast-xml-parser": { - "version": "5.0.9", - "resolved": "git+ssh://git@github.com/srl295/fast-xml-parser.git#985271d7c731a6777456c362849d106073feaddd", + "version": "5.1.0", + "resolved": "git+ssh://git@github.com/NaturalIntelligence/fast-xml-parser.git#ab8d40a3077f109f34d02d05653146b8621bd255", "funding": [ { "type": "github", "url": "https://github.com/sponsors/NaturalIntelligence" } ], - "license": "MIT", "dependencies": { "strnum": "^2.0.5" }, From 8d438283c61afb6f26a5a70d106ed41ec91d3a8c Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 3 Apr 2025 16:40:53 -0500 Subject: [PATCH 016/431] feat(developer): update for line numbers - split out LineFinder to separate class Fixes: #10622 --- developer/src/common/web/utils/src/index.ts | 1 + .../src/common/web/utils/src/line-utils.ts | 61 +++++++++++++++++++ .../src/types/ldml-keyboard/eventresolver.ts | 20 ++++-- .../src/common/web/utils/src/xml-utils.ts | 28 --------- .../common/web/utils/test/line-utils.tests.ts | 28 +++++++++ .../common/web/utils/test/xml-utils.tests.ts | 10 +-- 6 files changed, 110 insertions(+), 38 deletions(-) create mode 100644 developer/src/common/web/utils/src/line-utils.ts create mode 100644 developer/src/common/web/utils/test/line-utils.tests.ts diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index 95ef39ffcb..dc51720b25 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -69,6 +69,7 @@ export { CommonTypesMessages } from './common-messages.js'; export * as SourceFilenamePatterns from './source-filename-patterns.js'; export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, KeymanXMLMetadata, XML_FILENAME_SYMBOL } from './xml-utils.js'; export { SymbolUtils } from './symbol-utils.js'; +export { LineFinder, LineColumn } from './line-utils.js'; export * as GitHubUrls from './github-urls.js'; export * as CloudUrls from './cloud-urls.js'; diff --git a/developer/src/common/web/utils/src/line-utils.ts b/developer/src/common/web/utils/src/line-utils.ts new file mode 100644 index 0000000000..5c9a4dec31 --- /dev/null +++ b/developer/src/common/web/utils/src/line-utils.ts @@ -0,0 +1,61 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by srl on 2025-04-03 + * + * Abstraction for line number processing + */ + +/** line number with an optional column number */ +export interface LineColumn { + line: number; + column?: number; +} + +export type LineList = number[]; + +/** a LineFinder is able to convert from an offset to a line/column */ +export class LineFinder { + private list: LineList; + constructor(public text: string) { + this.list = LineFinder.textToLines(text); + } + + /** + * Convert an offset into line:column + * @param offset input offset into the text + * @returns line:column information + */ + public findOffset(offset: number): LineColumn { + return LineFinder.offsetToLineColumn(offset, this.list); + } + + /** + * preprocess text to turn it into arrays of line lengths. + * This is effectively a 1-based line length, since line 0 has length of + * 0. + */ + public static textToLines(text: string): LineList { + return [ + 0, // "line 0" is empty + ...text.replaceAll("\r\n", "\n").split("\n") + .map(l => l.length + 1) // line length (counting the trailing newline) + ]; + } + + /** + * convert a line number array to a line/column + */ + public static offsetToLineColumn(offset: number, list: LineList): LineColumn { + for (let line = 1; line < list.length; line++) { // 1-based (assume the first row is 0) + if (list[line] > offset) { + return { line, column: offset }; + } + offset = offset - (list[line]); // count newline at end + } + // default: line 0, error + return { line: 0 } + } +} + + diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts index 3299169642..d155cf27d2 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/eventresolver.ts @@ -1,18 +1,26 @@ -import { CompilerEvent, EventResolver, KeymanXMLReader } from "../../index.js"; +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by srl on 2025-03-03 + * + * Implementation of a hook on LDML events + */ +import { CompilerEvent, EventResolver, LineFinder } from "../../index.js"; export class LdmlEventResolver implements EventResolver { - contentsCache: Map = new Map(); + contentsCache: Map = new Map(); - /** add or update a file */ + /** add or update a source file */ addFile(filename: string, xml: any) { - this.contentsCache.set(filename, KeymanXMLReader.textToLines(xml)); + this.contentsCache.set(filename, new LineFinder(xml)); } + /** resolve a CompilerEvent by expanding the line numbers */ resolve(event: CompilerEvent) { if (event.offset && !event.line && event.filename) { - const lines = this.contentsCache.get(event.filename); - const loc = KeymanXMLReader.offsetToLineColumn(event.offset, lines); + const lf = this.contentsCache.get(event.filename); + const loc = lf.findOffset(event.offset); event.line = loc.line; event.column = loc.column; } diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 435fbb4c20..9ec8872975 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -115,11 +115,6 @@ const GENERATOR_OPTIONS: KeymanXMLGeneratorOptionsBag = { }, }; -export interface LineColumn { - line: number; - column?: number; -} - export interface KeymanXMLMetadata extends XMLMetaData { /** override of name of XML file */ [XML_FILENAME_SYMBOL]?: string; @@ -151,29 +146,6 @@ export class KeymanXMLReader { return m; } - /** - * preprocess text to turn it into arrays of line lengths. - * This is effectively a 1-based line length, since line 0 has length of - * 0. - */ - public static textToLines(text: string) : number[] { - return [ - 0, // "line 0" is empty - ...text.replaceAll("\r\n", "\n").split("\n") - .map(l => l.length + 1) // line length (counting the trailing newline) - ]; - } - public static offsetToLineColumn(offset: number, lines: number[]) : LineColumn { - for (let line = 1; line < lines.length; line++) { // 1-based (assume the first row is 0) - if (lines[line] > offset) { - return { line, column: offset }; - } - offset = offset - (lines[line]); // count newline at end - } - // default: line 0, error - return { line: 0 } - } - /** move `{ $abc: 4 }` into `{ $: { abc: 4 } }` */ private static fixupDollarAttributes(data: any) : any { if (typeof data === 'object') { diff --git a/developer/src/common/web/utils/test/line-utils.tests.ts b/developer/src/common/web/utils/test/line-utils.tests.ts new file mode 100644 index 0000000000..9c7b5da01c --- /dev/null +++ b/developer/src/common/web/utils/test/line-utils.tests.ts @@ -0,0 +1,28 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by srl on 2025-04-03 + * + * Test for LineFinder + */ + +import { assert } from 'chai'; +import { readFileSync } from 'node:fs'; +import 'mocha'; +import { LineFinder } from '../src/line-utils.js'; +import { makePathToFixture } from './helpers/index.js'; + +describe(`LineFinder test`, () => { + const path = 'tran_fail-empty.xml'; + + it(`Should be able to process ${path}`, () => { + const xmlPath = makePathToFixture('xml', `${path}`); + const data = readFileSync(xmlPath, 'utf-8'); + assert.ok(data); + const lf = new LineFinder(data); + assert.ok(lf); + assert.deepEqual(lf.findOffset(40), {line: 3, column: 0}); + assert.deepEqual(lf.findOffset(136), {line: 4, column: 2}); + assert.deepEqual(lf.findOffset(186), {line: 8, column: 2}); + }); +}); diff --git a/developer/src/common/web/utils/test/xml-utils.tests.ts b/developer/src/common/web/utils/test/xml-utils.tests.ts index 8728eec207..4b3df736b6 100644 --- a/developer/src/common/web/utils/test/xml-utils.tests.ts +++ b/developer/src/common/web/utils/test/xml-utils.tests.ts @@ -13,6 +13,7 @@ import { readFileSync, writeFileSync } from 'node:fs'; import { KeymanXMLType, KeymanXMLReader, KeymanXMLWriter } from '../src/xml-utils.js'; +import { LineFinder } from '../src/line-utils.js'; import { makePathToFixture } from './helpers/index.js'; import { SymbolUtils } from '../src/symbol-utils.js'; @@ -175,7 +176,7 @@ describe(`XML Reader line number test`, () => { // now, parse. subsitute endings for Win const actual = reader.parse(xml); - const lines = KeymanXMLReader.textToLines(xml); + const lines = LineFinder.textToLines(xml); assert.ok(actual, `Parser failed on ${xmlPath}`); // now, assert char offset @@ -188,13 +189,14 @@ describe(`XML Reader line number test`, () => { assert.equal( getMetaData(actual.keyboard3.transforms)?.startIndex, 186); assert.deepEqual( - KeymanXMLReader.offsetToLineColumn( + LineFinder.offsetToLineColumn( getMetaData(actual.keyboard3).startIndex, lines), { line: 3, column: 0 }); assert.deepEqual( - KeymanXMLReader.offsetToLineColumn( + LineFinder.offsetToLineColumn( getMetaData(actual.keyboard3.info).startIndex, lines), { line: 4, column: 2 }); assert.deepEqual( - KeymanXMLReader.offsetToLineColumn( + LineFinder.offsetToLineColumn( getMetaData(actual.keyboard3.transforms).startIndex, lines), { line: 8, column: 2 }); }); }); + From 11f77ed87647ff4e1e32c90388d664a939168092 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 3 Apr 2025 16:47:04 -0500 Subject: [PATCH 017/431] feat(developer): update for line numbers - fix ts issue on reader Fixes: #10622 --- .../utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index 84e82c62ea..2b5b0fc746 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -281,7 +281,7 @@ export class LDMLKeyboardXMLSourceFileReader { // Mark all children as an import subsubval.forEach(o => { o[ImportStatus.import] = basePath; - KeymanXMLReader.setMetaData(o, {[XML_FILENAME_SYMBOL]: importPath}); // mark overriding import path + KeymanXMLReader.setMetaData(o, {[XML_FILENAME_SYMBOL as any]: importPath}); // mark overriding import path }); if (implied) { From 53fd228d0889da1fb1f3aabf791af3fae9bb9d2b Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 3 Apr 2025 17:24:57 -0500 Subject: [PATCH 018/431] feat(developer): update for line numbers - fix windows path specific issue in test Fixes: #10622 --- .../web/utils/test/ldml/ldml-keyboard-xml-reader.tests.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/src/common/web/utils/test/ldml/ldml-keyboard-xml-reader.tests.ts b/developer/src/common/web/utils/test/ldml/ldml-keyboard-xml-reader.tests.ts index 6da13e0645..af0005cb6d 100644 --- a/developer/src/common/web/utils/test/ldml/ldml-keyboard-xml-reader.tests.ts +++ b/developer/src/common/web/utils/test/ldml/ldml-keyboard-xml-reader.tests.ts @@ -151,7 +151,7 @@ describe('ldml keyboard xml reader tests', function () { assert.ok(metadata); const snailFilename = (metadata)[XML_FILENAME_SYMBOL]; assert.ok(snailFilename); - assert.ok(/\/keys-Zyyy-morepunctuation.xml$/.test(snailFilename) + assert.ok(/keys-Zyyy-morepunctuation.xml$/.test(snailFilename) , `snail key filename is ${snailFilename}`); }, }, From 34a58ca9f92bc17f73dc583f090c6ccf3b7fb63a Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 7 Apr 2025 13:31:00 +0700 Subject: [PATCH 019/431] fix(web): base key should emitted even if at reset-recentering trigger --- .../internal/gestures/keyman-keyboard-gesture-specs.md | 6 +++--- web/src/engine/osk/src/input/gestures/specsForLayout.ts | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/web/docs/internal/gestures/keyman-keyboard-gesture-specs.md b/web/docs/internal/gestures/keyman-keyboard-gesture-specs.md index c3d8955eb1..acbd3a1369 100644 --- a/web/docs/internal/gestures/keyman-keyboard-gesture-specs.md +++ b/web/docs/internal/gestures/keyman-keyboard-gesture-specs.md @@ -336,8 +336,8 @@ graph TD; end recenter-condition -- Yes --> flick-restart - recenter-condition -- No + input finishes --> reject - %% comment %% (should --> flick-reset-end) + recenter-condition -- No + input finishes --> flick-reset-end + %% comment %% ... could treat as a --> reject if we wanted cancellation per #10876... subgraph Stage6[" "] flick-restart --> restart-move-condition @@ -345,7 +345,7 @@ graph TD; restart-move-condition -- Yes --> flick-mid restart-move-condition -- No + input finishes --> flick-reset-end - %% comment %% ... could treat as a --> reject if we wanted a semblance of cancellation... + %% comment %% ... could treat as a --> reject if we wanted cancellation per #10876... ``` Note that the early cases from 'flick-start' that lead to 'reject' are all valid inputs for gesture types handled by previously-defined FSMs. \ No newline at end of file diff --git a/web/src/engine/osk/src/input/gestures/specsForLayout.ts b/web/src/engine/osk/src/input/gestures/specsForLayout.ts index b0de9fece2..8d1d1a0958 100644 --- a/web/src/engine/osk/src/input/gestures/specsForLayout.ts +++ b/web/src/engine/osk/src/input/gestures/specsForLayout.ts @@ -959,6 +959,13 @@ export function flickResetCenteringModel(params: FullGestureParams): GestureMode type: 'chain', next: 'flick-restart' }, + rejectionActions: { + // Only 'rejects' in this form if the path is completed before direction-locking state. + path: { + type: 'replace', + replace: 'flick-reset-end' + } + }, sustainWhenNested: true }; } From e003612ea05b5bdcb048868b7900250088e82ca7 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 10 Apr 2025 17:22:51 -0500 Subject: [PATCH 020/431] feat(developer): update for line numbers - remove resolver complexity in callback subclasses - call LDMLKeyboardXMLSourceFileReader.eventResolver.resolve() statically from ldml-compiler-messages Fixes: #10622 --- .../web/test-helpers/TestCompilerCallbacks.ts | 13 +------------ .../common/web/utils/src/compiler-callbacks.ts | 16 ---------------- .../ldml-keyboard/ldml-keyboard-xml-reader.ts | 1 - .../src/compiler/ldml-compiler-messages.ts | 9 +++++++-- .../src/kmc/src/util/NodeCompilerCallbacks.ts | 13 +------------ 5 files changed, 9 insertions(+), 43 deletions(-) diff --git a/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts b/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts index cd1c816aaa..893e3363ed 100644 --- a/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts +++ b/developer/src/common/web/test-helpers/TestCompilerCallbacks.ts @@ -2,7 +2,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { CompilerEvent, CompilerCallbacks, CompilerPathCallbacks, CompilerFileSystemCallbacks, CompilerError, CompilerNetAsyncCallbacks, DefaultCompilerFileSystemAsyncCallbacks, - CompilerFileSystemAsyncCallbacks, EventResolver, NullEventResolver, + CompilerFileSystemAsyncCallbacks, CompilerErrorSeverity} from '@keymanapp/developer-utils'; import { fileURLToPath } from 'url'; @@ -111,18 +111,7 @@ export class TestCompilerCallbacks implements CompilerCallbacks { return filename; } - private eventResolver = new NullEventResolver(); - - setEventResolver(eventResolver: EventResolver): void { - this.eventResolver = eventResolver; - } - reportMessage(event: CompilerEvent): void { - this.eventResolver.resolve(event); - this.handleReportMessage(event); - } - - handleReportMessage(event: CompilerEvent): void { // console.log(event.message); this.messages.push(event); } diff --git a/developer/src/common/web/utils/src/compiler-callbacks.ts b/developer/src/common/web/utils/src/compiler-callbacks.ts index f8b42afb7a..aabf1794e5 100644 --- a/developer/src/common/web/utils/src/compiler-callbacks.ts +++ b/developer/src/common/web/utils/src/compiler-callbacks.ts @@ -55,12 +55,8 @@ export interface CompilerCallbacks { */ resolveFilename(baseFilename: string, filename: string): string; - /** change the EventResolver to use. If called more than once, the last resolver replaces a prior one. */ - setEventResolver(eventResolver: EventResolver): void; - /** * Report a message from the compiler back to the caller. - * It is expected that eventResolver.resolve() will be called on the message before processing. * @param event */ reportMessage(event: CompilerEvent): void; @@ -225,19 +221,7 @@ export class CompilerFileCallbacks implements CompilerCallbacks { return this.parent.resolveFilename(baseFilename, filename); } - private eventResolver = new NullEventResolver(); - - setEventResolver(eventResolver: EventResolver): void { - this.eventResolver = eventResolver; - this.parent.setEventResolver(eventResolver); - } - reportMessage(event: CompilerEvent): void { - this.eventResolver.resolve(event); - this.handleReportMessage(event); - } - - handleReportMessage(event: CompilerEvent): void { const disable = CompilerFileCallbacks.applyMessageOverridesToEvent(event, this.options.messageOverrides); this.messages.push(event); if (!disable) { diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index 36dab45202..2d099849cd 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -30,7 +30,6 @@ export class LDMLKeyboardXMLSourceFileReaderOptions { export class LDMLKeyboardXMLSourceFileReader { static eventResolver: LdmlEventResolver = new LdmlEventResolver(); constructor(private options: LDMLKeyboardXMLSourceFileReaderOptions, private callbacks : CompilerCallbacks) { - callbacks.setEventResolver(LDMLKeyboardXMLSourceFileReader.eventResolver); } static get defaultImportsURL(): [string,string] { diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 3a26c9a5fe..6deeedfceb 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -1,5 +1,5 @@ import { util } from "@keymanapp/common-types"; -import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_FILENAME_SYMBOL, CompilerEvent, KeymanXMLReader } from '@keymanapp/developer-utils'; +import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_FILENAME_SYMBOL, CompilerEvent, KeymanXMLReader, LDMLKeyboardXMLSourceFileReader } from '@keymanapp/developer-utils'; // const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.LdmlKeyboardCompiler; @@ -10,7 +10,12 @@ const SevError = CompilerErrorSeverity.Error | CompilerErrorNamespace.LdmlKeyboa const SevErrorTransform = SevError | 0xF00; // like m() but takes an XML object for line numbers -const mx = (x: any, code: number, message: string, detail?: string): CompilerEvent => LdmlCompilerMessages.offset(m(code, message, detail), x); +const mx = (x: any, code: number, message: string, detail?: string): CompilerEvent => { + let evt = m(code, message, detail); // raw message + evt = LdmlCompilerMessages.offset(evt, x); // with offset + evt = LDMLKeyboardXMLSourceFileReader.eventResolver.resolve(evt); // offset expanded to line number + return evt; +}; /** * @internal diff --git a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts index d42e3b6545..2ec3854c3e 100644 --- a/developer/src/kmc/src/util/NodeCompilerCallbacks.ts +++ b/developer/src/kmc/src/util/NodeCompilerCallbacks.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { platform } from 'os'; -import { CompilerCallbacks, CompilerEvent, EventResolver, NullEventResolver, +import { CompilerCallbacks, CompilerEvent, CompilerPathCallbacks, CompilerFileSystemCallbacks, compilerLogLevelToSeverity, CompilerErrorSeverity, CompilerError, @@ -148,18 +148,7 @@ export class NodeCompilerCallbacks implements CompilerCallbacks { return fileURLToPath(url); } - private eventResolver = new NullEventResolver(); - - setEventResolver(eventResolver: EventResolver): void { - this.eventResolver = eventResolver; - } - reportMessage(event: CompilerEvent): void { - this.eventResolver.resolve(event); - this.handleReportMessage(event); - } - - handleReportMessage(event: CompilerEvent): void { if(!event.filename) { event.filename = this.messageFilename; } From ddefb79b709d77c6f9c0dcf296921ba1c1b6aeed Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 21 Apr 2025 08:18:56 +0700 Subject: [PATCH 021/431] fix(web): reconnect unit tests for low-level web utility classes/methods --- web/build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/web/build.sh b/web/build.sh index 0dd9b5ef53..49160de690 100755 --- a/web/build.sh +++ b/web/build.sh @@ -26,6 +26,7 @@ builder_describe "Builds engine modules for Keyman Engine for Web (KMW)." \ ":app/webview A puppetable version of KMW designed for use in a host app's WebView" \ ":app/ui Builds KMW's desktop form-factor keyboard-selection UI modules" \ ":engine/attachment Subset used for detecting valid page contexts for use in text editing " \ + ":engine/common/web-utils Low-level, headless utility methods and classes used across multiple modules" \ ":engine/dom-utils A common subset of function used for DOM calculations, layout, etc" \ ":engine/events Specialized classes utilized to support KMW API events" \ ":engine/element-wrappers Subset used to integrate with website elements" \ @@ -70,6 +71,7 @@ builder_describe_outputs \ build:engine/main "/web/build/engine/main/lib/index.mjs" \ build:engine/osk "/web/build/engine/osk/lib/index.mjs" \ build:engine/predictive-text "/web/src/engine/predictive-text/worker-main/build/lib/web/index.mjs" \ + build:engine/common/web-utils "/web/src/engine/common/web-utils/build/lib/index.mjs" \ build:samples "/web/src/samples/simplest/keymanweb.js" \ build:tools "/web/build/tools/building/sourcemap-root/index.js" \ build:test-pages "/web/build/test-resources/sentry-manager.js" @@ -163,6 +165,7 @@ coverage_action() { rm -rf build/coverage/tmp } +builder_run_child_actions build:engine/common/web-utils builder_run_child_actions build:engine/dom-utils builder_run_child_actions build:engine/keyboard From 5b14b858fbce372db09cd01e0a034662015cc1ae Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 21 Apr 2025 08:35:26 +0700 Subject: [PATCH 022/431] chore(web): replace kmwFromCharCode with its ES6 equivalent --- .../engine/common/web-utils/src/kmwstring.ts | 30 +------------------ web/src/engine/keyboard/src/defaultRules.ts | 6 ++-- .../keyboard/src/keyboards/activeLayout.ts | 4 +-- 3 files changed, 4 insertions(+), 36 deletions(-) diff --git a/web/src/engine/common/web-utils/src/kmwstring.ts b/web/src/engine/common/web-utils/src/kmwstring.ts index 4766bbc1d9..eb80e6121c 100644 --- a/web/src/engine/common/web-utils/src/kmwstring.ts +++ b/web/src/engine/common/web-utils/src/kmwstring.ts @@ -10,8 +10,6 @@ declare global { interface StringConstructor { - kmwFromCharCode(cp0: number): string, - _kmwFromCharCode(cp0: number): string, kmwEnableSupplementaryPlane(bEnable: boolean): void } @@ -50,31 +48,6 @@ declare global { } export default function extendString() { - /** - * Constructs a string from one or more Unicode character codepoint values - * passed as integer parameters. - * - * @param {number} cp0,... 1 or more Unicode codepoints, e.g. 0x0065, 0x10000 - * @return {string|null} The new String object. - */ - String.kmwFromCharCode = function(cp0) { - var chars = [], i; - for (i = 0; i < arguments.length; i++) { - var c = Number(arguments[i]); - if (!isFinite(c) || c < 0 || c > 0x10FFFF || Math.floor(c) !== c) { - throw new RangeError("Invalid code point " + c); - } - if (c < 0x10000) { - chars.push(c); - } else { - c -= 0x10000; - chars.push((c >> 10) + 0xD800); - chars.push((c % 0x400) + 0xDC00); - } - } - return String.fromCharCode.apply(undefined, chars); - } - /** * Returns a number indicating the Unicode value of the character at the given * code point index, with support for supplementary plane characters. @@ -456,7 +429,6 @@ export default function extendString() { String.kmwEnableSupplementaryPlane = function(bEnable) { var p=String.prototype; - String._kmwFromCharCode = bEnable ? String.kmwFromCharCode : String.fromCharCode; p._kmwCharAt = bEnable ? p.kmwCharAt : p.charAt; p._kmwCharCodeAt = bEnable ? p.kmwCharCodeAt : p.charCodeAt; p._kmwIndexOf = bEnable ? p.kmwIndexOf :p.indexOf; @@ -472,7 +444,7 @@ export default function extendString() { } // Ensure that _all_ String extensions are established, even if disabled by default. - if(!String._kmwFromCharCode) { + if(!""._kmwCharAt) { String.kmwEnableSupplementaryPlane(false); } } diff --git a/web/src/engine/keyboard/src/defaultRules.ts b/web/src/engine/keyboard/src/defaultRules.ts index a86d32a4be..c78c743e8b 100644 --- a/web/src/engine/keyboard/src/defaultRules.ts +++ b/web/src/engine/keyboard/src/defaultRules.ts @@ -146,7 +146,7 @@ export default class DefaultRules { } else { Lch = Lkc.Lcode-64; } - let ch = String._kmwFromCharCode(Lch); //I3319 + let ch = String.fromCodePoint(Lch); return ch; } else { return null; @@ -178,9 +178,7 @@ export default class DefaultRules { // We'll attempt to add valid chars continue; } else { - // String.fromCharCode() is inadequate to handle the entire range of Unicode - // Someday after upgrading to ES2015, can use String.fromCodePoint() - result += String.kmwFromCharCode(codePointValue); + result += String.fromCodePoint(codePointValue); } } return result ? result : null; diff --git a/web/src/engine/keyboard/src/keyboards/activeLayout.ts b/web/src/engine/keyboard/src/keyboards/activeLayout.ts index c440511f17..f04a22246c 100644 --- a/web/src/engine/keyboard/src/keyboards/activeLayout.ts +++ b/web/src/engine/keyboard/src/keyboards/activeLayout.ts @@ -284,9 +284,7 @@ export class ActiveKeyBase { } continue; } else { - // String.fromCharCode() is inadequate to handle the entire range of Unicode - // Someday after upgrading to ES2015, can use String.fromCodePoint() - result += String.kmwFromCharCode(codePointValue); + result += String.fromCodePoint(codePointValue); } } return result ? result : null; From be15fc1c3fa14732a0528fad8574d03222bd8f2d Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 21 Apr 2025 13:09:01 +0700 Subject: [PATCH 023/431] chore(web): add unit tests for SMP-aware string-handling helper methods --- .../web-utils/src/tests/stringHandling.js | 446 ++++++++++++++++++ 1 file changed, 446 insertions(+) create mode 100644 web/src/engine/common/web-utils/src/tests/stringHandling.js diff --git a/web/src/engine/common/web-utils/src/tests/stringHandling.js b/web/src/engine/common/web-utils/src/tests/stringHandling.js new file mode 100644 index 0000000000..bc7e250c77 --- /dev/null +++ b/web/src/engine/common/web-utils/src/tests/stringHandling.js @@ -0,0 +1,446 @@ +import { assert } from 'chai'; +// Trigger the String-extending methods. +import * as utils from '@keymanapp/web-utils'; + +describe.only('Unicode string handling', () => { + describe('disabled: BMP-mode, BMP-only strings', () => { + before(() => { + String.kmwEnableSupplementaryPlane(false); + }); + + it('_kmwCharAt', () => { + const str = "string"; + assert.equal(str._kmwCharAt(-1), ''); + assert.equal(str._kmwCharAt(6), ''); + assert.equal(str._kmwCharAt(1000), ''); + assert.equal(str._kmwCharAt(0), "s"); + assert.equal(str._kmwCharAt(2), "r"); + assert.equal(str._kmwCharAt(5), "g"); + }); + + it('_kmwCharCodeAt', () => { + const str = "string" + assert.isNaN(str._kmwCharCodeAt(-1)); + assert.isNaN(str._kmwCharCodeAt(6)); + assert.isNaN(str._kmwCharCodeAt(1000)); + assert.equal(str._kmwCharCodeAt(0), "s".charCodeAt(0)); + assert.equal(str._kmwCharCodeAt(2), "r".charCodeAt(0)); + assert.equal(str._kmwCharCodeAt(5), "g".charCodeAt(0)); + }); + + it('_kmwIndexOf', () => { + const banana_nation = "banana nation"; + assert.equal(banana_nation._kmwIndexOf("XYZ"), -1); + assert.equal(banana_nation._kmwIndexOf("na"), 2); + assert.equal(banana_nation._kmwIndexOf("na", -1), 2); + assert.equal(banana_nation._kmwIndexOf("na", 0), 2); + assert.equal(banana_nation._kmwIndexOf("na", 2), 2); + assert.equal(banana_nation._kmwIndexOf("na", 3), 4); + assert.equal(banana_nation._kmwIndexOf("na", 5), 7); + assert.equal(banana_nation._kmwIndexOf("na", 8), -1); + assert.equal(banana_nation._kmwIndexOf("na", 1000), -1); + }); + + it('_kmwLastIndexOf', () => { + const banana_nation = "banana nation"; + assert.equal(banana_nation._kmwLastIndexOf("XYZ"), -1); + assert.equal(banana_nation._kmwLastIndexOf("na"), 7); + assert.equal(banana_nation._kmwLastIndexOf("na", -1), -1); + assert.equal(banana_nation._kmwLastIndexOf("na", 0), -1); + assert.equal(banana_nation._kmwLastIndexOf("na", 2), 2); + assert.equal(banana_nation._kmwLastIndexOf("na", 3), 2); + assert.equal(banana_nation._kmwLastIndexOf("na", 5), 4); + assert.equal(banana_nation._kmwLastIndexOf("na", 8), 7); + assert.equal(banana_nation._kmwLastIndexOf("na", 1000), 7); + }); + + it('_kmwLength', () => { + assert.equal(""._kmwLength(), 0); + assert.equal("string"._kmwLength(), 6); + // From the current MDN example: + assert.equal("Life, the universe and everything. Answer:"._kmwLength(), 42); + }); + + it('_kmwSlice', () => { + // From https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/slice + const str = "The quick brown fox jumps over the lazy dog."; + + assert.equal(str._kmwSlice(31), "the lazy dog."); + assert.equal(str._kmwSlice(4, 19), "quick brown fox"); + assert.equal(str._kmwSlice(-4), "dog."); + assert.equal(str._kmwSlice(-9, -5), "lazy"); + assert.equal(str._kmwSlice(), str); + + // Per documented spec for slice... + assert.equal(str._kmwSlice(31, -5), "the lazy"); // " dog." = 5 chars + assert.equal(str._kmwSlice(-5, -9), ""); + }); + + it('_kmwSubstr', () => { + const str = "The quick brown fox jumps over the lazy dog."; + + assert.equal(str._kmwSubstr(1000), ''); + assert.equal(str._kmwSubstr(31), "the lazy dog."); + assert.equal(str._kmwSubstr(-4), "dog."); + assert.equal(str._kmwSubstr(31, 31), "the lazy dog."); + assert.equal(str._kmwSubstr(31, -5), ''); // The big difference from .slice: length must be non-negative. + }); + + it('_kmwSubstring', () => { + const str = "The quick brown fox jumps over the lazy dog."; + + assert.equal(str._kmwSubstring(1000), ''); + assert.equal(str._kmwSubstring(31), "the lazy dog."); + assert.equal(str._kmwSubstring(31, 1000), "the lazy dog."); + assert.equal(str._kmwSubstring(4, 9), "quick"); + + // negatives are coerced to 0 & parameters are used in order of ascending value. + assert.equal(str._kmwSubstring(9, 4), "quick"); + assert.equal(str._kmwSubstring(31, -4), str.substring(0, 31)); + assert.equal(str._kmwSubstring(-5, 9), str.substring(0, 9)); + assert.equal(str._kmwSubstring(-9, -5), ''); + }); + + it('kmwCodePointToCodeUnit', () => { + const str = "The quick brown fox jumps over the lazy dog."; + assert.equal(str.kmwCodePointToCodeUnit(2), 2); + assert.equal(str.kmwCodePointToCodeUnit(9), 9); + assert.equal(str.kmwCodePointToCodeUnit(31), 31); + }); + + it('kmwCodeUnitToCodePoint', () => { + const str = "The quick brown fox jumps over the lazy dog."; + assert.equal(str.kmwCodeUnitToCodePoint(2), 2); + assert.equal(str.kmwCodeUnitToCodePoint(9), 9); + assert.equal(str.kmwCodeUnitToCodePoint(31), 31); + }); + }); + + describe('enabled: SMP-mode, but BMP-only strings', () => { + before(() => { + String.kmwEnableSupplementaryPlane(true); + }); + + it('_kmwCharAt', () => { + const str = "string"; + assert.equal(str._kmwCharAt(-1), ''); + assert.equal(str._kmwCharAt(6), ''); + assert.equal(str._kmwCharAt(1000), ''); + assert.equal(str._kmwCharAt(0), "s"); + assert.equal(str._kmwCharAt(2), "r"); + assert.equal(str._kmwCharAt(5), "g"); + }); + + it('_kmwCharCodeAt', () => { + const str = "string"; + assert.isNaN(str._kmwCharCodeAt(-1)); + assert.isNaN(str._kmwCharCodeAt(6)); + assert.isNaN(str._kmwCharCodeAt(1000)); + assert.equal(str._kmwCharCodeAt(0), "s".charCodeAt(0)); + assert.equal(str._kmwCharCodeAt(2), "r".charCodeAt(0)); + assert.equal(str._kmwCharCodeAt(5), "g".charCodeAt(0)); + }); + + it('_kmwIndexOf', () => { + const banana_nation = "banana nation"; + assert.equal(banana_nation._kmwIndexOf("XYZ"), -1); + assert.equal(banana_nation._kmwIndexOf("na"), 2); + assert.equal(banana_nation._kmwIndexOf("na", -1), 2); + assert.equal(banana_nation._kmwIndexOf("na", 0), 2); + assert.equal(banana_nation._kmwIndexOf("na", 2), 2); + assert.equal(banana_nation._kmwIndexOf("na", 3), 4); + assert.equal(banana_nation._kmwIndexOf("na", 5), 7); + assert.equal(banana_nation._kmwIndexOf("na", 8), -1); + assert.equal(banana_nation._kmwIndexOf("na", 1000), -1); + }); + + it('_kmwLastIndexOf', () => { + const banana_nation = "banana nation"; + assert.equal(banana_nation._kmwLastIndexOf("XYZ"), -1); + assert.equal(banana_nation._kmwLastIndexOf("na"), 7); + assert.equal(banana_nation._kmwLastIndexOf("na", -1), -1); + assert.equal(banana_nation._kmwLastIndexOf("na", 0), -1); + assert.equal(banana_nation._kmwLastIndexOf("na", 2), 2); + assert.equal(banana_nation._kmwLastIndexOf("na", 3), 2); + assert.equal(banana_nation._kmwLastIndexOf("na", 5), 4); + assert.equal(banana_nation._kmwLastIndexOf("na", 8), 7); + assert.equal(banana_nation._kmwLastIndexOf("na", 1000), 7); + }); + + it('_kmwLength', () => { + assert.equal(""._kmwLength(), 0); + assert.equal("string"._kmwLength(), 6); + // From the current MDN example: + assert.equal("Life, the universe and everything. Answer:"._kmwLength(), 42); + }); + + it('_kmwSlice', () => { + // From https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/slice + const str = "The quick brown fox jumps over the lazy dog."; + + // FAILS: does not handle `undefined` length like the real .slice() method. + // assert.equal(str._kmwSlice(31), "the lazy dog."); + assert.equal(str._kmwSlice(4, 19), "quick brown fox"); + // FAILS: does not handle `undefined` length like the real .slice() method. + // assert.equal(str._kmwSlice(-4), "dog."); + assert.equal(str._kmwSlice(-9, -5), "lazy"); + // FAILS: does not handle `undefined` start like the real .slice() method. + // assert.equal(str._kmwSlice(), str); + + // Per documented spec for slice... + assert.equal(str._kmwSlice(31, -5), "the lazy"); // " dog." = 5 chars + assert.equal(str._kmwSlice(-5, -9), ""); + }); + + it('_kmwSubstr', () => { + const str = "The quick brown fox jumps over the lazy dog."; + + assert.equal(str._kmwSubstr(1000), ''); + assert.equal(str._kmwSubstr(31), "the lazy dog."); + assert.equal(str._kmwSubstr(-4), "dog."); + assert.equal(str._kmwSubstr(31, 31), "the lazy dog."); + assert.equal(str._kmwSubstr(31, -5), ''); // The big difference from .slice: length must be non-negative. + }); + + it('_kmwSubstring', () => { + const str = "The quick brown fox jumps over the lazy dog."; + + // FAILS - doesn't coerce the out-of-bounds value. + // assert.equal(str._kmwSubstring(1000), ''); + assert.equal(str._kmwSubstring(31), "the lazy dog."); + assert.equal(str._kmwSubstring(31, 1000), "the lazy dog."); + assert.equal(str._kmwSubstring(4, 9), "quick"); + + // negatives are coerced to 0 & parameters are used in order of ascending value. + assert.equal(str._kmwSubstring(9, 4), "quick"); + // FAILS: these do not handle negative-valued inputs the same way the real .substring does. + // assert.equal(str._kmwSubstring(31, -4), str.substring(0, 31)); + // assert.equal(str._kmwSubstring(-5, 9), str.substring(0, 9)); + // assert.equal(str._kmwSubstring(-9, -5), ''); + }); + + it('kmwCodePointToCodeUnit', () => { + const str = "The quick brown fox jumps over the lazy dog."; + assert.equal(str.kmwCodePointToCodeUnit(2), 2); + assert.equal(str.kmwCodePointToCodeUnit(9), 9); + assert.equal(str.kmwCodePointToCodeUnit(31), 31); + }); + + it('kmwCodeUnitToCodePoint', () => { + const str = "The quick brown fox jumps over the lazy dog."; + assert.equal(str.kmwCodeUnitToCodePoint(2), 2); + assert.equal(str.kmwCodeUnitToCodePoint(9), 9); + assert.equal(str.kmwCodeUnitToCodePoint(31), 31); + }); + }); + + + describe('enabled: SMP-mode, with mixed-plane strings', () => { + before(() => { + String.kmwEnableSupplementaryPlane(true); + }); + + it('_kmwCharAt', () => { + // 0x1d5c9: MATHEMATICAL SANS-SERIF SMALL p + // 0x1d5be: MATHEMATICAL SANS-SERIF SMALL e + const apples = 'a' + String.fromCodePoint(0x1d5c9) + 'p' + 'l' + String.fromCodePoint(0x1d5be) + 's'; + assert.equal(apples._kmwCharAt(-1), ''); + assert.equal(apples._kmwCharAt(6), ''); + assert.equal(apples._kmwCharAt(1000), ''); + assert.equal(apples._kmwCharAt(0), "a"); + assert.equal(apples._kmwCharAt(1), String.fromCodePoint(0x1d5c9)); + assert.equal(apples._kmwCharAt(2), "p"); + assert.equal(apples._kmwCharAt(5), "s"); + }); + + it('_kmwCharCodeAt', () => { + // 0x1d5c9: MATHEMATICAL SANS-SERIF SMALL p + // 0x1d5be: MATHEMATICAL SANS-SERIF SMALL e + const apples = 'a' + String.fromCodePoint(0x1d5c9) + 'p' + 'l' + String.fromCodePoint(0x1d5be) + 's'; + assert.isNaN(apples._kmwCharCodeAt(-1)); + assert.isNaN(apples._kmwCharCodeAt(6)); + assert.isNaN(apples._kmwCharCodeAt(1000)); + assert.equal(apples._kmwCharCodeAt(0), "a".charCodeAt(0)); + assert.equal(apples._kmwCharCodeAt(1), 0x1d5c9); + assert.equal(apples._kmwCharCodeAt(2), "p".charCodeAt(0)); + assert.equal(apples._kmwCharCodeAt(5), "s".charCodeAt(0)); + }); + + it('_kmwIndexOf', () => { + // 0x1d5ba: MATHEMATICAL SANS-SERIF SMALL a + const a = String.fromCodePoint(0x1d5ba); + const banana_nation = "ban" + a + "na n" + a + "tion"; + assert.equal(banana_nation._kmwIndexOf("XYZ"), -1); + assert.equal(banana_nation._kmwIndexOf("na"), 4); + assert.equal(banana_nation._kmwIndexOf("na", -1), 4); + assert.equal(banana_nation._kmwIndexOf("na", 0), 4); + assert.equal(banana_nation._kmwIndexOf("na", 2), 4); + assert.equal(banana_nation._kmwIndexOf("na", 3), 4); + // ERROR: returns 4 (!) + // assert.equal(banana_nation._kmwIndexOf("na", 5), 7); + assert.equal(banana_nation._kmwIndexOf("na", 8), -1); + assert.equal(banana_nation._kmwIndexOf("na", 1000), -1); + + const na = "n" + a; + assert.equal(banana_nation._kmwIndexOf(na), 2); + assert.equal(banana_nation._kmwIndexOf(na, -1), 2); + assert.equal(banana_nation._kmwIndexOf(na, 0), 2); + assert.equal(banana_nation._kmwIndexOf(na, 2), 2); + assert.equal(banana_nation._kmwIndexOf(na, 3), 7); + assert.equal(banana_nation._kmwIndexOf(na, 5), 7); + // ERROR: returns 7 (!) + // assert.equal(banana_nation._kmwIndexOf(na, 8), -1); + assert.equal(banana_nation._kmwIndexOf(na, 1000), -1); + }); + + it('_kmwLastIndexOf', () => { + // 0x1d5ba: MATHEMATICAL SANS-SERIF SMALL a + const a = String.fromCodePoint(0x1d5ba); + const banana_nation = "ban" + a + "na n" + a + "tion"; + assert.equal(banana_nation._kmwLastIndexOf("XYZ"), -1); + assert.equal(banana_nation._kmwLastIndexOf("na"), 4); + assert.equal(banana_nation._kmwLastIndexOf("na", -1), -1); + assert.equal(banana_nation._kmwLastIndexOf("na", 0), -1); + assert.equal(banana_nation._kmwLastIndexOf("na", 2), -1); + assert.equal(banana_nation._kmwLastIndexOf("na", 3), -1); + + // ERROR: Can't find the instance at the starting index? + // assert.equal(banana_nation._kmwLastIndexOf("na", 4), 4); + assert.equal(banana_nation._kmwLastIndexOf("na", 5), 4); + assert.equal(banana_nation._kmwLastIndexOf("na", 8), 4); + assert.equal(banana_nation._kmwLastIndexOf("na", 1000), 4); + + const na = "n" + a; + assert.equal(banana_nation._kmwLastIndexOf(na), 7); + assert.equal(banana_nation._kmwLastIndexOf(na, -1), -1); + assert.equal(banana_nation._kmwLastIndexOf(na, 0), -1); + assert.equal(banana_nation._kmwLastIndexOf(na, 2), 2); + assert.equal(banana_nation._kmwLastIndexOf(na, 3), 2); + assert.equal(banana_nation._kmwLastIndexOf(na, 5), 2); + // ERROR: Inconsistent with the index-2 case above. + // assert.equal(banana_nation._kmwLastIndexOf(na, 7), 7); + assert.equal(banana_nation._kmwLastIndexOf(na, 8), 7); + assert.equal(banana_nation._kmwLastIndexOf(na, 1000), 7); + }); + + it('_kmwLength', () => { + assert.equal(""._kmwLength(), 0); + // 0x1d5ba: MATHEMATICAL SANS-SERIF SMALL a + const a = String.fromCodePoint(0x1d5ba); + const banana_nation = "ban" + a + "na n" + a + "tion"; + assert.equal(banana_nation._kmwLength(), 13); + + // 0x1d5c9: MATHEMATICAL SANS-SERIF SMALL p + // 0x1d5be: MATHEMATICAL SANS-SERIF SMALL e + const apples = 'a' + String.fromCodePoint(0x1d5c9) + 'p' + 'l' + String.fromCodePoint(0x1d5be) + 's'; + assert.equal(apples._kmwLength(), 6); + }); + + it('_kmwSlice', () => { + // 0x1d5ba: MATHEMATICAL SANS-SERIF SMALL a + // 0x1d5c9: MATHEMATICAL SANS-SERIF SMALL p + // 0x1d5be: MATHEMATICAL SANS-SERIF SMALL e + // 0x1d5c2: MATHEMATICAL SANS-SERIF SMALL i + const a = String.fromCodePoint(0x1d5ba); + const p = String.fromCodePoint(0x1d5c9); + const e = String.fromCodePoint(0x1d5be); + const i = String.fromCodePoint(0x1d5c2); + + // From https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/slice + const str = "Th" + e + " qu" + i + "ck brown fox jum" + p + "s over the l" + a + "zy dog."; + + // FAILS: does not handle `undefined` length like the real .slice() method. + // assert.equal(str._kmwSlice(31), "the l" + a + "zy dog."); + assert.equal(str._kmwSlice(4, 19), "qu" + i + "ck brown fox"); + // FAILS: does not handle `undefined` length like the real .slice() method. + // assert.equal(str._kmwSlice(-4), "dog."); + assert.equal(str._kmwSlice(-9, -5), "l" + a + "zy"); + // FAILS: does not handle `undefined` start like the real .slice() method. + // assert.equal(str._kmwSlice(), str); + + // Per documented spec for slice... + assert.equal(str._kmwSlice(31, -5), "the l" + a + "zy"); // " dog." = 5 chars + assert.equal(str._kmwSlice(-5, -9), ""); + }); + + it('_kmwSubstr', () => { + // 0x1d5ba: MATHEMATICAL SANS-SERIF SMALL a + // 0x1d5c9: MATHEMATICAL SANS-SERIF SMALL p + // 0x1d5be: MATHEMATICAL SANS-SERIF SMALL e + // 0x1d5c2: MATHEMATICAL SANS-SERIF SMALL i + const a = String.fromCodePoint(0x1d5ba); + const p = String.fromCodePoint(0x1d5c9); + const e = String.fromCodePoint(0x1d5be); + const i = String.fromCodePoint(0x1d5c2); + + const str = "Th" + e + " qu" + i + "ck brown fox jum" + p + "s over the l" + a + "zy dog."; + + assert.equal(str._kmwSubstr(1000), ''); + assert.equal(str._kmwSubstr(31), "the l" + a + "zy dog."); + assert.equal(str._kmwSubstr(-4), "dog."); + assert.equal(str._kmwSubstr(31, 31), "the l" + a + "zy dog."); + assert.equal(str._kmwSubstr(31, -5), ''); // The big difference from .slice: length must be non-negative. + }); + + it('_kmwSubstring', () => { + // 0x1d5ba: MATHEMATICAL SANS-SERIF SMALL a + // 0x1d5c9: MATHEMATICAL SANS-SERIF SMALL p + // 0x1d5be: MATHEMATICAL SANS-SERIF SMALL e + // 0x1d5c2: MATHEMATICAL SANS-SERIF SMALL i + const a = String.fromCodePoint(0x1d5ba); + const p = String.fromCodePoint(0x1d5c9); + const e = String.fromCodePoint(0x1d5be); + const i = String.fromCodePoint(0x1d5c2); + + const str = "Th" + e + " qu" + i + "ck brown fox jum" + p + "s over the l" + a + "zy dog."; + + // FAILS - doesn't coerce the out-of-bounds value. + // assert.equal(str._kmwSubstring(1000), ''); + assert.equal(str._kmwSubstring(31), "the l" + a + "zy dog."); + assert.equal(str._kmwSubstring(31, 1000), "the l" + a + "zy dog."); + assert.equal(str._kmwSubstring(4, 9), "qu" + i + "ck"); + + // negatives are coerced to 0 & parameters are used in order of ascending value. + assert.equal(str._kmwSubstring(9, 4), "qu" + i + "ck"); + // FAILS: these do not handle negative-valued inputs the same way the real .substring does. + // assert.equal(str._kmwSubstring(31, -4), str.substring(0, 31)); + // assert.equal(str._kmwSubstring(-5, 9), str.substring(0, 9)); + // assert.equal(str._kmwSubstring(-9, -5), ''); + }); + + it('kmwCodePointToCodeUnit', () => { + // 0x1d5ba: MATHEMATICAL SANS-SERIF SMALL a + // 0x1d5c9: MATHEMATICAL SANS-SERIF SMALL p + // 0x1d5be: MATHEMATICAL SANS-SERIF SMALL e + // 0x1d5c2: MATHEMATICAL SANS-SERIF SMALL i + const a = String.fromCodePoint(0x1d5ba); + const p = String.fromCodePoint(0x1d5c9); + const e = String.fromCodePoint(0x1d5be); + const i = String.fromCodePoint(0x1d5c2); + + const str = "Th" + e + " qu" + i + "ck brown fox jum" + p + "s over the l" + a + "zy dog."; + + assert.equal(str.kmwCodePointToCodeUnit(2), 2); + assert.equal(str.kmwCodePointToCodeUnit(9), 11); // "The quick" = 9; 2 SMP replacements before then. + assert.equal(str.kmwCodePointToCodeUnit(31), 34); // "the lazy dog" is the remnant; 3 SMP replacements before then. + }); + + it('kmwCodeUnitToCodePoint', () => { + // 0x1d5ba: MATHEMATICAL SANS-SERIF SMALL a + // 0x1d5c9: MATHEMATICAL SANS-SERIF SMALL p + // 0x1d5be: MATHEMATICAL SANS-SERIF SMALL e + // 0x1d5c2: MATHEMATICAL SANS-SERIF SMALL i + const a = String.fromCodePoint(0x1d5ba); + const p = String.fromCodePoint(0x1d5c9); + const e = String.fromCodePoint(0x1d5be); + const i = String.fromCodePoint(0x1d5c2); + + const str = "Th" + e + " qu" + i + "ck brown fox jum" + p + "s over the l" + a + "zy dog."; + + // Inverting the tests from the prior section on `kmwCodePointToCodeUnit`... + assert.equal(str.kmwCodeUnitToCodePoint(2), 2); + assert.equal(str.kmwCodeUnitToCodePoint(11), 9); // "The quick" = 9; 2 SMP replacements before then. + assert.equal(str.kmwCodeUnitToCodePoint(34), 31); // "the lazy dog" is the remnant; 3 SMP replacements before then. + }); + }); +}); \ No newline at end of file From e5a530594694c945d9a822fae9d23a50f1439e37 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 22 Apr 2025 09:12:35 +0700 Subject: [PATCH 024/431] chore(web): drop .only from test suite --- web/src/engine/common/web-utils/src/tests/stringHandling.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/engine/common/web-utils/src/tests/stringHandling.js b/web/src/engine/common/web-utils/src/tests/stringHandling.js index bc7e250c77..cda7df6819 100644 --- a/web/src/engine/common/web-utils/src/tests/stringHandling.js +++ b/web/src/engine/common/web-utils/src/tests/stringHandling.js @@ -2,7 +2,7 @@ import { assert } from 'chai'; // Trigger the String-extending methods. import * as utils from '@keymanapp/web-utils'; -describe.only('Unicode string handling', () => { +describe('Unicode string handling', () => { describe('disabled: BMP-mode, BMP-only strings', () => { before(() => { String.kmwEnableSupplementaryPlane(false); From b368f5fe6ac9cd61aeed8c9a95b7b6d399f48c05 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 22 Apr 2025 09:22:40 +0700 Subject: [PATCH 025/431] refactor(web): refactor _kmwNextChar, _kmwPrevChar helpers --- web/src/engine/common/web-utils/src/index.ts | 1 + .../engine/common/web-utils/src/kmwstring.ts | 187 ++++++++---------- .../web-utils/src/tests/stringHandling.js | 2 +- 3 files changed, 88 insertions(+), 102 deletions(-) diff --git a/web/src/engine/common/web-utils/src/index.ts b/web/src/engine/common/web-utils/src/index.ts index da09e38c83..25c0c0bc06 100644 --- a/web/src/engine/common/web-utils/src/index.ts +++ b/web/src/engine/common/web-utils/src/index.ts @@ -17,6 +17,7 @@ export { default as Version } from "./version.js"; export { default as globalObject } from "./globalObject.js"; export { default as extendString } from "./kmwstring.js"; +export * as KMWString from './kmwstring.js'; export { default as ManagedPromise } from "./managedPromise.js"; export { default as TimeoutPromise, timedPromise } from "./timeoutPromise.js"; diff --git a/web/src/engine/common/web-utils/src/kmwstring.ts b/web/src/engine/common/web-utils/src/kmwstring.ts index eb80e6121c..e037c87dfb 100644 --- a/web/src/engine/common/web-utils/src/kmwstring.ts +++ b/web/src/engine/common/web-utils/src/kmwstring.ts @@ -1,8 +1,83 @@ /*** - KeymanWeb 14.0 - Copyright 2020 SIL International + KeymanWeb 19.0 + Copyright 2025 SIL International ***/ +/** + * Indicates if functionality supporting Supplementary Multilingual Plane (SMP) + * characters should be enabled. + */ +let EXTENSION_ENABLED = false; + +/** + * Enables or disables support for SMP-aware string handling. + * @param bEnable + */ +export function enableSupplementaryPlane(bEnable: boolean) { + EXTENSION_ENABLED = bEnable; +} + +/** + * Returns the code unit index for the next code point in the string, accounting for + * supplementary pairs + * + * @param {string} s The string + * @param {number|null} codeUnitIndex The code unit position to increment + * @return {number|null} The index of the next code point in the string, + * in code units + */ +export function nextChar(s: string, codeUnitIndex: number) { // nextCharAfterUnit + const str = s; + if(codeUnitIndex === null || codeUnitIndex < 0 || codeUnitIndex >= str.length - 1) { + return null; + } + + if(!EXTENSION_ENABLED) { + return codeUnitIndex+1; + } + + const first = str.charCodeAt(codeUnitIndex); + if(first >= 0xD800 && first <= 0xDBFF && str.length > codeUnitIndex + 1) { + const second = str.charCodeAt(codeUnitIndex + 1); + if(second >= 0xDC00 && second <= 0xDFFF) { + if(codeUnitIndex == str.length - 2) { + return null; + } + return codeUnitIndex + 2; + } + } + return codeUnitIndex + 1; +} + +/** + * Returns the code unit index for the previous code point in the string, accounting + * for supplementary pairs + * + * @param {string} s The string + * @param {number|null} codeUnitIndex The code unit position to decrement + * @return {number|null} The index of the previous code point in the + * string, in code units +*/ +export function prevChar(s: string, codeUnitIndex: number) { // prevCharBeforeUnit + const str = s; + + if(codeUnitIndex == null || codeUnitIndex <= 0 || codeUnitIndex > str.length) { + return null; + } + + if(!EXTENSION_ENABLED) { + return codeUnitIndex-1; + } + + const second = str.charCodeAt(codeUnitIndex - 1); + if(second >= 0xDC00 && second <= 0xDFFF && codeUnitIndex > 1) { + const first = str.charCodeAt(codeUnitIndex - 2); + if(first >= 0xD800 && first <= 0xDBFF) { + return codeUnitIndex - 2; + } + } + return codeUnitIndex - 1; +} /* * TODO: Remove this file as part of addressing https://github.com/keymanapp/keyman/issues/2492. @@ -24,10 +99,6 @@ declare global { kmwBMPSubstr(start: number, length?: number) : string, kmwLength(): number, kmwBMPLength(): number, - kmwNextChar(codeUnitIndex: number): number, - kmwBMPNextChar(codeUnitIndex: number): number, - kmwPrevChar(codeUnitIndex: number): number, - kmwBMPPrevChar(codeUnitIndex: number): number, kmwCodePointToCodeUnit(codePointIndex: number) : number, kmwBMPCodePointToCodeUnit(codePointIndex: number) : number, kmwCodeUnitToCodePoint(codeUnitIndex: number) : number, @@ -40,8 +111,6 @@ declare global { _kmwSubstring(start: number, length?: number) : string, _kmwSubstr(start: number, length?: number) : string, _kmwLength(): number, - _kmwNextChar(codeUnitIndex: number): number, - _kmwPrevChar(codeUnitIndex: number): number, _kmwCodePointToCodeUnit(codePointIndex: number) : number, _kmwCodeUnitToCodePoint(codeUnitIndex: number) : number } @@ -65,7 +134,7 @@ export default function extendString() { } for(var i = 0; i < codePointIndex; i++) { - codeUnitIndex = str.kmwNextChar(codeUnitIndex); + codeUnitIndex = nextChar(str, codeUnitIndex); if(codeUnitIndex === null) return NaN; } @@ -96,7 +165,7 @@ export default function extendString() { } var codePointIndex = 0; - for(var i = 0; i !== null && i < codeUnitIndex; i = str.kmwNextChar(i)) codePointIndex++; + for(var i = 0; i !== null && i < codeUnitIndex; i = nextChar(str, i)) codePointIndex++; return codePointIndex; } @@ -118,7 +187,7 @@ export default function extendString() { } var codePointIndex = 0; - for(var i = 0; i !== null && i < codeUnitIndex; i = str.kmwNextChar(i)) codePointIndex++; + for(var i = 0; i !== null && i < codeUnitIndex; i = nextChar(str, i)) codePointIndex++; return codePointIndex; } @@ -133,7 +202,7 @@ export default function extendString() { if(str.length == 0) return 0; for(var i = 0, codeUnitIndex = 0; codeUnitIndex !== null; i++) - codeUnitIndex = str.kmwNextChar(codeUnitIndex); + codeUnitIndex = nextChar(str, codeUnitIndex); return i; } @@ -182,7 +251,7 @@ export default function extendString() { if(arguments.length < 2) { endCodeUnit = str.length; } else { - for(var i = 0; i < length; i++) endCodeUnit = str.kmwNextChar(endCodeUnit); + for(var i = 0; i < length; i++) endCodeUnit = nextChar(str, endCodeUnit); } if(endCodeUnit === null) return str.substring(startCodeUnit); @@ -225,59 +294,6 @@ export default function extendString() { Helper functions */ - /** - * Returns the code unit index for the next code point in the string, accounting for - * supplementary pairs - * - * @param {number|null} codeUnitIndex The code unit position to increment - * @return {number|null} The index of the next code point in the string, - * in code units - */ - String.prototype.kmwNextChar = function(codeUnitIndex) { - var str = String(this); - - if(codeUnitIndex === null || codeUnitIndex < 0 || codeUnitIndex >= str.length - 1) { - return null; - } - - var first = str.charCodeAt(codeUnitIndex); - if (first >= 0xD800 && first <= 0xDBFF && str.length > codeUnitIndex + 1) { - var second = str.charCodeAt(codeUnitIndex + 1); - if (second >= 0xDC00 && second <= 0xDFFF) { - if(codeUnitIndex == str.length - 2) { - return null; - } - return codeUnitIndex + 2; - } - } - return codeUnitIndex + 1; - } - - /** - * Returns the code unit index for the previous code point in the string, accounting - * for supplementary pairs - * - * @param {number|null} codeUnitIndex The code unit position to decrement - * @return {number|null} The index of the previous code point in the - * string, in code units - */ - String.prototype.kmwPrevChar = function(codeUnitIndex) { - var str = String(this); - - if(codeUnitIndex == null || codeUnitIndex <= 0 || codeUnitIndex > str.length) { - return null; - } - - var second = str.charCodeAt(codeUnitIndex - 1); - if (second >= 0xDC00 && second <= 0xDFFF && codeUnitIndex > 1) { - var first = str.charCodeAt(codeUnitIndex - 2); - if(first >= 0xD800 && first <= 0xDBFF) { - return codeUnitIndex - 2; - } - } - return codeUnitIndex - 1; - } - /** * Returns the corresponding code unit index to the code point index passed * @@ -294,14 +310,14 @@ export default function extendString() { if(codePointIndex < 0) { codeUnitIndex = str.length; for(var i = 0; i > codePointIndex; i--) - codeUnitIndex = str.kmwPrevChar(codeUnitIndex); + codeUnitIndex = prevChar(str, codeUnitIndex); return codeUnitIndex; } if(codePointIndex == str.kmwLength()) return str.length; for(var i = 0; i < codePointIndex; i++) - codeUnitIndex = str.kmwNextChar(codeUnitIndex); + codeUnitIndex = nextChar(str, codeUnitIndex); return codeUnitIndex; } @@ -341,37 +357,6 @@ export default function extendString() { * to simplify enabling or disabling supplementary plane functionality (I3319) */ - /** - * Returns the code unit index for the next code point in the string - * - * @param {number} codeUnitIndex A code point index in the string - * @return {number|null} The corresponding character - */ - String.prototype.kmwBMPNextChar = function(codeUnitIndex) - { - var str = String(this); - if(codeUnitIndex < 0 || codeUnitIndex >= str.length - 1) { - return null; - } - return codeUnitIndex + 1; - } - - /** - * Returns the code unit index for the previous code point in the string - * - * @param {number} codeUnitIndex A code unit index in the string - * @return {number|null} The corresponding character - */ - String.prototype.kmwBMPPrevChar = function(codeUnitIndex) - { - var str = String(this); - - if(codeUnitIndex <= 0 || codeUnitIndex > str.length) { - return null; - } - return codeUnitIndex - 1; - } - /** * Returns the code unit index for a code point index * @@ -437,10 +422,10 @@ export default function extendString() { p._kmwSubstring = bEnable ? p.kmwSubstring : p.substring; p._kmwSubstr = bEnable ? p.kmwSubstr : p.kmwBMPSubstr; p._kmwLength = bEnable ? p.kmwLength : p.kmwBMPLength; - p._kmwNextChar = bEnable ? p.kmwNextChar : p.kmwBMPNextChar; - p._kmwPrevChar = bEnable ? p.kmwPrevChar : p.kmwBMPPrevChar; p._kmwCodePointToCodeUnit = bEnable ? p.kmwCodePointToCodeUnit : p.kmwBMPCodePointToCodeUnit; p._kmwCodeUnitToCodePoint = bEnable ? p.kmwCodeUnitToCodePoint : p.kmwBMPCodeUnitToCodePoint; + + enableSupplementaryPlane(bEnable); } // Ensure that _all_ String extensions are established, even if disabled by default. diff --git a/web/src/engine/common/web-utils/src/tests/stringHandling.js b/web/src/engine/common/web-utils/src/tests/stringHandling.js index cda7df6819..501c2916d4 100644 --- a/web/src/engine/common/web-utils/src/tests/stringHandling.js +++ b/web/src/engine/common/web-utils/src/tests/stringHandling.js @@ -1,6 +1,6 @@ import { assert } from 'chai'; // Trigger the String-extending methods. -import * as utils from '@keymanapp/web-utils'; +import { KMWString } from '@keymanapp/web-utils'; describe('Unicode string handling', () => { describe('disabled: BMP-mode, BMP-only strings', () => { From f934de23b5e1b36e77e1e0b0b528d503d6839d6b Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 22 Apr 2025 09:28:30 +0700 Subject: [PATCH 026/431] refactor(web): refactor _kmwLength --- .../engine/common/web-utils/src/kmwstring.ts | 70 +++++++++---------- .../web-utils/src/tests/stringHandling.js | 24 +++---- 2 files changed, 45 insertions(+), 49 deletions(-) diff --git a/web/src/engine/common/web-utils/src/kmwstring.ts b/web/src/engine/common/web-utils/src/kmwstring.ts index e037c87dfb..daa8e4b1c0 100644 --- a/web/src/engine/common/web-utils/src/kmwstring.ts +++ b/web/src/engine/common/web-utils/src/kmwstring.ts @@ -17,6 +17,31 @@ export function enableSupplementaryPlane(bEnable: boolean) { EXTENSION_ENABLED = bEnable; } +/** + * Returns the length of the string in code points, as opposed to code units. + * + * @param {string} s The string + * @return {number} The length of the string in code points + */ +export function length(s: string) { + const str = s; + + if(!EXTENSION_ENABLED) { + return str.length; + } + + if(str.length == 0) { + return 0; + } + + let i: number; + let codeUnitIndex: number; + for(i = 0, codeUnitIndex = 0; codeUnitIndex !== null; i++) { + codeUnitIndex = nextChar(str, codeUnitIndex); + } + return i; +} + /** * Returns the code unit index for the next code point in the string, accounting for * supplementary pairs @@ -97,8 +122,6 @@ declare global { kmwSubstring(start: number, length: number) : string, kmwSubstr(start: number, length?: number) : string, kmwBMPSubstr(start: number, length?: number) : string, - kmwLength(): number, - kmwBMPLength(): number, kmwCodePointToCodeUnit(codePointIndex: number) : number, kmwBMPCodePointToCodeUnit(codePointIndex: number) : number, kmwCodeUnitToCodePoint(codeUnitIndex: number) : number, @@ -110,7 +133,6 @@ declare global { _kmwSlice(beginSlice: number, endSlice: number) : string, _kmwSubstring(start: number, length?: number) : string, _kmwSubstr(start: number, length?: number) : string, - _kmwLength(): number, _kmwCodePointToCodeUnit(codePointIndex: number) : number, _kmwCodeUnitToCodePoint(codeUnitIndex: number) : number } @@ -191,21 +213,6 @@ export default function extendString() { return codePointIndex; } - /** - * Returns the length of the string in code points, as opposed to code units. - * - * @return {number} The length of the string in code points - */ - String.prototype.kmwLength = function() { - var str = String(this); - - if(str.length == 0) return 0; - - for(var i = 0, codeUnitIndex = 0; codeUnitIndex !== null; i++) - codeUnitIndex = nextChar(str, codeUnitIndex); - return i; - } - /** * Extracts a section of a string and returns a new string. * @@ -232,15 +239,15 @@ export default function extendString() { * * @param {number} start The start code point index in the string to * extract from - * @param {number=} length Optional length to extract + * @param {number=} len Optional length to extract * @return {string} The substring as selected by start and length */ - String.prototype.kmwSubstr = function(start, length?) + String.prototype.kmwSubstr = function(start, len?) { var str = String(this); if(start < 0) { - start = str.kmwLength() + start; + start = length(str) + start; } if(start < 0) start = 0; var startCodeUnit = str.kmwCodePointToCodeUnit(start); @@ -251,7 +258,7 @@ export default function extendString() { if(arguments.length < 2) { endCodeUnit = str.length; } else { - for(var i = 0; i < length; i++) endCodeUnit = nextChar(str, endCodeUnit); + for(var i = 0; i < len; i++) endCodeUnit = nextChar(str, endCodeUnit); } if(endCodeUnit === null) return str.substring(startCodeUnit); @@ -314,7 +321,7 @@ export default function extendString() { return codeUnitIndex; } - if(codePointIndex == str.kmwLength()) return str.length; + if(codePointIndex == length(str)) return str.length; for(var i = 0; i < codePointIndex; i++) codeUnitIndex = nextChar(str, codeUnitIndex); @@ -335,9 +342,9 @@ export default function extendString() { else if(codeUnitIndex == 0) return 0; else if(codeUnitIndex < 0) - return str.substr(codeUnitIndex).kmwLength(); + return length(str.substr(codeUnitIndex)); else - return str.substr(0,codeUnitIndex).kmwLength(); + return length(str.substr(0,codeUnitIndex)); } /** @@ -357,6 +364,7 @@ export default function extendString() { * to simplify enabling or disabling supplementary plane functionality (I3319) */ + /** * Returns the code unit index for a code point index * @@ -379,17 +387,6 @@ export default function extendString() { return codeUnitIndex; } - /** - * Returns the length of a BMP string - * - * @return {number} The length in code points - */ - String.prototype.kmwBMPLength = function() - { - var str = String(this); - return str.length; - } - /** * Returns a substring * @@ -421,7 +418,6 @@ export default function extendString() { p._kmwSlice = bEnable ? p.kmwSlice : p.slice; p._kmwSubstring = bEnable ? p.kmwSubstring : p.substring; p._kmwSubstr = bEnable ? p.kmwSubstr : p.kmwBMPSubstr; - p._kmwLength = bEnable ? p.kmwLength : p.kmwBMPLength; p._kmwCodePointToCodeUnit = bEnable ? p.kmwCodePointToCodeUnit : p.kmwBMPCodePointToCodeUnit; p._kmwCodeUnitToCodePoint = bEnable ? p.kmwCodeUnitToCodePoint : p.kmwBMPCodeUnitToCodePoint; diff --git a/web/src/engine/common/web-utils/src/tests/stringHandling.js b/web/src/engine/common/web-utils/src/tests/stringHandling.js index 501c2916d4..92488b30d4 100644 --- a/web/src/engine/common/web-utils/src/tests/stringHandling.js +++ b/web/src/engine/common/web-utils/src/tests/stringHandling.js @@ -54,11 +54,11 @@ describe('Unicode string handling', () => { assert.equal(banana_nation._kmwLastIndexOf("na", 1000), 7); }); - it('_kmwLength', () => { - assert.equal(""._kmwLength(), 0); - assert.equal("string"._kmwLength(), 6); + it('length', () => { + assert.equal(KMWString.length(""), 0); + assert.equal(KMWString.length("string"), 6); // From the current MDN example: - assert.equal("Life, the universe and everything. Answer:"._kmwLength(), 42); + assert.equal(KMWString.length("Life, the universe and everything. Answer:"), 42); }); it('_kmwSlice', () => { @@ -167,11 +167,11 @@ describe('Unicode string handling', () => { assert.equal(banana_nation._kmwLastIndexOf("na", 1000), 7); }); - it('_kmwLength', () => { - assert.equal(""._kmwLength(), 0); - assert.equal("string"._kmwLength(), 6); + it('length', () => { + assert.equal(KMWString.length(""), 0); + assert.equal(KMWString.length("string"), 6); // From the current MDN example: - assert.equal("Life, the universe and everything. Answer:"._kmwLength(), 42); + assert.equal(KMWString.length("Life, the universe and everything. Answer:"), 42); }); it('_kmwSlice', () => { @@ -323,17 +323,17 @@ describe('Unicode string handling', () => { assert.equal(banana_nation._kmwLastIndexOf(na, 1000), 7); }); - it('_kmwLength', () => { - assert.equal(""._kmwLength(), 0); + it('length', () => { + assert.equal(KMWString.length(""), 0); // 0x1d5ba: MATHEMATICAL SANS-SERIF SMALL a const a = String.fromCodePoint(0x1d5ba); const banana_nation = "ban" + a + "na n" + a + "tion"; - assert.equal(banana_nation._kmwLength(), 13); + assert.equal(KMWString.length(banana_nation), 13); // 0x1d5c9: MATHEMATICAL SANS-SERIF SMALL p // 0x1d5be: MATHEMATICAL SANS-SERIF SMALL e const apples = 'a' + String.fromCodePoint(0x1d5c9) + 'p' + 'l' + String.fromCodePoint(0x1d5be) + 's'; - assert.equal(apples._kmwLength(), 6); + assert.equal(KMWString.length(apples), 6); }); it('_kmwSlice', () => { From 4863e95c465519b8a28304b434b3d596648257af Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 22 Apr 2025 09:31:59 +0700 Subject: [PATCH 027/431] refactor(web): refactor _kmwCodePointToCodeUnit, _kmwCodeUnitToCodePoint --- .../engine/common/web-utils/src/kmwstring.ts | 152 ++++++++---------- .../web-utils/src/tests/stringHandling.js | 48 +++--- 2 files changed, 92 insertions(+), 108 deletions(-) diff --git a/web/src/engine/common/web-utils/src/kmwstring.ts b/web/src/engine/common/web-utils/src/kmwstring.ts index daa8e4b1c0..f86e4d5fa6 100644 --- a/web/src/engine/common/web-utils/src/kmwstring.ts +++ b/web/src/engine/common/web-utils/src/kmwstring.ts @@ -104,6 +104,67 @@ export function prevChar(s: string, codeUnitIndex: number) { // prevCharBeforeUn return codeUnitIndex - 1; } +/** + * Returns the corresponding code unit index to the code point index passed + * + * @param {string} s The string + * @param {number|null} codePointIndex A code point index in the string + * @return {number|null} The corresponding code unit index + */ +export function codePointToCodeUnit(s: string, codePointIndex: number) { // + if(codePointIndex === null) { + return null; + } + + if(!EXTENSION_ENABLED) { + return codePointIndex; + } + + const str = s; + let codeUnitIndex = 0; + + if(codePointIndex < 0) { + codeUnitIndex = str.length; + for(let i = 0; i > codePointIndex; i--) { + codeUnitIndex = prevChar(str, codeUnitIndex); + } + return codeUnitIndex; + } + + if(codePointIndex == length(str)) { + return str.length; + } + + for(let i = 0; i < codePointIndex; i++) { + codeUnitIndex = nextChar(str, codeUnitIndex); + } + return codeUnitIndex; +} + +/** + * Returns the corresponding code point index to the code unit index passed + * + * @param {number|null} codeUnitIndex A code unit index in the string + * @return {number|null} The corresponding code point index + */ +export function codeUnitToCodePoint(s: string, codeUnitIndex: number) { // + const str = s; + + if(!EXTENSION_ENABLED) { + return codeUnitIndex; + } + + if(codeUnitIndex === null) { + return null; + } else if(codeUnitIndex == 0) { + return 0; + } else if(codeUnitIndex < 0) { + return length(str.substr(codeUnitIndex)); + } else { + return length(str.substr(0,codeUnitIndex)); + } +} + /* * TODO: Remove this file as part of addressing https://github.com/keymanapp/keyman/issues/2492. */ @@ -122,19 +183,13 @@ declare global { kmwSubstring(start: number, length: number) : string, kmwSubstr(start: number, length?: number) : string, kmwBMPSubstr(start: number, length?: number) : string, - kmwCodePointToCodeUnit(codePointIndex: number) : number, - kmwBMPCodePointToCodeUnit(codePointIndex: number) : number, - kmwCodeUnitToCodePoint(codeUnitIndex: number) : number, - kmwBMPCodeUnitToCodePoint(codeUnitIndex: number) : number, _kmwCharCodeAt(codePointIndex: number): number, _kmwCharAt(codePointIndex: number) : string, _kmwIndexOf(searchValue: string, fromIndex?: number) : number, _kmwLastIndexOf(searchValue: string, fromIndex?: number) : number, _kmwSlice(beginSlice: number, endSlice: number) : string, _kmwSubstring(start: number, length?: number) : string, - _kmwSubstr(start: number, length?: number) : string, - _kmwCodePointToCodeUnit(codePointIndex: number) : number, - _kmwCodeUnitToCodePoint(codeUnitIndex: number) : number + _kmwSubstr(start: number, length?: number) : string } } @@ -225,8 +280,8 @@ export default function extendString() { */ String.prototype.kmwSlice = function(beginSlice, endSlice) { var str = String(this); - var beginSliceCodeUnit = str.kmwCodePointToCodeUnit(beginSlice); - var endSliceCodeUnit = str.kmwCodePointToCodeUnit(endSlice); + var beginSliceCodeUnit = codePointToCodeUnit(str, beginSlice); + var endSliceCodeUnit = codePointToCodeUnit(str, endSlice); if(beginSliceCodeUnit === null || endSliceCodeUnit === null) return ''; else @@ -250,7 +305,7 @@ export default function extendString() { start = length(str) + start; } if(start < 0) start = 0; - var startCodeUnit = str.kmwCodePointToCodeUnit(start); + var startCodeUnit = codePointToCodeUnit(str, start); var endCodeUnit = startCodeUnit; if(startCodeUnit === null) return ''; @@ -281,15 +336,15 @@ export default function extendString() { if(typeof(indexB) == 'undefined') { - indexACodeUnit = str.kmwCodePointToCodeUnit(indexA); + indexACodeUnit = codePointToCodeUnit(str, indexA); indexBCodeUnit = str.length; } else { if(indexA > indexB) { var c = indexA; indexA = indexB; indexB = c; } - indexACodeUnit = str.kmwCodePointToCodeUnit(indexA); - indexBCodeUnit = str.kmwCodePointToCodeUnit(indexB); + indexACodeUnit = codePointToCodeUnit(str, indexA); + indexBCodeUnit = codePointToCodeUnit(str, indexB); } if(isNaN(indexACodeUnit) || indexACodeUnit === null) indexACodeUnit = 0; if(isNaN(indexBCodeUnit) || indexBCodeUnit === null) indexBCodeUnit = str.length; @@ -301,52 +356,6 @@ export default function extendString() { Helper functions */ - /** - * Returns the corresponding code unit index to the code point index passed - * - * @param {number|null} codePointIndex A code point index in the string - * @return {number|null} The corresponding code unit index - */ - String.prototype.kmwCodePointToCodeUnit = function(codePointIndex) { - - if(codePointIndex === null) return null; - - var str = String(this); - var codeUnitIndex = 0; - - if(codePointIndex < 0) { - codeUnitIndex = str.length; - for(var i = 0; i > codePointIndex; i--) - codeUnitIndex = prevChar(str, codeUnitIndex); - return codeUnitIndex; - } - - if(codePointIndex == length(str)) return str.length; - - for(var i = 0; i < codePointIndex; i++) - codeUnitIndex = nextChar(str, codeUnitIndex); - return codeUnitIndex; - } - - /** - * Returns the corresponding code point index to the code unit index passed - * - * @param {number|null} codeUnitIndex A code unit index in the string - * @return {number|null} The corresponding code point index - */ - String.prototype.kmwCodeUnitToCodePoint = function(codeUnitIndex) { - var str = String(this); - - if(codeUnitIndex === null) - return null; - else if(codeUnitIndex == 0) - return 0; - else if(codeUnitIndex < 0) - return length(str.substr(codeUnitIndex)); - else - return length(str.substr(0,codeUnitIndex)); - } - /** * Returns the character at a the code point index passed * @@ -364,29 +373,6 @@ export default function extendString() { * to simplify enabling or disabling supplementary plane functionality (I3319) */ - - /** - * Returns the code unit index for a code point index - * - * @param {number} codePointIndex A code point index in the string - * @return {number} The corresponding character - */ - String.prototype.kmwBMPCodePointToCodeUnit = function(codePointIndex) - { - return codePointIndex; - } - - /** - * Returns the code point index for a code unit index - * - * @param {number} codeUnitIndex A code point index in the string - * @return {number} The corresponding character - */ - String.prototype.kmwBMPCodeUnitToCodePoint = function(codeUnitIndex) - { - return codeUnitIndex; - } - /** * Returns a substring * @@ -418,8 +404,6 @@ export default function extendString() { p._kmwSlice = bEnable ? p.kmwSlice : p.slice; p._kmwSubstring = bEnable ? p.kmwSubstring : p.substring; p._kmwSubstr = bEnable ? p.kmwSubstr : p.kmwBMPSubstr; - p._kmwCodePointToCodeUnit = bEnable ? p.kmwCodePointToCodeUnit : p.kmwBMPCodePointToCodeUnit; - p._kmwCodeUnitToCodePoint = bEnable ? p.kmwCodeUnitToCodePoint : p.kmwBMPCodeUnitToCodePoint; enableSupplementaryPlane(bEnable); } diff --git a/web/src/engine/common/web-utils/src/tests/stringHandling.js b/web/src/engine/common/web-utils/src/tests/stringHandling.js index 92488b30d4..156e3b7ab4 100644 --- a/web/src/engine/common/web-utils/src/tests/stringHandling.js +++ b/web/src/engine/common/web-utils/src/tests/stringHandling.js @@ -101,18 +101,18 @@ describe('Unicode string handling', () => { assert.equal(str._kmwSubstring(-9, -5), ''); }); - it('kmwCodePointToCodeUnit', () => { + it('codePointToCodeUnit', () => { const str = "The quick brown fox jumps over the lazy dog."; - assert.equal(str.kmwCodePointToCodeUnit(2), 2); - assert.equal(str.kmwCodePointToCodeUnit(9), 9); - assert.equal(str.kmwCodePointToCodeUnit(31), 31); + assert.equal(KMWString.codePointToCodeUnit(str, 2), 2); + assert.equal(KMWString.codePointToCodeUnit(str, 9), 9); + assert.equal(KMWString.codePointToCodeUnit(str, 31), 31); }); - it('kmwCodeUnitToCodePoint', () => { + it('codeUnitToCodePoint', () => { const str = "The quick brown fox jumps over the lazy dog."; - assert.equal(str.kmwCodeUnitToCodePoint(2), 2); - assert.equal(str.kmwCodeUnitToCodePoint(9), 9); - assert.equal(str.kmwCodeUnitToCodePoint(31), 31); + assert.equal(KMWString.codeUnitToCodePoint(str, 2), 2); + assert.equal(KMWString.codeUnitToCodePoint(str, 9), 9); + assert.equal(KMWString.codeUnitToCodePoint(str, 31), 31); }); }); @@ -219,18 +219,18 @@ describe('Unicode string handling', () => { // assert.equal(str._kmwSubstring(-9, -5), ''); }); - it('kmwCodePointToCodeUnit', () => { + it('codePointToCodeUnit', () => { const str = "The quick brown fox jumps over the lazy dog."; - assert.equal(str.kmwCodePointToCodeUnit(2), 2); - assert.equal(str.kmwCodePointToCodeUnit(9), 9); - assert.equal(str.kmwCodePointToCodeUnit(31), 31); + assert.equal(KMWString.codePointToCodeUnit(str, 2), 2); + assert.equal(KMWString.codePointToCodeUnit(str, 9), 9); + assert.equal(KMWString.codePointToCodeUnit(str, 31), 31); }); - it('kmwCodeUnitToCodePoint', () => { + it('codeUnitToCodePoint', () => { const str = "The quick brown fox jumps over the lazy dog."; - assert.equal(str.kmwCodeUnitToCodePoint(2), 2); - assert.equal(str.kmwCodeUnitToCodePoint(9), 9); - assert.equal(str.kmwCodeUnitToCodePoint(31), 31); + assert.equal(KMWString.codeUnitToCodePoint(str, 2), 2); + assert.equal(KMWString.codeUnitToCodePoint(str, 9), 9); + assert.equal(KMWString.codeUnitToCodePoint(str, 31), 31); }); }); @@ -408,7 +408,7 @@ describe('Unicode string handling', () => { // assert.equal(str._kmwSubstring(-9, -5), ''); }); - it('kmwCodePointToCodeUnit', () => { + it('codePointToCodeUnit', () => { // 0x1d5ba: MATHEMATICAL SANS-SERIF SMALL a // 0x1d5c9: MATHEMATICAL SANS-SERIF SMALL p // 0x1d5be: MATHEMATICAL SANS-SERIF SMALL e @@ -420,12 +420,12 @@ describe('Unicode string handling', () => { const str = "Th" + e + " qu" + i + "ck brown fox jum" + p + "s over the l" + a + "zy dog."; - assert.equal(str.kmwCodePointToCodeUnit(2), 2); - assert.equal(str.kmwCodePointToCodeUnit(9), 11); // "The quick" = 9; 2 SMP replacements before then. - assert.equal(str.kmwCodePointToCodeUnit(31), 34); // "the lazy dog" is the remnant; 3 SMP replacements before then. + assert.equal(KMWString.codePointToCodeUnit(str, 2), 2); + assert.equal(KMWString.codePointToCodeUnit(str, 9), 11); // "The quick" = 9; 2 SMP replacements before then. + assert.equal(KMWString.codePointToCodeUnit(str, 31), 34); // "the lazy dog" is the remnant; 3 SMP replacements before then. }); - it('kmwCodeUnitToCodePoint', () => { + it('codeUnitToCodePoint', () => { // 0x1d5ba: MATHEMATICAL SANS-SERIF SMALL a // 0x1d5c9: MATHEMATICAL SANS-SERIF SMALL p // 0x1d5be: MATHEMATICAL SANS-SERIF SMALL e @@ -438,9 +438,9 @@ describe('Unicode string handling', () => { const str = "Th" + e + " qu" + i + "ck brown fox jum" + p + "s over the l" + a + "zy dog."; // Inverting the tests from the prior section on `kmwCodePointToCodeUnit`... - assert.equal(str.kmwCodeUnitToCodePoint(2), 2); - assert.equal(str.kmwCodeUnitToCodePoint(11), 9); // "The quick" = 9; 2 SMP replacements before then. - assert.equal(str.kmwCodeUnitToCodePoint(34), 31); // "the lazy dog" is the remnant; 3 SMP replacements before then. + assert.equal(KMWString.codeUnitToCodePoint(str, 2), 2); + assert.equal(KMWString.codeUnitToCodePoint(str, 11), 9); // "The quick" = 9; 2 SMP replacements before then. + assert.equal(KMWString.codeUnitToCodePoint(str, 34), 31); // "the lazy dog" is the remnant; 3 SMP replacements before then. }); }); }); \ No newline at end of file From 418f3d6aec60f0c9de37c4be74f1e3f79a2b447f Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 22 Apr 2025 10:09:10 +0700 Subject: [PATCH 028/431] chore(web): update all Web engine uses of kmwLength, kmwCodePointToCodeUnit, kmwCodeUnitToCodePoint --- common/test/resources/model-helpers.mjs | 1 - web/src/app/webview/src/contextManager.ts | 9 ++--- .../engine/common/web-utils/src/kmwstring.ts | 8 ++--- .../element-wrappers/src/contentEditable.ts | 7 ++-- .../element-wrappers/src/designIFrame.ts | 7 ++-- web/src/engine/element-wrappers/src/input.ts | 15 +++++---- .../engine/element-wrappers/src/textarea.ts | 15 +++++---- .../engine/js-processor/src/kbdInterface.ts | 8 ++--- .../js-processor/src/keyboardProcessor.ts | 4 +-- web/src/engine/js-processor/src/mock.ts | 17 +++++----- .../engine/js-processor/src/outputTarget.ts | 8 ++--- .../engine/main/src/headless/contextWindow.ts | 7 ++-- .../predictive-text/templates/src/common.ts | 8 ++--- .../templates/src/trie-model.ts | 6 ++-- .../templates/tests/helpers.js | 1 - .../src/main/correction/context-tracker.ts | 33 ++++++++++--------- .../worker-thread/src/main/index.ts | 4 +++ .../src/main/model-compositor.ts | 6 ++-- .../worker-thread/src/main/model-helpers.ts | 4 ++- .../worker-thread/src/main/predict-helpers.ts | 7 ++-- .../element_interfaces.tests.ts | 4 +-- .../specialized-backspace.tests.js | 5 +-- .../main/headless/inputProcessor.tests.js | 9 ++--- 23 files changed, 105 insertions(+), 88 deletions(-) diff --git a/common/test/resources/model-helpers.mjs b/common/test/resources/model-helpers.mjs index b2ad083f96..27eebddff9 100644 --- a/common/test/resources/model-helpers.mjs +++ b/common/test/resources/model-helpers.mjs @@ -22,7 +22,6 @@ import { fileURLToPath } from 'url'; // Ensure that our KMW string-extensions activate. extendString(); -assert.ok('💩'.kmwLength); /** * Creates a MessageEvent (for inter-worker communication), with the given data payload. diff --git a/web/src/app/webview/src/contextManager.ts b/web/src/app/webview/src/contextManager.ts index 4507f44f55..9c4d51a820 100644 --- a/web/src/app/webview/src/contextManager.ts +++ b/web/src/app/webview/src/contextManager.ts @@ -4,6 +4,7 @@ import { KeyboardStub } from 'keyman/engine/keyboard-storage'; import { ContextManagerBase } from 'keyman/engine/main'; import { WebviewConfiguration } from './configuration.js'; import { LexicalModelTypes } from '@keymanapp/common-types'; +import { KMWString } from '@keymanapp/web-utils'; export type OnInsertTextFunc = (deleteLeft: number, text: string, deleteRight: number) => void; @@ -64,7 +65,7 @@ export class ContextHost extends Mock { updateContext(text: string, selStart: number, selEnd: number): boolean { let shouldResetContext = false; - let tempMock = new Mock(text, selStart ?? text._kmwLength(), selEnd ?? text._kmwLength()); + let tempMock = new Mock(text, selStart ?? KMWString.length(text), selEnd ?? KMWString.length(text)); let newLeft = tempMock.getTextBeforeCaret(); let oldLeft = this.getTextBeforeCaret(); @@ -79,7 +80,7 @@ export class ContextHost extends Mock { this.selEnd = selEnd; } else { // Transform selection coordinates to their location within the longform context window. - let delta = oldLeft._kmwLength() - newLeft._kmwLength(); + let delta = KMWString.length(oldLeft) - KMWString.length(newLeft); this.selStart = selStart - delta; this.selEnd = selEnd - delta; } @@ -89,7 +90,7 @@ export class ContextHost extends Mock { // Our host app will not know whether or not the keyboard uses SMP chars, // and we want a consistent interface for context synchronization between // host app + app/webview KMW. - this.setSelection(this.text._kmwLength()); + this.setSelection(KMWString.length(this.text)); } this.saveState(); @@ -105,7 +106,7 @@ export class ContextHost extends Mock { // Our host app will not know whether or not the keyboard uses SMP chars, // and we want a consistent interface for context synchronization between // host app + app/webview KMW. - this.setSelection(this.text._kmwLength()); + this.setSelection(KMWString.length(this.text)); this.savedState = Mock.from(this); } } diff --git a/web/src/engine/common/web-utils/src/kmwstring.ts b/web/src/engine/common/web-utils/src/kmwstring.ts index f86e4d5fa6..666eed9509 100644 --- a/web/src/engine/common/web-utils/src/kmwstring.ts +++ b/web/src/engine/common/web-utils/src/kmwstring.ts @@ -51,7 +51,7 @@ export function length(s: string) { * @return {number|null} The index of the next code point in the string, * in code units */ -export function nextChar(s: string, codeUnitIndex: number) { // nextCharAfterUnit +export function nextChar(s: string, codeUnitIndex: number) { const str = s; if(codeUnitIndex === null || codeUnitIndex < 0 || codeUnitIndex >= str.length - 1) { return null; @@ -83,7 +83,7 @@ export function nextChar(s: string, codeUnitIndex: number) { // nextCharAfterUni * @return {number|null} The index of the previous code point in the * string, in code units */ -export function prevChar(s: string, codeUnitIndex: number) { // prevCharBeforeUnit +export function prevChar(s: string, codeUnitIndex: number) { const str = s; if(codeUnitIndex == null || codeUnitIndex <= 0 || codeUnitIndex > str.length) { @@ -111,7 +111,7 @@ export function prevChar(s: string, codeUnitIndex: number) { // prevCharBeforeUn * @param {number|null} codePointIndex A code point index in the string * @return {number|null} The corresponding code unit index */ -export function codePointToCodeUnit(s: string, codePointIndex: number) { // +export function codePointToCodeUnit(s: string, codePointIndex: number) { if(codePointIndex === null) { return null; } @@ -147,7 +147,7 @@ export function codePointToCodeUnit(s: string, codePointIndex: number) { // +export function codeUnitToCodePoint(s: string, codeUnitIndex: number) { const str = s; if(!EXTENSION_ENABLED) { diff --git a/web/src/engine/element-wrappers/src/contentEditable.ts b/web/src/engine/element-wrappers/src/contentEditable.ts index cc532d226b..a8c2cf38fe 100644 --- a/web/src/engine/element-wrappers/src/contentEditable.ts +++ b/web/src/engine/element-wrappers/src/contentEditable.ts @@ -1,4 +1,5 @@ import OutputTarget from './outputTarget.js'; +import { KMWString } from '@keymanapp/web-utils'; class SelectionCaret { node: Node; @@ -103,7 +104,7 @@ export default class ContentEditable extends OutputTarget<{}> { } getDeadkeyCaret(): number { - return this.getTextBeforeCaret().kmwLength(); + return KMWString.length(this.getTextBeforeCaret()); } getTextBeforeCaret(): string { @@ -179,7 +180,7 @@ export default class ContentEditable extends OutputTarget<{}> { } let start = this.getCarets().start; - let delta = s._kmwLength(); + let delta = KMWString.length(s); let Lsel = this.root.ownerDocument.getSelection(); if(delta == 0) { @@ -243,7 +244,7 @@ export default class ContentEditable extends OutputTarget<{}> { } let caret = this.getCarets().end; - let delta = s._kmwLength(); + let delta = KMWString.length(s); if(delta == 0) { return; diff --git a/web/src/engine/element-wrappers/src/designIFrame.ts b/web/src/engine/element-wrappers/src/designIFrame.ts index 4fdfc5c103..b0f84fbf19 100644 --- a/web/src/engine/element-wrappers/src/designIFrame.ts +++ b/web/src/engine/element-wrappers/src/designIFrame.ts @@ -1,4 +1,5 @@ import OutputTarget from './outputTarget.js'; +import { KMWString } from '@keymanapp/web-utils'; class SelectionCaret { node: Node; @@ -125,7 +126,7 @@ export default class DesignIFrame extends OutputTarget<{}> { } getDeadkeyCaret(): number { - return this.getTextBeforeCaret().kmwLength(); + return KMWString.length(this.getTextBeforeCaret()); } getTextBeforeCaret(): string { @@ -201,7 +202,7 @@ export default class DesignIFrame extends OutputTarget<{}> { } let start = this.getCarets().start; - let delta = s._kmwLength(); + let delta = KMWString.length(s); let Lsel = this.doc.getSelection(); if(delta == 0) { @@ -265,7 +266,7 @@ export default class DesignIFrame extends OutputTarget<{}> { } let caret = this.getCarets().end; - let delta = s._kmwLength(); + let delta = KMWString.length(s); if(delta == 0) { return; diff --git a/web/src/engine/element-wrappers/src/input.ts b/web/src/engine/element-wrappers/src/input.ts index e70bb6bdfb..2c61da8eb1 100644 --- a/web/src/engine/element-wrappers/src/input.ts +++ b/web/src/engine/element-wrappers/src/input.ts @@ -1,4 +1,5 @@ import OutputTarget from './outputTarget.js'; +import { KMWString } from '@keymanapp/web-utils'; interface EventMap { /** @@ -91,8 +92,8 @@ export default class Input extends OutputTarget { getCaret(): number { if(this.root.selectionStart != this._cachedSelectionStart) { this._cachedSelectionStart = this.root.selectionStart; // KMW-1 - this.processedSelectionStart = this.root.value._kmwCodeUnitToCodePoint(this.root.selectionStart); // I3319 - this.processedSelectionEnd = this.root.value._kmwCodeUnitToCodePoint(this.root.selectionEnd); // I3319 + this.processedSelectionStart = KMWString.codeUnitToCodePoint(this.root.value, this.root.selectionStart); // I3319 + this.processedSelectionEnd = KMWString.codeUnitToCodePoint(this.root.value, this.root.selectionEnd); // I3319 } return this.root.selectionDirection == 'forward' ? this.processedSelectionEnd : this.processedSelectionStart; } @@ -106,8 +107,8 @@ export default class Input extends OutputTarget { } setSelection(start: number, end: number, direction: "forward" | "backward" | "none") { - let domStart = this.root.value._kmwCodePointToCodeUnit(start); - let domEnd = this.root.value._kmwCodePointToCodeUnit(end); + let domStart = KMWString.codePointToCodeUnit(this.root.value, start); + let domEnd = KMWString.codePointToCodeUnit(this.root.value, end); this.root.setSelectionRange(domStart, domEnd, direction); this.processedSelectionStart = start; @@ -164,7 +165,7 @@ export default class Input extends OutputTarget { this.getCaret(); let selectionLength = this.processedSelectionEnd - this.processedSelectionStart; let direction = this.getSelectionDirection(); - let newCaret = text._kmwLength(); + let newCaret = KMWString.length(text); this.root.value = text + this.getText()._kmwSubstring(this.processedSelectionStart); this.setSelection(newCaret, newCaret + selectionLength, direction); @@ -210,9 +211,9 @@ export default class Input extends OutputTarget { let front = this.getTextBeforeCaret(); let back = this.getText()._kmwSubstring(this.processedSelectionStart); - this.adjustDeadkeys(s._kmwLength()); + this.adjustDeadkeys(KMWString.length(s)); this.root.value = front + s + back; - this.setCaret(caret + s._kmwLength()); + this.setCaret(caret + KMWString.length(s)); } handleNewlineAtCaret(): void { diff --git a/web/src/engine/element-wrappers/src/textarea.ts b/web/src/engine/element-wrappers/src/textarea.ts index 04a8ea3ae8..e9a9495698 100644 --- a/web/src/engine/element-wrappers/src/textarea.ts +++ b/web/src/engine/element-wrappers/src/textarea.ts @@ -1,4 +1,5 @@ import OutputTarget from './outputTarget.js'; +import { KMWString } from '@keymanapp/web-utils'; export default class TextArea extends OutputTarget<{}> { root: HTMLTextAreaElement; @@ -66,8 +67,8 @@ export default class TextArea extends OutputTarget<{}> { getCaret(): number { if(this.root.selectionStart != this._cachedSelectionStart) { this._cachedSelectionStart = this.root.selectionStart; // KMW-1 - this.processedSelectionStart = this.root.value._kmwCodeUnitToCodePoint(this.root.selectionStart); // I3319 - this.processedSelectionEnd = this.root.value._kmwCodeUnitToCodePoint(this.root.selectionEnd); // I3319 + this.processedSelectionStart = KMWString.codeUnitToCodePoint(this.root.value, this.root.selectionStart); // I3319 + this.processedSelectionEnd = KMWString.codeUnitToCodePoint(this.root.value, this.root.selectionEnd); // I3319 } return this.root.selectionDirection == 'forward' ? this.processedSelectionEnd : this.processedSelectionStart; } @@ -81,8 +82,8 @@ export default class TextArea extends OutputTarget<{}> { } setSelection(start: number, end: number, direction: "forward" | "backward" | "none") { - let domStart = this.root.value._kmwCodePointToCodeUnit(start); - let domEnd = this.root.value._kmwCodePointToCodeUnit(end); + let domStart = KMWString.codePointToCodeUnit(this.root.value, start); + let domEnd = KMWString.codePointToCodeUnit(this.root.value, end); this.root.setSelectionRange(domStart, domEnd, direction); this.processedSelectionStart = start; @@ -134,7 +135,7 @@ export default class TextArea extends OutputTarget<{}> { this.getCaret(); let selectionLength = this.processedSelectionEnd - this.processedSelectionStart; let direction = this.getSelectionDirection(); - let newCaret = text._kmwLength(); + let newCaret = KMWString.length(text); this.root.value = text + this.getText()._kmwSubstring(this.processedSelectionStart); this.setSelection(newCaret, newCaret + selectionLength, direction); @@ -185,9 +186,9 @@ export default class TextArea extends OutputTarget<{}> { let front = this.getTextBeforeCaret(); let back = this.getText()._kmwSubstring(this.processedSelectionStart); - this.adjustDeadkeys(s._kmwLength()); + this.adjustDeadkeys(KMWString.length(s)); this.root.value = front + s + back; - this.setCaret(caret + s._kmwLength()); + this.setCaret(caret + KMWString.length(s)); } handleNewlineAtCaret(): void { diff --git a/web/src/engine/js-processor/src/kbdInterface.ts b/web/src/engine/js-processor/src/kbdInterface.ts index efca7b8a80..8848f3ccde 100644 --- a/web/src/engine/js-processor/src/kbdInterface.ts +++ b/web/src/engine/js-processor/src/kbdInterface.ts @@ -5,7 +5,7 @@ //#region Imports -import { type DeviceSpec } from "@keymanapp/web-utils"; +import { type DeviceSpec, KMWString } from "@keymanapp/web-utils"; import { ModifierKeyConstants } from '@keymanapp/common-types'; import { Codes, type KeyEvent, KeyMapping, Keyboard, KeyboardHarness, KeyboardKeymanGlobal, VariableStoreDictionary } from "keyman/engine/keyboard"; import type OutputTarget from './outputTarget.js'; @@ -279,8 +279,8 @@ export default class KeyboardInterface extends KeyboardHarness { // If we have a selection, we have an empty context tempContext = outputTarget.isSelectionEmpty() ? outputTarget.getTextBeforeCaret() : ""; - if(tempContext._kmwLength() < n) { - tempContext = Array(n-tempContext._kmwLength()+1).join("\uFFFE") + tempContext; + if(KMWString.length(tempContext) < n) { + tempContext = Array(n-KMWString.length(tempContext)+1).join("\uFFFE") + tempContext; } return tempContext._kmwSubstr(-n)._kmwSubstr(0,ln); @@ -642,7 +642,7 @@ export default class KeyboardInterface extends KeyboardHarness { // Nope, so let's build its cache. var result: ComplexKeyboardStore = []; - for(var i=0; i < store._kmwLength(); i++) { + for(var i=0; i < KMWString.length(store); i++) { result.push(store._kmwCharAt(i)); } diff --git a/web/src/engine/js-processor/src/keyboardProcessor.ts b/web/src/engine/js-processor/src/keyboardProcessor.ts index dea537dfdf..7679fd772a 100644 --- a/web/src/engine/js-processor/src/keyboardProcessor.ts +++ b/web/src/engine/js-processor/src/keyboardProcessor.ts @@ -16,7 +16,7 @@ import { Mock } from "./mock.js"; import type OutputTarget from "./outputTarget.js"; import RuleBehavior from "./ruleBehavior.js"; import KeyboardInterface from './kbdInterface.js'; -import { DeviceSpec, globalObject } from "@keymanapp/web-utils"; +import { DeviceSpec, globalObject, KMWString } from "@keymanapp/web-utils"; import { type MutableSystemStore, SystemStoreIDs } from "./systemStores.js"; // #endregion @@ -214,7 +214,7 @@ export default class KeyboardProcessor extends EventEmitter { var matchBehavior: RuleBehavior; // Before keyboard rules apply, check if the left-context is empty. - const nothingDeletable = outputTarget.getTextBeforeCaret().kmwLength() == 0 && outputTarget.isSelectionEmpty(); + const nothingDeletable = KMWString.length(outputTarget.getTextBeforeCaret()) == 0 && outputTarget.isSelectionEmpty(); // Pass this key code and state to the keyboard program if(this.activeKeyboard && keyEvent.Lcode != 0) { diff --git a/web/src/engine/js-processor/src/mock.ts b/web/src/engine/js-processor/src/mock.ts index e44b76e759..5910ba554b 100644 --- a/web/src/engine/js-processor/src/mock.ts +++ b/web/src/engine/js-processor/src/mock.ts @@ -1,4 +1,5 @@ import OutputTarget from './outputTarget.js'; +import { KMWString } from '@keymanapp/web-utils'; export class Mock extends OutputTarget { text: string; @@ -13,7 +14,7 @@ export class Mock extends OutputTarget { super(); this.text = text ? text : ""; - var defaultLength = this.text._kmwLength(); + var defaultLength = KMWString.length(this.text); // Ensures that `caretPos == 0` is handled correctly. this.selStart = typeof selStart == "number" ? selStart : defaultLength; @@ -29,13 +30,13 @@ export class Mock extends OutputTarget { let clone: Mock; if (outputTarget instanceof Mock) { - // Avoids the need to run expensive kmwstring.ts / `_kmwLength()` + // Avoids the need to run expensive kmwstring.ts `length()` // calculations when deep-copying Mock instances. let priorMock = outputTarget as Mock; clone = new Mock(priorMock.text, priorMock.selStart, priorMock.selEnd); } else { const text = outputTarget.getText(); - const textLen = text._kmwLength(); + const textLen = KMWString.length(text); // If !hasSelection() let selectionStart: number = textLen; @@ -44,8 +45,8 @@ export class Mock extends OutputTarget { if (outputTarget.hasSelection()) { let beforeText = outputTarget.getTextBeforeCaret(); let afterText = outputTarget.getTextAfterCaret(); - selectionStart = beforeText._kmwLength(); - selectionEnd = textLen - afterText._kmwLength(); + selectionStart = KMWString.length(beforeText); + selectionEnd = textLen - KMWString.length(afterText); } // readonly group or not, the returned Mock remains the same. @@ -123,10 +124,10 @@ export class Mock extends OutputTarget { } insertTextBeforeCaret(s: string): void { - this.adjustDeadkeys(s._kmwLength()); + this.adjustDeadkeys(KMWString.length(s)); this.text = this.getTextBeforeCaret() + s + this.text.kmwSubstr(this.selStart); - this.selStart += s.kmwLength(); - this.selEnd += s.kmwLength(); + this.selStart += KMWString.length(s); + this.selEnd += KMWString.length(s); } handleNewlineAtCaret(): void { diff --git a/web/src/engine/js-processor/src/outputTarget.ts b/web/src/engine/js-processor/src/outputTarget.ts index 4b6aac6af6..d4f1285568 100644 --- a/web/src/engine/js-processor/src/outputTarget.ts +++ b/web/src/engine/js-processor/src/outputTarget.ts @@ -1,4 +1,4 @@ -import { extendString } from "@keymanapp/web-utils"; +import { extendString, KMWString } from "@keymanapp/web-utils"; import { findCommonSubstringEndIndex } from "./stringDivergence.js"; import { Mock } from "./mock.js"; import { OutputTarget as OutputTargetInterface } from 'keyman/engine/keyboard'; @@ -129,7 +129,7 @@ export default abstract class OutputTarget implements OutputTargetInterface { const fromLeft = original.getTextBeforeCaret(); const leftDivergenceIndex = findCommonSubstringEndIndex(fromLeft, toLeft, false); - const deletedLeft = fromLeft.substring(leftDivergenceIndex)._kmwLength(); + const deletedLeft = KMWString.length(fromLeft.substring(leftDivergenceIndex)); // No need for our specialized variant here. const insertedText = toLeft.substring(leftDivergenceIndex); @@ -140,7 +140,7 @@ export default abstract class OutputTarget implements OutputTargetInterface { // Right insertions aren't supported, but right deletions will matter in some scenarios. // In particular, once we allow right-deletion for pred-text suggestions applied with the // caret mid-word.. - const deletedRight = fromRight.substring(0, rightDivergenceIndex + 1)._kmwLength(); + const deletedRight = KMWString.length(fromRight.substring(0, rightDivergenceIndex + 1)); return new TextTransform(insertedText, deletedLeft, deletedRight, original.getSelectedText() && !this.getSelectedText()); } @@ -202,7 +202,7 @@ export default abstract class OutputTarget implements OutputTargetInterface { */ protected setTextBeforeCaret(s: string): void { // This one's easy enough to provide a default implementation for. - this.deleteCharsBeforeCaret(this.getTextBeforeCaret()._kmwLength()); + this.deleteCharsBeforeCaret(KMWString.length(this.getTextBeforeCaret())); this.insertTextBeforeCaret(s); } diff --git a/web/src/engine/main/src/headless/contextWindow.ts b/web/src/engine/main/src/headless/contextWindow.ts index 85baa603bd..b37014fc5b 100644 --- a/web/src/engine/main/src/headless/contextWindow.ts +++ b/web/src/engine/main/src/headless/contextWindow.ts @@ -1,5 +1,6 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import { Mock } from "keyman/engine/js-processor"; +import { KMWString } from '@keymanapp/web-utils'; export default class ContextWindow implements LexicalModelTypes.Context { // Used to limit the range of context replicated for use of keyboard rules within @@ -19,14 +20,14 @@ export default class ContextWindow implements LexicalModelTypes.Context { constructor(mock: Mock, config: LexicalModelTypes.Configuration, layerId: string) { this.left = mock.getTextBeforeCaret(); - this.startOfBuffer = this.left._kmwLength() <= config.leftContextCodePoints; + this.startOfBuffer = KMWString.length(this.left) <= config.leftContextCodePoints; if(!this.startOfBuffer) { // Our custom substring version will return the last n characters if param #1 is given -n. this.left = this.left._kmwSubstr(-config.leftContextCodePoints); } this.right = mock.getTextAfterCaret(); - this.endOfBuffer = this.right._kmwLength() <= config.rightContextCodePoints; + this.endOfBuffer = KMWString.length(this.right) <= config.rightContextCodePoints; if(!this.endOfBuffer) { this.right = this.right._kmwSubstr(0, config.rightContextCodePoints); } @@ -38,7 +39,7 @@ export default class ContextWindow implements LexicalModelTypes.Context { } public toMock(): Mock { - let caretPos = this.left._kmwLength(); + let caretPos = KMWString.length(this.left); return new Mock(this.left + (this.right || ""), caretPos); } diff --git a/web/src/engine/predictive-text/templates/src/common.ts b/web/src/engine/predictive-text/templates/src/common.ts index 5e1134dd8a..26e3a55338 100644 --- a/web/src/engine/predictive-text/templates/src/common.ts +++ b/web/src/engine/predictive-text/templates/src/common.ts @@ -6,7 +6,7 @@ import Outcome = LexicalModelTypes.Outcome; import Suggestion = LexicalModelTypes.Suggestion; import Transform = LexicalModelTypes.Transform; import WithOutcome = LexicalModelTypes.WithOutcome; -import { extendString } from "@keymanapp/web-utils"; +import { extendString, KMWString } from "@keymanapp/web-utils"; extendString(); @@ -15,13 +15,13 @@ export const SENTINEL_CODE_UNIT = '\uFDD0'; export function applyTransform(transform: Transform, context: Context): Context { // First, get the current context let fullLeftContext = context.left || ''; - let lLen = fullLeftContext.kmwLength(); + let lLen = KMWString.length(fullLeftContext); let lDel = lLen < transform.deleteLeft ? lLen : transform.deleteLeft; let leftContext = fullLeftContext.kmwSubstr(0, lLen - lDel) + (transform.insert || ''); let fullRightContext = context.right || ''; - let rLen = fullRightContext.kmwLength(); + let rLen = KMWString.length(fullRightContext); let rDel = (rLen < (transform.deleteRight ?? 0)) ? rLen : (transform.deleteRight ?? 0); let rightContext = fullRightContext.kmwSubstr(rDel); @@ -47,7 +47,7 @@ export function buildMergedTransform(first: Transform, second: Transform): Trans // The 'fun' case: the second Transform wants to delete something from the first. if(second.deleteLeft) { - let firstLength = first.insert.kmwLength(); + let firstLength = KMWString.length(first.insert); if(firstLength <= second.deleteLeft) { mergedFirstInsert = ''; mergedSecondDelete = second.deleteLeft - firstLength; diff --git a/web/src/engine/predictive-text/templates/src/trie-model.ts b/web/src/engine/predictive-text/templates/src/trie-model.ts index 2c5e6a194e..f5cdb2223e 100644 --- a/web/src/engine/predictive-text/templates/src/trie-model.ts +++ b/web/src/engine/predictive-text/templates/src/trie-model.ts @@ -26,7 +26,7 @@ // Should probably make a 'lm-utils' submodule. // Allows the kmwstring bindings to resolve. -import { extendString, PriorityQueue } from "@keymanapp/web-utils"; +import { extendString, KMWString, PriorityQueue } from "@keymanapp/web-utils"; import { default as defaultWordBreaker } from "@keymanapp/models-wordbreakers"; import { applyTransform, isHighSurrogate, isSentinel, SENTINEL_CODE_UNIT, transformToSuggestion } from "./common.js"; @@ -323,7 +323,7 @@ export default class TrieModel implements LexicalModel { let newContext = applyTransform(transform, context); // Computes the different in word length after applying the transform above. - let leftDelOffset = transform.deleteLeft - transform.insert.kmwLength(); + let leftDelOffset = transform.deleteLeft - KMWString.length(transform.insert); // All text to the left of the cursor INCLUDING anything that has // just been typed. @@ -334,7 +334,7 @@ export default class TrieModel implements LexicalModel { transformToSuggestion({ insert: text, // Delete whatever the prefix that the user wrote. - deleteLeft: leftDelOffset + prefix.kmwLength() + deleteLeft: leftDelOffset + KMWString.length(prefix) // Note: a separate capitalization/orthography engine can take this // result and transform it as needed. }, diff --git a/web/src/engine/predictive-text/templates/tests/helpers.js b/web/src/engine/predictive-text/templates/tests/helpers.js index c4376b613f..1bf4fabcfb 100644 --- a/web/src/engine/predictive-text/templates/tests/helpers.js +++ b/web/src/engine/predictive-text/templates/tests/helpers.js @@ -13,7 +13,6 @@ import { createRequire } from "module"; import { fileURLToPath } from 'url'; extendString(); -assert.ok('💩'.kmwLength); /** * Load JSON fixtures from a well-known place. diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts index 9bbc7f19c2..548341ef60 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts @@ -1,4 +1,5 @@ import { applyTransform, buildMergedTransform, Token } from '@keymanapp/models-templates'; +import { KMWString } from '@keymanapp/web-utils'; import { ClassicalDistanceCalculation, EditOperation } from './classical-calculation.js'; import { SearchSpace } from './distance-modeler.js'; @@ -16,7 +17,7 @@ import USVString = LexicalModelTypes.USVString; function textToCharTransforms(text: string, transformId?: number) { let perCharTransforms: Transform[] = []; - for(let i=0; i < text.kmwLength(); i++) { + for(let i=0; i < KMWString.length(text); i++) { let char = text.kmwCharAt(i); // is SMP-aware let transform: Transform = { @@ -40,7 +41,7 @@ export function getEditPathLastMatch(editPath: EditOperation[]) { return editPath.lastIndexOf('match', editLength - 2); } else { return editPath.lastIndexOf('match'); - } + } } export class TrackedContextSuggestion { @@ -62,7 +63,7 @@ export class TrackedContextToken { constructor(instance?: TrackedContextToken) { if(instance) { Object.assign(this, instance); - // We don't alter the values in replacements, but we do wish to prevent aliasing + // We don't alter the values in replacements, but we do wish to prevent aliasing // of the array containing them. this.replacements = instance.replacements.slice(); } @@ -109,7 +110,7 @@ export class TrackedContextToken { }); this.raw = tokenText; - this.transformDistributions = backspacedTokenContext; + this.transformDistributions = backspacedTokenContext; this.clearReplacements(); } @@ -329,14 +330,14 @@ interface ContextMatchResult { /** * Indicates the portion of the incoming keystroke data, if any, that applies to * tokens before the last pre-caret token and thus should not be replaced by predictions - * based upon `state`. If the provided context state + the incoming transform do not + * based upon `state`. If the provided context state + the incoming transform do not * adequately match the current context, the match attempt will fail with a `null` result. * * Should generally be non-null if the token before the caret did not previously exist. - * - * The result may be null if it does not match the prior context state or if bookkeeping - * based upon it is problematic - say, if wordbreaking effects shift due to new input, - * causing a mismatch with the prior state's tokenization. + * + * The result may be null if it does not match the prior context state or if bookkeeping + * based upon it is problematic - say, if wordbreaking effects shift due to new input, + * causing a mismatch with the prior state's tokenization. * (Refer to #12494 for an example case.) */ preservationTransform?: Transform; @@ -591,7 +592,7 @@ export class ContextTracker extends CircularArray { // multitaps makes it tricky to reliably use in all situations. // It's best to handle `delete` cases directly for this reason. for(let j = i + 1; j < editPath.length; j++) { - // If something _other_ than delete follows a 'delete' on the edit path, + // If something _other_ than delete follows a 'delete' on the edit path, // we probably have a context mismatch. // // It's possible to construct cases where this isn't true, but it's likely not @@ -620,12 +621,12 @@ export class ContextTracker extends CircularArray { priorEdit = editPath[i]; } - return { - state, - baseState: matchState, - preservationTransform, - headTokensRemoved: poppedTokenCount, - tailTokensAdded: pushedTokenCount + return { + state, + baseState: matchState, + preservationTransform, + headTokensRemoved: poppedTokenCount, + tailTokensAdded: pushedTokenCount }; } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/index.ts b/web/src/engine/predictive-text/worker-thread/src/main/index.ts index d0676992e0..db12195738 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/index.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/index.ts @@ -37,6 +37,7 @@ extendString(); import * as models from './models/index.js'; import * as correction from './correction/index.js'; import * as wordBreakers from '@keymanapp/models-wordbreakers'; +import { KMWString } from "@keymanapp/web-utils"; import ModelCompositor from './model-compositor.js'; import { ImportScripts, IncomingMessage, LMLayerWorkerState, LoadMessage, ModelEval, ModelFile, ModelSourceSpec, PostMessage } from './worker-interfaces.js'; @@ -108,6 +109,9 @@ export default class LMLayerWorker { importScripts: null, postMessage: null }) { + // Within the worker, we can't infer this from the keyboard. + // Additionally, we always work with small text windows, so it's not _too_ expensive to keep on. + KMWString.enableSupplementaryPlane(true); this._postMessage = options.postMessage || postMessage; this._importScripts = options.importScripts || importScripts; this.setupConfigState(); diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts index d0b4658b13..3b9e049b77 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts @@ -1,6 +1,8 @@ import * as models from '@keymanapp/models-templates'; import * as correction from './correction/index.js' +import { KMWString } from '@keymanapp/web-utils'; + import TransformUtils from './transformUtils.js'; import { correctAndEnumerate, dedupeSuggestions, finalizeSuggestions, predictionAutoSelect, processSimilarity, toAnnotatedSuggestion, tupleDisplayOrderSort } from './predict-helpers.js'; import { detectCurrentCasing, determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js'; @@ -194,7 +196,7 @@ export class ModelCompositor { // Responsible for applying casing rules to suggestions. private applySuggestionCasing(suggestion: Suggestion, baseWord: USVString, casingForm: CasingForm) { // Step 1: does the suggestion replace the whole word? If not, we should extend the suggestion to do so. - let unchangedLength = baseWord.kmwLength() - suggestion.transform.deleteLeft; + let unchangedLength = KMWString.length(baseWord) - suggestion.transform.deleteLeft; if(unchangedLength > 0) { suggestion.transform.deleteLeft += unchangedLength; @@ -210,7 +212,7 @@ export class ModelCompositor { // Step 1: generate and save the reversion's Transform. let sourceTransform = suggestion.transform; let deletedLeftChars = context.left.kmwSubstr(-sourceTransform.deleteLeft, sourceTransform.deleteLeft); - let insertedLength = sourceTransform.insert.kmwLength(); + let insertedLength = KMWString.length(sourceTransform.insert); let reversionTransform: Transform = { insert: deletedLeftChars, diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-helpers.ts index d5f9b4397f..071cad588c 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/model-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/model-helpers.ts @@ -6,6 +6,8 @@ import LexicalModelPunctuation = LexicalModelTypes.LexicalModelPunctuation; import * as models from '@keymanapp/models-templates'; import * as wordBreakers from '@keymanapp/models-wordbreakers'; +import { KMWString } from '@keymanapp/web-utils'; + /** * The default punctuation and spacing produced by the model. */ @@ -99,7 +101,7 @@ export function detectCurrentCasing(lexicalModel: LexicalModel, context: Context return 'lower'; } else if(model.applyCasing('upper', text) == text) { // If only a single character has been input, assume we're in 'initial' mode. - return text.kmwLength() > 1 ? 'upper' : 'initial'; + return KMWString.length(text) > 1 ? 'upper' : 'initial'; } else if(model.applyCasing('initial', text) == text) { // We check 'initial' last, as upper-case input is indistinguishable. return 'initial'; diff --git a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts index ab07355d01..87e6abf189 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/predict-helpers.ts @@ -1,4 +1,5 @@ import * as models from '@keymanapp/models-templates'; +import { KMWString } from '@keymanapp/web-utils'; import TransformUtils from './transformUtils.js'; import { determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js'; @@ -228,11 +229,11 @@ export async function correctAndEnumerate( * Since we replace a word being corrected/predicted, we take length of the remaining * context's tail token in addition to however far was deleted to reach that state. */ - deleteLeft = wordbreak(postContext).kmwLength() + inputTransform.deleteLeft; + deleteLeft = KMWString.length(wordbreak(postContext)) + inputTransform.deleteLeft; } else { // Suggestions are applied to the pre-input context, so get the token's original length. // We're on the same token, so just delete its text for the replacement op. - deleteLeft = wordbreak(context).kmwLength(); + deleteLeft = KMWString.length(wordbreak(context)); } // Is the token under construction newly-constructed / is there no pre-existing root? @@ -583,7 +584,7 @@ export function processSimilarity( // Generate a full-word 'keep' replacement like other suggestions when one is not otherwise // produced; we want to replace the full token in the same manner used for other suggestions. - const basePrefixLength = truePrefix.kmwLength() - inputTransform.insert.kmwLength() + inputTransform.deleteLeft; + const basePrefixLength = KMWString.length(truePrefix) - KMWString.length(inputTransform.insert) + inputTransform.deleteLeft; const keepTransform = { insert: truePrefix, deleteLeft: basePrefixLength diff --git a/web/src/test/auto/dom/cases/element-wrappers/element_interfaces.tests.ts b/web/src/test/auto/dom/cases/element-wrappers/element_interfaces.tests.ts index cdfc8814af..8ccb0ce428 100644 --- a/web/src/test/auto/dom/cases/element-wrappers/element_interfaces.tests.ts +++ b/web/src/test/auto/dom/cases/element-wrappers/element_interfaces.tests.ts @@ -1,6 +1,6 @@ import { assert } from 'chai'; -import { extendString } from 'keyman/engine/keyboard'; +import { extendString, KMWString } from 'keyman/engine/keyboard'; import { Mock } from 'keyman/engine/js-processor'; import * as wrappers from 'keyman/engine/element-wrappers'; @@ -346,7 +346,7 @@ class MockTestHelper implements TestHelper { } setSelectionRange(pair: ElementPair, start: number, end: number) { - const convert = (index: number) => pair.wrapper.text.kmwCodeUnitToCodePoint(index); + const convert = (index: number) => KMWString.codeUnitToCodePoint(pair.wrapper.text, index); pair.wrapper.setSelection(convert(start), convert(end)); } diff --git a/web/src/test/auto/headless/engine/js-processor/specialized-backspace.tests.js b/web/src/test/auto/headless/engine/js-processor/specialized-backspace.tests.js index 7e35817f07..005a197b0f 100644 --- a/web/src/test/auto/headless/engine/js-processor/specialized-backspace.tests.js +++ b/web/src/test/auto/headless/engine/js-processor/specialized-backspace.tests.js @@ -4,6 +4,7 @@ import fs from 'fs'; import { createRequire } from 'module'; const require = createRequire(import.meta.url); +import { KMWString } from '@keymanapp/web-utils'; import { Codes, KeyEvent, MinimalKeymanGlobal } from 'keyman/engine/keyboard'; import { KeyboardInterface, KeyboardProcessor, Mock } from 'keyman/engine/js-processor'; import { NodeKeyboardLoader } from 'keyman/engine/keyboard/node-keyboard-loader'; @@ -51,7 +52,7 @@ const DOUBLED_BKSP_KEYBOARD_SCRIPT = function keyboard_core () { this.g_main=function(t,e) { var k=KeymanWeb; // Standard compiled keyboards do not directly use any methods on `t`. - if(e.Lcode == Codes.keyCodes.K_BKSP && t.getTextBeforeCaret().kmwLength() >= 2){ + if(e.Lcode == Codes.keyCodes.K_BKSP && KMWString.length(t.getTextBeforeCaret()) >= 2){ // If the context has at least two characters, delete two. k.KO(2, t, '') // Our rule matched, so signal that. @@ -189,7 +190,7 @@ describe('Engine - specialized backspace handling', function() { it('empty context, positional keyboard, but text is selected', () => { let contextSource = new Mock('selected text', 0); - contextSource.setSelection(0, contextSource.getText().kmwLength()); + contextSource.setSelection(0, KMWString.length(contextSource.getText())); let event = new KeyEvent({ Lcode: Codes.keyCodes.K_BKSP, diff --git a/web/src/test/auto/headless/engine/main/headless/inputProcessor.tests.js b/web/src/test/auto/headless/engine/main/headless/inputProcessor.tests.js index 91e5410b3a..a23e06dfdf 100644 --- a/web/src/test/auto/headless/engine/main/headless/inputProcessor.tests.js +++ b/web/src/test/auto/headless/engine/main/headless/inputProcessor.tests.js @@ -12,6 +12,7 @@ import { KeyboardTest } from '@keymanapp/recorder-core'; import { Worker } from '@keymanapp/lexical-model-layer/node'; import * as utils from '@keymanapp/web-utils'; +const KMWString = utils.KMWString; // Required initialization setup. global.keyman = {}; // So that keyboard-based checks against the global `keyman` succeed. @@ -115,9 +116,9 @@ describe('InputProcessor', function() { assert.isNotNull(behavior); }); - it('with extremely long context (' + coreSourceCode._kmwLength() + ' chars, no fat-fingers)', function() { + it('with extremely long context (' + KMWString.length(coreSourceCode) + ' chars, no fat-fingers)', function() { // Assumes no SMP chars in the source, which is fine. - let context = new Mock(coreSourceCode, coreSourceCode._kmwLength()); + let context = new Mock(coreSourceCode, KMWString.length(coreSourceCode)); this.timeout(500); // 500 ms, excluding text import. // These often run on VMs, so we'll be a bit generous. @@ -153,9 +154,9 @@ describe('InputProcessor', function() { assert.isNotNull(behavior); }); - it('with extremely long context (' + coreSourceCode._kmwLength() + ' chars, with fat-fingers)', function() { + it('with extremely long context (' + KMWString.length(coreSourceCode) + ' chars, with fat-fingers)', function() { // Assumes no SMP chars in the source, which is fine. - let context = new Mock(coreSourceCode, coreSourceCode._kmwLength()); + let context = new Mock(coreSourceCode, KMWString.length(coreSourceCode)); this.timeout(500); // 500 ms, excluding text import. // These often run on VMs, so we'll be a bit generous. From a53f2dcd1033402e44387782e43dbcbb7214a64b Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 22 Apr 2025 10:12:40 +0700 Subject: [PATCH 029/431] chore(web): refactor _kmwCharCodeAt --- .../common/web-utils/src/codepointString.ts | 410 ++++++++++++++++++ .../engine/common/web-utils/src/kmwstring.ts | 75 ++-- .../web-utils/src/tests/stringHandling.js | 44 +- .../js-processor/src/keyboardProcessor.ts | 2 +- 4 files changed, 474 insertions(+), 57 deletions(-) create mode 100644 web/src/engine/common/web-utils/src/codepointString.ts diff --git a/web/src/engine/common/web-utils/src/codepointString.ts b/web/src/engine/common/web-utils/src/codepointString.ts new file mode 100644 index 0000000000..b489da87da --- /dev/null +++ b/web/src/engine/common/web-utils/src/codepointString.ts @@ -0,0 +1,410 @@ +/* + * NOTE: in previous versions of Keyman Engine for Web, these methods were set on the standard + * JS String type in a manner considered a "side-effect". (When inspecting older versions, it may help + * to know that they were originally implemented within a file named kmwstring.ts.) + * + * These methods seek to support SMP-aware string functionality within the Web engine, as many + * keyboards exist that utilize characters within such ranges. + */ + +// utf16helpers.ts (?) + +/** + * Indicates if functionality supporting Supplementary Multilingual Plane (SMP) + * characters should be enabled. + */ +let EXTENSION_ENABLED = false; + +/** + * Enables or disables support for SMP-aware string handling. + * @param flag + */ +export function enableSupplementaryPlane(flag: boolean) { // enableSMPProcessing + EXTENSION_ENABLED = flag; +} + +/** + * Returns a number indicating the Unicode value of the character at the given + * code point index, with support for supplementary plane characters. + * + * @param {string} s The string + * @param {number} index The code point index into the string (not + the code unit index) to return + * @return {number} The Unicode character value + */ +export function charCodeAt(s: string, index: number) { + // While String.codepointAt exists, it does not use code-point indexing and so is + // unsuited to our needs here. + const str = s; + + if(!EXTENSION_ENABLED) { + return s.charCodeAt(index); + } + + let codeUnitIndex = 0; + + if(index < 0 || index >= str.length) { + return NaN; + } + + for(let i = 0; i < index; i++) { + codeUnitIndex = nextChar(str, codeUnitIndex); + if(codeUnitIndex === null) { + return NaN; + } + } + + let first = str.charCodeAt(codeUnitIndex); + if(first >= 0xD800 && first <= 0xDBFF && str.length > codeUnitIndex + 1) { + let second = str.charCodeAt(codeUnitIndex + 1); + if(second >= 0xDC00 && second <= 0xDFFF) { + return ((first - 0xD800) << 10) + (second - 0xDC00) + 0x10000; + } + } + return first; +} + +/** + * Returns the code point index within the calling String object of the first occurrence + * of the specified value, or -1 if not found. + * + * @param {string} s The string + * @param {string} searchValue The value to search for + * @param {number=} fromIndex Optional code point index to start searching from + * @return {number} The code point index of the specified search value + */ +export function indexOf(s: string, searchValue: string, fromIndex?: number) { // pointIndexOf() + const str = s; + + if(!EXTENSION_ENABLED) { + return s.indexOf(searchValue, fromIndex); + } + + const codeUnitIndex = str.indexOf(searchValue, fromIndex); + + if(codeUnitIndex < 0) { + return codeUnitIndex; + } + + let codePointIndex = 0; + for(let i = 0; i !== null && i < codeUnitIndex; i = nextChar(str, i)) { + codePointIndex++; + } + return codePointIndex; +} + +/** + * Returns the code point index within the calling String object of the last occurrence + * of the specified value, or -1 if not found. + * + * @param {string} searchValue The value to search for + * @param {number=} fromIndex Optional code point index to start searching from + * @return {number} The code point index of the specified search value + */ +export function lastIndexOf(s: string, searchValue: string, fromIndex?: number) { // lastPointIndexOf() + const str = s; + + if(!EXTENSION_ENABLED) { + return str.lastIndexOf(searchValue, fromIndex); + } + + const codeUnitIndex = str.lastIndexOf(searchValue, fromIndex); + + if(codeUnitIndex < 0) { + return codeUnitIndex; + } + + let codePointIndex = 0; + for(let i = 0; i !== null && i < codeUnitIndex; i = nextChar(str, i)) { + codePointIndex++; + } + return codePointIndex; +} + +/** + * Returns the length of the string in code points, as opposed to code units. + * + * @param {string} s The string + * @return {number} The length of the string in code points + */ +export function length(s: string) { // codepointLength() + const str = s; + + if(!EXTENSION_ENABLED) { + return str.length; + } + + if(str.length == 0) { + return 0; + } + + let i: number; + let codeUnitIndex: number; + for(i = 0, codeUnitIndex = 0; codeUnitIndex !== null; i++) { + codeUnitIndex = nextChar(str, codeUnitIndex); + } + return i; +} + +/** + * Extracts a section of a string and returns a new string. + * + * @param {string} s The string + * @param {number} beginSlice The start code point index in the string to + * extract from + * @param {number} endSlice Optional end code point index in the string + * to extract to + * @return {string} The substring as selected by beginSlice and + * endSlice + */ +export function slice(s: string, beginSlice: number, endSlice?: number) { // sliceAtPoints() + const str = s; + + if(!EXTENSION_ENABLED) { + return str.slice(beginSlice, endSlice); + } + + const beginSliceCodeUnit = codePointToCodeUnit(str, beginSlice); + const endSliceCodeUnit = codePointToCodeUnit(str, endSlice); + if(beginSliceCodeUnit === null || endSliceCodeUnit === null) { + return ''; + } else { + return str.slice(beginSliceCodeUnit, endSliceCodeUnit); + } +} + +/** + * Returns the characters in a string beginning at the specified location through + * the specified number of characters. + * + * @param {string} s The string + * @param {number} start The start code point index in the string to + * extract from + * @param {number=} len Optional length to extract + * @return {string} The substring as selected by start and length + */ +export function substr(s: string, start: number, len?: number) { // substrFromPoint() + const str = s; + + if(!EXTENSION_ENABLED) { + if(start > -1) { + return str.substr(start,len); + } else { + return str.substr(str.length+start,-start); + } + } + + if(start < 0) { + start = length(str) + start; + } + if(start < 0) { + start = 0; + } + const startCodeUnit = codePointToCodeUnit(str, start); + let endCodeUnit = startCodeUnit; + + if(startCodeUnit === null) { + return ''; + } + + if(arguments.length < 3) { + endCodeUnit = str.length; + } else { + for(let i = 0; i < len; i++) { + endCodeUnit = nextChar(str, endCodeUnit); + } + } + + if(endCodeUnit === null) { + return str.substring(startCodeUnit); + } else { + return str.substring(startCodeUnit, endCodeUnit); + } +} + +/** + * Returns the characters in a string between two indexes into the string. + * + * @param {string} s The string + * @param {number} indexA The start code point index in the string to + * extract from + * @param {number} indexB The end code point index in the string to + * extract to + * @return {string} The substring as selected by indexA and indexB + */ +export function substring(s: string, indexA: number, indexB: number) { // substringBetweenPoints() + const str = s; + + if(!EXTENSION_ENABLED) { + return str.substring(indexA, indexB); + } + + let indexACodeUnit, indexBCodeUnit; + + if(typeof(indexB) == 'undefined') { + indexACodeUnit = codePointToCodeUnit(str, indexA); + indexBCodeUnit = str.length; + } else { + if(indexA > indexB) { + let c = indexA; + indexA = indexB; + indexB = c; + } + + indexACodeUnit = codePointToCodeUnit(str, indexA); + indexBCodeUnit = codePointToCodeUnit(str, indexB); + } + + if(isNaN(indexACodeUnit) || indexACodeUnit === null) { + indexACodeUnit = 0; + } + if(isNaN(indexBCodeUnit) || indexBCodeUnit === null) { + indexBCodeUnit = str.length; + } + + return str.substring(indexACodeUnit, indexBCodeUnit); +} + +/** + * Returns the code unit index for the next code point in the string, accounting for + * supplementary pairs + * + * @param {string} s The string + * @param {number|null} codeUnitIndex The code unit position to increment + * @return {number|null} The index of the next code point in the string, + * in code units + */ +export function nextChar(s: string, codeUnitIndex: number) { // nextCharAfterUnit + const str = s; + if(codeUnitIndex === null || codeUnitIndex < 0 || codeUnitIndex >= str.length - 1) { + return null; + } + + if(!EXTENSION_ENABLED) { + return codeUnitIndex+1; + } + + const first = str.charCodeAt(codeUnitIndex); + if(first >= 0xD800 && first <= 0xDBFF && str.length > codeUnitIndex + 1) { + const second = str.charCodeAt(codeUnitIndex + 1); + if(second >= 0xDC00 && second <= 0xDFFF) { + if(codeUnitIndex == str.length - 2) { + return null; + } + return codeUnitIndex + 2; + } + } + return codeUnitIndex + 1; +} + +/** + * Returns the code unit index for the previous code point in the string, accounting + * for supplementary pairs + * + * @param {string} s The string + * @param {number|null} codeUnitIndex The code unit position to decrement + * @return {number|null} The index of the previous code point in the + * string, in code units +*/ +export function prevChar(s: string, codeUnitIndex: number) { // prevCharBeforeUnit + const str = s; + + if(codeUnitIndex == null || codeUnitIndex <= 0 || codeUnitIndex > str.length) { + return null; + } + + if(!EXTENSION_ENABLED) { + return codeUnitIndex-1; + } + + const second = str.charCodeAt(codeUnitIndex - 1); + if(second >= 0xDC00 && second <= 0xDFFF && codeUnitIndex > 1) { + const first = str.charCodeAt(codeUnitIndex - 2); + if(first >= 0xD800 && first <= 0xDBFF) { + return codeUnitIndex - 2; + } + } + return codeUnitIndex - 1; +} + +/** + * Returns the corresponding code unit index to the code point index passed + * + * @param {string} s The string + * @param {number|null} codePointIndex A code point index in the string + * @return {number|null} The corresponding code unit index + */ +export function codePointToCodeUnit(s: string, codePointIndex: number) { // + if(codePointIndex === null) { + return null; + } + + if(!EXTENSION_ENABLED) { + return codePointIndex; + } + + const str = s; + let codeUnitIndex = 0; + + if(codePointIndex < 0) { + codeUnitIndex = str.length; + for(let i = 0; i > codePointIndex; i--) { + codeUnitIndex = prevChar(str, codeUnitIndex); + } + return codeUnitIndex; + } + + if(codePointIndex == length(str)) { + return str.length; + } + + for(let i = 0; i < codePointIndex; i++) { + codeUnitIndex = nextChar(str, codeUnitIndex); + } + return codeUnitIndex; +} + +/** + * Returns the corresponding code point index to the code unit index passed + * + * @param {number|null} codeUnitIndex A code unit index in the string + * @return {number|null} The corresponding code point index + */ +export function codeUnitToCodePoint(s: string, codeUnitIndex: number) { // + const str = s; + + if(!EXTENSION_ENABLED) { + return codeUnitIndex; + } + + if(codeUnitIndex === null) { + return null; + } else if(codeUnitIndex == 0) { + return 0; + } else if(codeUnitIndex < 0) { + return length(str.substr(codeUnitIndex)); + } else { + return length(str.substr(0,codeUnitIndex)); + } +} + +/** + * Returns the character at a the code point index passed + * + * @param {string} s The string + * @param {number} index A code point index in the string + * @return {string} The corresponding character + */ +export function charAt(s: string, index: number) { // charAtPoint + const str = s; + + if(!EXTENSION_ENABLED) { + return str.charAt(index); + } + + if(index >= 0) { + return substr(str, index, 1); + } else { + return '' + }; +} \ No newline at end of file diff --git a/web/src/engine/common/web-utils/src/kmwstring.ts b/web/src/engine/common/web-utils/src/kmwstring.ts index 666eed9509..b844534b42 100644 --- a/web/src/engine/common/web-utils/src/kmwstring.ts +++ b/web/src/engine/common/web-utils/src/kmwstring.ts @@ -17,6 +17,47 @@ export function enableSupplementaryPlane(bEnable: boolean) { EXTENSION_ENABLED = bEnable; } +/** + * Returns a number indicating the Unicode value of the character at the given + * code point index, with support for supplementary plane characters. + * + * @param {string} s The string + * @param {number} index The code point index into the string (not + the code unit index) to return + * @return {number} The Unicode character value + */ +export function charCodeAt(s: string, index: number) { + // While String.codepointAt exists, it does not use code-point indexing and so is + // unsuited to our needs here. + const str = s; + + if(!EXTENSION_ENABLED) { + return s.charCodeAt(index); + } + + let codeUnitIndex = 0; + + if(index < 0 || index >= str.length) { + return NaN; + } + + for(let i = 0; i < index; i++) { + codeUnitIndex = nextChar(str, codeUnitIndex); + if(codeUnitIndex === null) { + return NaN; + } + } + + let first = str.charCodeAt(codeUnitIndex); + if(first >= 0xD800 && first <= 0xDBFF && str.length > codeUnitIndex + 1) { + let second = str.charCodeAt(codeUnitIndex + 1); + if(second >= 0xDC00 && second <= 0xDFFF) { + return ((first - 0xD800) << 10) + (second - 0xDC00) + 0x10000; + } + } + return first; +} + /** * Returns the length of the string in code points, as opposed to code units. * @@ -175,7 +216,6 @@ declare global { } interface String { - kmwCharCodeAt(codePointIndex: number): number, kmwCharAt(codePointIndex: number) : string, kmwIndexOf(searchValue: string, fromIndex?: number) : number, kmwLastIndexOf(searchValue: string, fromIndex?: number) : number, @@ -183,7 +223,6 @@ declare global { kmwSubstring(start: number, length: number) : string, kmwSubstr(start: number, length?: number) : string, kmwBMPSubstr(start: number, length?: number) : string, - _kmwCharCodeAt(codePointIndex: number): number, _kmwCharAt(codePointIndex: number) : string, _kmwIndexOf(searchValue: string, fromIndex?: number) : number, _kmwLastIndexOf(searchValue: string, fromIndex?: number) : number, @@ -194,37 +233,6 @@ declare global { } export default function extendString() { - /** - * Returns a number indicating the Unicode value of the character at the given - * code point index, with support for supplementary plane characters. - * - * @param {number} codePointIndex The code point index into the string (not - the code unit index) to return - * @return {number} The Unicode character value - */ - String.prototype.kmwCharCodeAt = function(codePointIndex) { - var str = String(this); - var codeUnitIndex = 0; - - if (codePointIndex < 0 || codePointIndex >= str.length) { - return NaN; - } - - for(var i = 0; i < codePointIndex; i++) { - codeUnitIndex = nextChar(str, codeUnitIndex); - if(codeUnitIndex === null) return NaN; - } - - var first = str.charCodeAt(codeUnitIndex); - if (first >= 0xD800 && first <= 0xDBFF && str.length > codeUnitIndex + 1) { - var second = str.charCodeAt(codeUnitIndex + 1); - if (second >= 0xDC00 && second <= 0xDFFF) { - return ((first - 0xD800) << 10) + (second - 0xDC00) + 0x10000; - } - } - return first; - } - /** * Returns the code point index within the calling String object of the first occurrence * of the specified value, or -1 if not found. @@ -398,7 +406,6 @@ export default function extendString() { { var p=String.prototype; p._kmwCharAt = bEnable ? p.kmwCharAt : p.charAt; - p._kmwCharCodeAt = bEnable ? p.kmwCharCodeAt : p.charCodeAt; p._kmwIndexOf = bEnable ? p.kmwIndexOf :p.indexOf; p._kmwLastIndexOf = bEnable ? p.kmwLastIndexOf : p.lastIndexOf ; p._kmwSlice = bEnable ? p.kmwSlice : p.slice; diff --git a/web/src/engine/common/web-utils/src/tests/stringHandling.js b/web/src/engine/common/web-utils/src/tests/stringHandling.js index 156e3b7ab4..677be1d176 100644 --- a/web/src/engine/common/web-utils/src/tests/stringHandling.js +++ b/web/src/engine/common/web-utils/src/tests/stringHandling.js @@ -18,14 +18,14 @@ describe('Unicode string handling', () => { assert.equal(str._kmwCharAt(5), "g"); }); - it('_kmwCharCodeAt', () => { + it('charCodeAt', () => { const str = "string" - assert.isNaN(str._kmwCharCodeAt(-1)); - assert.isNaN(str._kmwCharCodeAt(6)); - assert.isNaN(str._kmwCharCodeAt(1000)); - assert.equal(str._kmwCharCodeAt(0), "s".charCodeAt(0)); - assert.equal(str._kmwCharCodeAt(2), "r".charCodeAt(0)); - assert.equal(str._kmwCharCodeAt(5), "g".charCodeAt(0)); + assert.isNaN(KMWString.charCodeAt(str, -1)); + assert.isNaN(KMWString.charCodeAt(str, 6)); + assert.isNaN(KMWString.charCodeAt(str, 1000)); + assert.equal(KMWString.charCodeAt(str, 0), "s".charCodeAt(0)); + assert.equal(KMWString.charCodeAt(str, 2), "r".charCodeAt(0)); + assert.equal(KMWString.charCodeAt(str, 5), "g".charCodeAt(0)); }); it('_kmwIndexOf', () => { @@ -131,14 +131,14 @@ describe('Unicode string handling', () => { assert.equal(str._kmwCharAt(5), "g"); }); - it('_kmwCharCodeAt', () => { + it('charCodeAt', () => { const str = "string"; - assert.isNaN(str._kmwCharCodeAt(-1)); - assert.isNaN(str._kmwCharCodeAt(6)); - assert.isNaN(str._kmwCharCodeAt(1000)); - assert.equal(str._kmwCharCodeAt(0), "s".charCodeAt(0)); - assert.equal(str._kmwCharCodeAt(2), "r".charCodeAt(0)); - assert.equal(str._kmwCharCodeAt(5), "g".charCodeAt(0)); + assert.isNaN(KMWString.charCodeAt(str, -1)); + assert.isNaN(KMWString.charCodeAt(str, 6)); + assert.isNaN(KMWString.charCodeAt(str, 1000)); + assert.equal(KMWString.charCodeAt(str, 0), "s".charCodeAt(0)); + assert.equal(KMWString.charCodeAt(str, 2), "r".charCodeAt(0)); + assert.equal(KMWString.charCodeAt(str, 5), "g".charCodeAt(0)); }); it('_kmwIndexOf', () => { @@ -253,17 +253,17 @@ describe('Unicode string handling', () => { assert.equal(apples._kmwCharAt(5), "s"); }); - it('_kmwCharCodeAt', () => { + it('charCodeAt', () => { // 0x1d5c9: MATHEMATICAL SANS-SERIF SMALL p // 0x1d5be: MATHEMATICAL SANS-SERIF SMALL e const apples = 'a' + String.fromCodePoint(0x1d5c9) + 'p' + 'l' + String.fromCodePoint(0x1d5be) + 's'; - assert.isNaN(apples._kmwCharCodeAt(-1)); - assert.isNaN(apples._kmwCharCodeAt(6)); - assert.isNaN(apples._kmwCharCodeAt(1000)); - assert.equal(apples._kmwCharCodeAt(0), "a".charCodeAt(0)); - assert.equal(apples._kmwCharCodeAt(1), 0x1d5c9); - assert.equal(apples._kmwCharCodeAt(2), "p".charCodeAt(0)); - assert.equal(apples._kmwCharCodeAt(5), "s".charCodeAt(0)); + assert.isNaN(KMWString.charCodeAt(apples, -1)); + assert.isNaN(KMWString.charCodeAt(apples, 6)); + assert.isNaN(KMWString.charCodeAt(apples, 1000)); + assert.equal(KMWString.charCodeAt(apples, 0), "a".charCodeAt(0)); + assert.equal(KMWString.charCodeAt(apples, 1), 0x1d5c9); + assert.equal(KMWString.charCodeAt(apples, 2), "p".charCodeAt(0)); + assert.equal(KMWString.charCodeAt(apples, 5), "s".charCodeAt(0)); }); it('_kmwIndexOf', () => { diff --git a/web/src/engine/js-processor/src/keyboardProcessor.ts b/web/src/engine/js-processor/src/keyboardProcessor.ts index 7679fd772a..a0e5725905 100644 --- a/web/src/engine/js-processor/src/keyboardProcessor.ts +++ b/web/src/engine/js-processor/src/keyboardProcessor.ts @@ -158,7 +158,7 @@ export default class KeyboardProcessor extends EventEmitter { // // Would recommend (conceptually) equaling K_RIGHT + K_BKSP, the former of which would technically be a 'command'. default: // In case we extend the allowed set, but forget to implement its handling case above. - ruleBehavior.errorLog = "Unexpected 'special emulation' character (\\u" + (special as String).kmwCharCodeAt(0).toString(16) + ") went unhandled!"; + ruleBehavior.errorLog = "Unexpected 'special emulation' character (\\u" + KMWString.charCodeAt(special as string, 0).toString(16) + ") went unhandled!"; } } else { // Back to the standard default, pending normal matching. From 2cb6e5a191c3b23a8d0ec43d2af80191ddb17c82 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 22 Apr 2025 10:15:41 +0700 Subject: [PATCH 030/431] refactor(web): refactor _kmwIndexOf, _kmwLastIndexOf --- .../common/web-utils/src/codepointString.ts | 410 ------------------ .../engine/common/web-utils/src/kmwstring.ts | 107 ++--- .../web-utils/src/tests/stringHandling.js | 156 +++---- 3 files changed, 136 insertions(+), 537 deletions(-) delete mode 100644 web/src/engine/common/web-utils/src/codepointString.ts diff --git a/web/src/engine/common/web-utils/src/codepointString.ts b/web/src/engine/common/web-utils/src/codepointString.ts deleted file mode 100644 index b489da87da..0000000000 --- a/web/src/engine/common/web-utils/src/codepointString.ts +++ /dev/null @@ -1,410 +0,0 @@ -/* - * NOTE: in previous versions of Keyman Engine for Web, these methods were set on the standard - * JS String type in a manner considered a "side-effect". (When inspecting older versions, it may help - * to know that they were originally implemented within a file named kmwstring.ts.) - * - * These methods seek to support SMP-aware string functionality within the Web engine, as many - * keyboards exist that utilize characters within such ranges. - */ - -// utf16helpers.ts (?) - -/** - * Indicates if functionality supporting Supplementary Multilingual Plane (SMP) - * characters should be enabled. - */ -let EXTENSION_ENABLED = false; - -/** - * Enables or disables support for SMP-aware string handling. - * @param flag - */ -export function enableSupplementaryPlane(flag: boolean) { // enableSMPProcessing - EXTENSION_ENABLED = flag; -} - -/** - * Returns a number indicating the Unicode value of the character at the given - * code point index, with support for supplementary plane characters. - * - * @param {string} s The string - * @param {number} index The code point index into the string (not - the code unit index) to return - * @return {number} The Unicode character value - */ -export function charCodeAt(s: string, index: number) { - // While String.codepointAt exists, it does not use code-point indexing and so is - // unsuited to our needs here. - const str = s; - - if(!EXTENSION_ENABLED) { - return s.charCodeAt(index); - } - - let codeUnitIndex = 0; - - if(index < 0 || index >= str.length) { - return NaN; - } - - for(let i = 0; i < index; i++) { - codeUnitIndex = nextChar(str, codeUnitIndex); - if(codeUnitIndex === null) { - return NaN; - } - } - - let first = str.charCodeAt(codeUnitIndex); - if(first >= 0xD800 && first <= 0xDBFF && str.length > codeUnitIndex + 1) { - let second = str.charCodeAt(codeUnitIndex + 1); - if(second >= 0xDC00 && second <= 0xDFFF) { - return ((first - 0xD800) << 10) + (second - 0xDC00) + 0x10000; - } - } - return first; -} - -/** - * Returns the code point index within the calling String object of the first occurrence - * of the specified value, or -1 if not found. - * - * @param {string} s The string - * @param {string} searchValue The value to search for - * @param {number=} fromIndex Optional code point index to start searching from - * @return {number} The code point index of the specified search value - */ -export function indexOf(s: string, searchValue: string, fromIndex?: number) { // pointIndexOf() - const str = s; - - if(!EXTENSION_ENABLED) { - return s.indexOf(searchValue, fromIndex); - } - - const codeUnitIndex = str.indexOf(searchValue, fromIndex); - - if(codeUnitIndex < 0) { - return codeUnitIndex; - } - - let codePointIndex = 0; - for(let i = 0; i !== null && i < codeUnitIndex; i = nextChar(str, i)) { - codePointIndex++; - } - return codePointIndex; -} - -/** - * Returns the code point index within the calling String object of the last occurrence - * of the specified value, or -1 if not found. - * - * @param {string} searchValue The value to search for - * @param {number=} fromIndex Optional code point index to start searching from - * @return {number} The code point index of the specified search value - */ -export function lastIndexOf(s: string, searchValue: string, fromIndex?: number) { // lastPointIndexOf() - const str = s; - - if(!EXTENSION_ENABLED) { - return str.lastIndexOf(searchValue, fromIndex); - } - - const codeUnitIndex = str.lastIndexOf(searchValue, fromIndex); - - if(codeUnitIndex < 0) { - return codeUnitIndex; - } - - let codePointIndex = 0; - for(let i = 0; i !== null && i < codeUnitIndex; i = nextChar(str, i)) { - codePointIndex++; - } - return codePointIndex; -} - -/** - * Returns the length of the string in code points, as opposed to code units. - * - * @param {string} s The string - * @return {number} The length of the string in code points - */ -export function length(s: string) { // codepointLength() - const str = s; - - if(!EXTENSION_ENABLED) { - return str.length; - } - - if(str.length == 0) { - return 0; - } - - let i: number; - let codeUnitIndex: number; - for(i = 0, codeUnitIndex = 0; codeUnitIndex !== null; i++) { - codeUnitIndex = nextChar(str, codeUnitIndex); - } - return i; -} - -/** - * Extracts a section of a string and returns a new string. - * - * @param {string} s The string - * @param {number} beginSlice The start code point index in the string to - * extract from - * @param {number} endSlice Optional end code point index in the string - * to extract to - * @return {string} The substring as selected by beginSlice and - * endSlice - */ -export function slice(s: string, beginSlice: number, endSlice?: number) { // sliceAtPoints() - const str = s; - - if(!EXTENSION_ENABLED) { - return str.slice(beginSlice, endSlice); - } - - const beginSliceCodeUnit = codePointToCodeUnit(str, beginSlice); - const endSliceCodeUnit = codePointToCodeUnit(str, endSlice); - if(beginSliceCodeUnit === null || endSliceCodeUnit === null) { - return ''; - } else { - return str.slice(beginSliceCodeUnit, endSliceCodeUnit); - } -} - -/** - * Returns the characters in a string beginning at the specified location through - * the specified number of characters. - * - * @param {string} s The string - * @param {number} start The start code point index in the string to - * extract from - * @param {number=} len Optional length to extract - * @return {string} The substring as selected by start and length - */ -export function substr(s: string, start: number, len?: number) { // substrFromPoint() - const str = s; - - if(!EXTENSION_ENABLED) { - if(start > -1) { - return str.substr(start,len); - } else { - return str.substr(str.length+start,-start); - } - } - - if(start < 0) { - start = length(str) + start; - } - if(start < 0) { - start = 0; - } - const startCodeUnit = codePointToCodeUnit(str, start); - let endCodeUnit = startCodeUnit; - - if(startCodeUnit === null) { - return ''; - } - - if(arguments.length < 3) { - endCodeUnit = str.length; - } else { - for(let i = 0; i < len; i++) { - endCodeUnit = nextChar(str, endCodeUnit); - } - } - - if(endCodeUnit === null) { - return str.substring(startCodeUnit); - } else { - return str.substring(startCodeUnit, endCodeUnit); - } -} - -/** - * Returns the characters in a string between two indexes into the string. - * - * @param {string} s The string - * @param {number} indexA The start code point index in the string to - * extract from - * @param {number} indexB The end code point index in the string to - * extract to - * @return {string} The substring as selected by indexA and indexB - */ -export function substring(s: string, indexA: number, indexB: number) { // substringBetweenPoints() - const str = s; - - if(!EXTENSION_ENABLED) { - return str.substring(indexA, indexB); - } - - let indexACodeUnit, indexBCodeUnit; - - if(typeof(indexB) == 'undefined') { - indexACodeUnit = codePointToCodeUnit(str, indexA); - indexBCodeUnit = str.length; - } else { - if(indexA > indexB) { - let c = indexA; - indexA = indexB; - indexB = c; - } - - indexACodeUnit = codePointToCodeUnit(str, indexA); - indexBCodeUnit = codePointToCodeUnit(str, indexB); - } - - if(isNaN(indexACodeUnit) || indexACodeUnit === null) { - indexACodeUnit = 0; - } - if(isNaN(indexBCodeUnit) || indexBCodeUnit === null) { - indexBCodeUnit = str.length; - } - - return str.substring(indexACodeUnit, indexBCodeUnit); -} - -/** - * Returns the code unit index for the next code point in the string, accounting for - * supplementary pairs - * - * @param {string} s The string - * @param {number|null} codeUnitIndex The code unit position to increment - * @return {number|null} The index of the next code point in the string, - * in code units - */ -export function nextChar(s: string, codeUnitIndex: number) { // nextCharAfterUnit - const str = s; - if(codeUnitIndex === null || codeUnitIndex < 0 || codeUnitIndex >= str.length - 1) { - return null; - } - - if(!EXTENSION_ENABLED) { - return codeUnitIndex+1; - } - - const first = str.charCodeAt(codeUnitIndex); - if(first >= 0xD800 && first <= 0xDBFF && str.length > codeUnitIndex + 1) { - const second = str.charCodeAt(codeUnitIndex + 1); - if(second >= 0xDC00 && second <= 0xDFFF) { - if(codeUnitIndex == str.length - 2) { - return null; - } - return codeUnitIndex + 2; - } - } - return codeUnitIndex + 1; -} - -/** - * Returns the code unit index for the previous code point in the string, accounting - * for supplementary pairs - * - * @param {string} s The string - * @param {number|null} codeUnitIndex The code unit position to decrement - * @return {number|null} The index of the previous code point in the - * string, in code units -*/ -export function prevChar(s: string, codeUnitIndex: number) { // prevCharBeforeUnit - const str = s; - - if(codeUnitIndex == null || codeUnitIndex <= 0 || codeUnitIndex > str.length) { - return null; - } - - if(!EXTENSION_ENABLED) { - return codeUnitIndex-1; - } - - const second = str.charCodeAt(codeUnitIndex - 1); - if(second >= 0xDC00 && second <= 0xDFFF && codeUnitIndex > 1) { - const first = str.charCodeAt(codeUnitIndex - 2); - if(first >= 0xD800 && first <= 0xDBFF) { - return codeUnitIndex - 2; - } - } - return codeUnitIndex - 1; -} - -/** - * Returns the corresponding code unit index to the code point index passed - * - * @param {string} s The string - * @param {number|null} codePointIndex A code point index in the string - * @return {number|null} The corresponding code unit index - */ -export function codePointToCodeUnit(s: string, codePointIndex: number) { // - if(codePointIndex === null) { - return null; - } - - if(!EXTENSION_ENABLED) { - return codePointIndex; - } - - const str = s; - let codeUnitIndex = 0; - - if(codePointIndex < 0) { - codeUnitIndex = str.length; - for(let i = 0; i > codePointIndex; i--) { - codeUnitIndex = prevChar(str, codeUnitIndex); - } - return codeUnitIndex; - } - - if(codePointIndex == length(str)) { - return str.length; - } - - for(let i = 0; i < codePointIndex; i++) { - codeUnitIndex = nextChar(str, codeUnitIndex); - } - return codeUnitIndex; -} - -/** - * Returns the corresponding code point index to the code unit index passed - * - * @param {number|null} codeUnitIndex A code unit index in the string - * @return {number|null} The corresponding code point index - */ -export function codeUnitToCodePoint(s: string, codeUnitIndex: number) { // - const str = s; - - if(!EXTENSION_ENABLED) { - return codeUnitIndex; - } - - if(codeUnitIndex === null) { - return null; - } else if(codeUnitIndex == 0) { - return 0; - } else if(codeUnitIndex < 0) { - return length(str.substr(codeUnitIndex)); - } else { - return length(str.substr(0,codeUnitIndex)); - } -} - -/** - * Returns the character at a the code point index passed - * - * @param {string} s The string - * @param {number} index A code point index in the string - * @return {string} The corresponding character - */ -export function charAt(s: string, index: number) { // charAtPoint - const str = s; - - if(!EXTENSION_ENABLED) { - return str.charAt(index); - } - - if(index >= 0) { - return substr(str, index, 1); - } else { - return '' - }; -} \ No newline at end of file diff --git a/web/src/engine/common/web-utils/src/kmwstring.ts b/web/src/engine/common/web-utils/src/kmwstring.ts index b844534b42..c7e090ee5f 100644 --- a/web/src/engine/common/web-utils/src/kmwstring.ts +++ b/web/src/engine/common/web-utils/src/kmwstring.ts @@ -58,6 +58,64 @@ export function charCodeAt(s: string, index: number) { return first; } + +/** + * Returns the code point index within the calling String object of the first occurrence + * of the specified value, or -1 if not found. + * + * @param {string} s The string + * @param {string} searchValue The value to search for + * @param {number=} fromIndex Optional code point index to start searching from + * @return {number} The code point index of the specified search value + */ +export function indexOf(s: string, searchValue: string, fromIndex?: number) { // pointIndexOf() + const str = s; + + if(!EXTENSION_ENABLED) { + return s.indexOf(searchValue, fromIndex); + } + + const codeUnitIndex = str.indexOf(searchValue, fromIndex); + + if(codeUnitIndex < 0) { + return codeUnitIndex; + } + + let codePointIndex = 0; + for(let i = 0; i !== null && i < codeUnitIndex; i = nextChar(str, i)) { + codePointIndex++; + } + return codePointIndex; +} + +/** + * Returns the code point index within the calling String object of the last occurrence + * of the specified value, or -1 if not found. + * + * @param {string} searchValue The value to search for + * @param {number=} fromIndex Optional code point index to start searching from + * @return {number} The code point index of the specified search value + */ +export function lastIndexOf(s: string, searchValue: string, fromIndex?: number) { // lastPointIndexOf() + const str = s; + + if(!EXTENSION_ENABLED) { + return str.lastIndexOf(searchValue, fromIndex); + } + + const codeUnitIndex = str.lastIndexOf(searchValue, fromIndex); + + if(codeUnitIndex < 0) { + return codeUnitIndex; + } + + let codePointIndex = 0; + for(let i = 0; i !== null && i < codeUnitIndex; i = nextChar(str, i)) { + codePointIndex++; + } + return codePointIndex; +} + /** * Returns the length of the string in code points, as opposed to code units. * @@ -217,15 +275,11 @@ declare global { interface String { kmwCharAt(codePointIndex: number) : string, - kmwIndexOf(searchValue: string, fromIndex?: number) : number, - kmwLastIndexOf(searchValue: string, fromIndex?: number) : number, kmwSlice(beginSlice: number, endSlice: number) : string, kmwSubstring(start: number, length: number) : string, kmwSubstr(start: number, length?: number) : string, kmwBMPSubstr(start: number, length?: number) : string, _kmwCharAt(codePointIndex: number) : string, - _kmwIndexOf(searchValue: string, fromIndex?: number) : number, - _kmwLastIndexOf(searchValue: string, fromIndex?: number) : number, _kmwSlice(beginSlice: number, endSlice: number) : string, _kmwSubstring(start: number, length?: number) : string, _kmwSubstr(start: number, length?: number) : string @@ -233,49 +287,6 @@ declare global { } export default function extendString() { - /** - * Returns the code point index within the calling String object of the first occurrence - * of the specified value, or -1 if not found. - * - * @param {string} searchValue The value to search for - * @param {number} [fromIndex] Optional code point index to start searching from - * @return {number} The code point index of the specified search value - */ - String.prototype.kmwIndexOf = function(searchValue, fromIndex) { - var str = String(this); - var codeUnitIndex = str.indexOf(searchValue, fromIndex); - - if(codeUnitIndex < 0) { - return codeUnitIndex; - } - - var codePointIndex = 0; - for(var i = 0; i !== null && i < codeUnitIndex; i = nextChar(str, i)) codePointIndex++; - return codePointIndex; - } - - /** - * Returns the code point index within the calling String object of the last occurrence - * of the specified value, or -1 if not found. - * - * @param {string} searchValue The value to search for - * @param {number} fromIndex Optional code point index to start searching from - * @return {number} The code point index of the specified search value - */ - String.prototype.kmwLastIndexOf = function(searchValue, fromIndex) - { - var str = String(this); - var codeUnitIndex = str.lastIndexOf(searchValue, fromIndex); - - if(codeUnitIndex < 0) { - return codeUnitIndex; - } - - var codePointIndex = 0; - for(var i = 0; i !== null && i < codeUnitIndex; i = nextChar(str, i)) codePointIndex++; - return codePointIndex; - } - /** * Extracts a section of a string and returns a new string. * @@ -406,8 +417,6 @@ export default function extendString() { { var p=String.prototype; p._kmwCharAt = bEnable ? p.kmwCharAt : p.charAt; - p._kmwIndexOf = bEnable ? p.kmwIndexOf :p.indexOf; - p._kmwLastIndexOf = bEnable ? p.kmwLastIndexOf : p.lastIndexOf ; p._kmwSlice = bEnable ? p.kmwSlice : p.slice; p._kmwSubstring = bEnable ? p.kmwSubstring : p.substring; p._kmwSubstr = bEnable ? p.kmwSubstr : p.kmwBMPSubstr; diff --git a/web/src/engine/common/web-utils/src/tests/stringHandling.js b/web/src/engine/common/web-utils/src/tests/stringHandling.js index 677be1d176..a8427f03f2 100644 --- a/web/src/engine/common/web-utils/src/tests/stringHandling.js +++ b/web/src/engine/common/web-utils/src/tests/stringHandling.js @@ -28,30 +28,30 @@ describe('Unicode string handling', () => { assert.equal(KMWString.charCodeAt(str, 5), "g".charCodeAt(0)); }); - it('_kmwIndexOf', () => { + it('indexOf', () => { const banana_nation = "banana nation"; - assert.equal(banana_nation._kmwIndexOf("XYZ"), -1); - assert.equal(banana_nation._kmwIndexOf("na"), 2); - assert.equal(banana_nation._kmwIndexOf("na", -1), 2); - assert.equal(banana_nation._kmwIndexOf("na", 0), 2); - assert.equal(banana_nation._kmwIndexOf("na", 2), 2); - assert.equal(banana_nation._kmwIndexOf("na", 3), 4); - assert.equal(banana_nation._kmwIndexOf("na", 5), 7); - assert.equal(banana_nation._kmwIndexOf("na", 8), -1); - assert.equal(banana_nation._kmwIndexOf("na", 1000), -1); + assert.equal(KMWString.indexOf(banana_nation, "XYZ"), -1); + assert.equal(KMWString.indexOf(banana_nation, "na"), 2); + assert.equal(KMWString.indexOf(banana_nation, "na", -1), 2); + assert.equal(KMWString.indexOf(banana_nation, "na", 0), 2); + assert.equal(KMWString.indexOf(banana_nation, "na", 2), 2); + assert.equal(KMWString.indexOf(banana_nation, "na", 3), 4); + assert.equal(KMWString.indexOf(banana_nation, "na", 5), 7); + assert.equal(KMWString.indexOf(banana_nation, "na", 8), -1); + assert.equal(KMWString.indexOf(banana_nation, "na", 1000), -1); }); - it('_kmwLastIndexOf', () => { + it('lastIndexOf', () => { const banana_nation = "banana nation"; - assert.equal(banana_nation._kmwLastIndexOf("XYZ"), -1); - assert.equal(banana_nation._kmwLastIndexOf("na"), 7); - assert.equal(banana_nation._kmwLastIndexOf("na", -1), -1); - assert.equal(banana_nation._kmwLastIndexOf("na", 0), -1); - assert.equal(banana_nation._kmwLastIndexOf("na", 2), 2); - assert.equal(banana_nation._kmwLastIndexOf("na", 3), 2); - assert.equal(banana_nation._kmwLastIndexOf("na", 5), 4); - assert.equal(banana_nation._kmwLastIndexOf("na", 8), 7); - assert.equal(banana_nation._kmwLastIndexOf("na", 1000), 7); + assert.equal(KMWString.lastIndexOf(banana_nation, "XYZ"), -1); + assert.equal(KMWString.lastIndexOf(banana_nation, "na"), 7); + assert.equal(KMWString.lastIndexOf(banana_nation, "na", -1), -1); + assert.equal(KMWString.lastIndexOf(banana_nation, "na", 0), -1); + assert.equal(KMWString.lastIndexOf(banana_nation, "na", 2), 2); + assert.equal(KMWString.lastIndexOf(banana_nation, "na", 3), 2); + assert.equal(KMWString.lastIndexOf(banana_nation, "na", 5), 4); + assert.equal(KMWString.lastIndexOf(banana_nation, "na", 8), 7); + assert.equal(KMWString.lastIndexOf(banana_nation, "na", 1000), 7); }); it('length', () => { @@ -141,30 +141,30 @@ describe('Unicode string handling', () => { assert.equal(KMWString.charCodeAt(str, 5), "g".charCodeAt(0)); }); - it('_kmwIndexOf', () => { + it('indexOf', () => { const banana_nation = "banana nation"; - assert.equal(banana_nation._kmwIndexOf("XYZ"), -1); - assert.equal(banana_nation._kmwIndexOf("na"), 2); - assert.equal(banana_nation._kmwIndexOf("na", -1), 2); - assert.equal(banana_nation._kmwIndexOf("na", 0), 2); - assert.equal(banana_nation._kmwIndexOf("na", 2), 2); - assert.equal(banana_nation._kmwIndexOf("na", 3), 4); - assert.equal(banana_nation._kmwIndexOf("na", 5), 7); - assert.equal(banana_nation._kmwIndexOf("na", 8), -1); - assert.equal(banana_nation._kmwIndexOf("na", 1000), -1); + assert.equal(KMWString.indexOf(banana_nation, "XYZ"), -1); + assert.equal(KMWString.indexOf(banana_nation, "na"), 2); + assert.equal(KMWString.indexOf(banana_nation, "na", -1), 2); + assert.equal(KMWString.indexOf(banana_nation, "na", 0), 2); + assert.equal(KMWString.indexOf(banana_nation, "na", 2), 2); + assert.equal(KMWString.indexOf(banana_nation, "na", 3), 4); + assert.equal(KMWString.indexOf(banana_nation, "na", 5), 7); + assert.equal(KMWString.indexOf(banana_nation, "na", 8), -1); + assert.equal(KMWString.indexOf(banana_nation, "na", 1000), -1); }); - it('_kmwLastIndexOf', () => { + it('lastIndexOf', () => { const banana_nation = "banana nation"; - assert.equal(banana_nation._kmwLastIndexOf("XYZ"), -1); - assert.equal(banana_nation._kmwLastIndexOf("na"), 7); - assert.equal(banana_nation._kmwLastIndexOf("na", -1), -1); - assert.equal(banana_nation._kmwLastIndexOf("na", 0), -1); - assert.equal(banana_nation._kmwLastIndexOf("na", 2), 2); - assert.equal(banana_nation._kmwLastIndexOf("na", 3), 2); - assert.equal(banana_nation._kmwLastIndexOf("na", 5), 4); - assert.equal(banana_nation._kmwLastIndexOf("na", 8), 7); - assert.equal(banana_nation._kmwLastIndexOf("na", 1000), 7); + assert.equal(KMWString.lastIndexOf(banana_nation, "XYZ"), -1); + assert.equal(KMWString.lastIndexOf(banana_nation, "na"), 7); + assert.equal(KMWString.lastIndexOf(banana_nation, "na", -1), -1); + assert.equal(KMWString.lastIndexOf(banana_nation, "na", 0), -1); + assert.equal(KMWString.lastIndexOf(banana_nation, "na", 2), 2); + assert.equal(KMWString.lastIndexOf(banana_nation, "na", 3), 2); + assert.equal(KMWString.lastIndexOf(banana_nation, "na", 5), 4); + assert.equal(KMWString.lastIndexOf(banana_nation, "na", 8), 7); + assert.equal(KMWString.lastIndexOf(banana_nation, "na", 1000), 7); }); it('length', () => { @@ -266,61 +266,61 @@ describe('Unicode string handling', () => { assert.equal(KMWString.charCodeAt(apples, 5), "s".charCodeAt(0)); }); - it('_kmwIndexOf', () => { + it('indexOf', () => { // 0x1d5ba: MATHEMATICAL SANS-SERIF SMALL a const a = String.fromCodePoint(0x1d5ba); const banana_nation = "ban" + a + "na n" + a + "tion"; - assert.equal(banana_nation._kmwIndexOf("XYZ"), -1); - assert.equal(banana_nation._kmwIndexOf("na"), 4); - assert.equal(banana_nation._kmwIndexOf("na", -1), 4); - assert.equal(banana_nation._kmwIndexOf("na", 0), 4); - assert.equal(banana_nation._kmwIndexOf("na", 2), 4); - assert.equal(banana_nation._kmwIndexOf("na", 3), 4); + assert.equal(KMWString.indexOf(banana_nation, "XYZ"), -1); + assert.equal(KMWString.indexOf(banana_nation, "na"), 4); + assert.equal(KMWString.indexOf(banana_nation, "na", -1), 4); + assert.equal(KMWString.indexOf(banana_nation, "na", 0), 4); + assert.equal(KMWString.indexOf(banana_nation, "na", 2), 4); + assert.equal(KMWString.indexOf(banana_nation, "na", 3), 4); // ERROR: returns 4 (!) - // assert.equal(banana_nation._kmwIndexOf("na", 5), 7); - assert.equal(banana_nation._kmwIndexOf("na", 8), -1); - assert.equal(banana_nation._kmwIndexOf("na", 1000), -1); + // assert.equal(KMWString.indexOf(banana_nation, "na", 5), 7); + assert.equal(KMWString.indexOf(banana_nation, "na", 8), -1); + assert.equal(KMWString.indexOf(banana_nation, "na", 1000), -1); const na = "n" + a; - assert.equal(banana_nation._kmwIndexOf(na), 2); - assert.equal(banana_nation._kmwIndexOf(na, -1), 2); - assert.equal(banana_nation._kmwIndexOf(na, 0), 2); - assert.equal(banana_nation._kmwIndexOf(na, 2), 2); - assert.equal(banana_nation._kmwIndexOf(na, 3), 7); - assert.equal(banana_nation._kmwIndexOf(na, 5), 7); + assert.equal(KMWString.indexOf(banana_nation, na), 2); + assert.equal(KMWString.indexOf(banana_nation, na, -1), 2); + assert.equal(KMWString.indexOf(banana_nation, na, 0), 2); + assert.equal(KMWString.indexOf(banana_nation, na, 2), 2); + assert.equal(KMWString.indexOf(banana_nation, na, 3), 7); + assert.equal(KMWString.indexOf(banana_nation, na, 5), 7); // ERROR: returns 7 (!) - // assert.equal(banana_nation._kmwIndexOf(na, 8), -1); - assert.equal(banana_nation._kmwIndexOf(na, 1000), -1); + // assert.equal(KMWString.indexOf(banana_nation, na, 8), -1); + assert.equal(KMWString.indexOf(banana_nation, na, 1000), -1); }); - it('_kmwLastIndexOf', () => { + it('lastIndexOf', () => { // 0x1d5ba: MATHEMATICAL SANS-SERIF SMALL a const a = String.fromCodePoint(0x1d5ba); const banana_nation = "ban" + a + "na n" + a + "tion"; - assert.equal(banana_nation._kmwLastIndexOf("XYZ"), -1); - assert.equal(banana_nation._kmwLastIndexOf("na"), 4); - assert.equal(banana_nation._kmwLastIndexOf("na", -1), -1); - assert.equal(banana_nation._kmwLastIndexOf("na", 0), -1); - assert.equal(banana_nation._kmwLastIndexOf("na", 2), -1); - assert.equal(banana_nation._kmwLastIndexOf("na", 3), -1); + assert.equal(KMWString.lastIndexOf(banana_nation, "XYZ"), -1); + assert.equal(KMWString.lastIndexOf(banana_nation, "na"), 4); + assert.equal(KMWString.lastIndexOf(banana_nation, "na", -1), -1); + assert.equal(KMWString.lastIndexOf(banana_nation, "na", 0), -1); + assert.equal(KMWString.lastIndexOf(banana_nation, "na", 2), -1); + assert.equal(KMWString.lastIndexOf(banana_nation, "na", 3), -1); // ERROR: Can't find the instance at the starting index? - // assert.equal(banana_nation._kmwLastIndexOf("na", 4), 4); - assert.equal(banana_nation._kmwLastIndexOf("na", 5), 4); - assert.equal(banana_nation._kmwLastIndexOf("na", 8), 4); - assert.equal(banana_nation._kmwLastIndexOf("na", 1000), 4); + // assert.equal(KMWString.lastIndexOf(banana_nation, "na", 4), 4); + assert.equal(KMWString.lastIndexOf(banana_nation, "na", 5), 4); + assert.equal(KMWString.lastIndexOf(banana_nation, "na", 8), 4); + assert.equal(KMWString.lastIndexOf(banana_nation, "na", 1000), 4); const na = "n" + a; - assert.equal(banana_nation._kmwLastIndexOf(na), 7); - assert.equal(banana_nation._kmwLastIndexOf(na, -1), -1); - assert.equal(banana_nation._kmwLastIndexOf(na, 0), -1); - assert.equal(banana_nation._kmwLastIndexOf(na, 2), 2); - assert.equal(banana_nation._kmwLastIndexOf(na, 3), 2); - assert.equal(banana_nation._kmwLastIndexOf(na, 5), 2); + assert.equal(KMWString.lastIndexOf(banana_nation, na), 7); + assert.equal(KMWString.lastIndexOf(banana_nation, na, -1), -1); + assert.equal(KMWString.lastIndexOf(banana_nation, na, 0), -1); + assert.equal(KMWString.lastIndexOf(banana_nation, na, 2), 2); + assert.equal(KMWString.lastIndexOf(banana_nation, na, 3), 2); + assert.equal(KMWString.lastIndexOf(banana_nation, na, 5), 2); // ERROR: Inconsistent with the index-2 case above. - // assert.equal(banana_nation._kmwLastIndexOf(na, 7), 7); - assert.equal(banana_nation._kmwLastIndexOf(na, 8), 7); - assert.equal(banana_nation._kmwLastIndexOf(na, 1000), 7); + // assert.equal(KMWString.lastIndexOf(banana_nation, na, 7), 7); + assert.equal(KMWString.lastIndexOf(banana_nation, na, 8), 7); + assert.equal(KMWString.lastIndexOf(banana_nation, na, 1000), 7); }); it('length', () => { From 70c19208541c62be060958f600b74d210b51e5ca Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 22 Apr 2025 10:35:34 +0700 Subject: [PATCH 031/431] refactor(web): refactor _kmwSlice, _kmwSubstr, _kmwSubstring --- .../engine/common/web-utils/src/kmwstring.ts | 231 +++++++++--------- .../web-utils/src/tests/stringHandling.js | 138 +++++------ .../element-wrappers/src/contentEditable.ts | 2 +- .../element-wrappers/src/designIFrame.ts | 2 +- web/src/engine/element-wrappers/src/input.ts | 14 +- .../engine/element-wrappers/src/textarea.ts | 14 +- .../engine/js-processor/src/kbdInterface.ts | 2 +- web/src/engine/js-processor/src/mock.ts | 10 +- .../engine/js-processor/src/outputTarget.ts | 2 +- .../engine/main/src/headless/contextWindow.ts | 4 +- .../predictive-text/templates/src/common.ts | 6 +- .../src/main/model-compositor.ts | 4 +- 12 files changed, 220 insertions(+), 209 deletions(-) diff --git a/web/src/engine/common/web-utils/src/kmwstring.ts b/web/src/engine/common/web-utils/src/kmwstring.ts index c7e090ee5f..70ccb7157d 100644 --- a/web/src/engine/common/web-utils/src/kmwstring.ts +++ b/web/src/engine/common/web-utils/src/kmwstring.ts @@ -141,6 +141,126 @@ export function length(s: string) { return i; } + +/** + * Extracts a section of a string and returns a new string. + * + * @param {string} s The string + * @param {number} beginSlice The start code point index in the string to + * extract from + * @param {number} endSlice Optional end code point index in the string + * to extract to + * @return {string} The substring as selected by beginSlice and + * endSlice + */ +export function slice(s: string, beginSlice: number, endSlice?: number) { // sliceAtPoints() + const str = s; + + if(!EXTENSION_ENABLED) { + return str.slice(beginSlice, endSlice); + } + + const beginSliceCodeUnit = codePointToCodeUnit(str, beginSlice); + const endSliceCodeUnit = codePointToCodeUnit(str, endSlice); + if(beginSliceCodeUnit === null || endSliceCodeUnit === null) { + return ''; + } else { + return str.slice(beginSliceCodeUnit, endSliceCodeUnit); + } +} + +/** + * Returns the characters in a string beginning at the specified location through + * the specified number of characters. + * + * @param {string} s The string + * @param {number} start The start code point index in the string to + * extract from + * @param {number=} len Optional length to extract + * @return {string} The substring as selected by start and length + */ +export function substr(s: string, start: number, len?: number) { // substrFromPoint() + const str = s; + + if(!EXTENSION_ENABLED) { + if(start > -1) { + return str.substr(start,len); + } else { + return str.substr(str.length+start,-start); + } + } + + if(start < 0) { + start = length(str) + start; + } + if(start < 0) { + start = 0; + } + const startCodeUnit = codePointToCodeUnit(str, start); + let endCodeUnit = startCodeUnit; + + if(startCodeUnit === null) { + return ''; + } + + if(arguments.length < 3) { + endCodeUnit = str.length; + } else { + for(let i = 0; i < len; i++) { + endCodeUnit = nextChar(str, endCodeUnit); + } + } + + if(endCodeUnit === null) { + return str.substring(startCodeUnit); + } else { + return str.substring(startCodeUnit, endCodeUnit); + } +} + +/** + * Returns the characters in a string between two indexes into the string. + * + * @param {string} s The string + * @param {number} indexA The start code point index in the string to + * extract from + * @param {number=} indexB The end code point index in the string to + * extract to + * @return {string} The substring as selected by indexA and indexB + */ +export function substring(s: string, indexA: number, indexB?: number) { // substringBetweenPoints() + const str = s; + + if(!EXTENSION_ENABLED) { + return str.substring(indexA, indexB); + } + + let indexACodeUnit, indexBCodeUnit; + + if(typeof(indexB) == 'undefined') { + indexACodeUnit = codePointToCodeUnit(str, indexA); + indexBCodeUnit = str.length; + } else { + if(indexA > indexB) { + let c = indexA; + indexA = indexB; + indexB = c; + } + + indexACodeUnit = codePointToCodeUnit(str, indexA); + indexBCodeUnit = codePointToCodeUnit(str, indexB); + } + + if(isNaN(indexACodeUnit) || indexACodeUnit === null) { + indexACodeUnit = 0; + } + if(isNaN(indexBCodeUnit) || indexBCodeUnit === null) { + indexBCodeUnit = str.length; + } + + return str.substring(indexACodeUnit, indexBCodeUnit); +} + /** * Returns the code unit index for the next code point in the string, accounting for * supplementary pairs @@ -275,102 +395,11 @@ declare global { interface String { kmwCharAt(codePointIndex: number) : string, - kmwSlice(beginSlice: number, endSlice: number) : string, - kmwSubstring(start: number, length: number) : string, - kmwSubstr(start: number, length?: number) : string, - kmwBMPSubstr(start: number, length?: number) : string, _kmwCharAt(codePointIndex: number) : string, - _kmwSlice(beginSlice: number, endSlice: number) : string, - _kmwSubstring(start: number, length?: number) : string, - _kmwSubstr(start: number, length?: number) : string } } export default function extendString() { - /** - * Extracts a section of a string and returns a new string. - * - * @param {number} beginSlice The start code point index in the string to - * extract from - * @param {number} endSlice Optional end code point index in the string - * to extract to - * @return {string} The substring as selected by beginSlice and - * endSlice - */ - String.prototype.kmwSlice = function(beginSlice, endSlice) { - var str = String(this); - var beginSliceCodeUnit = codePointToCodeUnit(str, beginSlice); - var endSliceCodeUnit = codePointToCodeUnit(str, endSlice); - if(beginSliceCodeUnit === null || endSliceCodeUnit === null) - return ''; - else - return str.slice(beginSliceCodeUnit, endSliceCodeUnit); - } - - /** - * Returns the characters in a string beginning at the specified location through - * the specified number of characters. - * - * @param {number} start The start code point index in the string to - * extract from - * @param {number=} len Optional length to extract - * @return {string} The substring as selected by start and length - */ - String.prototype.kmwSubstr = function(start, len?) - { - var str = String(this); - if(start < 0) - { - start = length(str) + start; - } - if(start < 0) start = 0; - var startCodeUnit = codePointToCodeUnit(str, start); - var endCodeUnit = startCodeUnit; - - if(startCodeUnit === null) return ''; - - if(arguments.length < 2) { - endCodeUnit = str.length; - } else { - for(var i = 0; i < len; i++) endCodeUnit = nextChar(str, endCodeUnit); - } - if(endCodeUnit === null) - return str.substring(startCodeUnit); - else - return str.substring(startCodeUnit, endCodeUnit); - } - - /** - * Returns the characters in a string between two indexes into the string. - * - * @param {number} indexA The start code point index in the string to - * extract from - * @param {number} indexB The end code point index in the string to - * extract to - * @return {string} The substring as selected by indexA and indexB - */ - String.prototype.kmwSubstring = function(indexA, indexB) - { - var str = String(this),indexACodeUnit,indexBCodeUnit; - - if(typeof(indexB) == 'undefined') - { - indexACodeUnit = codePointToCodeUnit(str, indexA); - indexBCodeUnit = str.length; - } - else - { - if(indexA > indexB) { var c = indexA; indexA = indexB; indexB = c; } - - indexACodeUnit = codePointToCodeUnit(str, indexA); - indexBCodeUnit = codePointToCodeUnit(str, indexB); - } - if(isNaN(indexACodeUnit) || indexACodeUnit === null) indexACodeUnit = 0; - if(isNaN(indexBCodeUnit) || indexBCodeUnit === null) indexBCodeUnit = str.length; - - return str.substring(indexACodeUnit, indexBCodeUnit); - } - /* Helper functions */ @@ -384,7 +413,7 @@ export default function extendString() { String.prototype.kmwCharAt = function(codePointIndex) { var str = String(this); - if(codePointIndex >= 0) return str.kmwSubstr(codePointIndex,1); else return ''; + if(codePointIndex >= 0) return substr(str, codePointIndex,1); else return ''; } /** @@ -392,21 +421,6 @@ export default function extendString() { * to simplify enabling or disabling supplementary plane functionality (I3319) */ - /** - * Returns a substring - * - * @param {number} n - * @param {number=} ln - * @return {string} - */ - String.prototype.kmwBMPSubstr = function(n,ln?) - { - var str=String(this); - if(n > -1) - return str.substr(n,ln); - else - return str.substr(str.length+n,-n); - } /** * Enable or disable supplementary plane string handling @@ -417,9 +431,6 @@ export default function extendString() { { var p=String.prototype; p._kmwCharAt = bEnable ? p.kmwCharAt : p.charAt; - p._kmwSlice = bEnable ? p.kmwSlice : p.slice; - p._kmwSubstring = bEnable ? p.kmwSubstring : p.substring; - p._kmwSubstr = bEnable ? p.kmwSubstr : p.kmwBMPSubstr; enableSupplementaryPlane(bEnable); } diff --git a/web/src/engine/common/web-utils/src/tests/stringHandling.js b/web/src/engine/common/web-utils/src/tests/stringHandling.js index a8427f03f2..43d0d34ae5 100644 --- a/web/src/engine/common/web-utils/src/tests/stringHandling.js +++ b/web/src/engine/common/web-utils/src/tests/stringHandling.js @@ -61,44 +61,44 @@ describe('Unicode string handling', () => { assert.equal(KMWString.length("Life, the universe and everything. Answer:"), 42); }); - it('_kmwSlice', () => { + it('slice', () => { // From https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/slice const str = "The quick brown fox jumps over the lazy dog."; - assert.equal(str._kmwSlice(31), "the lazy dog."); - assert.equal(str._kmwSlice(4, 19), "quick brown fox"); - assert.equal(str._kmwSlice(-4), "dog."); - assert.equal(str._kmwSlice(-9, -5), "lazy"); - assert.equal(str._kmwSlice(), str); + assert.equal(KMWString.slice(str, 31), "the lazy dog."); + assert.equal(KMWString.slice(str, 4, 19), "quick brown fox"); + assert.equal(KMWString.slice(str, -4), "dog."); + assert.equal(KMWString.slice(str, -9, -5), "lazy"); + assert.equal(KMWString.slice(str), str); // Per documented spec for slice... - assert.equal(str._kmwSlice(31, -5), "the lazy"); // " dog." = 5 chars - assert.equal(str._kmwSlice(-5, -9), ""); + assert.equal(KMWString.slice(str, 31, -5), "the lazy"); // " dog." = 5 chars + assert.equal(KMWString.slice(str, -5, -9), ""); }); - it('_kmwSubstr', () => { + it('substr', () => { const str = "The quick brown fox jumps over the lazy dog."; - assert.equal(str._kmwSubstr(1000), ''); - assert.equal(str._kmwSubstr(31), "the lazy dog."); - assert.equal(str._kmwSubstr(-4), "dog."); - assert.equal(str._kmwSubstr(31, 31), "the lazy dog."); - assert.equal(str._kmwSubstr(31, -5), ''); // The big difference from .slice: length must be non-negative. + assert.equal(KMWString.substr(str, 1000), ''); + assert.equal(KMWString.substr(str, 31), "the lazy dog."); + assert.equal(KMWString.substr(str, -4), "dog."); + assert.equal(KMWString.substr(str, 31, 31), "the lazy dog."); + assert.equal(KMWString.substr(str, 31, -5), ''); // The big difference from .slice: length must be non-negative. }); - it('_kmwSubstring', () => { + it('substring', () => { const str = "The quick brown fox jumps over the lazy dog."; - assert.equal(str._kmwSubstring(1000), ''); - assert.equal(str._kmwSubstring(31), "the lazy dog."); - assert.equal(str._kmwSubstring(31, 1000), "the lazy dog."); - assert.equal(str._kmwSubstring(4, 9), "quick"); + assert.equal(KMWString.substring(str, 1000), ''); + assert.equal(KMWString.substring(str, 31), "the lazy dog."); + assert.equal(KMWString.substring(str, 31, 1000), "the lazy dog."); + assert.equal(KMWString.substring(str, 4, 9), "quick"); // negatives are coerced to 0 & parameters are used in order of ascending value. - assert.equal(str._kmwSubstring(9, 4), "quick"); - assert.equal(str._kmwSubstring(31, -4), str.substring(0, 31)); - assert.equal(str._kmwSubstring(-5, 9), str.substring(0, 9)); - assert.equal(str._kmwSubstring(-9, -5), ''); + assert.equal(KMWString.substring(str, 9, 4), "quick"); + assert.equal(KMWString.substring(str, 31, -4), str.substring(0, 31)); + assert.equal(KMWString.substring(str, -5, 9), str.substring(0, 9)); + assert.equal(KMWString.substring(str, -9, -5), ''); }); it('codePointToCodeUnit', () => { @@ -174,49 +174,49 @@ describe('Unicode string handling', () => { assert.equal(KMWString.length("Life, the universe and everything. Answer:"), 42); }); - it('_kmwSlice', () => { + it('slice', () => { // From https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/slice const str = "The quick brown fox jumps over the lazy dog."; // FAILS: does not handle `undefined` length like the real .slice() method. - // assert.equal(str._kmwSlice(31), "the lazy dog."); - assert.equal(str._kmwSlice(4, 19), "quick brown fox"); + // assert.equal(KMWString.slice(str, 31), "the lazy dog."); + assert.equal(KMWString.slice(str, 4, 19), "quick brown fox"); // FAILS: does not handle `undefined` length like the real .slice() method. - // assert.equal(str._kmwSlice(-4), "dog."); - assert.equal(str._kmwSlice(-9, -5), "lazy"); + // assert.equal(KMWString.slice(str, -4), "dog."); + assert.equal(KMWString.slice(str, -9, -5), "lazy"); // FAILS: does not handle `undefined` start like the real .slice() method. - // assert.equal(str._kmwSlice(), str); + // assert.equal(KMWString.slice(str), str); // Per documented spec for slice... - assert.equal(str._kmwSlice(31, -5), "the lazy"); // " dog." = 5 chars - assert.equal(str._kmwSlice(-5, -9), ""); + assert.equal(KMWString.slice(str, 31, -5), "the lazy"); // " dog." = 5 chars + assert.equal(KMWString.slice(str, -5, -9), ""); }); - it('_kmwSubstr', () => { + it('substr', () => { const str = "The quick brown fox jumps over the lazy dog."; - assert.equal(str._kmwSubstr(1000), ''); - assert.equal(str._kmwSubstr(31), "the lazy dog."); - assert.equal(str._kmwSubstr(-4), "dog."); - assert.equal(str._kmwSubstr(31, 31), "the lazy dog."); - assert.equal(str._kmwSubstr(31, -5), ''); // The big difference from .slice: length must be non-negative. + assert.equal(KMWString.substr(str, 1000), ''); + assert.equal(KMWString.substr(str, 31), "the lazy dog."); + assert.equal(KMWString.substr(str, -4), "dog."); + assert.equal(KMWString.substr(str, 31, 31), "the lazy dog."); + assert.equal(KMWString.substr(str, 31, -5), ''); // The big difference from .slice: length must be non-negative. }); - it('_kmwSubstring', () => { + it('substring', () => { const str = "The quick brown fox jumps over the lazy dog."; // FAILS - doesn't coerce the out-of-bounds value. - // assert.equal(str._kmwSubstring(1000), ''); - assert.equal(str._kmwSubstring(31), "the lazy dog."); - assert.equal(str._kmwSubstring(31, 1000), "the lazy dog."); - assert.equal(str._kmwSubstring(4, 9), "quick"); + // assert.equal(KMWString.substring(str, 1000), ''); + assert.equal(KMWString.substring(str, 31), "the lazy dog."); + assert.equal(KMWString.substring(str, 31, 1000), "the lazy dog."); + assert.equal(KMWString.substring(str, 4, 9), "quick"); // negatives are coerced to 0 & parameters are used in order of ascending value. - assert.equal(str._kmwSubstring(9, 4), "quick"); + assert.equal(KMWString.substring(str, 9, 4), "quick"); // FAILS: these do not handle negative-valued inputs the same way the real .substring does. - // assert.equal(str._kmwSubstring(31, -4), str.substring(0, 31)); - // assert.equal(str._kmwSubstring(-5, 9), str.substring(0, 9)); - // assert.equal(str._kmwSubstring(-9, -5), ''); + // assert.equal(KMWString.substring(str, 31, -4), str.substring(0, 31)); + // assert.equal(KMWString.substring(str, -5, 9), str.substring(0, 9)); + // assert.equal(KMWString.substring(str, -9, -5), ''); }); it('codePointToCodeUnit', () => { @@ -336,7 +336,7 @@ describe('Unicode string handling', () => { assert.equal(KMWString.length(apples), 6); }); - it('_kmwSlice', () => { + it('slice', () => { // 0x1d5ba: MATHEMATICAL SANS-SERIF SMALL a // 0x1d5c9: MATHEMATICAL SANS-SERIF SMALL p // 0x1d5be: MATHEMATICAL SANS-SERIF SMALL e @@ -350,20 +350,20 @@ describe('Unicode string handling', () => { const str = "Th" + e + " qu" + i + "ck brown fox jum" + p + "s over the l" + a + "zy dog."; // FAILS: does not handle `undefined` length like the real .slice() method. - // assert.equal(str._kmwSlice(31), "the l" + a + "zy dog."); - assert.equal(str._kmwSlice(4, 19), "qu" + i + "ck brown fox"); + // assert.equal(KMWString.slice(str, 31), "the l" + a + "zy dog."); + assert.equal(KMWString.slice(str, 4, 19), "qu" + i + "ck brown fox"); // FAILS: does not handle `undefined` length like the real .slice() method. - // assert.equal(str._kmwSlice(-4), "dog."); - assert.equal(str._kmwSlice(-9, -5), "l" + a + "zy"); + // assert.equal(KMWString.slice(str, (-4), "dog."); + assert.equal(KMWString.slice(str, -9, -5), "l" + a + "zy"); // FAILS: does not handle `undefined` start like the real .slice() method. - // assert.equal(str._kmwSlice(), str); + // assert.equal(KMWString.slice(str, (), str); // Per documented spec for slice... - assert.equal(str._kmwSlice(31, -5), "the l" + a + "zy"); // " dog." = 5 chars - assert.equal(str._kmwSlice(-5, -9), ""); + assert.equal(KMWString.slice(str, 31, -5), "the l" + a + "zy"); // " dog." = 5 chars + assert.equal(KMWString.slice(str, -5, -9), ""); }); - it('_kmwSubstr', () => { + it('substr', () => { // 0x1d5ba: MATHEMATICAL SANS-SERIF SMALL a // 0x1d5c9: MATHEMATICAL SANS-SERIF SMALL p // 0x1d5be: MATHEMATICAL SANS-SERIF SMALL e @@ -375,14 +375,14 @@ describe('Unicode string handling', () => { const str = "Th" + e + " qu" + i + "ck brown fox jum" + p + "s over the l" + a + "zy dog."; - assert.equal(str._kmwSubstr(1000), ''); - assert.equal(str._kmwSubstr(31), "the l" + a + "zy dog."); - assert.equal(str._kmwSubstr(-4), "dog."); - assert.equal(str._kmwSubstr(31, 31), "the l" + a + "zy dog."); - assert.equal(str._kmwSubstr(31, -5), ''); // The big difference from .slice: length must be non-negative. + assert.equal(KMWString.substr(str, 1000), ''); + assert.equal(KMWString.substr(str, 31), "the l" + a + "zy dog."); + assert.equal(KMWString.substr(str, -4), "dog."); + assert.equal(KMWString.substr(str, 31, 31), "the l" + a + "zy dog."); + assert.equal(KMWString.substr(str, 31, -5), ''); // The big difference from .slice: length must be non-negative. }); - it('_kmwSubstring', () => { + it('substring', () => { // 0x1d5ba: MATHEMATICAL SANS-SERIF SMALL a // 0x1d5c9: MATHEMATICAL SANS-SERIF SMALL p // 0x1d5be: MATHEMATICAL SANS-SERIF SMALL e @@ -395,17 +395,17 @@ describe('Unicode string handling', () => { const str = "Th" + e + " qu" + i + "ck brown fox jum" + p + "s over the l" + a + "zy dog."; // FAILS - doesn't coerce the out-of-bounds value. - // assert.equal(str._kmwSubstring(1000), ''); - assert.equal(str._kmwSubstring(31), "the l" + a + "zy dog."); - assert.equal(str._kmwSubstring(31, 1000), "the l" + a + "zy dog."); - assert.equal(str._kmwSubstring(4, 9), "qu" + i + "ck"); + // assert.equal(KMWString.substring(str, 1000), ''); + assert.equal(KMWString.substring(str, 31), "the l" + a + "zy dog."); + assert.equal(KMWString.substring(str, 31, 1000), "the l" + a + "zy dog."); + assert.equal(KMWString.substring(str, 4, 9), "qu" + i + "ck"); // negatives are coerced to 0 & parameters are used in order of ascending value. - assert.equal(str._kmwSubstring(9, 4), "qu" + i + "ck"); + assert.equal(KMWString.substring(str, 9, 4), "qu" + i + "ck"); // FAILS: these do not handle negative-valued inputs the same way the real .substring does. - // assert.equal(str._kmwSubstring(31, -4), str.substring(0, 31)); - // assert.equal(str._kmwSubstring(-5, 9), str.substring(0, 9)); - // assert.equal(str._kmwSubstring(-9, -5), ''); + // assert.equal(KMWString.substring(str, 31, -4), str.substring(0, 31)); + // assert.equal(KMWString.substring(str, -5, 9), str.substring(0, 9)); + // assert.equal(KMWString.substring(str, -9, -5), ''); }); it('codePointToCodeUnit', () => { diff --git a/web/src/engine/element-wrappers/src/contentEditable.ts b/web/src/engine/element-wrappers/src/contentEditable.ts index a8c2cf38fe..f847038268 100644 --- a/web/src/engine/element-wrappers/src/contentEditable.ts +++ b/web/src/engine/element-wrappers/src/contentEditable.ts @@ -163,7 +163,7 @@ export default class ContentEditable extends OutputTarget<{}> { } let range = this.root.ownerDocument.createRange(); - let dnOffset = start.offset - start.node.nodeValue.substr(0, start.offset)._kmwSubstr(-dn).length; + let dnOffset = start.offset - KMWString.substr(start.node.nodeValue.substr(0, start.offset), -dn).length; range.setStart(start.node, dnOffset); range.setEnd(start.node, start.offset); diff --git a/web/src/engine/element-wrappers/src/designIFrame.ts b/web/src/engine/element-wrappers/src/designIFrame.ts index b0f84fbf19..0f336a1b0b 100644 --- a/web/src/engine/element-wrappers/src/designIFrame.ts +++ b/web/src/engine/element-wrappers/src/designIFrame.ts @@ -185,7 +185,7 @@ export default class DesignIFrame extends OutputTarget<{}> { } let range = this.doc.createRange(); - let dnOffset = start.offset - start.node.nodeValue.substr(0, start.offset)._kmwSubstr(-dn).length; + let dnOffset = start.offset - KMWString.substr(start.node.nodeValue.substr(0, start.offset), -dn).length; range.setStart(start.node, dnOffset); range.setEnd(start.node, start.offset); diff --git a/web/src/engine/element-wrappers/src/input.ts b/web/src/engine/element-wrappers/src/input.ts index 2c61da8eb1..7c0561920a 100644 --- a/web/src/engine/element-wrappers/src/input.ts +++ b/web/src/engine/element-wrappers/src/input.ts @@ -70,7 +70,7 @@ export default class Input extends OutputTarget { clearSelection(): void { // Processes our codepoint-based variants of selectionStart and selectionEnd. this.getCaret(); // updates processedSelectionStart if required - this.root.value = this.root.value._kmwSubstring(0, this.processedSelectionStart) + this.root.value._kmwSubstring(this.processedSelectionEnd); //I3319 + this.root.value = KMWString.substr(this.root.value, 0, this.processedSelectionStart) + KMWString.substr(this.root.value, this.processedSelectionEnd); //I3319 this.setCaret(this.processedSelectionStart); } @@ -153,12 +153,12 @@ export default class Input extends OutputTarget { getTextBeforeCaret(): string { this.getCaret(); - return this.getText()._kmwSubstring(0, this.processedSelectionStart); + return KMWString.substring(this.getText(), 0, this.processedSelectionStart); } getSelectedText(): string { this.getCaret(); - return this.getText()._kmwSubstring(this.processedSelectionStart, this.processedSelectionEnd); + return KMWString.substring(this.getText(), this.processedSelectionStart, this.processedSelectionEnd); } setTextBeforeCaret(text: string) { @@ -166,7 +166,7 @@ export default class Input extends OutputTarget { let selectionLength = this.processedSelectionEnd - this.processedSelectionStart; let direction = this.getSelectionDirection(); let newCaret = KMWString.length(text); - this.root.value = text + this.getText()._kmwSubstring(this.processedSelectionStart); + this.root.value = text + KMWString.substring(this.getText(), this.processedSelectionStart); this.setSelection(newCaret, newCaret + selectionLength, direction); } @@ -180,7 +180,7 @@ export default class Input extends OutputTarget { getTextAfterCaret(): string { this.getCaret(); - return this.getText()._kmwSubstring(this.processedSelectionEnd); + return KMWString.substring(this.getText(), this.processedSelectionEnd); } getText(): string { @@ -197,7 +197,7 @@ export default class Input extends OutputTarget { } this.adjustDeadkeys(-dn); - this.setTextBeforeCaret(curText.kmwSubstring(0, caret - dn)); + this.setTextBeforeCaret(KMWString.substring(curText, 0, caret - dn)); this.setCaret(caret - dn); } } @@ -209,7 +209,7 @@ export default class Input extends OutputTarget { let caret = this.getCaret(); let front = this.getTextBeforeCaret(); - let back = this.getText()._kmwSubstring(this.processedSelectionStart); + let back = KMWString.substring(this.getText(), this.processedSelectionStart); this.adjustDeadkeys(KMWString.length(s)); this.root.value = front + s + back; diff --git a/web/src/engine/element-wrappers/src/textarea.ts b/web/src/engine/element-wrappers/src/textarea.ts index e9a9495698..cd50cc1e2b 100644 --- a/web/src/engine/element-wrappers/src/textarea.ts +++ b/web/src/engine/element-wrappers/src/textarea.ts @@ -45,7 +45,7 @@ export default class TextArea extends OutputTarget<{}> { clearSelection(): void { // Processes our codepoint-based variants of selectionStart and selectionEnd. this.getCaret(); // updates processedSelectionStart if required - this.root.value = this.root.value._kmwSubstring(0, this.processedSelectionStart) + this.root.value._kmwSubstring(this.processedSelectionEnd); //I3319 + this.root.value = KMWString.substr(this.root.value, 0, this.processedSelectionStart) + KMWString.substr(this.root.value, this.processedSelectionEnd); //I3319 this.setCaret(this.processedSelectionStart); } @@ -128,7 +128,7 @@ export default class TextArea extends OutputTarget<{}> { getTextBeforeCaret(): string { this.getCaret(); - return this.getText()._kmwSubstring(0, this.processedSelectionStart); + return KMWString.substring(this.getText(), 0, this.processedSelectionStart); } setTextBeforeCaret(text: string) { @@ -136,7 +136,7 @@ export default class TextArea extends OutputTarget<{}> { let selectionLength = this.processedSelectionEnd - this.processedSelectionStart; let direction = this.getSelectionDirection(); let newCaret = KMWString.length(text); - this.root.value = text + this.getText()._kmwSubstring(this.processedSelectionStart); + this.root.value = text + KMWString.substring(this.getText(), this.processedSelectionStart); this.setSelection(newCaret, newCaret + selectionLength, direction); } @@ -150,12 +150,12 @@ export default class TextArea extends OutputTarget<{}> { getTextAfterCaret(): string { this.getCaret(); - return this.getText()._kmwSubstring(this.processedSelectionEnd); + return KMWString.substring(this.getText(), this.processedSelectionEnd); } getSelectedText(): string { this.getCaret(); - return this.getText()._kmwSubstring(this.processedSelectionStart, this.processedSelectionEnd); + return KMWString.substring(this.getText(), this.processedSelectionStart, this.processedSelectionEnd); } getText(): string { @@ -172,7 +172,7 @@ export default class TextArea extends OutputTarget<{}> { } this.adjustDeadkeys(-dn); - this.setTextBeforeCaret(curText.kmwSubstring(0, caret - dn)); + this.setTextBeforeCaret(KMWString.substr(curText, 0, caret - dn)); this.setCaret(caret - dn); } } @@ -184,7 +184,7 @@ export default class TextArea extends OutputTarget<{}> { let caret = this.getCaret(); let front = this.getTextBeforeCaret(); - let back = this.getText()._kmwSubstring(this.processedSelectionStart); + let back = KMWString.substring(this.getText(), this.processedSelectionStart); this.adjustDeadkeys(KMWString.length(s)); this.root.value = front + s + back; diff --git a/web/src/engine/js-processor/src/kbdInterface.ts b/web/src/engine/js-processor/src/kbdInterface.ts index 8848f3ccde..1f0d967754 100644 --- a/web/src/engine/js-processor/src/kbdInterface.ts +++ b/web/src/engine/js-processor/src/kbdInterface.ts @@ -283,7 +283,7 @@ export default class KeyboardInterface extends KeyboardHarness { tempContext = Array(n-KMWString.length(tempContext)+1).join("\uFFFE") + tempContext; } - return tempContext._kmwSubstr(-n)._kmwSubstr(0,ln); + return KMWString.substr(KMWString.substr(tempContext, -n), 0, ln); } /** diff --git a/web/src/engine/js-processor/src/mock.ts b/web/src/engine/js-processor/src/mock.ts index 5910ba554b..c153829469 100644 --- a/web/src/engine/js-processor/src/mock.ts +++ b/web/src/engine/js-processor/src/mock.ts @@ -96,15 +96,15 @@ export class Mock extends OutputTarget { } getTextBeforeCaret(): string { - return this.text.kmwSubstr(0, this.selStart); + return KMWString.substr(this.text, 0, this.selStart); } getSelectedText(): string { - return this.text.kmwSubstr(this.selStart, this.selEnd - this.selStart); + return KMWString.substr(this.text, this.selStart, this.selEnd - this.selStart); } getTextAfterCaret(): string { - return this.text.kmwSubstr(this.selEnd); + return KMWString.substr(this.text, this.selEnd); } getText(): string { @@ -117,7 +117,7 @@ export class Mock extends OutputTarget { dn = this.selStart; } this.adjustDeadkeys(-dn); - this.text = this.text.kmwSubstr(0, this.selStart - dn) + this.text.kmwSubstr(this.selStart); + this.text = KMWString.substr(this.text, 0, this.selStart - dn) + KMWString.substr(this.text, this.selStart); this.selStart -= dn; this.selEnd -= dn; } @@ -125,7 +125,7 @@ export class Mock extends OutputTarget { insertTextBeforeCaret(s: string): void { this.adjustDeadkeys(KMWString.length(s)); - this.text = this.getTextBeforeCaret() + s + this.text.kmwSubstr(this.selStart); + this.text = this.getTextBeforeCaret() + s + KMWString.substr(this.text, this.selStart); this.selStart += KMWString.length(s); this.selEnd += KMWString.length(s); } diff --git a/web/src/engine/js-processor/src/outputTarget.ts b/web/src/engine/js-processor/src/outputTarget.ts index d4f1285568..c44ebe5080 100644 --- a/web/src/engine/js-processor/src/outputTarget.ts +++ b/web/src/engine/js-processor/src/outputTarget.ts @@ -179,7 +179,7 @@ export default abstract class OutputTarget implements OutputTargetInterface { this.clearSelection(); if(transform.deleteRight) { - this.setTextAfterCaret(this.getTextAfterCaret()._kmwSubstr(transform.deleteRight)); + this.setTextAfterCaret(KMWString.substr(this.getTextAfterCaret(), transform.deleteRight)); } if(transform.deleteLeft) { diff --git a/web/src/engine/main/src/headless/contextWindow.ts b/web/src/engine/main/src/headless/contextWindow.ts index b37014fc5b..7394add046 100644 --- a/web/src/engine/main/src/headless/contextWindow.ts +++ b/web/src/engine/main/src/headless/contextWindow.ts @@ -23,13 +23,13 @@ export default class ContextWindow implements LexicalModelTypes.Context { this.startOfBuffer = KMWString.length(this.left) <= config.leftContextCodePoints; if(!this.startOfBuffer) { // Our custom substring version will return the last n characters if param #1 is given -n. - this.left = this.left._kmwSubstr(-config.leftContextCodePoints); + this.left = KMWString.substr(this.left, -config.leftContextCodePoints); } this.right = mock.getTextAfterCaret(); this.endOfBuffer = KMWString.length(this.right) <= config.rightContextCodePoints; if(!this.endOfBuffer) { - this.right = this.right._kmwSubstr(0, config.rightContextCodePoints); + this.right = KMWString.substr(this.right, 0, config.rightContextCodePoints); } this.casingForm = diff --git a/web/src/engine/predictive-text/templates/src/common.ts b/web/src/engine/predictive-text/templates/src/common.ts index 26e3a55338..e9d4f39819 100644 --- a/web/src/engine/predictive-text/templates/src/common.ts +++ b/web/src/engine/predictive-text/templates/src/common.ts @@ -18,13 +18,13 @@ export function applyTransform(transform: Transform, context: Context): Context let lLen = KMWString.length(fullLeftContext); let lDel = lLen < transform.deleteLeft ? lLen : transform.deleteLeft; - let leftContext = fullLeftContext.kmwSubstr(0, lLen - lDel) + (transform.insert || ''); + let leftContext = KMWString.substr(fullLeftContext, 0, lLen - lDel) + (transform.insert || ''); let fullRightContext = context.right || ''; let rLen = KMWString.length(fullRightContext); let rDel = (rLen < (transform.deleteRight ?? 0)) ? rLen : (transform.deleteRight ?? 0); - let rightContext = fullRightContext.kmwSubstr(rDel); + let rightContext = KMWString.substr(fullRightContext, rDel); return { left: leftContext, @@ -52,7 +52,7 @@ export function buildMergedTransform(first: Transform, second: Transform): Trans mergedFirstInsert = ''; mergedSecondDelete = second.deleteLeft - firstLength; } else { - mergedFirstInsert = first.insert.kmwSubstr(0, firstLength - second.deleteLeft); + mergedFirstInsert = KMWString.substr(first.insert, 0, firstLength - second.deleteLeft); mergedSecondDelete = 0; } } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts index 3b9e049b77..2a526a0454 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts @@ -200,7 +200,7 @@ export class ModelCompositor { if(unchangedLength > 0) { suggestion.transform.deleteLeft += unchangedLength; - suggestion.transform.insert = baseWord.kmwSubstr(0, unchangedLength) + suggestion.transform.insert; + suggestion.transform.insert = KMWString.substr(baseWord, 0, unchangedLength) + suggestion.transform.insert; } // Step 2: Now that the transform affects the whole word, we may safely apply casing rules. @@ -211,7 +211,7 @@ export class ModelCompositor { acceptSuggestion(suggestion: Suggestion, context: Context, postTransform?: Transform): Reversion { // Step 1: generate and save the reversion's Transform. let sourceTransform = suggestion.transform; - let deletedLeftChars = context.left.kmwSubstr(-sourceTransform.deleteLeft, sourceTransform.deleteLeft); + let deletedLeftChars = KMWString.substr(context.left, -sourceTransform.deleteLeft, sourceTransform.deleteLeft); let insertedLength = KMWString.length(sourceTransform.insert); let reversionTransform: Transform = { From f8ea902fd918e713b15dd5c5ed8c21d45b7535ee Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 22 Apr 2025 10:44:54 +0700 Subject: [PATCH 032/431] refactor(web): refactor _kmwCharAt --- .../engine/common/web-utils/src/kmwstring.ts | 56 +++++++------------ .../web-utils/src/tests/stringHandling.js | 44 +++++++-------- .../engine/js-processor/src/kbdInterface.ts | 2 +- .../src/main/correction/context-tracker.ts | 2 +- 4 files changed, 45 insertions(+), 59 deletions(-) diff --git a/web/src/engine/common/web-utils/src/kmwstring.ts b/web/src/engine/common/web-utils/src/kmwstring.ts index 70ccb7157d..8e755feb07 100644 --- a/web/src/engine/common/web-utils/src/kmwstring.ts +++ b/web/src/engine/common/web-utils/src/kmwstring.ts @@ -384,6 +384,27 @@ export function codeUnitToCodePoint(s: string, codeUnitIndex: number) { } } +/** + * Returns the character at a the code point index passed + * + * @param {string} s The string + * @param {number} index A code point index in the string + * @return {string} The corresponding character + */ +export function charAt(s: string, index: number) { // charAtPoint + const str = s; + + if(!EXTENSION_ENABLED) { + return str.charAt(index); + } + + if(index >= 0) { + return substr(str, index, 1); + } else { + return '' + }; +} + /* * TODO: Remove this file as part of addressing https://github.com/keymanapp/keyman/issues/2492. */ @@ -392,36 +413,9 @@ declare global { interface StringConstructor { kmwEnableSupplementaryPlane(bEnable: boolean): void } - - interface String { - kmwCharAt(codePointIndex: number) : string, - _kmwCharAt(codePointIndex: number) : string, - } } export default function extendString() { - /* - Helper functions - */ - - /** - * Returns the character at a the code point index passed - * - * @param {number} codePointIndex A code point index in the string - * @return {string} The corresponding character - */ - String.prototype.kmwCharAt = function(codePointIndex) { - var str = String(this); - - if(codePointIndex >= 0) return substr(str, codePointIndex,1); else return ''; - } - - /** - * String prototype library extensions for basic plane characters, - * to simplify enabling or disabling supplementary plane functionality (I3319) - */ - - /** * Enable or disable supplementary plane string handling * @@ -429,16 +423,8 @@ export default function extendString() { */ String.kmwEnableSupplementaryPlane = function(bEnable) { - var p=String.prototype; - p._kmwCharAt = bEnable ? p.kmwCharAt : p.charAt; - enableSupplementaryPlane(bEnable); } - - // Ensure that _all_ String extensions are established, even if disabled by default. - if(!""._kmwCharAt) { - String.kmwEnableSupplementaryPlane(false); - } } // For side-effect imports: diff --git a/web/src/engine/common/web-utils/src/tests/stringHandling.js b/web/src/engine/common/web-utils/src/tests/stringHandling.js index 43d0d34ae5..89d99dded9 100644 --- a/web/src/engine/common/web-utils/src/tests/stringHandling.js +++ b/web/src/engine/common/web-utils/src/tests/stringHandling.js @@ -8,14 +8,14 @@ describe('Unicode string handling', () => { String.kmwEnableSupplementaryPlane(false); }); - it('_kmwCharAt', () => { + it('charAt', () => { const str = "string"; - assert.equal(str._kmwCharAt(-1), ''); - assert.equal(str._kmwCharAt(6), ''); - assert.equal(str._kmwCharAt(1000), ''); - assert.equal(str._kmwCharAt(0), "s"); - assert.equal(str._kmwCharAt(2), "r"); - assert.equal(str._kmwCharAt(5), "g"); + assert.equal(KMWString.charAt(str, -1), ''); + assert.equal(KMWString.charAt(str, 6), ''); + assert.equal(KMWString.charAt(str, 1000), ''); + assert.equal(KMWString.charAt(str, 0), "s"); + assert.equal(KMWString.charAt(str, 2), "r"); + assert.equal(KMWString.charAt(str, 5), "g"); }); it('charCodeAt', () => { @@ -121,14 +121,14 @@ describe('Unicode string handling', () => { String.kmwEnableSupplementaryPlane(true); }); - it('_kmwCharAt', () => { + it('charAt', () => { const str = "string"; - assert.equal(str._kmwCharAt(-1), ''); - assert.equal(str._kmwCharAt(6), ''); - assert.equal(str._kmwCharAt(1000), ''); - assert.equal(str._kmwCharAt(0), "s"); - assert.equal(str._kmwCharAt(2), "r"); - assert.equal(str._kmwCharAt(5), "g"); + assert.equal(KMWString.charAt(str, -1), ''); + assert.equal(KMWString.charAt(str, 6), ''); + assert.equal(KMWString.charAt(str, 1000), ''); + assert.equal(KMWString.charAt(str, 0), "s"); + assert.equal(KMWString.charAt(str, 2), "r"); + assert.equal(KMWString.charAt(str, 5), "g"); }); it('charCodeAt', () => { @@ -240,17 +240,17 @@ describe('Unicode string handling', () => { String.kmwEnableSupplementaryPlane(true); }); - it('_kmwCharAt', () => { + it('charAt', () => { // 0x1d5c9: MATHEMATICAL SANS-SERIF SMALL p // 0x1d5be: MATHEMATICAL SANS-SERIF SMALL e const apples = 'a' + String.fromCodePoint(0x1d5c9) + 'p' + 'l' + String.fromCodePoint(0x1d5be) + 's'; - assert.equal(apples._kmwCharAt(-1), ''); - assert.equal(apples._kmwCharAt(6), ''); - assert.equal(apples._kmwCharAt(1000), ''); - assert.equal(apples._kmwCharAt(0), "a"); - assert.equal(apples._kmwCharAt(1), String.fromCodePoint(0x1d5c9)); - assert.equal(apples._kmwCharAt(2), "p"); - assert.equal(apples._kmwCharAt(5), "s"); + assert.equal(KMWString.charAt(apples, -1), ''); + assert.equal(KMWString.charAt(apples, 6), ''); + assert.equal(KMWString.charAt(apples, 1000), ''); + assert.equal(KMWString.charAt(apples, 0), "a"); + assert.equal(KMWString.charAt(apples, 1), String.fromCodePoint(0x1d5c9)); + assert.equal(KMWString.charAt(apples, 2), "p"); + assert.equal(KMWString.charAt(apples, 5), "s"); }); it('charCodeAt', () => { diff --git a/web/src/engine/js-processor/src/kbdInterface.ts b/web/src/engine/js-processor/src/kbdInterface.ts index 1f0d967754..4a82b8e460 100644 --- a/web/src/engine/js-processor/src/kbdInterface.ts +++ b/web/src/engine/js-processor/src/kbdInterface.ts @@ -643,7 +643,7 @@ export default class KeyboardInterface extends KeyboardHarness { // Nope, so let's build its cache. var result: ComplexKeyboardStore = []; for(var i=0; i < KMWString.length(store); i++) { - result.push(store._kmwCharAt(i)); + result.push(KMWString.charAt(store, i)); } // Cache the result for later! diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts index 548341ef60..5dc4f7d347 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts @@ -18,7 +18,7 @@ function textToCharTransforms(text: string, transformId?: number) { let perCharTransforms: Transform[] = []; for(let i=0; i < KMWString.length(text); i++) { - let char = text.kmwCharAt(i); // is SMP-aware + let char = KMWString.charAt(text, i); // is SMP-aware let transform: Transform = { insert: char, From dc450cde9ed3e1241cb662c77f43212817526fb8 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 22 Apr 2025 11:25:19 +0700 Subject: [PATCH 033/431] chore(web): finalize removal of side-effect KMWString implementation --- common/test/resources/model-helpers.mjs | 7 - web/src/engine/common/web-utils/src/index.ts | 1 - .../engine/common/web-utils/src/kmwstring.ts | 39 +-- .../web-utils/src/tests/stringHandling.js | 6 +- .../engine/js-processor/src/outputTarget.ts | 4 +- web/src/engine/main/src/keymanEngine.ts | 3 +- .../predictive-text/templates/src/common.ts | 4 +- .../templates/src/trie-model.ts | 5 +- .../templates/tests/helpers.js | 7 +- .../worker-thread/src/main/index.ts | 3 - .../element_interfaces.tests.ts | 248 +++++++++--------- .../element-wrappers/target_mocks.tests.ts | 7 +- .../cases/keyboard/domKeyboardLoader.tests.ts | 5 +- .../engine/js-processor/basic-init.tests.js | 3 - .../js-processor/bundled-module.tests.js | 7 +- .../engine/notany_context.tests.js | 6 - .../js-processor/engine/stores.tests.js | 9 +- .../js-processor/transcriptions.tests.js | 14 +- .../main/headless/inputProcessor.tests.js | 2 +- .../main/headless/languageProcessor.tests.js | 3 - 20 files changed, 161 insertions(+), 222 deletions(-) diff --git a/common/test/resources/model-helpers.mjs b/common/test/resources/model-helpers.mjs index 27eebddff9..a3f141cd54 100644 --- a/common/test/resources/model-helpers.mjs +++ b/common/test/resources/model-helpers.mjs @@ -13,16 +13,9 @@ import vm from 'vm'; // Ensure that we can successfully load the module & apply kmwLength, as it's // needed for some of the unit tests. -// // Verify that the KMW string extensions are loaded via side-effect. - -import { extendString } from '@keymanapp/web-utils'; - import { createRequire } from "module"; import { fileURLToPath } from 'url'; -// Ensure that our KMW string-extensions activate. -extendString(); - /** * Creates a MessageEvent (for inter-worker communication), with the given data payload. * diff --git a/web/src/engine/common/web-utils/src/index.ts b/web/src/engine/common/web-utils/src/index.ts index 25c0c0bc06..30f4821590 100644 --- a/web/src/engine/common/web-utils/src/index.ts +++ b/web/src/engine/common/web-utils/src/index.ts @@ -16,7 +16,6 @@ export { default as Version } from "./version.js"; export { default as globalObject } from "./globalObject.js"; -export { default as extendString } from "./kmwstring.js"; export * as KMWString from './kmwstring.js'; export { default as ManagedPromise } from "./managedPromise.js"; diff --git a/web/src/engine/common/web-utils/src/kmwstring.ts b/web/src/engine/common/web-utils/src/kmwstring.ts index 8e755feb07..3f464a103d 100644 --- a/web/src/engine/common/web-utils/src/kmwstring.ts +++ b/web/src/engine/common/web-utils/src/kmwstring.ts @@ -1,6 +1,13 @@ /*** - KeymanWeb 19.0 - Copyright 2025 SIL International + KeymanWeb 19.0 + Copyright 2025 SIL International + + NOTE: in previous versions of Keyman Engine for Web, these methods were set on the standard + JS String type in a manner considered a "side-effect". (When inspecting older versions, it may help + to know that they were originally implemented within a file named kmwstring.ts.) + * + These methods seek to support SMP-aware string functionality within the Web engine, as many + keyboards exist that utilize characters within such ranges. ***/ /** @@ -58,7 +65,6 @@ export function charCodeAt(s: string, index: number) { return first; } - /** * Returns the code point index within the calling String object of the first occurrence * of the specified value, or -1 if not found. @@ -403,29 +409,4 @@ export function charAt(s: string, index: number) { // charAtPoint } else { return '' }; -} - -/* - * TODO: Remove this file as part of addressing https://github.com/keymanapp/keyman/issues/2492. - */ - -declare global { - interface StringConstructor { - kmwEnableSupplementaryPlane(bEnable: boolean): void - } -} - -export default function extendString() { - /** - * Enable or disable supplementary plane string handling - * - * @param {boolean} bEnable - */ - String.kmwEnableSupplementaryPlane = function(bEnable) - { - enableSupplementaryPlane(bEnable); - } -} - -// For side-effect imports: -extendString(); \ No newline at end of file +} \ No newline at end of file diff --git a/web/src/engine/common/web-utils/src/tests/stringHandling.js b/web/src/engine/common/web-utils/src/tests/stringHandling.js index 89d99dded9..a8555e57e9 100644 --- a/web/src/engine/common/web-utils/src/tests/stringHandling.js +++ b/web/src/engine/common/web-utils/src/tests/stringHandling.js @@ -5,7 +5,7 @@ import { KMWString } from '@keymanapp/web-utils'; describe('Unicode string handling', () => { describe('disabled: BMP-mode, BMP-only strings', () => { before(() => { - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); }); it('charAt', () => { @@ -118,7 +118,7 @@ describe('Unicode string handling', () => { describe('enabled: SMP-mode, but BMP-only strings', () => { before(() => { - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); }); it('charAt', () => { @@ -237,7 +237,7 @@ describe('Unicode string handling', () => { describe('enabled: SMP-mode, with mixed-plane strings', () => { before(() => { - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); }); it('charAt', () => { diff --git a/web/src/engine/js-processor/src/outputTarget.ts b/web/src/engine/js-processor/src/outputTarget.ts index c44ebe5080..df0ff1355d 100644 --- a/web/src/engine/js-processor/src/outputTarget.ts +++ b/web/src/engine/js-processor/src/outputTarget.ts @@ -1,10 +1,8 @@ -import { extendString, KMWString } from "@keymanapp/web-utils"; +import { KMWString } from "@keymanapp/web-utils"; import { findCommonSubstringEndIndex } from "./stringDivergence.js"; import { Mock } from "./mock.js"; import { OutputTarget as OutputTargetInterface } from 'keyman/engine/keyboard'; -extendString(); - // Defines deadkey management in a manner attachable to each element interface. import { type KeyEvent } from 'keyman/engine/keyboard'; import { Deadkey, DeadkeyTracker } from "./deadkeys.js"; diff --git a/web/src/engine/main/src/keymanEngine.ts b/web/src/engine/main/src/keymanEngine.ts index 8ef53a9008..651c9c8340 100644 --- a/web/src/engine/main/src/keymanEngine.ts +++ b/web/src/engine/main/src/keymanEngine.ts @@ -15,6 +15,7 @@ import { LegacyAPIEvents } from "./legacyAPIEvents.js"; import { EventNames, EventListener, LegacyEventEmitter } from "keyman/engine/events"; import DOMCloudRequester from "keyman/engine/keyboard-storage/dom-requester"; import KEYMAN_VERSION from "@keymanapp/keyman-version"; +import { KMWString } from "@keymanapp/web-utils"; // From https://stackoverflow.com/a/69328045 type WithRequired = T & { [P in K]-?: T[P] }; @@ -237,7 +238,7 @@ export default class KeymanEngine< config.initialize(optionSpec); // Initialize supplementary plane string extensions - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); // Since we're not sandboxing keyboard loads yet, we just use `window` as the jsGlobal object. // All components initialized below require a properly-configured `config.paths` or similar. diff --git a/web/src/engine/predictive-text/templates/src/common.ts b/web/src/engine/predictive-text/templates/src/common.ts index e9d4f39819..f23ad6095b 100644 --- a/web/src/engine/predictive-text/templates/src/common.ts +++ b/web/src/engine/predictive-text/templates/src/common.ts @@ -6,9 +6,7 @@ import Outcome = LexicalModelTypes.Outcome; import Suggestion = LexicalModelTypes.Suggestion; import Transform = LexicalModelTypes.Transform; import WithOutcome = LexicalModelTypes.WithOutcome; -import { extendString, KMWString } from "@keymanapp/web-utils"; - -extendString(); +import { KMWString } from "@keymanapp/web-utils"; export const SENTINEL_CODE_UNIT = '\uFDD0'; diff --git a/web/src/engine/predictive-text/templates/src/trie-model.ts b/web/src/engine/predictive-text/templates/src/trie-model.ts index f5cdb2223e..32825f8930 100644 --- a/web/src/engine/predictive-text/templates/src/trie-model.ts +++ b/web/src/engine/predictive-text/templates/src/trie-model.ts @@ -25,8 +25,7 @@ // Worth noting: we're starting to get quite a 'library' of common model/LMLayer functionality. // Should probably make a 'lm-utils' submodule. -// Allows the kmwstring bindings to resolve. -import { extendString, KMWString, PriorityQueue } from "@keymanapp/web-utils"; +import { KMWString, PriorityQueue } from "@keymanapp/web-utils"; import { default as defaultWordBreaker } from "@keymanapp/models-wordbreakers"; import { applyTransform, isHighSurrogate, isSentinel, SENTINEL_CODE_UNIT, transformToSuggestion } from "./common.js"; @@ -47,8 +46,6 @@ import USVString = LexicalModelTypes.USVString; import WithOutcome = LexicalModelTypes.WithOutcome; import WordBreakingFunction = LexicalModelTypes.WordBreakingFunction; -extendString(); - /** * @file trie-model.ts * diff --git a/web/src/engine/predictive-text/templates/tests/helpers.js b/web/src/engine/predictive-text/templates/tests/helpers.js index 1bf4fabcfb..94ca5ed1b2 100644 --- a/web/src/engine/predictive-text/templates/tests/helpers.js +++ b/web/src/engine/predictive-text/templates/tests/helpers.js @@ -7,13 +7,14 @@ var _ = global; // TODO: then mocha invocation is as follows: // TODO: mocha -r @keymanapp/models-test-helpers test/ -import { extendString } from '@keymanapp/web-utils'; +import { KMWString } from '@keymanapp/web-utils'; + +// The predictive-text engine always keeps these enabled. +KMWString.enableSupplementaryPlane(true); import { createRequire } from "module"; import { fileURLToPath } from 'url'; -extendString(); - /** * Load JSON fixtures from a well-known place. */ diff --git a/web/src/engine/predictive-text/worker-thread/src/main/index.ts b/web/src/engine/predictive-text/worker-thread/src/main/index.ts index db12195738..30a2d8cea3 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/index.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/index.ts @@ -30,9 +30,6 @@ */ /// -import { extendString } from "@keymanapp/web-utils"; - -extendString(); import * as models from './models/index.js'; import * as correction from './correction/index.js'; diff --git a/web/src/test/auto/dom/cases/element-wrappers/element_interfaces.tests.ts b/web/src/test/auto/dom/cases/element-wrappers/element_interfaces.tests.ts index 8ccb0ce428..47c82d37d9 100644 --- a/web/src/test/auto/dom/cases/element-wrappers/element_interfaces.tests.ts +++ b/web/src/test/auto/dom/cases/element-wrappers/element_interfaces.tests.ts @@ -1,14 +1,12 @@ import { assert } from 'chai'; -import { extendString, KMWString } from 'keyman/engine/keyboard'; +import { KMWString } from 'keyman/engine/keyboard'; import { Mock } from 'keyman/engine/js-processor'; import * as wrappers from 'keyman/engine/element-wrappers'; import { DynamicElements } from '../../test_utils.js'; import { DEFAULT_BROWSER_TIMEOUT } from '@keymanapp/common-test-resources/test-timeouts.mjs'; -extendString(); - const host = document.createElement('div'); host.id = 'DynamicElements'; document.body.appendChild(host); @@ -383,25 +381,25 @@ class InterfaceTests { const Apple = InterfaceTests.Strings.Apple; const pair = testObj.setupElement(); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); testObj.resetWithText(pair, Apple.normal); testObj.setCaret(pair, 4); assert.equal(pair.wrapper.getCaret(), 4, "Failed to correctly read the caret's position within a simple string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.smp); testObj.setCaret(pair, 8); assert.equal(pair.wrapper.getCaret(), 4, "Failed to correctly read the caret's position within an SMP string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.mixed); testObj.setCaret(pair, 5); assert.equal(pair.wrapper.getCaret(), 4, "Failed to correctly read the caret's position within a mixed SMP string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); } // Corresponds to a helper method for certain classes. @@ -409,61 +407,61 @@ class InterfaceTests { const Apple = InterfaceTests.Strings.Apple; const pair = testObj.setupElement(); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); testObj.resetWithText(pair, Apple.normal); testObj.setSelectionRange(pair, 3, 5); assert.equal(pair.wrapper.getCaret(), 5, "Failed to choose the caret's initial position for forward-selections within a simple string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); testObj.resetWithText(pair, Apple.normal); testObj.setSelectionRange(pair, 5, 3); assert.equal(pair.wrapper.getCaret(), 3, "Failed to choose the caret's initial position for backward-selections within a simple string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.smp); testObj.setSelectionRange(pair, 6, 10); assert.equal(pair.wrapper.getCaret(), 5, "Failed to caret's initial position for forward-selections within an SMP string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.smp); testObj.setSelectionRange(pair, 10, 6); assert.equal(pair.wrapper.getCaret(), 3, "Failed to caret's initial position for backward-selections within an SMP string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); } public static getSelectedText(testObj: TestHelper) { const Apple = InterfaceTests.Strings.Apple; const pair = testObj.setupElement(); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); testObj.resetWithText(pair, Apple.normal); testObj.setSelectionRange(pair, 3, 5); assert.equal(pair.wrapper.getSelectedText(), '45', "Failed to properly retrieve selected text"); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); testObj.resetWithText(pair, Apple.normal); testObj.setSelectionRange(pair, 5, 3); assert.equal(pair.wrapper.getSelectedText(), '45', "Failed to properly retrieve reverse-direction selected text"); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); testObj.resetWithText(pair, Apple.normal); testObj.setSelectionRange(pair, 3, 3); assert.equal(pair.wrapper.getSelectedText(), '', "No text was selected, but some was returned"); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.smp); testObj.setSelectionRange(pair, 6, 10); assert.equal(pair.wrapper.getSelectedText(), Apple.smp.substring(6, 10), "Failed to properly retrieve selected SMP text"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); } @@ -472,25 +470,25 @@ class InterfaceTests { const Apple = InterfaceTests.Strings.Apple; const pair = testObj.setupElement(); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); testObj.resetWithText(pair, Apple.normal); pair.wrapper.setCaret(4); assert.equal(testObj.getCaret(pair), 4, "Failed to correctly set the caret within a simple string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); testObj.resetWithText(pair, Apple.smp); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); pair.wrapper.setCaret(4); assert.equal(testObj.getCaret(pair), 8, "Failed to correctly set the caret within an SMP string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); testObj.resetWithText(pair, Apple.mixed); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); pair.wrapper.setCaret(4); assert.equal(testObj.getCaret(pair), 5, "Failed to correctly set the caret within a mixed SMP string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); } // Corresponds to a helper method for certain classes. @@ -498,247 +496,247 @@ class InterfaceTests { const Apple = InterfaceTests.Strings.Apple; const pair = testObj.setupElement(); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); testObj.resetWithText(pair, Apple.normal); testObj.setSelectionRange(pair, 3, 5); pair.wrapper.setCaret(4); assert.equal(testObj.getCaret(pair), 4, "Failed to correctly set the caret within a simple string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); testObj.resetWithText(pair, Apple.normal); testObj.setSelectionRange(pair, 5, 3); pair.wrapper.setCaret(4); assert.equal(testObj.getCaret(pair), 4, "Failed to correctly set the caret within a simple string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.smp); testObj.setSelectionRange(pair, 6, 10); pair.wrapper.setCaret(4); assert.equal(testObj.getCaret(pair), 8, "Failed to correctly set the caret within an SMP string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.smp); testObj.setSelectionRange(pair, 10, 6); pair.wrapper.setCaret(4); assert.equal(testObj.getCaret(pair), 8, "Failed to correctly set the caret within an SMP string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); } public static getTextNoSelection(testObj: TestHelper) { const Apple = InterfaceTests.Strings.Apple; const pair = testObj.setupElement(); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); testObj.resetWithText(pair, Apple.normal); assert.equal(pair.wrapper.getText(), Apple.normal, "Failed to properly return its stored value: a simple string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.smp); assert.equal(pair.wrapper.getText(), Apple.smp, "Failed to properly return its stored value: a string of SMP characters"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); testObj.resetWithText(pair, Apple.mixed); assert.equal(pair.wrapper.getText(), Apple.mixed, "Failed to properly return its stored value: a string with some SMP characters"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); } public static getTextWithSelection(testObj: TestHelper) { const Apple = InterfaceTests.Strings.Apple; const pair = testObj.setupElement(); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); testObj.resetWithText(pair, Apple.normal); testObj.setSelectionRange(pair, 3, 5); assert.equal(pair.wrapper.getText(), Apple.normal, "Failed to properly return its stored value: a simple string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.smp); testObj.setSelectionRange(pair, 6, 10); assert.equal(pair.wrapper.getText(), Apple.smp, "Failed to properly return its stored value: a string of SMP characters"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); testObj.resetWithText(pair, Apple.mixed); testObj.setSelectionRange(pair, 4, 7); assert.equal(pair.wrapper.getText(), Apple.mixed, "Failed to properly return its stored value: a string with some SMP characters"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); } public static getTextBeforeCaretNoSelection(testObj: TestHelper) { const Apple = InterfaceTests.Strings.Apple; const pair = testObj.setupElement(); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); testObj.resetWithText(pair, Apple.normal); testObj.setCaret(pair, 4); assert.equal(pair.wrapper.getTextBeforeCaret(), Apple.normal.substr(0, 4), "Failed to properly return correct substring for a simple string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.smp); testObj.setCaret(pair, 8); assert.equal(pair.wrapper.getTextBeforeCaret(), Apple.smp.substr(0, 8), "Failed to properly return correct substring for an SMP string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.mixed); testObj.setCaret(pair, 5); assert.equal(pair.wrapper.getTextBeforeCaret(), Apple.mixed.substr(0, 5), "Failed to properly return correct substring for a mixed SMP string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); } public static getTextBeforeCaretWithSelection(testObj: TestHelper) { const Apple = InterfaceTests.Strings.Apple; const pair = testObj.setupElement(); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); testObj.resetWithText(pair, Apple.normal); testObj.setSelectionRange(pair, 3, 5); assert.equal(pair.wrapper.getTextBeforeCaret(), Apple.normal.substr(0, 3), "Failed simple string, normal-order selection"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); testObj.resetWithText(pair, Apple.normal); testObj.setSelectionRange(pair, 5, 3); assert.equal(pair.wrapper.getTextBeforeCaret(), Apple.normal.substr(0, 3), "Failed simple string, reverse-order selection"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.smp); testObj.setSelectionRange(pair, 6, 10); assert.equal(pair.wrapper.getTextBeforeCaret(), Apple.smp.substr(0, 6), "Failed SMP string, normal-order selection"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.smp); testObj.setSelectionRange(pair, 10, 6); assert.equal(pair.wrapper.getTextBeforeCaret(), Apple.smp.substr(0, 6), "Failed SMP string, reverse-order selection"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.mixed); testObj.setSelectionRange(pair, 4, 7); assert.equal(pair.wrapper.getTextBeforeCaret(), Apple.mixed.substr(0, 4), "Failed mixed SMP string, forward-order selection"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.mixed); testObj.setSelectionRange(pair, 7, 4); assert.equal(pair.wrapper.getTextBeforeCaret(), Apple.mixed.substr(0, 4), "Failed mixed SMP string, reverse-order selection"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); } public static getTextAfterCaretNoSelection(testObj: TestHelper) { const Apple = InterfaceTests.Strings.Apple; const pair = testObj.setupElement(); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); testObj.resetWithText(pair, Apple.normal); testObj.setCaret(pair, 4); assert.equal(pair.wrapper.getTextAfterCaret(), Apple.normal.substr(4), "Failed to properly return correct substring for a simple string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.smp); testObj.setCaret(pair, 8); assert.equal(pair.wrapper.getTextAfterCaret(), Apple.smp.substr(8), "Failed to properly return correct substring for an SMP string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.mixed); testObj.setCaret(pair, 5); assert.equal(pair.wrapper.getTextAfterCaret(), Apple.mixed.substr(5), "Failed to properly return correct substring for a mixed SMP string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); } public static getTextAfterCaretWithSelection(testObj: TestHelper) { const Apple = InterfaceTests.Strings.Apple; const pair = testObj.setupElement(); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); testObj.resetWithText(pair, Apple.normal); testObj.setSelectionRange(pair, 2, 4); assert.equal(pair.wrapper.getTextAfterCaret(), Apple.normal.substr(4), "Failed simple string, normal-order selection"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); testObj.resetWithText(pair, Apple.normal); testObj.setSelectionRange(pair, 4, 2); assert.equal(pair.wrapper.getTextAfterCaret(), Apple.normal.substr(4), "Failed simple string, reverse-order selection"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.smp); testObj.setSelectionRange(pair, 4, 8); assert.equal(pair.wrapper.getTextAfterCaret(), Apple.smp.substr(8), "Failed SMP string, normal-order selection"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.smp); testObj.setSelectionRange(pair, 8, 4); assert.equal(pair.wrapper.getTextAfterCaret(), Apple.smp.substr(8), "Failed SMP string, reverse-order selection"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.mixed); testObj.setSelectionRange(pair, 3, 5); assert.equal(pair.wrapper.getTextAfterCaret(), Apple.mixed.substr(5), "Failed mixed SMP string, forward-order selection"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.mixed); testObj.setSelectionRange(pair, 5, 3); assert.equal(pair.wrapper.getTextAfterCaret(), Apple.mixed.substr(5), "Failed mixed SMP string, reverse-order selection"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); } public static clearSelection(testObj: TestHelper) { const Apple = InterfaceTests.Strings.Apple; const pair = testObj.setupElement(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.mixed); testObj.setSelectionRange(pair, 3, 5); assert.equal(pair.wrapper.getText(), Apple.mixed, "String should not be modified on selection: forward-order selection"); pair.wrapper.clearSelection(); assert.equal(pair.wrapper.getText(), Apple.mixed.substr(0, 3) + Apple.mixed.substr(5), "Selected text improperly removed: forward-order selection"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.mixed); testObj.setSelectionRange(pair, 5, 3); assert.equal(pair.wrapper.getText(), Apple.mixed, "String should not be modified on selection: reverse-order selection"); pair.wrapper.clearSelection(); assert.equal(pair.wrapper.getText(), Apple.mixed.substr(0, 3) + Apple.mixed.substr(5), "Selected text improperly removed: backward-order selection"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); } // Corresponds to a helper method for certain subclasses. @@ -746,91 +744,91 @@ class InterfaceTests { const Apple = InterfaceTests.Strings.Apple; const pair = testObj.setupElement(); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); testObj.resetWithText(pair, Apple.normal); testObj.setCaret(pair, 4); pair.wrapper.setTextBeforeCaret(Apple.normal.substr(0, 2)) assert.equal(pair.wrapper.getText(), Apple.normal.substr(0, 2) + Apple.normal.substr(4), "Error replacing text in a simple string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.smp); testObj.setCaret(pair, 8); pair.wrapper.setTextBeforeCaret(Apple.smp.substr(0, 4)); assert.equal(pair.wrapper.getText(), Apple.smp.substr(0, 4) + Apple.smp.substr(8), "Error replacing text in an SMP string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.mixed); testObj.setCaret(pair, 5); pair.wrapper.setTextBeforeCaret(Apple.smp.substr(0, 4)) assert.equal(pair.wrapper.getText(), Apple.smp.substr(0, 4) + Apple.mixed.substr(5), "Error replacing text in a mixed SMP string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); } public static setTextBeforeCaretWithSelection(testObj: TestHelper) { const Apple = InterfaceTests.Strings.Apple; const pair = testObj.setupElement(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.mixed); testObj.setSelectionRange(pair, 3, 5); pair.wrapper.setTextBeforeCaret(Apple.mixed.substr(0, 3)); assert.equal(pair.wrapper.getText(), Apple.mixed, "Actively-selected text was erroneously removed"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.mixed); testObj.setSelectionRange(pair, 3, 5); pair.wrapper.setTextBeforeCaret(Apple.smp.substr(0, 4)); assert.equal(pair.wrapper.getText(), Apple.smp.substr(0, 4) + Apple.mixed.substr(3), "Error with text replacement: forward-order selection"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.mixed); testObj.setSelectionRange(pair, 5, 3); pair.wrapper.setTextBeforeCaret(Apple.smp.substr(0, 4)); assert.equal(pair.wrapper.getText(), Apple.smp.substr(0, 4) + Apple.mixed.substr(3), "Error with text replacement: backward-order selection"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); } public static deleteCharsBeforeCaretNoSelection(testObj: TestHelper) { const Apple = InterfaceTests.Strings.Apple; const pair = testObj.setupElement(); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); testObj.resetWithText(pair, Apple.normal); testObj.setCaret(pair, 4); pair.wrapper.deleteCharsBeforeCaret(1); assert.equal(pair.wrapper.getText(), Apple.normal.substr(0, 3) + Apple.normal.substr(4), "Error deleting context chars from a simple string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.smp); testObj.setCaret(pair, 8); pair.wrapper.deleteCharsBeforeCaret(1); assert.equal(pair.wrapper.getText(), Apple.smp.substr(0, 6) + Apple.smp.substr(8), "Error deleting context chars from an SMP string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.mixed); testObj.setCaret(pair, 4); pair.wrapper.deleteCharsBeforeCaret(2); assert.equal(pair.wrapper.getText(), Apple.mixed.substr(0, 1) + Apple.mixed.substr(4), "Error deleting context chars from a mixed SMP string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); testObj.resetWithText(pair, Apple.normal); testObj.setCaret(pair, 4); pair.wrapper.deleteCharsBeforeCaret(7); assert.equal(pair.wrapper.getText(), "5", "Bounds-check on deletion range failed; context improperly deleted."); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); } @@ -838,12 +836,12 @@ class InterfaceTests { const Apple = InterfaceTests.Strings.Apple; const pair = testObj.setupElement(); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); testObj.resetWithText(pair, Apple.normal); testObj.setSelectionRange(pair, 2, 4); pair.wrapper.deleteCharsBeforeCaret(1); assert.equal(pair.wrapper.getText(), Apple.normal.substr(0, 1) + Apple.normal.substr(2), "Selected text erroneously deleted from selection"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); } @@ -851,56 +849,56 @@ class InterfaceTests { const Apple = InterfaceTests.Strings.Apple; const pair = testObj.setupElement(); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); testObj.resetWithText(pair, Apple.normal); testObj.setCaret(pair, 4); pair.wrapper.insertTextBeforeCaret(Apple.normal.substr(0, 2)) assert.equal(pair.wrapper.getText(), Apple.normal.substr(0, 4) + Apple.normal.substr(0, 2) + Apple.normal.substr(4), "Error inserting text in a simple string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.smp); testObj.setCaret(pair, 8); pair.wrapper.insertTextBeforeCaret(Apple.smp.substr(0, 4)); assert.equal(pair.wrapper.getText(), Apple.smp.substr(0, 8) + Apple.smp.substr(0, 4) + Apple.smp.substr(8), "Error inserting text in an SMP string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.mixed); testObj.setCaret(pair, 5); pair.wrapper.insertTextBeforeCaret(Apple.smp.substr(0, 4)); assert.equal(pair.wrapper.getText(), Apple.mixed.substr(0, 5) + Apple.smp.substr(0, 4) + Apple.mixed.substr(5), "Error inserting text in a mixed SMP string"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); } public static insertTextBeforeCaretWithSelection(testObj: TestHelper) { const Apple = InterfaceTests.Strings.Apple; const pair = testObj.setupElement(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.mixed); testObj.setSelectionRange(pair, 3, 5); pair.wrapper.insertTextBeforeCaret(""); assert.equal(pair.wrapper.getText(), Apple.mixed, "Actively-selected text was erroneously removed"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.mixed); testObj.setSelectionRange(pair, 3, 5); pair.wrapper.insertTextBeforeCaret(Apple.smp.substr(0, 4)); assert.equal(pair.wrapper.getText(), Apple.mixed.substr(0, 3) + Apple.smp.substr(0, 4) + Apple.mixed.substr(3), "Error with text replacement: forward-order selection"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); pair.wrapper.invalidateSelection(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.mixed); testObj.setSelectionRange(pair, 5, 3); pair.wrapper.insertTextBeforeCaret(Apple.smp.substr(0, 4)); assert.equal(pair.wrapper.getText(), Apple.mixed.substr(0, 3) + Apple.smp.substr(0, 4) + Apple.mixed.substr(3), "Error with text replacement: backward-order selection"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); } public static getSelectionOwned(testObj: ContentEditableTestHelper) { @@ -950,7 +948,7 @@ class InterfaceTests { const Apple = InterfaceTests.Strings.Apple; const pair = testObj.setupElement(); - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); testObj.resetWithText(pair, Apple.mixed); testObj.setCaret(pair, 1); @@ -979,7 +977,7 @@ class InterfaceTests { assert.isTrue(pair.wrapper.deadkeys().isMatch(1, 0, 1), "Failed to leave deadkey before caret unmoved after text insertion"); assert.isTrue(pair.wrapper.deadkeys().isMatch(8, 0, 3), "Failed to properly adjust position of post-caret deadkey after text insertion"); - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); } //#endregion @@ -992,7 +990,7 @@ describe('Element Input/Output Interfacing', function () { before(function () { // Make sure the basic SMP extension hooks exist to prevent errors later. - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); }); afterEach(function () { host.innerHTML = ''; diff --git a/web/src/test/auto/dom/cases/element-wrappers/target_mocks.tests.ts b/web/src/test/auto/dom/cases/element-wrappers/target_mocks.tests.ts index 707d863899..8e13bd9495 100644 --- a/web/src/test/auto/dom/cases/element-wrappers/target_mocks.tests.ts +++ b/web/src/test/auto/dom/cases/element-wrappers/target_mocks.tests.ts @@ -1,11 +1,10 @@ import { assert } from 'chai'; -import { extendString } from 'keyman/engine/keyboard'; +import { KMWString } from 'keyman/engine/keyboard'; import { Mock } from 'keyman/engine/js-processor'; import { Input } from 'keyman/engine/element-wrappers'; import { DEFAULT_BROWSER_TIMEOUT } from '@keymanapp/common-test-resources/test-timeouts.mjs'; -extendString(); const host = document.createElement('div'); document.body.appendChild(host); @@ -69,7 +68,7 @@ describe('OutputTarget Mocking', function() { before(function() { // Make sure the basic SMP extension hooks exist to prevent errors later. - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); }); afterEach(function() { @@ -77,7 +76,7 @@ describe('OutputTarget Mocking', function() { }); after(function() { - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); }) describe('The "Mock" output target', function() { diff --git a/web/src/test/auto/dom/cases/keyboard/domKeyboardLoader.tests.ts b/web/src/test/auto/dom/cases/keyboard/domKeyboardLoader.tests.ts index 95db91b0f8..507a4ad828 100644 --- a/web/src/test/auto/dom/cases/keyboard/domKeyboardLoader.tests.ts +++ b/web/src/test/auto/dom/cases/keyboard/domKeyboardLoader.tests.ts @@ -1,7 +1,7 @@ import { assert } from 'chai'; import { DOMKeyboardLoader } from 'keyman/engine/keyboard/dom-keyboard-loader'; -import { extendString, KeyboardHarness, Keyboard, MinimalKeymanGlobal, DeviceSpec, KeyboardKeymanGlobal } from 'keyman/engine/keyboard'; +import { KeyboardHarness, Keyboard, MinimalKeymanGlobal, DeviceSpec, KeyboardKeymanGlobal } from 'keyman/engine/keyboard'; import { KeyboardInterface, Mock } from 'keyman/engine/js-processor'; declare let window: typeof globalThis; @@ -10,9 +10,6 @@ declare let window: typeof globalThis; declare let keyman: KeyboardKeymanGlobal; declare let KeymanWeb: KeyboardInterface; -// Note: rule processing tests will fail if string extensions are not established beforehand. -extendString(); - const device: DeviceSpec = { touchable: false, formFactor: DeviceSpec.FormFactor.Phone, diff --git a/web/src/test/auto/headless/engine/js-processor/basic-init.tests.js b/web/src/test/auto/headless/engine/js-processor/basic-init.tests.js index cabdb9d432..2c0e6ee2f2 100644 --- a/web/src/test/auto/headless/engine/js-processor/basic-init.tests.js +++ b/web/src/test/auto/headless/engine/js-processor/basic-init.tests.js @@ -9,9 +9,6 @@ import { NodeKeyboardLoader } from 'keyman/engine/keyboard/node-keyboard-loader' global.keyman = {}; // So that keyboard-based checks against the global `keyman` succeed. // 10.0+ dependent keyboards, like khmer_angkor, will otherwise fail to load. -// Initialize supplementary plane string extensions -String.kmwEnableSupplementaryPlane(false); - // Test the KeyboardProcessor interface. describe('KeyboardProcessor', function() { describe('[[constructor]]', function () { diff --git a/web/src/test/auto/headless/engine/js-processor/bundled-module.tests.js b/web/src/test/auto/headless/engine/js-processor/bundled-module.tests.js index a6b80d5004..a78f8a6cda 100644 --- a/web/src/test/auto/headless/engine/js-processor/bundled-module.tests.js +++ b/web/src/test/auto/headless/engine/js-processor/bundled-module.tests.js @@ -1,6 +1,7 @@ import { assert } from "chai"; import * as Package from "keyman/engine/js-processor"; import * as Package2 from "keyman/engine/keyboard"; +const KMWString = Package2.KMWString; // A few small tests to ensure that the ES Module bundle was successfully constructed and is usable. @@ -29,15 +30,13 @@ describe('Bundled ES Module for js-processor', function() { }); it('smp test', () => { - // Is installed as a _side effect_ from importing the module. - // We could disable that and require a call of `extendString()` instead. - String.kmwEnableSupplementaryPlane(true); // Declared & defined in web-utils. + KMWString.enableSupplementaryPlane(true); // Declared & defined in web-utils. try { let target = new Package.Mock(u(0x1d5ba)+u(0x1d5c9)+u(0x1d5c5)+u(0x1d5be), 2); // ap | le target.insertTextBeforeCaret(u(0x1d5c9)); assert.equal(target.getText(), u(0x1d5ba)+u(0x1d5c9)+u(0x1d5c9)+u(0x1d5c5)+u(0x1d5be)); } finally { - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); } }); }); diff --git a/web/src/test/auto/headless/engine/js-processor/engine/notany_context.tests.js b/web/src/test/auto/headless/engine/js-processor/engine/notany_context.tests.js index 86a35058ae..2d414e2aef 100644 --- a/web/src/test/auto/headless/engine/js-processor/engine/notany_context.tests.js +++ b/web/src/test/auto/headless/engine/js-processor/engine/notany_context.tests.js @@ -7,12 +7,6 @@ import { MinimalKeymanGlobal } from 'keyman/engine/keyboard'; import { KeyboardInterface, Mock } from 'keyman/engine/js-processor'; import { NodeKeyboardLoader } from 'keyman/engine/keyboard/node-keyboard-loader'; import { NodeProctor, RecordedKeystrokeSequence } from '@keymanapp/recorder-core'; -import { extendString } from '@keymanapp/web-utils'; - -extendString(); // Ensure KMW's string-extension functionality is available. - -// Initialize supplementary plane string extensions -String.kmwEnableSupplementaryPlane(false); const device = { formFactor: 'desktop', diff --git a/web/src/test/auto/headless/engine/js-processor/engine/stores.tests.js b/web/src/test/auto/headless/engine/js-processor/engine/stores.tests.js index 8174cbe077..eb1ad139eb 100644 --- a/web/src/test/auto/headless/engine/js-processor/engine/stores.tests.js +++ b/web/src/test/auto/headless/engine/js-processor/engine/stores.tests.js @@ -1,10 +1,7 @@ import { assert } from 'chai'; -import { Keyboard } from 'keyman/engine/keyboard'; +import { Keyboard, KMWString } from 'keyman/engine/keyboard'; import { KeyboardProcessor } from 'keyman/engine/js-processor'; -import { extendString } from '@keymanapp/web-utils'; - -extendString(); let device = { formFactor: 'desktop', @@ -42,11 +39,11 @@ describe('Engine - Stores', function() { for(var i=0; i < STORES.length; i++) { var s = STORES[i]; - String.kmwEnableSupplementaryPlane(s.smp); + KMWString.enableSupplementaryPlane(s.smp); var result = processor.keyboardInterface._ExplodeStore(s.in); assert.sameOrderedMembers(result, s.out, "Failure exploding " + (s.smp ? "SMP" : "non-SMP") + " string value \"" + s.in + "\""); } - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); }); // TODO: Migrate a Node-based copy of the variable store tests here as well. diff --git a/web/src/test/auto/headless/engine/js-processor/transcriptions.tests.js b/web/src/test/auto/headless/engine/js-processor/transcriptions.tests.js index 45c09b3b80..d21cc54c44 100644 --- a/web/src/test/auto/headless/engine/js-processor/transcriptions.tests.js +++ b/web/src/test/auto/headless/engine/js-processor/transcriptions.tests.js @@ -1,11 +1,7 @@ import { assert } from 'chai'; import { Mock, findCommonSubstringEndIndex } from 'keyman/engine/js-processor'; -import { extendString } from '@keymanapp/web-utils'; - -extendString(); // Ensure KMW's string-extension functionality is available. - -String.kmwEnableSupplementaryPlane(false); +import { KMWString } from '@keymanapp/web-utils'; // A unicode-coding like alias for use in constructing non-BMP strings. const u = String.fromCodePoint; @@ -265,7 +261,7 @@ but not himself.`; // Sheev Palpatine, in the Star Wars prequels. it("handles deletions around the caret without text insertion (non-BMP text)", function() { try { - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); var target = new Mock(smpApple, 2); var original = Mock.from(target); target.setSelection(3); @@ -280,7 +276,7 @@ but not himself.`; // Sheev Palpatine, in the Star Wars prequels. assert.equal(transcription.transform.deleteLeft, 1, "Incorrect count for left-of-caret deletions"); assert.equal(transcription.transform.deleteRight, 1, "Incorrect count for right-of-caret deletions"); } finally { - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); } }); @@ -357,7 +353,7 @@ but not himself.`; // Sheev Palpatine, in the Star Wars prequels. it("handles deletions around the caret with text insertion (non-BMP text)", function() { try { - String.kmwEnableSupplementaryPlane(true); + KMWString.enableSupplementaryPlane(true); // We have other texts validating Mocks; by using them as our base 'element', this unit test file // could eventually run in 'headless' mode. @@ -433,7 +429,7 @@ but not himself.`; // Sheev Palpatine, in the Star Wars prequels. assert.equal(transcription.transform.deleteLeft, 1, "Incorrect count for left-of-caret deletions"); assert.equal(transcription.transform.deleteRight, 3, "Incorrect count for right-of-caret deletions"); } finally { - String.kmwEnableSupplementaryPlane(false); + KMWString.enableSupplementaryPlane(false); } }); diff --git a/web/src/test/auto/headless/engine/main/headless/inputProcessor.tests.js b/web/src/test/auto/headless/engine/main/headless/inputProcessor.tests.js index a23e06dfdf..39db5d8c6e 100644 --- a/web/src/test/auto/headless/engine/main/headless/inputProcessor.tests.js +++ b/web/src/test/auto/headless/engine/main/headless/inputProcessor.tests.js @@ -25,7 +25,7 @@ let device = { }; // Initialize supplementary plane string extensions -String.kmwEnableSupplementaryPlane(false); +KMWString.enableSupplementaryPlane(false); // Test the KeyboardProcessor interface. describe('InputProcessor', function() { diff --git a/web/src/test/auto/headless/engine/main/headless/languageProcessor.tests.js b/web/src/test/auto/headless/engine/main/headless/languageProcessor.tests.js index b073454dbd..6d866ef8ed 100644 --- a/web/src/test/auto/headless/engine/main/headless/languageProcessor.tests.js +++ b/web/src/test/auto/headless/engine/main/headless/languageProcessor.tests.js @@ -19,9 +19,6 @@ const KEYMAN_ROOT = env.KEYMAN_ROOT; global.keyman = {}; // So that keyboard-based checks against the global `keyman` succeed. // 10.0+ dependent keyboards, like khmer_angkor, will otherwise fail to load. -// Initialize supplementary plane string extensions -String.kmwEnableSupplementaryPlane(false); - // Test the KeyboardProcessor interface. describe('LanguageProcessor', function() { let languageProcessor; From 9993a33f1bfe0db1e4fd5abe94878f64f5143cc9 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 22 Apr 2025 11:52:44 +0700 Subject: [PATCH 034/431] chore(web): remove side-effect field from web-utils package.json --- web/src/engine/common/web-utils/package.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/web/src/engine/common/web-utils/package.json b/web/src/engine/common/web-utils/package.json index b99c7577fe..f66e28dba4 100644 --- a/web/src/engine/common/web-utils/package.json +++ b/web/src/engine/common/web-utils/package.json @@ -34,8 +34,5 @@ "type": "module", "paths": { "@keymanapp/keyman-version": "*" - }, - "sideEffects": [ - "./src/kmwstring.ts", "./build/obj/kmwstring.js" - ] + } } From eb7bd3adba5f29ae9b5c1728333aabe0a9aace8a Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 22 Apr 2025 11:59:16 +0700 Subject: [PATCH 035/431] chore(web): remove artifacts re: potential method renames --- web/src/engine/common/web-utils/src/kmwstring.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/web/src/engine/common/web-utils/src/kmwstring.ts b/web/src/engine/common/web-utils/src/kmwstring.ts index 3f464a103d..740e6e6ed8 100644 --- a/web/src/engine/common/web-utils/src/kmwstring.ts +++ b/web/src/engine/common/web-utils/src/kmwstring.ts @@ -74,7 +74,7 @@ export function charCodeAt(s: string, index: number) { * @param {number=} fromIndex Optional code point index to start searching from * @return {number} The code point index of the specified search value */ -export function indexOf(s: string, searchValue: string, fromIndex?: number) { // pointIndexOf() +export function indexOf(s: string, searchValue: string, fromIndex?: number) { const str = s; if(!EXTENSION_ENABLED) { @@ -102,7 +102,7 @@ export function indexOf(s: string, searchValue: string, fromIndex?: number) { // * @param {number=} fromIndex Optional code point index to start searching from * @return {number} The code point index of the specified search value */ -export function lastIndexOf(s: string, searchValue: string, fromIndex?: number) { // lastPointIndexOf() +export function lastIndexOf(s: string, searchValue: string, fromIndex?: number) { const str = s; if(!EXTENSION_ENABLED) { @@ -159,7 +159,7 @@ export function length(s: string) { * @return {string} The substring as selected by beginSlice and * endSlice */ -export function slice(s: string, beginSlice: number, endSlice?: number) { // sliceAtPoints() +export function slice(s: string, beginSlice: number, endSlice?: number) { const str = s; if(!EXTENSION_ENABLED) { @@ -185,7 +185,7 @@ export function slice(s: string, beginSlice: number, endSlice?: number) { // sli * @param {number=} len Optional length to extract * @return {string} The substring as selected by start and length */ -export function substr(s: string, start: number, len?: number) { // substrFromPoint() +export function substr(s: string, start: number, len?: number) { const str = s; if(!EXTENSION_ENABLED) { @@ -234,7 +234,7 @@ export function substr(s: string, start: number, len?: number) { // substrFromPo * extract to * @return {string} The substring as selected by indexA and indexB */ -export function substring(s: string, indexA: number, indexB?: number) { // substringBetweenPoints() +export function substring(s: string, indexA: number, indexB?: number) { const str = s; if(!EXTENSION_ENABLED) { @@ -397,7 +397,7 @@ export function codeUnitToCodePoint(s: string, codeUnitIndex: number) { * @param {number} index A code point index in the string * @return {string} The corresponding character */ -export function charAt(s: string, index: number) { // charAtPoint +export function charAt(s: string, index: number) { const str = s; if(!EXTENSION_ENABLED) { From 784b366b6967d500dfe66577c9c45a644efcd7b6 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 24 Apr 2025 18:44:25 +0700 Subject: [PATCH 036/431] chore: update labels for v19 Note: m:predictive-text and m:developer-ide mark a change where we have some modules with fuzzy boundaries and where it may not be easy to automatically determine all changes by pathname. I am not altogether comfortable with these so we may iterate a bit further yet. --- .github/labeler.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index 19c858cc81..a4b0001f57 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -16,16 +16,12 @@ android/app/: android/KMAPro/** android/engine/: android/KMEA/** android/samples/: android/Samples/** -common/: - - common/** - - resources/** +common/: common/** -common/models/templates/: web/src/engine/predictive-text/templates/** -common/models/wordbreakers/: web/src/engine/predictive-text/wordbreakers/** - -common/resources/: resources/** common/web/: common/web/** +m:predictive-text: web/src/engine/predictive-text/** + core/: - core/** @@ -37,7 +33,7 @@ developer/compilers/: - developer/src/kmcmplib/** - developer/src/kmc-*/** -developer/ide/: +m:developer-ide: - developer/src/server/** - developer/src/tike/** @@ -65,6 +61,8 @@ oem/fv/android/: oem/firstvoices/android/** oem/fv/ios/: oem/firstvoices/ios/** oem/fv/windows/: oem/firstvoices/windows/** +resources/: resources/** + web/: web/** # web/bookmarklet/ web/engine/: web/source/** From f11aa37a3c39c2b38278be2687e4cff30f5b3f91 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Thu, 24 Apr 2025 14:01:42 -0400 Subject: [PATCH 037/431] auto: increment master version to 19.0.27 --- HISTORY.md | 7 +++++++ VERSION.md | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 4278264505..2842d7a284 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,12 @@ # Keyman Version History +## 19.0.26 alpha 2025-04-24 + +* chore: merge beta version 18.0.235 history to master (#13735) +* chore(linux): Update debian changelog (#13738) +* chore: comment out CODEOWNERS (#13739) +* chore(linux): add a stub unit test to allow successful coverage reporting (#13740) + ## 19.0.25 alpha 2025-04-22 * chore: merge beta to master (pre-release prep) (#13730) diff --git a/VERSION.md b/VERSION.md index c966bc82e4..6ad8cf45ad 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -19.0.26 \ No newline at end of file +19.0.27 \ No newline at end of file From 45fb8df97d296d1fc07915a38f5837a4fd21d361 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 25 Apr 2025 13:13:48 +0700 Subject: [PATCH 038/431] docs(web): add a Web engine keystroke lifecycle internal doc Fixes: #12732 --- web/docs/internal/keystroke-lifecycle.md | 90 +++++++++++++++++++++++ web/docs/internal/keystroke-processing.md | 2 +- 2 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 web/docs/internal/keystroke-lifecycle.md diff --git a/web/docs/internal/keystroke-lifecycle.md b/web/docs/internal/keystroke-lifecycle.md new file mode 100644 index 0000000000..18582459a8 --- /dev/null +++ b/web/docs/internal/keystroke-lifecycle.md @@ -0,0 +1,90 @@ +# The Keyman Engine for Web keystroke lifecycle + +Depending on the variant of Keyman Engine for Web in play, the engine integrates either with a host app or with the current browser page to interpret and/or intercept keystroke events and produce the desired context manipulations corresponding to them. While some details diverge between `app/browser` and `app/webview` builds, the general structure and pattern for this process holds. + +This document will break down the full keystroke lifecycle into four distinct phases, as follows: + +1. Keystroke pre-processing: event interception + interpretation (pre-processing) +2. Keystroke processing: keystroke main-group rule evaluation +3. Keystroke post-processing: keystroke post-keystroke evaluation + side-effects +4. Keystroke resolution: UI state updating + +## Keystroke pre-processing + +Both the `app/browser` and `app/webview` products are capable of handling simulated keystrokes from the on-screen keyboard as well as hardware keystrokes. For hardware keystrokes, this varies between the two, while the majority of on-screen keyboard keystrokes are handled identically. + +### `app/browser` + +The `app/browser` version of the engine is designed to integrate with the active page of the user's browser, giving it far more direct access to the user's true context in addition to direct access to handling DOM events resulting from hardware keystrokes. Its `HardwareEventKeyboard` class is the primary module for establishing the needed engine event hooks and for interpreting the corresponding DOM events as they arise. + +The bulk of DOM key events trigger keystroke processing on key-down, though modifier keys also receive key-up handling. Key-press events are handled selectively for held backspaces and possibly certain keystrokes for mnemonic keyboards. In all cases, DOM keyboard event objects are preprocessed by the centralized `preprocessKeyboardEvent` method, which converts the DOM event format into Web's `KeyEvent` format, which is the form required by the keyboard rules within compiled JS keyboards. This includes mnemonic keystroke preprocessing and remapping. + +### `app/webview` + +The `app/webview` version of the engine, which is designed to be used while embedded in a platform-specific host app, does not have direct access to standard hardware keystroke events. It's run in a WebView that does not take input - a WebView owned by the host app. It is the responsibility of the host app to handle hardware keystroke events and preprocess them on behalf of the Web engine, then forward them to the Web engine via a carefully-constructed JS call from the host upon the WebView. The `PassthroughKeyboard` class within `app/webview` space provides the method `raiseKeyEvent` as an internal API for this purpose, which converts the mobile-app format for hardware keystroke into the internal `KeyEvent` format. This includes mnemonic keystroke processing and remapping. + +### On-screen keyboard + +For both `app/browser` and `app/webview`, the on-screen keyboard delegates mouse and touch processing to [our internal gesture engine](./gestures/index.md) for most keyboards. (Keyboards that instead display "help pages", such as `sil_euro_latin` on desktop, have specialized handling.) The gesture engine is responsible for interpreting all DOM mouse and touch events for OSKs based on the touch-layout format, which are managed by the `VisualKeyboard` class. The gesture engine will raise events about the gesture lifecycle that, when appropriate, will emit `KeyElement` instances - elements of the constructed OSK corresponding to elements of the OSK directly representing keyboard keys. + +OSK keystroke preprocessing operates a little differently from hardware keystrokes. The `ActiveLayout` object for the current form-factor of the `Keyboard` maintains a "base" `KeyEvent` object, per key, that is lazy-constructed (by the `.baseKeyEvent` property) when first needed - the meat of which is performed by the `ActiveKeyBase.constructBaseKeyEvent` method. This is only evaluated up to once per key per activation of the keyboard. As keystrokes are received during OSK use, `VisualKeyboard.keyEventFromSpec` calls the `Keyboard.constructKeyEvent` method to clone that base `KeyEvent` object for the incoming keystroke and then adjust it to match current modifier and state-key settings. + +### Final steps + +Regardless of the path taken to generate the `KeyEvent` object, _all_ keystroke handling modules of Keyman Engine for Web then raise that `KeyEvent` object via their `keyevent` event as specified by the `KeyEventSourceInterface` type. This event also permits a callback that is used by the final phase of the lifecycle - keystroke resolution - as needed. The main `KeymanEngine` base class listens for these events and forwards them for next phase of the lifecycle - keystroke processing - to the `InputProcessor` module, which passes control to whichever `KeystrokeProcessor` type is active for the keyboard when appropriate. + +## Keystroke processing + +Once the `KeyEvent` object for a keystroke has been passed into the `InputProcessor` module and its `processKeyEvent` method, the keystroke-processing stage has been reached. There are a number of subprocesses that trigger here in sequence - keystrokes don't go _straight_ to evaluation against keyboard rules. + +### Pre-keystroke state management + +First, the engine checks the keyboard associated with the `KeyEvent`, ensuring that it is still in place. (See #7543 for more details.) With that assertion in place, it then performs a check needed to support multitaps - it checks to see if the key event has an associated "base context" and reverts the current context to match it if so. Once those checks are fulfilled, the engine then checks to see if the key event corresponds to a pure layer-switching or modifier/state-key interaction. If those are detected, rule-evaluation is bypassed. + +On a similar note, backspace and spacebar keystrokes receive special handling next in associated with predictive-text functionality. When autocorrect is enabled, the spacebar in particular can be used to accept the current suggestion instead of acting as whitespace; if this is triggered, its `KeyEvent` will be swallowed - no rule processing will occur for the it. The backspace may trigger display of a "reversion" suggestion when applicable but will not be swallowed. + +Once this phase is reached, the engine will then snapshot the current state of the context, then trigger standard rule processing against the original context by calling the current `KeyboardProcessor`'s `processKeystroke` method. Any operations against the context will be persisted and take full effect; DOM input elements (for `app/browser`) are updated during rule evaluation. + +### Determining direct keystroke effects + +The `KeyboardProcessor` classes are responsible for determining the effects that should arise when evaluating the incoming keystroke with the current context state. While the exact mechanisms of this may vary depending upon the type of keyboard, the role is the same - interpreting the keystroke in context. + +### JS keyboards + +Before proceeding further, `KeyboardProcessor` does a context check to see whether or not the context contains anything considered deletable - in the case of a BKSP, we need to ensure that the keystroke passes through to the engine's host cleanly, as it may have special implications for the site or host app. + +With that out of the way, the JS keyboard's main group is then provided with the `KeyEvent` object in order to evaluate the keyboard's rules against the keystroke. This involves constructing a `RuleBehavior` object to track any and all needed side-effects for the keystroke. Such changes are generally not applied to the engine at large at this time in order to prevent unwanted side effects that may arise to and from predictive text operations that follow shortly afterward. + +The `KeyboardInterface` class defines the internal API used by keyboards for their evaluation of keyboard rules. + +Once the keyboard's script returns control to the main engine, the engine then determines the net effects to the context that resulted from applying the keyboard rule and preserves that as a `Transcription`, part of includes a `Transform` that most directly represents the changes to text surrounding the text insertion point. + +For JS keyboards, if no keyboard rule is defined for a non-frame key on the `default` or `shift` layer, there is an additional set of implied "default" keyboard rules that is applied. (A keyboard's source must _explicitly_ specify to not emit anything for such keys if such an outcome is desired.) This occurs regardless of the keystroke's source, as such effects are necessary to incorporate for predictive-text operations. If no such rule exists, the processor will add a note to the `RuleBehavior` object that default processing could not be provided and is thus needed from the engine's host. + +### KMX keyboards & KMX+ (LDML) keyboards + +For KMX and KMX+ keyboards, keystrokes are evaluated by using the embedded WASM version of Keyman Core. + +TODO: add link to specific mechanisms here once available. + +### "Alternate" processing (predictive-text) + +Once the incoming `KeyEvent` has been processed, if predictive-text is active and the `KeyEvent` includes fat-finger oriented data, the engine will then build a set of "alternates" - of "phantom" keystrokes - and process _them_ against headless clones of the original context in the same manner as the original keystroke. + +Once all relevant "alternates" have been evaluated, the engine will then signal the predictive-text engine with all relevant data, including the original, snapshotted state of the context. It does not actually trigger predictive-text at this time; processing of alternates is done before keystroke post-processing in order to avoid cross-effects from JS-keyboard variable store manipulations and similar mechanisms. Fat-finger "alternate" `KeyEvent`s are evaluated as if they occurred under the exact same conditions as the keystroke that was actually detected. + +### Keystroke post-processing + +At this stage, the side-effects desired for the original keystroke, as saved to its `RuleBehavior`, are then applied. Note that this includes changes to the `&layer` store - this update will also trigger an event that updates the state of the OSK to match the desired layer. + +Once that is done, the engine will then save data relevant to the original context state, the `KeyEvent`, and the `RuleBehavior` to the "context cache" supporting multitap context reversion. These effects include layer-change effects that result from JS keyboard rules, which modify the KMN-style keyboard stores `&oldlayer` and `&newlayer` in preparation for `begin PostKeystroke` group processing, which is then executed (and which can trigger another `&layer` & OSK-layer update). + +Once post-keystroke effects have been determined, the engine then signals predictive-text to generate new predictions associated with the keystroke. For `app/browser` variants of the engine, changes to the context will then also trigger the DOM `input` event on the context's element in case third parties wish to leverage such events for their website's pages. + +### Keystroke resolution + +At this stage, `InputProcessor` passes control back to the main `KeymanEngine` module, which then triggers the "on rule finalization" setting if configured. This setting is mostly used for page-integration within `app/browser`, supporting DOM element `change` event emulation whenever focus shifts away from the active context element. + +For OSK keystrokes, an identifier associated with the `KeyEvent`'s context snapshot will be made accessible through the `keyevent` event's callback parameter - a feature utilized by multitap gesture support. + +For hardware-keyboard keystrokes, the `keyevent` callback is used to determine whether or not native handling for the keystroke should be triggered - such as for TAB and ENTER keys, which may have semantic command-like functionality depending on the state of the Web engine's host - be that a web page or an Android/iOS app utilizing the keyboard. ENTER keys are often used to submit forms, and this helps ensure we trigger such functionality when appropriate. \ No newline at end of file diff --git a/web/docs/internal/keystroke-processing.md b/web/docs/internal/keystroke-processing.md index 4406b1dcab..8606d18cba 100644 --- a/web/docs/internal/keystroke-processing.md +++ b/web/docs/internal/keystroke-processing.md @@ -1,6 +1,6 @@ # Keystroke Processing -In addition to handling keystroke events produced from hardware keyboards, Keyman Engine for Web manages an on-screen keyboard that can also be used to generate such events. This document will enumerate the various components of data management and control flow used to handle and process keystrokes. +In addition to handling keystroke events produced from hardware keyboards, Keyman Engine for Web manages an on-screen keyboard that can also be used to generate such events. This document will enumerate the various components of data management and control flow used to handle and process keystrokes. For a more thorough examination of the keystroke lifecycle, see the [keystroke-lifecycle](./keystroke-lifecycle.md) page. ## Input: KeyEvent - the pre-processed keystroke From dcc9638198d2e6707e9ab5f123c27f9344dcebf9 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Fri, 25 Apr 2025 13:16:31 +0700 Subject: [PATCH 039/431] docs(web): minor nit regarding AltGr emulation --- web/docs/internal/keystroke-lifecycle.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/docs/internal/keystroke-lifecycle.md b/web/docs/internal/keystroke-lifecycle.md index 18582459a8..ff3dca47df 100644 --- a/web/docs/internal/keystroke-lifecycle.md +++ b/web/docs/internal/keystroke-lifecycle.md @@ -17,7 +17,7 @@ Both the `app/browser` and `app/webview` products are capable of handling simula The `app/browser` version of the engine is designed to integrate with the active page of the user's browser, giving it far more direct access to the user's true context in addition to direct access to handling DOM events resulting from hardware keystrokes. Its `HardwareEventKeyboard` class is the primary module for establishing the needed engine event hooks and for interpreting the corresponding DOM events as they arise. -The bulk of DOM key events trigger keystroke processing on key-down, though modifier keys also receive key-up handling. Key-press events are handled selectively for held backspaces and possibly certain keystrokes for mnemonic keyboards. In all cases, DOM keyboard event objects are preprocessed by the centralized `preprocessKeyboardEvent` method, which converts the DOM event format into Web's `KeyEvent` format, which is the form required by the keyboard rules within compiled JS keyboards. This includes mnemonic keystroke preprocessing and remapping. +The bulk of DOM key events trigger keystroke processing on key-down, though modifier keys also receive key-up handling. Key-press events are handled selectively for held backspaces and possibly certain keystrokes for mnemonic keyboards. In all cases, DOM keyboard event objects are preprocessed by the centralized `preprocessKeyboardEvent` method, which converts the DOM event format into Web's `KeyEvent` format, which is the form required by the keyboard rules within compiled JS keyboards. This includes mnemonic keystroke preprocessing and remapping in addition to AltGr emulation. ### `app/webview` From 84b3c182829fc9169665676d627fa7147fa5aae4 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 28 Apr 2025 07:45:56 +0700 Subject: [PATCH 040/431] chore: update scopes to match labeler --- resources/scopes/scopes.json | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/resources/scopes/scopes.json b/resources/scopes/scopes.json index d547588ecc..fc723f8b92 100644 --- a/resources/scopes/scopes.json +++ b/resources/scopes/scopes.json @@ -2,44 +2,31 @@ "scopes": { "android": { "app": null, - "browser": null, "engine": null, - "resources": null, "samples": null }, "common": { - "models": { - "types": null, - "templates": null, - "wordbreakers": null - }, "resources": null, "web": null }, "core": null, "developer": { "compilers": null, - "ide": null, - "resources": null, - "tools": null + "ide": null }, "ios": { "app": null, - "browser": null, "engine": null, - "resources": null, "samples": null }, "linux": { "config": null, "engine": null, - "resources": null, "samples": null }, "mac": { "config": null, "engine": null, - "resources": null, "samples": null }, "oem": { @@ -52,14 +39,12 @@ "web": { "bookmarklet": null, "engine": null, - "resources": null, "ui": null, "samples": null }, "windows": { "config": null, "engine": null, - "resources": null, "samples": null } } From b8cef2853875163c4287dbd630277ed4fff01fae Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 28 Apr 2025 09:21:12 +0700 Subject: [PATCH 041/431] chore: additional tweaks to scopes/labels --- .github/labeler.yml | 7 ++----- .github/multi-labeler.yml | 8 +++----- resources/scopes/scopes.json | 5 +++-- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index a4b0001f57..6876d8298e 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -17,11 +17,8 @@ android/engine/: android/KMEA/** android/samples/: android/Samples/** common/: common/** - common/web/: common/web/** -m:predictive-text: web/src/engine/predictive-text/** - core/: - core/** @@ -33,11 +30,10 @@ developer/compilers/: - developer/src/kmcmplib/** - developer/src/kmc-*/** -m:developer-ide: +developer/ide/: - developer/src/server/** - developer/src/tike/** - ios/: ios/** ios/app/: ios/keyman/** ios/engine/: ios/engine/** @@ -68,6 +64,7 @@ web/: web/** web/engine/: web/source/** web/ui/: web/source/kmwui* web/samples/: web/samples/** +web/predictive-text/: web/src/engine/predictive-text/** windows/: windows/** windows/config/: windows/src/desktop/** diff --git a/.github/multi-labeler.yml b/.github/multi-labeler.yml index 5b494a8569..aa12f9b2cb 100644 --- a/.github/multi-labeler.yml +++ b/.github/multi-labeler.yml @@ -57,11 +57,6 @@ labels: matcher: title: '(🍒|:cherries:)' - # long-lived feature branches - - label: 'feature-branch' - matcher: - branch: '^feature-.+' - # # Scopes that we look for in the PR title # @@ -90,6 +85,9 @@ labels: - label: 'oem/' matcher: title: '\(.*oem.*\):' + - label: 'resources/' + matcher: + title: '\(.*resources.*\):' - label: 'web/' matcher: title: '\(.*web.*\):' diff --git a/resources/scopes/scopes.json b/resources/scopes/scopes.json index fc723f8b92..1ceab19967 100644 --- a/resources/scopes/scopes.json +++ b/resources/scopes/scopes.json @@ -6,7 +6,6 @@ "samples": null }, "common": { - "resources": null, "web": null }, "core": null, @@ -36,11 +35,13 @@ "windows": null } }, + "resources": null, "web": { "bookmarklet": null, "engine": null, "ui": null, - "samples": null + "samples": null, + "predictive-text": null }, "windows": { "config": null, From b9f1f51a33f1bb24524c1cc4049e18309ab3270c Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Mon, 28 Apr 2025 10:42:52 +0700 Subject: [PATCH 042/431] chore(android): Use `builder_is_ci_build` rather than `--ci` flag --- android/KMAPro/build.sh | 4 ++-- android/KMEA/build.sh | 8 ++++---- android/Samples/KMSample1/build.sh | 6 +++--- android/Samples/KMSample2/build.sh | 6 +++--- android/Tests/KeyboardHarness/build.sh | 6 +++--- android/build.sh | 2 +- oem/firstvoices/android/build.sh | 6 +++--- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/android/KMAPro/build.sh b/android/KMAPro/build.sh index 1eafe1e667..bfe81bba5b 100755 --- a/android/KMAPro/build.sh +++ b/android/KMAPro/build.sh @@ -7,6 +7,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" ## END STANDARD BUILD SCRIPT INCLUDE . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" +. "$KEYMAN_ROOT/resources/build/build-utils-ci.inc.sh" . "$KEYMAN_ROOT/resources/build/build-help.inc.sh" . "$KEYMAN_ROOT/resources/build/build-download-resources.sh" @@ -28,7 +29,6 @@ builder_describe "Builds Keyman for Android app." \ "build" \ "test Runs lint and unit tests." \ "publish Publishes symbols to Sentry and the APK to the Play Store." \ - "--ci Don't start the Gradle daemon. For CI" \ "--upload-sentry Upload to sentry" # parse before describe_outputs to check debug flags @@ -50,7 +50,7 @@ builder_describe_outputs \ # Parse args -if builder_has_option --ci; then +if builder_is_ci_build; then DAEMON_FLAG=--no-daemon fi diff --git a/android/KMEA/build.sh b/android/KMEA/build.sh index 29cb519cf5..978353f9d0 100755 --- a/android/KMEA/build.sh +++ b/android/KMEA/build.sh @@ -8,6 +8,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" ## END STANDARD BUILD SCRIPT INCLUDE . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" +. "$KEYMAN_ROOT/resources/build/build-utils-ci.inc.sh" # ################################ Main script ################################ @@ -28,8 +29,7 @@ builder_describe "Builds Keyman Engine for Android." \ "configure" \ "build" \ "test Runs lint and unit tests." \ - ":engine Builds Engine" \ - "--ci Don't start the Gradle daemon. For CI" + ":engine Builds Engine" # parse before describe_outputs to check debug flags builder_parse "$@" @@ -51,7 +51,7 @@ builder_describe_outputs \ # Parse args DAEMON_FLAG= -if builder_has_option --ci; then +if builder_is_ci_build; then DAEMON_FLAG=--no-daemon fi @@ -98,7 +98,7 @@ fi if builder_start_action test:engine; then - if builder_has_option --ci; then + if builder_is_ci_test_build; then # Report JUnit test results to CI echo "$JUNIT_RESULTS" fi diff --git a/android/Samples/KMSample1/build.sh b/android/Samples/KMSample1/build.sh index 66a04b8d7c..172ef6c5e7 100755 --- a/android/Samples/KMSample1/build.sh +++ b/android/Samples/KMSample1/build.sh @@ -8,6 +8,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" ## END STANDARD BUILD SCRIPT INCLUDE . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" +. "$KEYMAN_ROOT/resources/build/build-utils-ci.inc.sh" ################################ Main script ################################ @@ -22,8 +23,7 @@ builder_describe "Build KMSample1 app for Android." \ "configure" \ "build" \ "test" \ - ":app KMSample1" \ - "--ci Don't start the Gradle daemon. Use for CI" + ":app KMSample1" # parse before describe_outputs to check debug flags builder_parse "$@" @@ -46,7 +46,7 @@ builder_describe_outputs \ # Parse args -if builder_has_option --ci; then +if builder_is_ci_build; then SAMPLE_FLAGS="$SAMPLE_FLAGS -no-daemon" fi diff --git a/android/Samples/KMSample2/build.sh b/android/Samples/KMSample2/build.sh index 75f985aa1f..2f7a99c764 100755 --- a/android/Samples/KMSample2/build.sh +++ b/android/Samples/KMSample2/build.sh @@ -8,6 +8,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" ## END STANDARD BUILD SCRIPT INCLUDE . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" +. "$KEYMAN_ROOT/resources/build/build-utils-ci.inc.sh" ################################ Main script ################################ @@ -22,8 +23,7 @@ builder_describe "Build KMSample2 app for Android." \ "configure" \ "build" \ "test" \ - ":app KMSample2" \ - "--ci Don't start the Gradle daemon. Use for CI" + ":app KMSample2" # parse before describe_outputs to check debug flags builder_parse "$@" @@ -47,7 +47,7 @@ builder_describe_outputs \ # Parse args -if builder_has_option --ci; then +if builder_is_ci_build; then SAMPLE_FLAGS="$SAMPLE_FLAGS -no-daemon" fi diff --git a/android/Tests/KeyboardHarness/build.sh b/android/Tests/KeyboardHarness/build.sh index 71b2deddf2..65671872f8 100755 --- a/android/Tests/KeyboardHarness/build.sh +++ b/android/Tests/KeyboardHarness/build.sh @@ -8,6 +8,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" ## END STANDARD BUILD SCRIPT INCLUDE . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" +. "$KEYMAN_ROOT/resources/build/build-utils-ci.inc.sh" ################################ Main script ################################ @@ -22,8 +23,7 @@ builder_describe "Build KeyboardHarness test app for Android." \ "configure" \ "build" \ "test" \ - ":app KeyboardHarness" \ - "--ci Don't start the Gradle daemon. Use for CI" + ":app KeyboardHarness" # parse before describe outputs to check debug flags builder_parse "$@" @@ -55,7 +55,7 @@ SHLVL=0 # Parse args # Build flags that apply to all targets -if builder_has_option --ci; then +if builder_is_ci_build; then BUILD_FLAGS="$BUILD_FLAGS -no-daemon" TEST_FLAGS="$TEST_FLAGS -no-daemon" fi diff --git a/android/build.sh b/android/build.sh index bb4fa0637c..05d47e207c 100755 --- a/android/build.sh +++ b/android/build.sh @@ -14,6 +14,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" ## END STANDARD BUILD SCRIPT INCLUDE . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" +. "$KEYMAN_ROOT/resources/build/build-utils-ci.inc.sh" ################################ Main script ################################ @@ -36,7 +37,6 @@ builder_describe \ build \ test \ "publish Publishes symbols to Sentry and the APKs to the Play Store." \ - --ci+ \ --upload-sentry+ \ ":engine=KMEA Keyman Engine for Android" \ ":app=KMAPro Keyman for Android" \ diff --git a/oem/firstvoices/android/build.sh b/oem/firstvoices/android/build.sh index 260f882368..f7cb441f00 100755 --- a/oem/firstvoices/android/build.sh +++ b/oem/firstvoices/android/build.sh @@ -8,6 +8,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" ## END STANDARD BUILD SCRIPT INCLUDE . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" +. "$KEYMAN_ROOT/resources/build/build-utils-ci.inc.sh" . "$KEYMAN_ROOT/resources/build/build-download-resources.sh" # ################################ Main script ################################ @@ -25,8 +26,7 @@ builder_describe "Builds FirstVoices for Android app." \ "configure" \ "build" \ "test Runs lint and tests." \ - "publish Publishes symbols to Sentry and the APK to the Play Store." \ - "--ci Don't start the Gradle daemon. For CI" + "publish Publishes symbols to Sentry and the APK to the Play Store." # parse before describe_outputs to check debug flags builder_parse "$@" @@ -48,7 +48,7 @@ builder_describe_outputs \ # Parse args -if builder_has_option --ci; then +if builder_is_ci_build; then DAEMON_FLAG=--no-daemon fi From a701b97888dedab84b7981dd894c00d16bd19d0f Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 28 Apr 2025 13:05:02 +0700 Subject: [PATCH 043/431] fix(web): add robustness for bad touch-layout fontsize specs Fixes: #13766 Fixes: KEYMAN-WEB-QX It seems that at some point, Developer has been compiling raw "12" values in the touch-layout JSON as a numeric 12 for the fontsize field. This clearly happened for the inuktitut_pirurvik keyboard; we should also inspect urdu_phonetic, which has the same raw "12". This change ensures that Web can handle this case without spewing tons of warning/error messages. It also adds error-logging to help with diagnosis for any related issue in the future. --- web/src/engine/osk/src/lengthStyle.ts | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/web/src/engine/osk/src/lengthStyle.ts b/web/src/engine/osk/src/lengthStyle.ts index b5fa17bb41..b0161d63bb 100644 --- a/web/src/engine/osk/src/lengthStyle.ts +++ b/web/src/engine/osk/src/lengthStyle.ts @@ -12,6 +12,24 @@ export class ParsedLengthStyle implements LengthStyle { public constructor(style: LengthStyle | string) { let parsed: LengthStyle = (typeof style == 'string') ? ParsedLengthStyle.parseLengthStyle(style) : style; + const fallback: LengthStyle = { + val: 1, + absolute: false + }; + + if(typeof style == 'number') { + // Addresses #13766 for compiled layouts with numbers in the touch-layout `fontsize` field + + // Do not actually emit console warning; we handle those as "errors" in the mobile apps, and + // this is more an issue with the compiled keyboard. + parsed = fallback; + } + + if(isNaN(parsed.val)) { + console.error("NaN appeared while trying to calculate style scaling effects"); + parsed = fallback + } + // While Object.assign would be nice (and previously, was used), it will break // on old but still supported versions of Android if their Chrome isn't updated. // Requires mobile Chrome 45+, but API 21 (5.0) launches with an older browser. @@ -38,6 +56,11 @@ export class ParsedLengthStyle implements LengthStyle { } public scaledBy(scalar: number): ParsedLengthStyle { + if(isNaN(scalar)) { + console.error("NaN appeared while trying to calculate style scaling effects"); + return this; + } + return new ParsedLengthStyle({ val: scalar * this.val, absolute: this.absolute From cf85aca1042ed1a763123464410a6e5fda0afb9e Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 28 Apr 2025 11:01:55 +0700 Subject: [PATCH 044/431] fix(ios): restore keyboard-scale settings for older, still-supported iOS devices Fixes: #13769 Fixes: KEYMAN-IOS-JC Fixes: KEYMAN-IOS-JG Fixes: KEYMAN-IOS-JH Fixes: KEYMAN-IOS-JJ Fixes: KEYMAN-IOS-JY Fixes: KEYMAN-IOS-JS --- .../Classes/Keyboard/KeyboardScaleMap.swift | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/ios/engine/KMEI/KeymanEngine/Classes/Keyboard/KeyboardScaleMap.swift b/ios/engine/KMEI/KeymanEngine/Classes/Keyboard/KeyboardScaleMap.swift index d224c2f790..0560224f93 100644 --- a/ios/engine/KMEI/KeymanEngine/Classes/Keyboard/KeyboardScaleMap.swift +++ b/ios/engine/KMEI/KeymanEngine/Classes/Keyboard/KeyboardScaleMap.swift @@ -123,6 +123,20 @@ class KeyboardScaleMap { // Slight difference here on landscape width from the iOS 13 measurement, but that's all. scalings[KeyboardScaleMap.hashKey(for: Device.iPhoneXR)] = Scaling(portrait: KeyboardSize(w: 414, h: 226, b: 45), landscape: KeyboardSize(w: 800, h: 150, b: 38)) + // Old settings we can't update, but that still affect supported devices. + // Kept based on https://iosref.com/ios docs of iOS device max versions. + scalings[KeyboardScaleMap.hashKey(for: Device.iPhone6s)] = Scaling(portrait: KeyboardSize(w: 375, h: 216, b: 44), landscape: KeyboardSize(w: 667, h: 162, b: 38)) // does not use full width in landscape + scalings[KeyboardScaleMap.hashKey(for: Device.iPhone6sPlus)] = Scaling(portrait: KeyboardSize(w: 414, h: 226, b: 45), landscape: KeyboardSize(w: 736, h: 162, b: 38)) // does not use full width in landscape + + scalings[KeyboardScaleMap.hashKey(for: Device.iPhoneSE)] = Scaling(portrait: KeyboardSize(w: 320, h: 216, b: 38), landscape: KeyboardSize(w: 568, h: 162, b: 38)) // landscape banner in 10.3.1: 31, not 37 (portrait) or 38! + scalings[KeyboardScaleMap.hashKey(for: Device.iPhone7)] = Scaling(portrait: KeyboardSize(w: 375, h: 216, b: 44), landscape: KeyboardSize(w: 667, h: 162, b: 38)) // does not use full width in landscape + scalings[KeyboardScaleMap.hashKey(for: Device.iPhone7Plus)] = Scaling(portrait: KeyboardSize(w: 414, h: 226, b: 45), landscape: KeyboardSize(w: 736, h: 162, b: 38)) // does not use full width in landscape + scalings[KeyboardScaleMap.hashKey(for: Device.iPhone8)] = Scaling(portrait: KeyboardSize(w: 375, h: 216, b: 44), landscape: KeyboardSize(w: 667, h: 162, b: 38)) // does not use full width in landscape + scalings[KeyboardScaleMap.hashKey(for: Device.iPhone8Plus)] = Scaling(portrait: KeyboardSize(w: 414, h: 226, b: 45), landscape: KeyboardSize(w: 736, h: 162, b: 38)) // does not use full width in landscape + scalings[KeyboardScaleMap.hashKey(for: Device.iPhoneX)] = Scaling(portrait: KeyboardSize(w: 375, h: 216, b: 45), landscape: KeyboardSize(w: 724, h: 150, b: 38)) // insets: p: 34, l: 21 + + + // ----------- // Note: tablets have an "assistant" bar that iOS also uses for its system @@ -153,6 +167,19 @@ class KeyboardScaleMap { scalings[KeyboardScaleMap.hashKey(for: Device.iPadPro12Inch5)] = Scaling(portrait: KeyboardSize(w: 1024, h: 328-5, b: 55), landscape: KeyboardSize(w: 1366, h: 423-5, b: 55)) scalings[KeyboardScaleMap.hashKey(for: Device.iPadPro12Inch6)] = Scaling(portrait: KeyboardSize(w: 1024, h: 328-5, b: 55), landscape: KeyboardSize(w: 1366, h: 423-5, b: 55)) scalings[KeyboardScaleMap.hashKey(for: Device.iPadPro13M4)] = Scaling(portrait: KeyboardSize(w: 1032, h: 330.5-5, b: 55), landscape: KeyboardSize(w: 1376, h: 426-5, b: 55)) + + // Old settings we can't update, but that still affect supported devices. + // Kept based on https://iosref.com/ios docs of iOS device max versions. + scalings[KeyboardScaleMap.hashKey(for: Device.iPadPro9Inch)] = Scaling(portrait: KeyboardSize(w: 768, h: 265, b: 55), landscape: KeyboardSize(w: 1024, h: 353, b: 55)) // Banner always shows; merges copy ctrls with pred-banner. + scalings[KeyboardScaleMap.hashKey(for: Device.iPadPro10Inch)] = Scaling(portrait: KeyboardSize(w: 834, h: 258, b: 55), landscape: KeyboardSize(w: 1112, h: 343, b: 55)) // Banner always shows; merges copy ctrls with pred-banner. + scalings[KeyboardScaleMap.hashKey(for: Device.iPadPro12Inch)] = Scaling(portrait: KeyboardSize(w: 1024, h: 323, b: 55), landscape: KeyboardSize(w: 1366, h: 416, b: 55)) + scalings[KeyboardScaleMap.hashKey(for: Device.iPadPro12Inch2)] = Scaling(portrait: KeyboardSize(w: 1024, h: 323, b: 55), landscape: KeyboardSize(w: 1366, h: 416, b: 55)) + scalings[KeyboardScaleMap.hashKey(for: Device.iPadMini4)] = Scaling(portrait: KeyboardSize(w: 768, h: 265, b: 55), landscape: KeyboardSize(w: 1024, h: 343, b: 55)) // Banner always shows; merges copy ctrls with pred-banner. + scalings[KeyboardScaleMap.hashKey(for: Device.iPad5)] = Scaling(portrait: KeyboardSize(w: 768, h: 258, b: 55), landscape: KeyboardSize(w: 1024, h: 343, b: 55)) // Banner always shows; merges copy ctrls with pred-banner. + scalings[KeyboardScaleMap.hashKey(for: Device.iPad6)] = Scaling(portrait: KeyboardSize(w: 768, h: 258, b: 55), landscape: KeyboardSize(w: 1024, h: 343, b: 55)) // Banner always shows; merges copy ctrls with pred-banner. + + + // The following definitions are in sorted order of increasing device resolution. // This tends to correlates with the underlying devices' release dates, From ff1fb147684206c331c4d85454645f406948ab85 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Mon, 28 Apr 2025 13:49:01 +0700 Subject: [PATCH 045/431] fix(android/engine): Use build_is_ci_build for test action --- android/KMEA/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/android/KMEA/build.sh b/android/KMEA/build.sh index 978353f9d0..c8c79c13de 100755 --- a/android/KMEA/build.sh +++ b/android/KMEA/build.sh @@ -98,7 +98,7 @@ fi if builder_start_action test:engine; then - if builder_is_ci_test_build; then + if builder_is_ci_build; then # Report JUnit test results to CI echo "$JUNIT_RESULTS" fi From 40ae5ed2f5017148b64ddb1e9a8c46a061f1955a Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Mon, 28 Apr 2025 13:49:27 +0700 Subject: [PATCH 046/431] fix(android): Revert ci build option. To deprecate in 20.0 --- android/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/android/build.sh b/android/build.sh index 05d47e207c..27ba447b4c 100755 --- a/android/build.sh +++ b/android/build.sh @@ -37,6 +37,7 @@ builder_describe \ build \ test \ "publish Publishes symbols to Sentry and the APKs to the Play Store." \ + "--ci+ Deprecated build option. Remove in 20.0" \ --upload-sentry+ \ ":engine=KMEA Keyman Engine for Android" \ ":app=KMAPro Keyman for Android" \ From 809e7bb66e6f7cc36605674839c5f791f132e6c9 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Mon, 28 Apr 2025 16:42:49 +0700 Subject: [PATCH 047/431] chore(common): Update dependencies in crowdin GitHub action --- .github/workflows/crowdin.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/crowdin.yml b/.github/workflows/crowdin.yml index 7568a292cd..d5a25aacb6 100644 --- a/.github/workflows/crowdin.yml +++ b/.github/workflows/crowdin.yml @@ -13,10 +13,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: crowdin action - uses: crowdin/github-action@1.3.2 + uses: crowdin/github-action@2.7.0 with: upload_sources: true From 2134d8a46617f7b27c7595b0108176a9a2f326c5 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Mon, 28 Apr 2025 16:42:46 +0200 Subject: [PATCH 048/431] change(linux): add link to issue --- .github/workflows/deb-packaging.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deb-packaging.yml b/.github/workflows/deb-packaging.yml index 96ad55ca1a..c026acafeb 100644 --- a/.github/workflows/deb-packaging.yml +++ b/.github/workflows/deb-packaging.yml @@ -198,7 +198,7 @@ jobs: # autopkgtest -B *.deb keyman_*.dsc -- qemu autopkgtest-jammy-amd64.img - name: Ignore run: | - echo "Ignored for now - until working solution is in place" + echo "Ignored for now - until working solution is in place (#13777)" deb_signing: name: Sign source and binary packages From 8a37b5f5b0624e639d2d7a1a6e7b52958f9ab112 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Mon, 28 Apr 2025 18:30:54 +0200 Subject: [PATCH 049/431] chore(common): rename `chore` GH label to `maint` During the "M3: Maintenance Sprints" session of the April 2025 planning conference we discussed that we'd rename the `chore` GH label to `maint`. However, this change doesn't remove the existing `chore` label because that is also used for merge commits. --- .github/multi-labeler.yml | 3 +++ resources/scopes/commit-types.json | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/multi-labeler.yml b/.github/multi-labeler.yml index 5b494a8569..cd4c6bc1de 100644 --- a/.github/multi-labeler.yml +++ b/.github/multi-labeler.yml @@ -33,6 +33,9 @@ labels: - label: 'fix' matcher: title: '^fix(\(|:)' + - label: 'maint' + matcher: + title: '^maint(\(|:)' - label: 'refactor' matcher: title: '^refactor(\(|:)' diff --git a/resources/scopes/commit-types.json b/resources/scopes/commit-types.json index ec1d2bff0a..958b577175 100644 --- a/resources/scopes/commit-types.json +++ b/resources/scopes/commit-types.json @@ -6,8 +6,9 @@ "docs", "feat", "fix", + "maint", "refactor", "style", "test" ] -} \ No newline at end of file +} From 7371c2378b1c37e66b11f962a595942f536ffcf6 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Mon, 28 Apr 2025 14:01:18 -0400 Subject: [PATCH 050/431] auto: increment master version to 19.0.28 --- HISTORY.md | 4 ++++ VERSION.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 2842d7a284..7716a47d8a 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,9 @@ # Keyman Version History +## 19.0.27 alpha 2025-04-28 + +* chore: update labels for v19 (#13744) + ## 19.0.26 alpha 2025-04-24 * chore: merge beta version 18.0.235 history to master (#13735) diff --git a/VERSION.md b/VERSION.md index 6ad8cf45ad..b55ce0b2f4 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -19.0.27 \ No newline at end of file +19.0.28 \ No newline at end of file From b3633ec4d0b4136557c9ddc4974c1027eedce4b6 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 29 Apr 2025 10:12:10 +0700 Subject: [PATCH 051/431] feat(web): add additional edge-case unit tests Co-authored-by: Eberhard Beilharz --- .../web-utils/src/tests/stringHandling.js | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/web/src/engine/common/web-utils/src/tests/stringHandling.js b/web/src/engine/common/web-utils/src/tests/stringHandling.js index cda7df6819..54a5ea8657 100644 --- a/web/src/engine/common/web-utils/src/tests/stringHandling.js +++ b/web/src/engine/common/web-utils/src/tests/stringHandling.js @@ -82,6 +82,8 @@ describe('Unicode string handling', () => { assert.equal(str._kmwSubstr(1000), ''); assert.equal(str._kmwSubstr(31), "the lazy dog."); assert.equal(str._kmwSubstr(-4), "dog."); + // Standard string substr can handle this, but _kmwBMPSubstr doesn't handle it correctly! + // assert.equal(str._kmwSubstr(-4, 2), "do"); assert.equal(str._kmwSubstr(31, 31), "the lazy dog."); assert.equal(str._kmwSubstr(31, -5), ''); // The big difference from .slice: length must be non-negative. }); @@ -106,6 +108,7 @@ describe('Unicode string handling', () => { assert.equal(str.kmwCodePointToCodeUnit(2), 2); assert.equal(str.kmwCodePointToCodeUnit(9), 9); assert.equal(str.kmwCodePointToCodeUnit(31), 31); + assert.equal(str.kmwCodePointToCodeUnit(-4), 40); }); it('kmwCodeUnitToCodePoint', () => { @@ -113,6 +116,8 @@ describe('Unicode string handling', () => { assert.equal(str.kmwCodeUnitToCodePoint(2), 2); assert.equal(str.kmwCodeUnitToCodePoint(9), 9); assert.equal(str.kmwCodeUnitToCodePoint(31), 31); + // Wait, why the wild functionality mismatch in contrast to kmwCodePointToCodeUnit? + assert.equal(str.kmwCodeUnitToCodePoint(-2), 2); }); }); @@ -198,6 +203,7 @@ describe('Unicode string handling', () => { assert.equal(str._kmwSubstr(1000), ''); assert.equal(str._kmwSubstr(31), "the lazy dog."); assert.equal(str._kmwSubstr(-4), "dog."); + assert.equal(str._kmwSubstr(-4, 2), "do"); assert.equal(str._kmwSubstr(31, 31), "the lazy dog."); assert.equal(str._kmwSubstr(31, -5), ''); // The big difference from .slice: length must be non-negative. }); @@ -224,6 +230,9 @@ describe('Unicode string handling', () => { assert.equal(str.kmwCodePointToCodeUnit(2), 2); assert.equal(str.kmwCodePointToCodeUnit(9), 9); assert.equal(str.kmwCodePointToCodeUnit(31), 31); + assert.equal(str.kmwCodePointToCodeUnit(-4), 40); + // This certainly doesn't seem right... for now, we're just verifying existing behaviors. + assert.equal(str.kmwCodePointToCodeUnit(1000), null); }); it('kmwCodeUnitToCodePoint', () => { @@ -231,6 +240,8 @@ describe('Unicode string handling', () => { assert.equal(str.kmwCodeUnitToCodePoint(2), 2); assert.equal(str.kmwCodeUnitToCodePoint(9), 9); assert.equal(str.kmwCodeUnitToCodePoint(31), 31); + // Again, odd mismatch in contrast to kmwCodePointToCodeUnit + assert.equal(str.kmwCodeUnitToCodePoint(-2), 2); }); }); @@ -276,8 +287,9 @@ describe('Unicode string handling', () => { assert.equal(banana_nation._kmwIndexOf("na", 0), 4); assert.equal(banana_nation._kmwIndexOf("na", 2), 4); assert.equal(banana_nation._kmwIndexOf("na", 3), 4); - // ERROR: returns 4 (!) - // assert.equal(banana_nation._kmwIndexOf("na", 5), 7); + assert.equal(banana_nation._kmwIndexOf("na", 4), 4); + // ERROR: returns 4, not -1 as it should! + // assert.equal(banana_nation._kmwIndexOf("na", 5), -1); assert.equal(banana_nation._kmwIndexOf("na", 8), -1); assert.equal(banana_nation._kmwIndexOf("na", 1000), -1); @@ -378,6 +390,7 @@ describe('Unicode string handling', () => { assert.equal(str._kmwSubstr(1000), ''); assert.equal(str._kmwSubstr(31), "the l" + a + "zy dog."); assert.equal(str._kmwSubstr(-4), "dog."); + assert.equal(str._kmwSubstr(-9, 4), "l" + a + "zy"); assert.equal(str._kmwSubstr(31, 31), "the l" + a + "zy dog."); assert.equal(str._kmwSubstr(31, -5), ''); // The big difference from .slice: length must be non-negative. }); @@ -422,7 +435,8 @@ describe('Unicode string handling', () => { assert.equal(str.kmwCodePointToCodeUnit(2), 2); assert.equal(str.kmwCodePointToCodeUnit(9), 11); // "The quick" = 9; 2 SMP replacements before then. - assert.equal(str.kmwCodePointToCodeUnit(31), 34); // "the lazy dog" is the remnant; 3 SMP replacements before then. + assert.equal(str.kmwCodePointToCodeUnit(31), 34); // "the lazy dog." is the remnant; 3 SMP replacements before then. + assert.equal(str.kmwCodePointToCodeUnit(-13), 34); // "the lazy dog." is the remnant, 13 codepoints long. }); it('kmwCodeUnitToCodePoint', () => { @@ -441,6 +455,8 @@ describe('Unicode string handling', () => { assert.equal(str.kmwCodeUnitToCodePoint(2), 2); assert.equal(str.kmwCodeUnitToCodePoint(11), 9); // "The quick" = 9; 2 SMP replacements before then. assert.equal(str.kmwCodeUnitToCodePoint(34), 31); // "the lazy dog" is the remnant; 3 SMP replacements before then. + // Clearly needs work in the future; only verifying behavioral stability for now. + assert.equal(str.kmwCodeUnitToCodePoint(-2), 2); }); }); }); \ No newline at end of file From 43eb195ee1a9dc136ab2d60baee18d1153cef5c0 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 29 Apr 2025 10:19:38 +0700 Subject: [PATCH 052/431] fix(web): patch up detected behavior change --- web/src/engine/common/web-utils/src/kmwstring.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/web/src/engine/common/web-utils/src/kmwstring.ts b/web/src/engine/common/web-utils/src/kmwstring.ts index 740e6e6ed8..d901f4adf8 100644 --- a/web/src/engine/common/web-utils/src/kmwstring.ts +++ b/web/src/engine/common/web-utils/src/kmwstring.ts @@ -341,10 +341,6 @@ export function codePointToCodeUnit(s: string, codePointIndex: number) { return null; } - if(!EXTENSION_ENABLED) { - return codePointIndex; - } - const str = s; let codeUnitIndex = 0; @@ -375,10 +371,6 @@ export function codePointToCodeUnit(s: string, codePointIndex: number) { export function codeUnitToCodePoint(s: string, codeUnitIndex: number) { const str = s; - if(!EXTENSION_ENABLED) { - return codeUnitIndex; - } - if(codeUnitIndex === null) { return null; } else if(codeUnitIndex == 0) { From e601d59b598529d45d8a3bbd21279dfc7e2142c5 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 29 Apr 2025 10:46:33 +0700 Subject: [PATCH 053/431] change(web): replace USVString with string in predictive-text engine Fixes: #12046 It was only ever an alias (for `string`!) for its intended use, with actual `USVString` typing being unavailable in the browser. --- common/web/types/src/lexical-model-types.ts | 26 ++++--------- .../types/tests/lexical-model-types.tests.ts | 2 +- .../templates/src/tokenization.ts | 2 +- .../templates/src/trie-model.ts | 9 ++--- .../engine/predictive-text/types/message.d.ts | 3 +- .../worker-main/src/lmlayer.ts | 5 +-- .../src/main/correction/context-tracker.ts | 39 +++++++++---------- .../src/main/correction/distance-modeler.ts | 11 +++--- .../src/main/model-compositor.ts | 3 +- 9 files changed, 41 insertions(+), 59 deletions(-) diff --git a/common/web/types/src/lexical-model-types.ts b/common/web/types/src/lexical-model-types.ts index 7c5cfdf643..e2670d2e21 100644 --- a/common/web/types/src/lexical-model-types.ts +++ b/common/web/types/src/lexical-model-types.ts @@ -5,18 +5,6 @@ /****************************** Lexical Models ******************************/ -/** - * A JavaScript string with the restriction that it must only - * contain Unicode scalar values. - * - * This means that any lone high surrogate must be paired with - * a low surrogate, if it exists. Lone surrogate code units are - * forbidden. - * - * See also: https://developer.mozilla.org/en-US/docs/Web/API/USVString - */ -export type USVString = string; - export type CasingForm = 'lower' | 'initial' | 'upper'; /** @@ -67,7 +55,7 @@ export interface LexiconTraversal { * - If `char` = 'e', the child represents a prefix of 'the'. * - Then `traversal` allows traversing the part of the lexicon prefixed by 'the'. */ - children(): Generator<{char: USVString, traversal: () => LexiconTraversal}>; + children(): Generator<{char: string, traversal: () => LexiconTraversal}>; /** * Allows direct access to the traversal state that results when appending one @@ -81,7 +69,7 @@ export interface LexiconTraversal { * That is, if a model "keys" `è` to `e`, there will be no `è` child. * @param char */ - child(char: USVString): LexiconTraversal | undefined; + child(char: string): LexiconTraversal | undefined; /** * Any entries directly keyed by the currently-represented lookup prefix. Entries and @@ -178,7 +166,7 @@ export interface LexicalModel { * @param text The original input text. * @returns The 'keyed' form of that text. */ - toKey?(text: USVString): USVString; + toKey?(text: string): string; /** * Generates predictive suggestions corresponding to the state of context after the proposed @@ -248,7 +236,7 @@ export interface LexicalModel { * @param context * @deprecated */ - wordbreak?(context: Context): USVString; + wordbreak?(context: Context): string; /** * Lexical models _may_ provide a LexiconTraversal object usable to enhance @@ -279,7 +267,7 @@ export interface Transform { * * Corresponds to `s` in com.keyman.KeyboardInterface.output. */ - insert: USVString; + insert: string; /** * The number of code units to delete to the left of the cursor. @@ -380,7 +368,7 @@ export interface Context { * buffer. If there is nothing to the left of the buffer, this is * an empty string. */ - readonly left: USVString; + readonly left: string; /** * Up to maxRightContextCodeUnits code units of Unicode scalar value @@ -390,7 +378,7 @@ export interface Context { * * This property may be missing entirely. */ - readonly right?: USVString; + readonly right?: string; /** * Whether the insertion point is at the start of the buffer. diff --git a/common/web/types/tests/lexical-model-types.tests.ts b/common/web/types/tests/lexical-model-types.tests.ts index ef8bed28a5..3075582a39 100644 --- a/common/web/types/tests/lexical-model-types.tests.ts +++ b/common/web/types/tests/lexical-model-types.tests.ts @@ -7,7 +7,7 @@ import { KMXPlus, LdmlKeyboardTypes, LexicalModelTypes } from "@keymanapp/common-types"; -export let u: LexicalModelTypes.USVString; +export let u: string; export let l: LexicalModelTypes.Transform; export let s: LexicalModelTypes.Suggestion; export let st: LexicalModelTypes.SuggestionTag; diff --git a/web/src/engine/predictive-text/templates/src/tokenization.ts b/web/src/engine/predictive-text/templates/src/tokenization.ts index a9bd03c5d3..fd8ed28d5c 100644 --- a/web/src/engine/predictive-text/templates/src/tokenization.ts +++ b/web/src/engine/predictive-text/templates/src/tokenization.ts @@ -214,6 +214,6 @@ export function getLastPreCaretToken(wordBreaker: LexicalModelTypes.WordBreaking // While it is currently identical to getLastWord, this may change in the future. // It's best not to write ourselves into a corner on this one, as disambiguating later // would likely be pretty painful. -export function wordbreak(wordBreaker: LexicalModelTypes.WordBreakingFunction, context: LexicalModelTypes.Context): LexicalModelTypes.USVString { +export function wordbreak(wordBreaker: LexicalModelTypes.WordBreakingFunction, context: LexicalModelTypes.Context): string { return getLastPreCaretToken(wordBreaker, context); } diff --git a/web/src/engine/predictive-text/templates/src/trie-model.ts b/web/src/engine/predictive-text/templates/src/trie-model.ts index 2c5e6a194e..f83b304cb3 100644 --- a/web/src/engine/predictive-text/templates/src/trie-model.ts +++ b/web/src/engine/predictive-text/templates/src/trie-model.ts @@ -43,7 +43,6 @@ import LexiconTraversal = LexicalModelTypes.LexiconTraversal; import Suggestion = LexicalModelTypes.Suggestion; import TextWithProbability = LexicalModelTypes.TextWithProbability; import Transform = LexicalModelTypes.Transform; -import USVString = LexicalModelTypes.USVString; import WithOutcome = LexicalModelTypes.WithOutcome; import WordBreakingFunction = LexicalModelTypes.WordBreakingFunction; @@ -112,7 +111,7 @@ class Traversal implements LexiconTraversal { this.totalWeight = totalWeight; } - child(char: USVString): LexiconTraversal | undefined { + child(char: string): LexiconTraversal | undefined { // May result for blank tokens resulting immediately after whitespace. if(char == '') { return this; @@ -131,7 +130,7 @@ class Traversal implements LexiconTraversal { } // Handles one code unit at a time. - private _child(char: USVString): Traversal | undefined { + private _child(char: string): Traversal | undefined { const root = this.root; const totalWeight = this.totalWeight; const nextPrefix = this.prefix + char; @@ -157,7 +156,7 @@ class Traversal implements LexiconTraversal { } } - *children(): Generator<{char: USVString, traversal: () => LexiconTraversal}> { + *children(): Generator<{char: string, traversal: () => LexiconTraversal}> { let root = this.root; // We refer to the field multiple times in this method, and it doesn't change. @@ -302,7 +301,7 @@ export default class TrieModel implements LexicalModel { }; } - toKey(text: USVString): USVString { + toKey(text: string): string { return this._trie.toKey(text); } diff --git a/web/src/engine/predictive-text/types/message.d.ts b/web/src/engine/predictive-text/types/message.d.ts index a0ca7c9cb7..39ff92d6ae 100644 --- a/web/src/engine/predictive-text/types/message.d.ts +++ b/web/src/engine/predictive-text/types/message.d.ts @@ -24,7 +24,6 @@ import { LexicalModelTypes } from '@keymanapp/common-types'; import Configuration = LexicalModelTypes.Configuration; import Reversion = LexicalModelTypes.Reversion; import Suggestion = LexicalModelTypes.Suggestion; -import USVString = LexicalModelTypes.USVString; /** * Tokens are signed 31-bit integers! @@ -87,7 +86,7 @@ interface CurrentWordMessage { * Contains the 'current word' left of the caret given the Context * of its source message - the 'wordbreak' message with matching Token value. */ - word: USVString; + word: string; } /** diff --git a/web/src/engine/predictive-text/worker-main/src/lmlayer.ts b/web/src/engine/predictive-text/worker-main/src/lmlayer.ts index 2f59917d0e..5727dabd94 100644 --- a/web/src/engine/predictive-text/worker-main/src/lmlayer.ts +++ b/web/src/engine/predictive-text/worker-main/src/lmlayer.ts @@ -28,7 +28,6 @@ import Distribution = LexicalModelTypes.Distribution; import Reversion = LexicalModelTypes.Reversion; import Suggestion = LexicalModelTypes.Suggestion; import Transform = LexicalModelTypes.Transform; -import USVString = LexicalModelTypes.USVString; import PromiseStore from "./promise-store.js"; import { OutgoingMessage } from '@keymanapp/lm-message-types'; @@ -62,7 +61,7 @@ export default class LMLayer { /** Call this when the LMLayer has sent us the 'ready' message! */ private _declareLMLayerReady: (conf: Configuration) => void; private _predictPromises: PromiseStore; - private _wordbreakPromises: PromiseStore; + private _wordbreakPromises: PromiseStore; private _acceptPromises: PromiseStore; private _revertPromises: PromiseStore; private _nextToken: number; @@ -152,7 +151,7 @@ export default class LMLayer { }); } - wordbreak(context: Context): Promise { + wordbreak(context: Context): Promise { let token = this._nextToken++; return new Promise((resolve, reject) => { this._wordbreakPromises.make(token, resolve, reject); diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts index 9bbc7f19c2..4c589ec358 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tracker.ts @@ -11,7 +11,6 @@ import Distribution = LexicalModelTypes.Distribution; import LexicalModel = LexicalModelTypes.LexicalModel; import Suggestion = LexicalModelTypes.Suggestion; import Transform = LexicalModelTypes.Transform; -import USVString = LexicalModelTypes.USVString; function textToCharTransforms(text: string, transformId?: number) { let perCharTransforms: Transform[] = []; @@ -40,7 +39,7 @@ export function getEditPathLastMatch(editPath: EditOperation[]) { return editPath.lastIndexOf('match', editLength - 2); } else { return editPath.lastIndexOf('match'); - } + } } export class TrackedContextSuggestion { @@ -62,7 +61,7 @@ export class TrackedContextToken { constructor(instance?: TrackedContextToken) { if(instance) { Object.assign(this, instance); - // We don't alter the values in replacements, but we do wish to prevent aliasing + // We don't alter the values in replacements, but we do wish to prevent aliasing // of the array containing them. this.replacements = instance.replacements.slice(); } @@ -94,7 +93,7 @@ export class TrackedContextToken { * @param tokenText * @param transformId */ - updateWithBackspace(tokenText: USVString, transformId: number) { + updateWithBackspace(tokenText: string, transformId: number) { // It's a backspace transform; time for special handling! // // For now, with 14.0, we simply compress all remaining Transforms for the token into @@ -109,11 +108,11 @@ export class TrackedContextToken { }); this.raw = tokenText; - this.transformDistributions = backspacedTokenContext; + this.transformDistributions = backspacedTokenContext; this.clearReplacements(); } - update(transformDistribution: Distribution, tokenText?: USVString) { + update(transformDistribution: Distribution, tokenText?: string) { // Preserve existing text if new text isn't specified. tokenText = tokenText || (tokenText === '' ? '' : this.raw); @@ -210,7 +209,7 @@ export class TrackedContextState { } toRawTokenization() { - let sequence: USVString[] = []; + let sequence: string[] = []; for(let token of this.tokens) { // Hide any tokens representing wordbreaks. (Thinking ahead to phrase-level possibilities) @@ -329,14 +328,14 @@ interface ContextMatchResult { /** * Indicates the portion of the incoming keystroke data, if any, that applies to * tokens before the last pre-caret token and thus should not be replaced by predictions - * based upon `state`. If the provided context state + the incoming transform do not + * based upon `state`. If the provided context state + the incoming transform do not * adequately match the current context, the match attempt will fail with a `null` result. * * Should generally be non-null if the token before the caret did not previously exist. - * - * The result may be null if it does not match the prior context state or if bookkeeping - * based upon it is problematic - say, if wordbreaking effects shift due to new input, - * causing a mismatch with the prior state's tokenization. + * + * The result may be null if it does not match the prior context state or if bookkeeping + * based upon it is problematic - say, if wordbreaking effects shift due to new input, + * causing a mismatch with the prior state's tokenization. * (Refer to #12494 for an example case.) */ preservationTransform?: Transform; @@ -352,7 +351,7 @@ export class ContextTracker extends CircularArray { transformSequenceDistribution?: Distribution ): ContextMatchResult { // Map the previous tokenized state to an edit-distance friendly version. - let matchContext: USVString[] = matchState.toRawTokenization(); + let matchContext: string[] = matchState.toRawTokenization(); // Inverted order, since 'match' existed before our new context. let mapping = ClassicalDistanceCalculation.computeDistance( @@ -591,7 +590,7 @@ export class ContextTracker extends CircularArray { // multitaps makes it tricky to reliably use in all situations. // It's best to handle `delete` cases directly for this reason. for(let j = i + 1; j < editPath.length; j++) { - // If something _other_ than delete follows a 'delete' on the edit path, + // If something _other_ than delete follows a 'delete' on the edit path, // we probably have a context mismatch. // // It's possible to construct cases where this isn't true, but it's likely not @@ -620,12 +619,12 @@ export class ContextTracker extends CircularArray { priorEdit = editPath[i]; } - return { - state, - baseState: matchState, - preservationTransform, - headTokensRemoved: poppedTokenCount, - tailTokensAdded: pushedTokenCount + return { + state, + baseState: matchState, + preservationTransform, + headTokensRemoved: poppedTokenCount, + tailTokensAdded: pushedTokenCount }; } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts index 555e896c39..316d3bc3e4 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/distance-modeler.ts @@ -9,7 +9,6 @@ import LexicalModel = LexicalModelTypes.LexicalModel; import LexiconTraversal = LexicalModelTypes.LexiconTraversal; import ProbabilityMass = LexicalModelTypes.ProbabilityMass; import Transform = LexicalModelTypes.Transform; -import USVString = LexicalModelTypes.USVString; type RealizedInput = ProbabilityMass[]; // NOT Distribution - they're masses from separate distributions. @@ -48,15 +47,15 @@ export class SearchNode { calculation: ClassicalDistanceCalculation, TraversableToken>; currentTraversal: LexiconTraversal; - toKey: (wordform: USVString) => USVString = str => str; + toKey: (wordform: string) => string = str => str; priorInput: RealizedInput; // Internal lazy-cache for .inputSamplingCost, as it's a bit expensive to re-compute. private _inputCost?: number; - constructor(rootTraversal: LexiconTraversal, toKey?: (arg0: USVString) => USVString); + constructor(rootTraversal: LexiconTraversal, toKey?: (arg0: string) => string); constructor(node: SearchNode); - constructor(rootTraversal: LexiconTraversal | SearchNode, toKey?: (arg0: USVString) => USVString) { + constructor(rootTraversal: LexiconTraversal | SearchNode, toKey?: (arg0: string) => string) { toKey = toKey || (x => x); if(rootTraversal instanceof SearchNode) { @@ -294,11 +293,11 @@ export class SearchResult { return this.resultNode.priorInput; } - get matchSequence(): TraversableToken[] { + get matchSequence(): TraversableToken[] { return this.resultNode.calculation.matchSequence; }; - get matchString(): USVString { + get matchString(): string { return this.resultNode.resultKey; } diff --git a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts index d0b4658b13..7b04c7a0ad 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts @@ -13,7 +13,6 @@ import LexicalModelPunctuation = LexicalModelTypes.LexicalModelPunctuation; import Reversion = LexicalModelTypes.Reversion; import Suggestion = LexicalModelTypes.Suggestion; import Transform = LexicalModelTypes.Transform; -import USVString = LexicalModelTypes.USVString; export class ModelCompositor { private lexicalModel: LexicalModel; @@ -192,7 +191,7 @@ export class ModelCompositor { } // Responsible for applying casing rules to suggestions. - private applySuggestionCasing(suggestion: Suggestion, baseWord: USVString, casingForm: CasingForm) { + private applySuggestionCasing(suggestion: Suggestion, baseWord: string, casingForm: CasingForm) { // Step 1: does the suggestion replace the whole word? If not, we should extend the suggestion to do so. let unchangedLength = baseWord.kmwLength() - suggestion.transform.deleteLeft; From 7b1cfe94ff5c08e5f5aca6cf1efb3c0b222059ed Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 29 Apr 2025 10:52:35 +0700 Subject: [PATCH 054/431] change(web): address engine/dom-utils linting errors --- .../engine/dom-utils/src/arrayFromNodeList.ts | 2 +- .../engine/dom-utils/src/cookieSerializer.ts | 20 +++++++++---------- web/src/engine/dom-utils/src/getAbsolute.ts | 16 +++++++-------- web/src/engine/dom-utils/src/landscapeView.ts | 2 +- web/src/engine/dom-utils/src/stylesheets.ts | 10 +++++----- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/web/src/engine/dom-utils/src/arrayFromNodeList.ts b/web/src/engine/dom-utils/src/arrayFromNodeList.ts index d99e6eddbc..aa6297d7ab 100644 --- a/web/src/engine/dom-utils/src/arrayFromNodeList.ts +++ b/web/src/engine/dom-utils/src/arrayFromNodeList.ts @@ -6,7 +6,7 @@ * @return {Array} */ export function arrayFromNodeList(nl: NodeList|HTMLCollectionOf): HTMLElement[] { - let res: (HTMLElement)[] = []; + const res: (HTMLElement)[] = []; for(let i=0; i < nl.length; i++) { // Typing says we could get Node instances; it's up to use to use this method responsibly. res.push(nl[i] as HTMLElement); diff --git a/web/src/engine/dom-utils/src/cookieSerializer.ts b/web/src/engine/dom-utils/src/cookieSerializer.ts index f8f830bd5c..daa6c59667 100644 --- a/web/src/engine/dom-utils/src/cookieSerializer.ts +++ b/web/src/engine/dom-utils/src/cookieSerializer.ts @@ -24,11 +24,11 @@ export default class CookieSerializer { - let v: Record = {}; + const v: Record = {}; if(typeof(document.cookie) != 'undefined' && document.cookie != '') { - let c = document.cookie.split(/;\s*/); + const c = document.cookie.split(/;\s*/); for(let i = 0; i < c.length; i++) { - let d = c[i].split('='); + const d = c[i].split('='); if(d.length == 2) { v[d[0]] = d[1]; } @@ -45,19 +45,19 @@ export default class CookieSerializer { - let cookie: Record = {}; - let allCookies = this._loadRawCookies(); + const cookie: Record = {}; + const allCookies = this._loadRawCookies(); const encodedCookie = allCookies[cookieName]; if(encodedCookie) { - let rawDecode = decodeURIComponent(encodedCookie).split(';'); + const rawDecode = decodeURIComponent(encodedCookie).split(';'); for(let i=0; i 1) { const [key, value] = record; // key, value @@ -79,12 +79,12 @@ export default class CookieSerializer, encoder: FilteredRecordEncoder) { let serialization=''; - for(let key in cookieValueMap) { + for(const key in cookieValueMap) { serialization += key + '=' + encoder(cookieValueMap[key], key) + ";"; } - let d = new Date(new Date().valueOf() + 1000 * 60 * 60 * 24 * 30).toUTCString(); - let cookieConfig = ' path=/; expires=' + d; //Fri, 31 Dec 2099 23:59:59 GMT;'; + const d = new Date(new Date().valueOf() + 1000 * 60 * 60 * 24 * 30).toUTCString(); + const cookieConfig = ' path=/; expires=' + d; //Fri, 31 Dec 2099 23:59:59 GMT;'; document.cookie = `${cookieName}=${encodeURIComponent(serialization)}; ${cookieConfig}`; } } \ No newline at end of file diff --git a/web/src/engine/dom-utils/src/getAbsolute.ts b/web/src/engine/dom-utils/src/getAbsolute.ts index 27a9b38eab..aa4a4f4c6d 100644 --- a/web/src/engine/dom-utils/src/getAbsolute.ts +++ b/web/src/engine/dom-utils/src/getAbsolute.ts @@ -6,13 +6,13 @@ * Description Returns x-coordinate of Pobj element absolute position with respect to page */ export function getAbsoluteX(Pobj: HTMLElement): number { // I1476 - Handle SELECT overlapping END - var Lobj: HTMLElement + let Lobj: HTMLElement if(!Pobj) { return 0; } - var Lcurleft = Pobj.offsetLeft ? Pobj.offsetLeft : 0; + let Lcurleft = Pobj.offsetLeft ? Pobj.offsetLeft : 0; Lobj = Pobj; // I2404 - Support for IFRAMEs if (Lobj.offsetParent) { @@ -22,7 +22,7 @@ export function getAbsoluteX(Pobj: HTMLElement): number { // I1476 - Handle SELE } // On mobile devices, the OSK uses 'fixed' - this requires some extra offset work to handle. - let Ldoc = Lobj.ownerDocument; + const Ldoc = Lobj.ownerDocument; if(Lobj.style.position == 'fixed' && Ldoc && Ldoc.scrollingElement) { Lcurleft += Ldoc.scrollingElement.scrollLeft; } @@ -30,7 +30,7 @@ export function getAbsoluteX(Pobj: HTMLElement): number { // I1476 - Handle SELE // Correct position if element is within a frame (but not if the controller is in document within that frame) // We used to reference a KMW state variable `this.keyman._MasterDocument`, but it was only ever set to `window.document`. if(Lobj && Lobj.ownerDocument && (Pobj.ownerDocument != window.document)) { - var Ldoc=Lobj.ownerDocument; // I2404 - Support for IFRAMEs + const Ldoc=Lobj.ownerDocument; // I2404 - Support for IFRAMEs if(Ldoc && Ldoc.defaultView && Ldoc.defaultView.frameElement) { return Lcurleft + getAbsoluteX(Ldoc.defaultView.frameElement) - Ldoc.documentElement.scrollLeft; @@ -47,12 +47,12 @@ export function getAbsoluteX(Pobj: HTMLElement): number { // I1476 - Handle SELE * Description Returns y-coordinate of Pobj element absolute position with respect to page */ export function getAbsoluteY(Pobj: HTMLElement): number { - var Lobj: HTMLElement + let Lobj: HTMLElement if(!Pobj) { return 0; } - var Lcurtop = Pobj.offsetTop ? Pobj.offsetTop : 0; + let Lcurtop = Pobj.offsetTop ? Pobj.offsetTop : 0; Lobj = Pobj; // I2404 - Support for IFRAMEs if (Lobj.ownerDocument && Lobj instanceof Lobj.ownerDocument.defaultView.HTMLElement) { @@ -62,7 +62,7 @@ export function getAbsoluteY(Pobj: HTMLElement): number { } // On mobile devices, the OSK uses 'fixed' - this requires some extra offset work to handle. - let Ldoc = Lobj.ownerDocument; + const Ldoc = Lobj.ownerDocument; if(Lobj.style.position == 'fixed' && Ldoc && Ldoc.scrollingElement) { Lcurtop += Ldoc.scrollingElement.scrollTop; } @@ -71,7 +71,7 @@ export function getAbsoluteY(Pobj: HTMLElement): number { // Correct position if element is within a frame (but not if the controller is in document within that frame) // We used to reference a KMW state variable `this.keyman._MasterDocument`, but it was only ever set to `window.document`. if(Lobj && Lobj.ownerDocument && (Pobj.ownerDocument != window.document)) { - var Ldoc=Lobj.ownerDocument; // I2404 - Support for IFRAMEs + const Ldoc=Lobj.ownerDocument; // I2404 - Support for IFRAMEs if(Ldoc && Ldoc.defaultView && Ldoc.defaultView.frameElement) { return Lcurtop + getAbsoluteY(Ldoc.defaultView.frameElement); diff --git a/web/src/engine/dom-utils/src/landscapeView.ts b/web/src/engine/dom-utils/src/landscapeView.ts index d12b347cbc..06873b3da2 100644 --- a/web/src/engine/dom-utils/src/landscapeView.ts +++ b/web/src/engine/dom-utils/src/landscapeView.ts @@ -4,7 +4,7 @@ * @return {boolean} */ export default function landscapeView(): boolean { // new for I3363 (Build 301) - var orientation: number; + let orientation: number; // Assume portrait mode if orientation undefined if(typeof window.orientation != 'undefined') { // Used by iOS Safari diff --git a/web/src/engine/dom-utils/src/stylesheets.ts b/web/src/engine/dom-utils/src/stylesheets.ts index 5318e28f61..eb6bbae2bb 100644 --- a/web/src/engine/dom-utils/src/stylesheets.ts +++ b/web/src/engine/dom-utils/src/stylesheets.ts @@ -20,7 +20,7 @@ export class StylesheetManager { public constructor(linkNode?: Node, doCacheBusting?: boolean) { if(!linkNode) { - let _ElemHead=document.getElementsByTagName('HEAD'); + const _ElemHead=document.getElementsByTagName('HEAD'); if(_ElemHead.length > 0) { linkNode = _ElemHead[0]; } else { @@ -138,7 +138,7 @@ export class StylesheetManager { // Build the font-face definition according to the browser being used // The OSK will similarly remove double-quotes from font-family names and double-quote // the name itself. (#13018, #13022) - var s='@font-face {\nfont-family:"' + let s='@font-face {\nfont-family:"' + fd.family.replace(/\u0022/g, '') + '";\nfont-style:normal;\nfont-weight:normal;\n'; // Build the font source string according to the browser, @@ -191,7 +191,7 @@ export class StylesheetManager { */ const fontFace = new FontFace(fd.family, source); - let loadPromise = fontFace.load(); + const loadPromise = fontFace.load(); const clearPromise = () => { this.fontPromises = this.fontPromises.filter((entry) => entry != loadPromise); } @@ -248,7 +248,7 @@ export class StylesheetManager { } public unlinkAll() { - for(let tuple of this.linkedSheets) { + for(const tuple of this.linkedSheets) { const sheet = tuple.sheet; if(sheet.parentNode) { sheet.parentNode.removeChild(sheet); @@ -268,7 +268,7 @@ export class StylesheetManager { * @return {Object} returns the object reference **/ export function createStyleSheet(styleString: string): HTMLStyleElement { - var _ElemStyle: HTMLStyleElement = document.createElement<'style'>('style'); + const _ElemStyle: HTMLStyleElement = document.createElement<'style'>('style'); _ElemStyle.type = 'text/css'; _ElemStyle.appendChild(document.createTextNode(styleString)); From db6b61e8fca942fb6bd6093cd693b95cd674f4da Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 29 Apr 2025 10:54:37 +0700 Subject: [PATCH 055/431] change(web): address engine/js-processor linter errors --- web/src/engine/js-processor/src/deadkeys.ts | 22 ++-- .../engine/js-processor/src/kbdInterface.ts | 104 +++++++++--------- .../js-processor/src/keyboardProcessor.ts | 40 +++---- web/src/engine/js-processor/src/mock.ts | 10 +- .../engine/js-processor/src/outputTarget.ts | 6 +- .../engine/js-processor/src/ruleBehavior.ts | 14 +-- .../engine/js-processor/src/systemStores.ts | 8 +- 7 files changed, 102 insertions(+), 102 deletions(-) diff --git a/web/src/engine/js-processor/src/deadkeys.ts b/web/src/engine/js-processor/src/deadkeys.ts index 31bfa1100d..53cf0d5995 100644 --- a/web/src/engine/js-processor/src/deadkeys.ts +++ b/web/src/engine/js-processor/src/deadkeys.ts @@ -14,7 +14,7 @@ export class Deadkey { } match(p: number, d: number): boolean { - var result:boolean = (this.p == p && this.d == d); + const result:boolean = (this.p == p && this.d == d); return result; } @@ -32,7 +32,7 @@ export class Deadkey { } clone(): Deadkey { - let dk = new Deadkey(this.p, this.d); + const dk = new Deadkey(this.p, this.d); dk.o = this.o; return dk; @@ -65,8 +65,8 @@ export class DeadkeyTracker { } clone(): DeadkeyTracker { - let dkt = new DeadkeyTracker(); - let dks = this.toSortedArray(); + const dkt = new DeadkeyTracker(); + const dks = this.toSortedArray(); // Make sure to clone the deadkeys themselves - the Deadkey object is mutable. dkt.dks = []; @@ -91,9 +91,9 @@ export class DeadkeyTracker { return false; // I3318 } - var sp=caretPos; + const sp=caretPos; n = sp - n; - for(var i = 0; i < this.dks.length; i++) { + for(let i = 0; i < this.dks.length; i++) { // Don't re-match an already-matched deadkey. It's possible to have two identical // entries, and they should be kept separately. if(this.dks[i].match(n, d) && !this.dks[i].matched) { @@ -114,7 +114,7 @@ export class DeadkeyTracker { } remove(dk: Deadkey) { - var index = this.dks.indexOf(dk); + const index = this.dks.indexOf(dk); this.dks.splice(index, 1); } @@ -123,13 +123,13 @@ export class DeadkeyTracker { } resetMatched() { - for(let dk of this.dks) { + for(const dk of this.dks) { dk.reset(); } } deleteMatched(): void { - for(var Li = 0; Li < this.dks.length; Li++) { + for(let Li = 0; Li < this.dks.length; Li++) { if(this.dks[Li].matched) { this.dks.splice(Li--, 1); // Don't forget to decrement! } @@ -148,7 +148,7 @@ export class DeadkeyTracker { return; } - for(let dk of this.dks) { + for(const dk of this.dks) { if(dk.p > Lstart) { dk.p += Ldelta; } @@ -163,7 +163,7 @@ export class DeadkeyTracker { const otherDks = other.dks; const matchedDks: Deadkey[] = []; - for(let dk of this.dks) { + for(const dk of this.dks) { const match = otherDks.find((otherDk) => dk.equal(otherDk)); if(!match) { return false; diff --git a/web/src/engine/js-processor/src/kbdInterface.ts b/web/src/engine/js-processor/src/kbdInterface.ts index efca7b8a80..d3c2593807 100644 --- a/web/src/engine/js-processor/src/kbdInterface.ts +++ b/web/src/engine/js-processor/src/kbdInterface.ts @@ -157,7 +157,7 @@ class CachedContextEx { } clone(): CachedContextEx { - let r = new CachedContextEx(); + const r = new CachedContextEx(); r._cache = this._cache; return r; } @@ -231,7 +231,7 @@ export default class KeyboardInterface extends KeyboardHarness { registerKeyboard(Pk: any): void { // NOTE: This implementation is web-core specific and is intentionally replaced, whole-sale, // by DOM-aware code. - let keyboard = new Keyboard(Pk); + const keyboard = new Keyboard(Pk); this.loadedKeyboard = keyboard; } @@ -250,12 +250,12 @@ export default class KeyboardInterface extends KeyboardHarness { */ context(n: number, ln: number, outputTarget: OutputTarget): string { - var v = this.cachedContext.get(n, ln); + const v = this.cachedContext.get(n, ln); if(v !== null) { return v; } - var r = this.KC_(n, ln, outputTarget); + const r = this.KC_(n, ln, outputTarget); this.cachedContext.set(n, ln, r); return r; } @@ -274,7 +274,7 @@ export default class KeyboardInterface extends KeyboardHarness { * KC(10,10,Pelem) == "XXXXabcdef" i.e. return as much as possible of the requested string, where X = \uFFFE */ private KC_(n: number, ln: number, outputTarget: OutputTarget): string { - var tempContext = ''; + let tempContext = ''; // If we have a selection, we have an empty context tempContext = outputTarget.isSelectionEmpty() ? outputTarget.getTextBeforeCaret() : ""; @@ -300,7 +300,7 @@ export default class KeyboardInterface extends KeyboardHarness { * KN(4,Pelem) == TRUE */ nul(n: number, outputTarget: OutputTarget): boolean { - var cx=this.context(n+1, 1, outputTarget); + const cx=this.context(n+1, 1, outputTarget); // With #31, the result will be a replacement character if context is empty. return cx === "\uFFFE"; @@ -317,7 +317,7 @@ export default class KeyboardInterface extends KeyboardHarness { * Description Test keyboard context for match */ contextMatch(n: number, outputTarget: OutputTarget, val: string, ln: number): boolean { - var cx=this.context(n, ln, outputTarget); + const cx=this.context(n, ln, outputTarget); if(cx === val) { return true; // I3318 } @@ -334,7 +334,7 @@ export default class KeyboardInterface extends KeyboardHarness { * @return {Array} Context array (of strings and numbers) */ private _BuildExtendedContext(n: number, ln: number, outputTarget: OutputTarget): CachedExEntry { - var cache: CachedExEntry = this.cachedContextEx.get(n, ln); + let cache: CachedExEntry = this.cachedContextEx.get(n, ln); if(cache !== null) { return cache; } else { @@ -343,15 +343,15 @@ export default class KeyboardInterface extends KeyboardHarness { cache = this.cachedContextEx.get(n, n); if(cache === null) { // First, let's make sure we have a cloned, sorted copy of the deadkey array. - let unmatchedDeadkeys = outputTarget.deadkeys().toSortedArray(); // Is reverse-order sorted for us already. + const unmatchedDeadkeys = outputTarget.deadkeys().toSortedArray(); // Is reverse-order sorted for us already. // Time to build from scratch! - var index = 0; + let index = 0; cache = { valContext: [], deadContext: []}; while(cache.valContext.length < n) { // As adapted from `deadkeyMatch`. - var sp = outputTarget.getDeadkeyCaret(); - var deadPos = sp - index; + const sp = outputTarget.getDeadkeyCaret(); + const deadPos = sp - index; if(unmatchedDeadkeys.length > 0 && unmatchedDeadkeys[0].p > deadPos) { // We have deadkeys at the right-hand side of the caret! They don't belong in the context, so pop 'em off. unmatchedDeadkeys.splice(0, 1); @@ -363,7 +363,7 @@ export default class KeyboardInterface extends KeyboardHarness { unmatchedDeadkeys.splice(0, 1); } else { // Take the character. We get "\ufffe" if it doesn't exist. - var kc = this.context(++index, 1, outputTarget); + const kc = this.context(++index, 1, outputTarget); cache.valContext = ([kc] as (string|number)[]).concat(cache.valContext); } } @@ -371,7 +371,7 @@ export default class KeyboardInterface extends KeyboardHarness { } // Now that we have the cache... - var subCache = cache; + const subCache = cache; subCache.valContext = subCache.valContext.slice(0, ln); this.cachedContextEx.set(n, ln, subCache); @@ -392,32 +392,32 @@ export default class KeyboardInterface extends KeyboardHarness { */ fullContextMatch(n: number, outputTarget: OutputTarget, rule: ContextEntry[]): boolean { // Stage one: build the context index map. - var fullContext = this._BuildExtendedContext(n, rule.length, outputTarget); + const fullContext = this._BuildExtendedContext(n, rule.length, outputTarget); this.ruleContextEx = this.cachedContextEx.clone(); - var context = fullContext.valContext; - var deadContext = fullContext.deadContext; + const context = fullContext.valContext; + const deadContext = fullContext.deadContext; - var mismatch = false; + let mismatch = false; // This symbol internally indicates lack of context in a position. (See KC_) const NUL_CONTEXT = "\uFFFE"; - var assertNever = function(x: never): never { + const assertNever = function(x: never): never { // Could be accessed by improperly handwritten calls to `fullContextMatch`. throw new Error("Unexpected object in fullContextMatch specification: " + x); } // Stage two: time to match against the rule specified. - for(var i=0; i < rule.length; i++) { + for(let i=0; i < rule.length; i++) { if(typeof rule[i] == 'string') { - var str = rule[i] as string; + const str = rule[i] as string; if(str !== context[i]) { mismatch = true; break; } } else { // TypeScript needs a cast to this intermediate type to do its discriminated union magic. - var r = rule[i] as ContextNonCharEntry; + const r = rule[i] as ContextNonCharEntry; switch(r.t) { case 'd': // We still need to set a flag here; @@ -428,7 +428,7 @@ export default class KeyboardInterface extends KeyboardHarness { } break; case 'a': - var lookup: KeyboardStoreElement; + let lookup: KeyboardStoreElement; if(typeof context[i] == 'string') { lookup = context[i] as string; @@ -436,7 +436,7 @@ export default class KeyboardInterface extends KeyboardHarness { lookup = {'t': 'd', 'd': context[i] as number}; } - var result = this.any(i, lookup, r.a); + const result = this.any(i, lookup, r.a); if(!r.n) { // If it's a standard 'any'... if(!result) { @@ -454,7 +454,7 @@ export default class KeyboardInterface extends KeyboardHarness { break; case 'i': // The context will never hold a 'beep.' - var ch = this._Index(r.i, r.o) as string | RuleDeadkey; + const ch = this._Index(r.i, r.o) as string | RuleDeadkey; if(ch !== undefined && (typeof(ch) == 'string' ? ch : ch.d) !== context[i]) { mismatch = true; @@ -552,12 +552,12 @@ export default class KeyboardInterface extends KeyboardHarness { * Description Test keystroke with modifiers against rule */ keyMatch(e: KeyEvent, Lruleshift:number, Lrulekey:number): boolean { - var retVal = false; // I3318 - var keyCode = (e.Lcode == 173 ? 189 : e.Lcode); //I3555 (Firefox hyphen issue) + let retVal = false; // I3318 + let keyCode = (e.Lcode == 173 ? 189 : e.Lcode); //I3555 (Firefox hyphen issue) - let bitmask = this.activeKeyboard.modifierBitmask; - var modifierBitmask = bitmask & Codes.modifierBitmasks["ALL"]; - var stateBitmask = bitmask & Codes.stateBitmasks["ALL"]; + const bitmask = this.activeKeyboard.modifierBitmask; + const modifierBitmask = bitmask & Codes.modifierBitmasks["ALL"]; + const stateBitmask = bitmask & Codes.stateBitmasks["ALL"]; const eventModifiers = KeyboardInterface.matchModifiersToRuleChirality(e.Lmodifiers, Lruleshift); @@ -598,7 +598,7 @@ export default class KeyboardInterface extends KeyboardHarness { * Description Get object with extended key event information */ keyInformation(e: KeyEvent): KeyInformation { - var ei = new KeyInformation(); + const ei = new KeyInformation(); ei['vk'] = e.LisVirtualKey; ei['code'] = e.Lcode; ei['modifiers'] = e.Lmodifiers; @@ -633,7 +633,7 @@ export default class KeyboardInterface extends KeyboardHarness { _ExplodeStore(store: KeyboardStore): ComplexKeyboardStore { if(typeof(store) == 'string') { - let cachedStores = this.activeKeyboard.explodedStores; + const cachedStores = this.activeKeyboard.explodedStores; // Is the result cached? if(cachedStores[store]) { @@ -641,8 +641,8 @@ export default class KeyboardInterface extends KeyboardHarness { } // Nope, so let's build its cache. - var result: ComplexKeyboardStore = []; - for(var i=0; i < store._kmwLength(); i++) { + const result: ComplexKeyboardStore = []; + for(let i=0; i < store._kmwLength(); i++) { result.push(store._kmwCharAt(i)); } @@ -669,8 +669,8 @@ export default class KeyboardInterface extends KeyboardHarness { } s = this._ExplodeStore(s); - var Lix = -1; - for(var i=0; i < s.length; i++) { + let Lix = -1; + for(let i=0; i < s.length; i++) { const entry = s[i]; if(typeof(entry) == 'string') { if(s[i] == ch) { @@ -720,12 +720,12 @@ export default class KeyboardInterface extends KeyboardHarness { indexOutput(Pdn: number, Ps: KeyboardStore, Pn: number, outputTarget: OutputTarget): void { this.resetContextCache(); - var assertNever = function(x: never): never { + const assertNever = function(x: never): never { // Could be accessed by improperly handwritten calls to `fullContextMatch`. throw new Error("Unexpected object in fullContextMatch specification: " + x); } - var indexChar = this._Index(Ps, Pn); + const indexChar = this._Index(Ps, Pn); if(indexChar !== "") { if(typeof indexChar == 'string' ) { this.output(Pdn, outputTarget, indexChar); //I3319 @@ -756,15 +756,15 @@ export default class KeyboardInterface extends KeyboardHarness { * Description Keyboard output */ deleteContext(dn: number, outputTarget: OutputTarget): void { - var context: CachedExEntry; + let context: CachedExEntry; // We want to control exactly which deadkeys get removed. if(dn > 0) { context = this._BuildExtendedContext(dn, dn, outputTarget); let nulCount = 0; - for(var i=0; i < context.valContext.length; i++) { - var dk = context.deadContext[i]; + for(let i=0; i < context.valContext.length; i++) { + const dk = context.deadContext[i]; if(dk) { // Remove deadkey in context. @@ -780,7 +780,7 @@ export default class KeyboardInterface extends KeyboardHarness { // Prevent attempts to delete nul sentinels, as they don't exist in the actual context. // (Addresses regression from KMW v 12.0 paired with Developer bug through same version) - let contextLength = context.valContext.length - nulCount; + const contextLength = context.valContext.length - nulCount; if(dn > contextLength) { dn = contextLength; } @@ -874,8 +874,8 @@ export default class KeyboardInterface extends KeyboardHarness { * @return {boolean} True if the test succeeds */ ifStore(systemId: number, strValue: string, outputTarget: OutputTarget): boolean { - var result=true; - let store = this.systemStores[systemId]; + let result=true; + const store = this.systemStores[systemId]; if(store) { result = store.matches(strValue); } @@ -919,7 +919,7 @@ export default class KeyboardInterface extends KeyboardHarness { loadStore(kbdName: string, storeName:string, dfltValue:string): string { this.resetContextCache(); if(this.variableStoreSerializer) { - let cValue = this.variableStoreSerializer.loadStore(kbdName, storeName); + const cValue = this.variableStoreSerializer.loadStore(kbdName, storeName); return cValue[storeName] || dfltValue; } else { return dfltValue; @@ -939,13 +939,13 @@ export default class KeyboardInterface extends KeyboardHarness { */ saveStore(storeName:string, optValue:string): boolean { this.resetContextCache(); - var kbd=this.activeKeyboard; + const kbd=this.activeKeyboard; if(!kbd || typeof kbd.id == 'undefined' || kbd.id == '') { return false; } // And the lookup under that entry looks for the value under the store name, again. - let valueObj: VariableStore = {}; + const valueObj: VariableStore = {}; valueObj[storeName] = optValue; // Null-check in case of invocation during unit-test @@ -1035,7 +1035,7 @@ export default class KeyboardInterface extends KeyboardHarness { this.resetContextCache(); // Capture the initial state of the OutputTarget before any rules are matched. - let preInput = Mock.from(outputTarget, true); + const preInput = Mock.from(outputTarget, true); // Capture the initial state of any variable stores const cachedVariableStores = this.activeKeyboard.variableStores; @@ -1049,7 +1049,7 @@ export default class KeyboardInterface extends KeyboardHarness { // Calls the start-group of the active keyboard. this.activeTargetOutput = outputTarget; - var matched = callee(outputTarget, keystroke); + const matched = callee(outputTarget, keystroke); this.activeTargetOutput = null; // Finalize the rule's results. @@ -1069,7 +1069,7 @@ export default class KeyboardInterface extends KeyboardHarness { this.ruleBehavior.triggerKeyDefault = !matched; // Clear our result-tracking variable to prevent any possible pollution for future processing. - let behavior = this.ruleBehavior; + const behavior = this.ruleBehavior; this.ruleBehavior = null; return behavior; @@ -1098,9 +1098,9 @@ export default class KeyboardInterface extends KeyboardHarness { */ static __publishShorthandAPI() { // Keyboard callbacks - let prototype = this.prototype; + const prototype = this.prototype; - var exportKBCallback = function(miniName: string, longName: keyof KeyboardInterface) { + const exportKBCallback = function(miniName: string, longName: keyof KeyboardInterface) { if(prototype[longName]) { // @ts-ignore prototype[miniName] = prototype[longName]; diff --git a/web/src/engine/js-processor/src/keyboardProcessor.ts b/web/src/engine/js-processor/src/keyboardProcessor.ts index dea537dfdf..9b2f656e30 100644 --- a/web/src/engine/js-processor/src/keyboardProcessor.ts +++ b/web/src/engine/js-processor/src/keyboardProcessor.ts @@ -129,12 +129,12 @@ export default class KeyboardProcessor extends EventEmitter { * @return {string} */ defaultRuleBehavior(Lkc: KeyEvent, outputTarget: OutputTarget, readonly: boolean): RuleBehavior { - let preInput = Mock.from(outputTarget, readonly); - let ruleBehavior = new RuleBehavior(); + const preInput = Mock.from(outputTarget, readonly); + const ruleBehavior = new RuleBehavior(); let matched = false; - var char = ''; - var special: EmulationKeystrokes; + let char = ''; + let special: EmulationKeystrokes; if(Lkc.isSynthetic || outputTarget.isSynthetic) { matched = true; // All the conditions below result in matches until the final else, which restores the expected default // if no match occurs. @@ -166,7 +166,7 @@ export default class KeyboardProcessor extends EventEmitter { } } - let isMnemonic = this.activeKeyboard && this.activeKeyboard.isMnemonic; + const isMnemonic = this.activeKeyboard && this.activeKeyboard.isMnemonic; if(!matched) { if((char = this.defaultRules.forAny(Lkc, isMnemonic, ruleBehavior)) != null) { @@ -192,7 +192,7 @@ export default class KeyboardProcessor extends EventEmitter { return ruleBehavior; } - let transcription = outputTarget.buildTranscriptionFrom(preInput, Lkc, readonly); + const transcription = outputTarget.buildTranscriptionFrom(preInput, Lkc, readonly); ruleBehavior.transcription = transcription; return ruleBehavior; @@ -211,7 +211,7 @@ export default class KeyboardProcessor extends EventEmitter { } processKeystroke(keyEvent: KeyEvent, outputTarget: OutputTarget): RuleBehavior { - var matchBehavior: RuleBehavior; + let matchBehavior: RuleBehavior; // Before keyboard rules apply, check if the left-context is empty. const nothingDeletable = outputTarget.getTextBeforeCaret().kmwLength() == 0 && outputTarget.isSelectionEmpty(); @@ -244,7 +244,7 @@ export default class KeyboardProcessor extends EventEmitter { // Match against the 'default keyboard' - rules to mimic the default string output when typing in a browser. // Many keyboards rely upon these 'implied rules'. - let defaultBehavior = this.defaultRuleBehavior(keyEvent, outputTarget, false); + const defaultBehavior = this.defaultRuleBehavior(keyEvent, outputTarget, false); if(defaultBehavior) { if(!matchBehavior) { matchBehavior = defaultBehavior; @@ -355,9 +355,9 @@ export default class KeyboardProcessor extends EventEmitter { * @return {boolean} return true if keyboard layer changed */ selectLayer(keyEvent: KeyEvent): boolean { - let keyName = keyEvent.kName; - var nextLayer = keyEvent.kNextLayer; - var isChiral = this.activeKeyboard && this.activeKeyboard.isChiral; + const keyName = keyEvent.kName; + let nextLayer = keyEvent.kNextLayer; + const isChiral = this.activeKeyboard && this.activeKeyboard.isChiral; // Layer must be identified by name, not number (27/08/2015) if(typeof nextLayer == 'number') { @@ -438,20 +438,20 @@ export default class KeyboardProcessor extends EventEmitter { * @param {string} id layer id (e.g. ctrlshift) */ updateLayer(keyEvent: KeyEvent, id: string) { - let activeLayer = this.layerId; - var s = activeLayer; + const activeLayer = this.layerId; + let s = activeLayer; // Do not change layer unless needed (27/08/2015) if(id == activeLayer && keyEvent.device.formFactor != DeviceSpec.FormFactor.Desktop) { return; } - var idx=id; - var i; + let idx=id; + let i; if(keyEvent.device.formFactor == DeviceSpec.FormFactor.Desktop) { // Need to test if target layer is a standard layer (based on the plain 'default') - var replacements= ['leftctrl', 'rightctrl', 'ctrl', 'leftalt', 'rightalt', 'alt', 'shift']; + const replacements= ['leftctrl', 'rightctrl', 'ctrl', 'leftalt', 'rightalt', 'alt', 'shift']; for(i=0; i < replacements.length; i++) { // Don't forget to remove the kebab-case hyphens! @@ -471,7 +471,7 @@ export default class KeyboardProcessor extends EventEmitter { // if(idx == '') with accompanying if-else structural shift would be a far better test here. else { // Save our current modifier state. - var modifier=Codes.getModifierState(s); + let modifier=Codes.getModifierState(s); // Strip down to the base modifiable layer. for(i=0; i < replacements.length; i++) { @@ -526,14 +526,14 @@ export default class KeyboardProcessor extends EventEmitter { s = id; } - let layout = this.activeKeyboard.layout(keyEvent.device.formFactor); + const layout = this.activeKeyboard.layout(keyEvent.device.formFactor); if(layout.getLayer(s)) { this.layerId = s; } else { this.layerId = 'default'; } - let baseModifierState = Codes.getModifierState(this.layerId); + const baseModifierState = Codes.getModifierState(this.layerId); this.modStateFlags = baseModifierState | keyEvent.Lstates; } @@ -603,7 +603,7 @@ export default class KeyboardProcessor extends EventEmitter { setNumericLayer(device: DeviceSpec) { if (this.activeKeyboard) { - let layout = this.activeKeyboard.layout(device.formFactor); + const layout = this.activeKeyboard.layout(device.formFactor); if(layout.getLayer('numeric')) { this.layerId = 'numeric'; } diff --git a/web/src/engine/js-processor/src/mock.ts b/web/src/engine/js-processor/src/mock.ts index e44b76e759..dacdbe2e15 100644 --- a/web/src/engine/js-processor/src/mock.ts +++ b/web/src/engine/js-processor/src/mock.ts @@ -13,7 +13,7 @@ export class Mock extends OutputTarget { super(); this.text = text ? text : ""; - var defaultLength = this.text._kmwLength(); + const defaultLength = this.text._kmwLength(); // Ensures that `caretPos == 0` is handled correctly. this.selStart = typeof selStart == "number" ? selStart : defaultLength; @@ -31,7 +31,7 @@ export class Mock extends OutputTarget { if (outputTarget instanceof Mock) { // Avoids the need to run expensive kmwstring.ts / `_kmwLength()` // calculations when deep-copying Mock instances. - let priorMock = outputTarget as Mock; + const priorMock = outputTarget as Mock; clone = new Mock(priorMock.text, priorMock.selStart, priorMock.selEnd); } else { const text = outputTarget.getText(); @@ -42,8 +42,8 @@ export class Mock extends OutputTarget { let selectionEnd: number = 0; if (outputTarget.hasSelection()) { - let beforeText = outputTarget.getTextBeforeCaret(); - let afterText = outputTarget.getTextAfterCaret(); + const beforeText = outputTarget.getTextBeforeCaret(); + const afterText = outputTarget.getTextAfterCaret(); selectionStart = beforeText._kmwLength(); selectionEnd = textLen - afterText._kmwLength(); } @@ -88,7 +88,7 @@ export class Mock extends OutputTarget { this.selForward = end >= start; if (!this.selForward) { - let temp = this.selStart; + const temp = this.selStart; this.selStart = this.selEnd; this.selEnd = temp; } diff --git a/web/src/engine/js-processor/src/outputTarget.ts b/web/src/engine/js-processor/src/outputTarget.ts index 4b6aac6af6..59773d6f35 100644 --- a/web/src/engine/js-processor/src/outputTarget.ts +++ b/web/src/engine/js-processor/src/outputTarget.ts @@ -46,7 +46,7 @@ export class Transcription { private static tokenSeed: number = 0; constructor(keystroke: KeyEvent, transform: TextTransform, preInput: Mock, alternates?: Alternate[]/*, removedDks: Deadkey[], insertedDks: Deadkey[]*/) { - let token = this.token = Transcription.tokenSeed++; + const token = this.token = Transcription.tokenSeed++; this.keystroke = keystroke; this.transform = transform; @@ -94,7 +94,7 @@ export default abstract class OutputTarget implements OutputTargetInterface { } insertDeadkeyBeforeCaret(d: number) { - var dk: Deadkey = new Deadkey(this.getDeadkeyCaret(), d); + const dk: Deadkey = new Deadkey(this.getDeadkeyCaret(), d); this.deadkeys().add(dk); } @@ -146,7 +146,7 @@ export default abstract class OutputTarget implements OutputTargetInterface { } buildTranscriptionFrom(original: OutputTarget, keyEvent: KeyEvent, readonly: boolean, alternates?: Alternate[]): Transcription { - let transform = this.buildTransformFrom(original); + const transform = this.buildTransformFrom(original); // If we ever decide to re-add deadkey tracking, this is the place for it. diff --git a/web/src/engine/js-processor/src/ruleBehavior.ts b/web/src/engine/js-processor/src/ruleBehavior.ts index fd736f0fd8..16e11cbe55 100644 --- a/web/src/engine/js-processor/src/ruleBehavior.ts +++ b/web/src/engine/js-processor/src/ruleBehavior.ts @@ -74,8 +74,8 @@ export default class RuleBehavior { processor.beepHandler(outputTarget); } - for(let storeID in this.setStore) { - let sysStore = processor.keyboardInterface.systemStores[storeID]; + for(const storeID in this.setStore) { + const sysStore = processor.keyboardInterface.systemStores[storeID]; if(sysStore) { try { sysStore.set(this.setStore[storeID]); @@ -92,13 +92,13 @@ export default class RuleBehavior { processor.keyboardInterface.applyVariableStores(this.variableStores); if(processor.keyboardInterface.variableStoreSerializer) { - for(let storeID in this.saveStore) { + for(const storeID in this.saveStore) { processor.keyboardInterface.variableStoreSerializer.saveStore(processor.activeKeyboard.id, storeID, this.saveStore[storeID]); } } if(this.triggersDefaultCommand) { - let keyEvent = this.transcription.keystroke; + const keyEvent = this.transcription.keystroke; processor.defaultRules.applyCommand(keyEvent, outputTarget); } @@ -121,15 +121,15 @@ export default class RuleBehavior { * @param other */ mergeInDefaults(other: RuleBehavior) { - let keystroke = this.transcription.keystroke; - let keyFromOther = other.transcription.keystroke; + const keystroke = this.transcription.keystroke; + const keyFromOther = other.transcription.keystroke; if(keystroke.Lcode != keyFromOther.Lcode || keystroke.Lmodifiers != keyFromOther.Lmodifiers) { throw "RuleBehavior default-merge not supported unless keystrokes are identical!"; } this.triggersDefaultCommand = this.triggersDefaultCommand || other.triggersDefaultCommand; - let mergingMock = Mock.from(this.transcription.preInput, false); + const mergingMock = Mock.from(this.transcription.preInput, false); mergingMock.apply(this.transcription.transform); mergingMock.apply(other.transcription.transform); diff --git a/web/src/engine/js-processor/src/systemStores.ts b/web/src/engine/js-processor/src/systemStores.ts index 3288c9feff..e191e78bd2 100644 --- a/web/src/engine/js-processor/src/systemStores.ts +++ b/web/src/engine/js-processor/src/systemStores.ts @@ -96,11 +96,11 @@ export class PlatformSystemStore extends SystemStore { } matches(value: string) { - var i,constraint,constraints=value.split(' '); - let device = this.kbdInterface.activeDevice; + const constraints=value.split(' '); + const device = this.kbdInterface.activeDevice; - for(i=0; i Date: Tue, 29 Apr 2025 10:55:30 +0700 Subject: [PATCH 056/431] change(web): address engine/element-wrappers linter errors --- .../element-wrappers/src/contentEditable.ts | 46 +++++++-------- .../element-wrappers/src/designIFrame.ts | 58 +++++++++---------- web/src/engine/element-wrappers/src/input.ts | 26 ++++----- .../engine/element-wrappers/src/textarea.ts | 26 ++++----- web/src/engine/element-wrappers/src/utils.ts | 2 +- .../element-wrappers/src/wrapElement.ts | 2 +- 6 files changed, 80 insertions(+), 80 deletions(-) diff --git a/web/src/engine/element-wrappers/src/contentEditable.ts b/web/src/engine/element-wrappers/src/contentEditable.ts index cc532d226b..c998f88049 100644 --- a/web/src/engine/element-wrappers/src/contentEditable.ts +++ b/web/src/engine/element-wrappers/src/contentEditable.ts @@ -49,7 +49,7 @@ export default class ContentEditable extends OutputTarget<{}> { } hasSelection(): boolean { - let Lsel = this.root.ownerDocument.getSelection(); + const Lsel = this.root.ownerDocument.getSelection(); if(this.root != Lsel.anchorNode && !this.root.contains(Lsel.anchorNode)) { return false; @@ -64,7 +64,7 @@ export default class ContentEditable extends OutputTarget<{}> { clearSelection(): void { if(this.hasSelection()) { - let Lsel = this.root.ownerDocument.getSelection(); + const Lsel = this.root.ownerDocument.getSelection(); if(!Lsel.isCollapsed) { Lsel.deleteFromDocument(); // I2134, I2192 @@ -79,15 +79,15 @@ export default class ContentEditable extends OutputTarget<{}> { */ } getCarets(): SelectionRange { - let Lsel = this.root.ownerDocument.getSelection(); + const Lsel = this.root.ownerDocument.getSelection(); let code = Lsel.anchorNode.compareDocumentPosition(Lsel.focusNode); if(Lsel.isCollapsed) { - let caret = new SelectionCaret(Lsel.anchorNode, Lsel.anchorOffset); + const caret = new SelectionCaret(Lsel.anchorNode, Lsel.anchorOffset); return new SelectionRange(caret, caret); } else { - let anchor = new SelectionCaret(Lsel.anchorNode, Lsel.anchorOffset); - let focus = new SelectionCaret(Lsel.focusNode, Lsel.focusOffset); + const anchor = new SelectionCaret(Lsel.anchorNode, Lsel.anchorOffset); + const focus = new SelectionCaret(Lsel.focusNode, Lsel.focusOffset); if(anchor.node == focus.node) { code = (focus.offset - anchor.offset > 0) ? 2 : 4; @@ -111,7 +111,7 @@ export default class ContentEditable extends OutputTarget<{}> { return this.getText(); } - let caret = this.getCarets().start; + const caret = this.getCarets().start; if(caret.node.nodeType != 3) { return ''; // Must be a text node to provide a context. @@ -131,7 +131,7 @@ export default class ContentEditable extends OutputTarget<{}> { return ''; } - let caret = this.getCarets().end; + const caret = this.getCarets().end; if(caret.node.nodeType != 3) { return ''; // Must be a text node to provide a context. @@ -149,7 +149,7 @@ export default class ContentEditable extends OutputTarget<{}> { return; } - let start = this.getCarets().start; + const start = this.getCarets().start; // Bounds-check on the number of chars to delete. if(dn > start.offset) { @@ -161,8 +161,8 @@ export default class ContentEditable extends OutputTarget<{}> { return; // No context to delete characters from. } - let range = this.root.ownerDocument.createRange(); - let dnOffset = start.offset - start.node.nodeValue.substr(0, start.offset)._kmwSubstr(-dn).length; + const range = this.root.ownerDocument.createRange(); + const dnOffset = start.offset - start.node.nodeValue.substr(0, start.offset)._kmwSubstr(-dn).length; range.setStart(start.node, dnOffset); range.setEnd(start.node, start.offset); @@ -178,9 +178,9 @@ export default class ContentEditable extends OutputTarget<{}> { return; } - let start = this.getCarets().start; - let delta = s._kmwLength(); - let Lsel = this.root.ownerDocument.getSelection(); + const start = this.getCarets().start; + const delta = s._kmwLength(); + const Lsel = this.root.ownerDocument.getSelection(); if(delta == 0) { return; @@ -193,17 +193,17 @@ export default class ContentEditable extends OutputTarget<{}> { // able to manage the caret properly. // // TODO: double-check that it was only IE-motivated, re-implement with Selection.extend(). - let finalCaret = this.root.ownerDocument.createRange(); + const finalCaret = this.root.ownerDocument.createRange(); if(start.node.nodeType == 3) { - let textStart = start.node; + const textStart = start.node; textStart.insertData(start.offset, s); finalCaret.setStart(textStart, start.offset + s.length); } else { // Create a new text node - empty control - var n = start.node.ownerDocument.createTextNode(s); + const n = start.node.ownerDocument.createTextNode(s); - let range = this.root.ownerDocument.createRange(); + const range = this.root.ownerDocument.createRange(); range.setStart(start.node, start.offset); range.collapse(true); range.insertNode(n); @@ -242,8 +242,8 @@ export default class ContentEditable extends OutputTarget<{}> { return; } - let caret = this.getCarets().end; - let delta = s._kmwLength(); + const caret = this.getCarets().end; + const delta = s._kmwLength(); if(delta == 0) { return; @@ -253,13 +253,13 @@ export default class ContentEditable extends OutputTarget<{}> { // will be handled after this method. if(caret.node.nodeType == 3) { - let textStart = caret.node; + const textStart = caret.node; textStart.replaceData(caret.offset, textStart.length, s); } else { // Create a new text node - empty control - var n = caret.node.ownerDocument.createTextNode(s); + const n = caret.node.ownerDocument.createTextNode(s); - let range = this.root.ownerDocument.createRange(); + const range = this.root.ownerDocument.createRange(); range.setStart(caret.node, caret.offset); range.collapse(true); range.insertNode(n); diff --git a/web/src/engine/element-wrappers/src/designIFrame.ts b/web/src/engine/element-wrappers/src/designIFrame.ts index 4fdfc5c103..ec711d27ab 100644 --- a/web/src/engine/element-wrappers/src/designIFrame.ts +++ b/web/src/engine/element-wrappers/src/designIFrame.ts @@ -71,8 +71,8 @@ export default class DesignIFrame extends OutputTarget<{}> { } hasSelection(): boolean { - let Lsel = this.doc.getSelection(); - let outerSel = document.getSelection(); + const Lsel = this.doc.getSelection(); + const outerSel = document.getSelection(); // If the outer doc's selection matches, we're active. if(outerSel.anchorNode == Lsel.anchorNode && outerSel.focusNode == Lsel.focusNode) { @@ -86,7 +86,7 @@ export default class DesignIFrame extends OutputTarget<{}> { clearSelection(): void { if(this.hasSelection()) { - let Lsel = this.doc.getSelection(); + const Lsel = this.doc.getSelection(); if(!Lsel.isCollapsed) { Lsel.deleteFromDocument(); // I2134, I2192 @@ -101,15 +101,15 @@ export default class DesignIFrame extends OutputTarget<{}> { */ } getCarets(): SelectionRange { - let Lsel = this.doc.getSelection(); + const Lsel = this.doc.getSelection(); let code = Lsel.anchorNode.compareDocumentPosition(Lsel.focusNode); if(Lsel.isCollapsed) { - let caret = new SelectionCaret(Lsel.anchorNode, Lsel.anchorOffset); + const caret = new SelectionCaret(Lsel.anchorNode, Lsel.anchorOffset); return new SelectionRange(caret, caret); } else { - let anchor = new SelectionCaret(Lsel.anchorNode, Lsel.anchorOffset); - let focus = new SelectionCaret(Lsel.focusNode, Lsel.focusOffset); + const anchor = new SelectionCaret(Lsel.anchorNode, Lsel.anchorOffset); + const focus = new SelectionCaret(Lsel.focusNode, Lsel.focusOffset); if(anchor.node == focus.node) { code = (focus.offset - anchor.offset > 0) ? 2 : 4; @@ -133,7 +133,7 @@ export default class DesignIFrame extends OutputTarget<{}> { return this.getText(); } - let caret = this.getCarets().start; + const caret = this.getCarets().start; if(caret.node.nodeType != 3) { return ''; // Must be a text node to provide a context. @@ -153,7 +153,7 @@ export default class DesignIFrame extends OutputTarget<{}> { return ''; } - let caret = this.getCarets().end; + const caret = this.getCarets().end; if(caret.node.nodeType != 3) { return ''; // Must be a text node to provide a context. @@ -171,7 +171,7 @@ export default class DesignIFrame extends OutputTarget<{}> { return; } - let start = this.getCarets().start; + const start = this.getCarets().start; // Bounds-check on the number of chars to delete. if(dn > start.offset) { @@ -183,8 +183,8 @@ export default class DesignIFrame extends OutputTarget<{}> { return; // No context to delete characters from. } - let range = this.doc.createRange(); - let dnOffset = start.offset - start.node.nodeValue.substr(0, start.offset)._kmwSubstr(-dn).length; + const range = this.doc.createRange(); + const dnOffset = start.offset - start.node.nodeValue.substr(0, start.offset)._kmwSubstr(-dn).length; range.setStart(start.node, dnOffset); range.setEnd(start.node, start.offset); @@ -200,9 +200,9 @@ export default class DesignIFrame extends OutputTarget<{}> { return; } - let start = this.getCarets().start; - let delta = s._kmwLength(); - let Lsel = this.doc.getSelection(); + const start = this.getCarets().start; + const delta = s._kmwLength(); + const Lsel = this.doc.getSelection(); if(delta == 0) { return; @@ -215,17 +215,17 @@ export default class DesignIFrame extends OutputTarget<{}> { // able to manage the caret properly. // // TODO: double-check that it was only IE-motivated, re-implement with Selection.extend(). - let finalCaret = this.root.ownerDocument.createRange(); + const finalCaret = this.root.ownerDocument.createRange(); if(start.node.nodeType == 3) { - let textStart = start.node; + const textStart = start.node; textStart.insertData(start.offset, s); finalCaret.setStart(textStart, start.offset + s.length); } else { // Create a new text node - empty control - var n = this.doc.createTextNode(s); + const n = this.doc.createTextNode(s); - let range = this.doc.createRange(); + const range = this.doc.createRange(); range.setStart(start.node, start.offset); range.collapse(true); range.insertNode(n); @@ -264,8 +264,8 @@ export default class DesignIFrame extends OutputTarget<{}> { return; } - let caret = this.getCarets().end; - let delta = s._kmwLength(); + const caret = this.getCarets().end; + const delta = s._kmwLength(); if(delta == 0) { return; @@ -275,13 +275,13 @@ export default class DesignIFrame extends OutputTarget<{}> { // will be handled after this method. if(caret.node.nodeType == 3) { - let textStart = caret.node; + const textStart = caret.node; textStart.replaceData(caret.offset, textStart.length, s); } else { // Create a new text node - empty control - var n = caret.node.ownerDocument.createTextNode(s); + const n = caret.node.ownerDocument.createTextNode(s); - let range = this.root.ownerDocument.createRange(); + const range = this.root.ownerDocument.createRange(); range.setStart(caret.node, caret.offset); range.collapse(true); range.insertNode(n); @@ -295,7 +295,7 @@ export default class DesignIFrame extends OutputTarget<{}> { */ saveProperties() { // Formerly _CacheCommands. - var _CacheableCommands=[ + const _CacheableCommands=[ new StyleCommand('backcolor',1), new StyleCommand('fontname',1), new StyleCommand('fontsize',1), new StyleCommand('forecolor',1), new StyleCommand('bold',0), new StyleCommand('italic',0), new StyleCommand('strikethrough',0), new StyleCommand('subscript',0), @@ -306,8 +306,8 @@ export default class DesignIFrame extends OutputTarget<{}> { _CacheableCommands.push(new StyleCommand('hilitecolor',1)); } - for(var n=0; n < _CacheableCommands.length; n++) { // I1511 - array prototype extended - let cmd = _CacheableCommands[n]; + for(let n=0; n < _CacheableCommands.length; n++) { // I1511 - array prototype extended + const cmd = _CacheableCommands[n]; //KeymanWeb._Debug('Command:'+_CacheableCommands[n][0]); if(cmd.stateType == 1) { cmd.cache = this.doc.queryCommandValue(cmd.cmd); @@ -329,8 +329,8 @@ export default class DesignIFrame extends OutputTarget<{}> { console.error("No command cache exists to restore!"); } - for(var n=0; n < this.commandCache.length; n++) { // I1511 - array prototype extended - let cmd = this.commandCache[n]; + for(let n=0; n < this.commandCache.length; n++) { // I1511 - array prototype extended + const cmd = this.commandCache[n]; //KeymanWeb._Debug('ResetCacheCommand:'+_CacheableCommands[n][0]+'='+_CacheableCommands[n][2]); if(cmd.stateType == 1) { diff --git a/web/src/engine/element-wrappers/src/input.ts b/web/src/engine/element-wrappers/src/input.ts index e70bb6bdfb..8f93476757 100644 --- a/web/src/engine/element-wrappers/src/input.ts +++ b/web/src/engine/element-wrappers/src/input.ts @@ -106,8 +106,8 @@ export default class Input extends OutputTarget { } setSelection(start: number, end: number, direction: "forward" | "backward" | "none") { - let domStart = this.root.value._kmwCodePointToCodeUnit(start); - let domEnd = this.root.value._kmwCodePointToCodeUnit(end); + const domStart = this.root.value._kmwCodePointToCodeUnit(start); + const domEnd = this.root.value._kmwCodePointToCodeUnit(end); this.root.setSelectionRange(domStart, domEnd, direction); this.processedSelectionStart = start; @@ -124,8 +124,8 @@ export default class Input extends OutputTarget { // We bypass this whenever operating in the embedded format. const element = this.getElement(); - let selectionStart = element.selectionStart; - let selectionEnd = element.selectionEnd; + const selectionStart = element.selectionStart; + const selectionEnd = element.selectionEnd; this._activeForcedScroll = true; @@ -162,16 +162,16 @@ export default class Input extends OutputTarget { setTextBeforeCaret(text: string) { this.getCaret(); - let selectionLength = this.processedSelectionEnd - this.processedSelectionStart; - let direction = this.getSelectionDirection(); - let newCaret = text._kmwLength(); + const selectionLength = this.processedSelectionEnd - this.processedSelectionStart; + const direction = this.getSelectionDirection(); + const newCaret = text._kmwLength(); this.root.value = text + this.getText()._kmwSubstring(this.processedSelectionStart); this.setSelection(newCaret, newCaret + selectionLength, direction); } protected setTextAfterCaret(s: string) { - let direction = this.getSelectionDirection(); + const direction = this.getSelectionDirection(); this.root.value = this.getTextBeforeCaret() + s; this.setSelection(this.processedSelectionStart, this.processedSelectionEnd, direction); @@ -188,8 +188,8 @@ export default class Input extends OutputTarget { deleteCharsBeforeCaret(dn: number) { if(dn > 0) { - let curText = this.getTextBeforeCaret(); - let caret = this.processedSelectionStart; + const curText = this.getTextBeforeCaret(); + const caret = this.processedSelectionStart; if(dn > caret) { dn = caret; @@ -206,9 +206,9 @@ export default class Input extends OutputTarget { return; } - let caret = this.getCaret(); - let front = this.getTextBeforeCaret(); - let back = this.getText()._kmwSubstring(this.processedSelectionStart); + const caret = this.getCaret(); + const front = this.getTextBeforeCaret(); + const back = this.getText()._kmwSubstring(this.processedSelectionStart); this.adjustDeadkeys(s._kmwLength()); this.root.value = front + s + back; diff --git a/web/src/engine/element-wrappers/src/textarea.ts b/web/src/engine/element-wrappers/src/textarea.ts index 04a8ea3ae8..e72cfb0681 100644 --- a/web/src/engine/element-wrappers/src/textarea.ts +++ b/web/src/engine/element-wrappers/src/textarea.ts @@ -81,8 +81,8 @@ export default class TextArea extends OutputTarget<{}> { } setSelection(start: number, end: number, direction: "forward" | "backward" | "none") { - let domStart = this.root.value._kmwCodePointToCodeUnit(start); - let domEnd = this.root.value._kmwCodePointToCodeUnit(end); + const domStart = this.root.value._kmwCodePointToCodeUnit(start); + const domEnd = this.root.value._kmwCodePointToCodeUnit(end); this.root.setSelectionRange(domStart, domEnd, direction); this.processedSelectionStart = start; @@ -99,8 +99,8 @@ export default class TextArea extends OutputTarget<{}> { // We bypass this whenever operating in the embedded format. const element = this.getElement(); - let selectionStart = element.selectionStart; - let selectionEnd = element.selectionEnd; + const selectionStart = element.selectionStart; + const selectionEnd = element.selectionEnd; this._activeForcedScroll = true; @@ -132,16 +132,16 @@ export default class TextArea extends OutputTarget<{}> { setTextBeforeCaret(text: string) { this.getCaret(); - let selectionLength = this.processedSelectionEnd - this.processedSelectionStart; - let direction = this.getSelectionDirection(); - let newCaret = text._kmwLength(); + const selectionLength = this.processedSelectionEnd - this.processedSelectionStart; + const direction = this.getSelectionDirection(); + const newCaret = text._kmwLength(); this.root.value = text + this.getText()._kmwSubstring(this.processedSelectionStart); this.setSelection(newCaret, newCaret + selectionLength, direction); } protected setTextAfterCaret(s: string) { - let direction = this.getSelectionDirection(); + const direction = this.getSelectionDirection(); this.root.value = this.getTextBeforeCaret() + s; this.setSelection(this.processedSelectionStart, this.processedSelectionEnd, direction); @@ -163,8 +163,8 @@ export default class TextArea extends OutputTarget<{}> { deleteCharsBeforeCaret(dn: number) { if(dn > 0) { - let curText = this.getTextBeforeCaret(); - let caret = this.processedSelectionStart; + const curText = this.getTextBeforeCaret(); + const caret = this.processedSelectionStart; if(dn > caret) { dn = caret; @@ -181,9 +181,9 @@ export default class TextArea extends OutputTarget<{}> { return; } - let caret = this.getCaret(); - let front = this.getTextBeforeCaret(); - let back = this.getText()._kmwSubstring(this.processedSelectionStart); + const caret = this.getCaret(); + const front = this.getTextBeforeCaret(); + const back = this.getText()._kmwSubstring(this.processedSelectionStart); this.adjustDeadkeys(s._kmwLength()); this.root.value = front + s + back; diff --git a/web/src/engine/element-wrappers/src/utils.ts b/web/src/engine/element-wrappers/src/utils.ts index c5fe24c63e..739f11a982 100644 --- a/web/src/engine/element-wrappers/src/utils.ts +++ b/web/src/engine/element-wrappers/src/utils.ts @@ -10,7 +10,7 @@ * @return {boolean} */ export function nestedInstanceOf(Pelem: EventTarget, className: string): boolean { - var scopedClass; + let scopedClass; if(!Pelem) { // If we're bothering to check something's type, null references don't match diff --git a/web/src/engine/element-wrappers/src/wrapElement.ts b/web/src/engine/element-wrappers/src/wrapElement.ts index 3a3e4d89e5..91d482e16e 100644 --- a/web/src/engine/element-wrappers/src/wrapElement.ts +++ b/web/src/engine/element-wrappers/src/wrapElement.ts @@ -13,7 +13,7 @@ export default function wrapElement(e: HTMLElement): OutputTarget { } else if(nestedInstanceOf(e, "HTMLTextAreaElement")) { return new TextArea( e); } else if(nestedInstanceOf(e, "HTMLIFrameElement")) { - let iframe = e; + const iframe = e; if(iframe.contentWindow && iframe.contentWindow.document && iframe.contentWindow.document.designMode == "on") { return new DesignIFrame(iframe); From f1ca263da28c9d3571d99c808afd54b270d19801 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 29 Apr 2025 10:57:28 +0700 Subject: [PATCH 057/431] change(web): address engine/events linter errors --- web/src/engine/events/src/domEventTracker.ts | 8 ++++---- web/src/engine/events/src/legacyEventEmitter.ts | 7 ++++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/web/src/engine/events/src/domEventTracker.ts b/web/src/engine/events/src/domEventTracker.ts index 9b6c7d1e25..5ddf0f1e04 100644 --- a/web/src/engine/events/src/domEventTracker.ts +++ b/web/src/engine/events/src/domEventTracker.ts @@ -61,7 +61,7 @@ export class DomEventTracker { Pelem.addEventListener(Peventname, Phandler, PuseCapture ? true : false); // Since we're attaching to the DOM, these events should be tracked for detachment during shutdown. - var event = new DomEventTracking(Pelem, Peventname, Phandler, PuseCapture); + const event = new DomEventTracking(Pelem, Peventname, Phandler, PuseCapture); this.domEvents.push(event); } @@ -96,8 +96,8 @@ export class DomEventTracker { Pelem.removeEventListener(Peventname, Phandler, PuseCapture); // Since we're detaching, we should drop the tracking data from the old event. - var event = new DomEventTracking(Pelem, Peventname, Phandler, PuseCapture); - for(var i = 0; i < this.domEvents.length; i++) { + const event = new DomEventTracking(Pelem, Peventname, Phandler, PuseCapture); + for(let i = 0; i < this.domEvents.length; i++) { if(this.domEvents[i].equals(event)) { this.domEvents.splice(i, 1); break; @@ -109,7 +109,7 @@ export class DomEventTracker { // Remove all events linking to elements of the original, unaltered page. // This should sever any still-existing page ties to this instance of KMW, // allowing browser GC to do its thing. - for(let event of this.domEvents) { + for(const event of this.domEvents) { // @ts-ignore // since it's simpler this way and doesn't earn us much to re-check types. this.detachDOMEvent(event.Pelem, event.Peventname, event.Phandler, event.PuseCapture); } diff --git a/web/src/engine/events/src/legacyEventEmitter.ts b/web/src/engine/events/src/legacyEventEmitter.ts index 05b02052ee..4ea893509b 100644 --- a/web/src/engine/events/src/legacyEventEmitter.ts +++ b/web/src/engine/events/src/legacyEventEmitter.ts @@ -107,7 +107,7 @@ export class LegacyEventEmitter { this.events[event] = []; } - for(var i=0; i { this.currentEvents.push(event); - for(var i=0; i, result=false; + for(let i=0; i; + let result=false; try { result=func(params as any); } catch(strExcept) { From 6ba37bc61bba9083412cbd096f2f5ed24b01c9b5 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 29 Apr 2025 10:57:52 +0700 Subject: [PATCH 058/431] change(web): address engine/interfaces linter errors --- .../interfaces/src/prediction/predictionContext.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/web/src/engine/interfaces/src/prediction/predictionContext.ts b/web/src/engine/interfaces/src/prediction/predictionContext.ts index ec632a5058..902566eed2 100644 --- a/web/src/engine/interfaces/src/prediction/predictionContext.ts +++ b/web/src/engine/interfaces/src/prediction/predictionContext.ts @@ -82,7 +82,7 @@ export default class PredictionContext extends EventEmitter { if(validSuggestionState()) { - let suggestions = await langProcessor.applyReversion(reversion, this.currentTarget); + const suggestions = await langProcessor.applyReversion(reversion, this.currentTarget); // We want to avoid altering flags that indicate our post-reversion state. this.swallowPrediction = true; this.updateSuggestions(new ReadySuggestions(suggestions, reversion.id ? -reversion.id : undefined)); @@ -110,7 +110,7 @@ export default class PredictionContext extends EventEmitter`; else, `null`. */ public accept(suggestion: Suggestion): Promise | Promise { - let _this = this; + const _this = this; // Selecting a suggestion or a reversion should both clear selection // and clear the reversion-displaying state of the banner. @@ -311,7 +311,7 @@ export default class PredictionContext extends EventEmitter { - let suggestions = prediction.suggestions; + const suggestions = prediction.suggestions; this._currentSuggestions = suggestions; this.selected = null; @@ -319,7 +319,7 @@ export default class PredictionContext extends EventEmitter