From e2360a9ad0ee5f7e42860fb3f33c1278b9a6b980 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 7 Feb 2024 10:12:32 +0700 Subject: [PATCH 01/17] fix(web): outputTarget.apply() now clears selection; adds unit tests --- .../src/text/outputTarget.ts | 4 +++ .../tests/node/transcriptions.js | 29 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/common/web/keyboard-processor/src/text/outputTarget.ts b/common/web/keyboard-processor/src/text/outputTarget.ts index 97a4792847..306dc2170d 100644 --- a/common/web/keyboard-processor/src/text/outputTarget.ts +++ b/common/web/keyboard-processor/src/text/outputTarget.ts @@ -223,6 +223,10 @@ export default abstract class OutputTarget { } apply(transform: Transform) { + // Selected text should disappear on any text edit; application of a transform + // certainly qualifies. + this.clearSelection(); + if(transform.deleteRight) { this.setTextAfterCaret(this.getTextAfterCaret()._kmwSubstr(transform.deleteRight)); } diff --git a/common/web/keyboard-processor/tests/node/transcriptions.js b/common/web/keyboard-processor/tests/node/transcriptions.js index 073e0dc6b3..3d8519ec12 100644 --- a/common/web/keyboard-processor/tests/node/transcriptions.js +++ b/common/web/keyboard-processor/tests/node/transcriptions.js @@ -297,6 +297,35 @@ but not himself.`; // Sheev Palpatine, in the Star Wars prequels. String.kmwEnableSupplementaryPlane(false); } }); + + it.only('from targets with existing selection', () => { + // | | + const target = new Mock("testing testing one two three"); + target.setSelection(8, 20) + const original = Mock.from(target); + target.clearSelection(); + + const transform = target.buildTransformFrom(original); + assert.deepEqual(transform, { + insert: '', + deleteLeft: 0, + deleteRight: 0 + }); + }); + + it.only('to targets with existing selection', () => { + // | | + const target = new Mock("testing testing one two three"); + target.setSelection(8, 20) + const transform = { + insert: '', + deleteLeft: 0, + deleteRight: 0 + }; + + target.apply(transform); + assert.equal(target.getText(), 'testing two three'); + }); }); /*describe("Operations with deadkeys", function() { From 5fe64b0cd42e38e1e6315c50bb7bf80d4cf8f24f Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 7 Feb 2024 12:08:19 +0700 Subject: [PATCH 02/17] fix(web): transcription-construction from contexts with selections --- common/web/keyboard-processor/src/index.ts | 1 + .../src/text/outputTarget.ts | 90 ++---------- .../src/text/stringDivergence.ts | 68 +++++++++ .../tests/node/transcriptions.js | 129 ++++++++++++++++-- web/src/app/browser/src/keymanEngine.ts | 3 - web/src/engine/main/src/keymanEngine.ts | 3 + 6 files changed, 206 insertions(+), 88 deletions(-) create mode 100644 common/web/keyboard-processor/src/text/stringDivergence.ts diff --git a/common/web/keyboard-processor/src/index.ts b/common/web/keyboard-processor/src/index.ts index ad7aa49523..f800691cfa 100644 --- a/common/web/keyboard-processor/src/index.ts +++ b/common/web/keyboard-processor/src/index.ts @@ -39,6 +39,7 @@ export { default as KeyMapping } from "./text/keyMapping.js"; export { default as OutputTarget } from "./text/outputTarget.js"; export * from "./text/outputTarget.js"; export { default as RuleBehavior } from "./text/ruleBehavior.js"; +export * from "./text/stringDivergence.js"; export * from "./text/systemStores.js"; export * from "@keymanapp/web-utils"; diff --git a/common/web/keyboard-processor/src/text/outputTarget.ts b/common/web/keyboard-processor/src/text/outputTarget.ts index 306dc2170d..1ef67b1f7b 100644 --- a/common/web/keyboard-processor/src/text/outputTarget.ts +++ b/common/web/keyboard-processor/src/text/outputTarget.ts @@ -1,6 +1,7 @@ /// import { extendString } from "@keymanapp/web-utils"; +import { searchStringDivergence } from "./stringDivergence.js"; extendString(); @@ -120,85 +121,24 @@ export default abstract class OutputTarget { * @param from An output target (preferably a Mock) representing the prior state of the input/output system. */ buildTransformFrom(original: OutputTarget): Transform { - let to = this.getText(); - let from = original.getText(); + const toLeft = this.getTextBeforeCaret(); + const fromLeft = original.getTextBeforeCaret(); - let fromCaret = original.getDeadkeyCaret(); - let toCaret = this.getDeadkeyCaret(); + const leftDivergenceIndex = searchStringDivergence(fromLeft, toLeft, false)[0]; + const deletedLeft = fromLeft.substring(leftDivergenceIndex)._kmwLength(); + // No need for our specialized variant here. + const insertedText = toLeft.substring(leftDivergenceIndex); - // Step 1: Determine the number of left-deletions. - let maxSMPLeftMatch = fromCaret < toCaret ? fromCaret : toCaret; + const toRight = this.getTextAfterCaret(); + const fromRight = original.getTextAfterCaret(); + const rightDivergence1 = searchStringDivergence(fromRight, toRight, true)[0]; - // We need the corresponding non-SMP caret location in order to binary-search efficiently. - // (Examining code units is much more computationally efficient.) - let maxLeftMatch = to._kmwCodePointToCodeUnit(maxSMPLeftMatch); + // Right insertions aren't supported, but right deletions will matter in some scenarios. + // In particular, once we allow right-deletion for pred-text suggestions applied with the + // caret mid-word.. + const deletedRight = fromRight.substring(0, rightDivergence1 + 1)._kmwLength(); - // 1.1: use a non-SMP-aware binary search to determine the divergence point. - let start = 0; - let end = maxLeftMatch; // the index AFTER the last possible matching char. - - // This search is O(maxLeftMatch). 1/2 + 1/4 + 1/8 + ... converges to = 1. - while(start < end) { - let mid = Math.floor((end+start+1) / 2); // round up (compare more) - let fromLeft = from.substr(start, mid-start); - let toLeft = to.substr(start, mid-start); - - if(fromLeft == toLeft) { - start = mid; - } else { - end = mid - 1; - } - } - - // At the loop's end: `end` now holds the non-SMP-aware divergence point. - // The 'caret' is after the last matching code unit. - - // 1.2: detect a possible surrogate-pair split scenario, correcting for it - // (by moving the split before the high-surrogate) if detected. - - // If the split location is precisely on either end of the context, we can't - // have split a surrogate pair. - if(end > 0 && end < maxLeftMatch) { - let potentialHigh = from.charCodeAt(end-1); - let potentialFromLow = from.charCodeAt(end); - let potentialToLow = to.charCodeAt(end); - - // if potentialHigh is a possible high surrogate... - if(potentialHigh >= 0xD800 && potentialHigh <= 0xDBFF) { - // and at least one potential 'low' is a possible low surrogate... - let flag = potentialFromLow >= 0xDC00 && potentialFromLow <= 0xDFFF; - flag = flag || (potentialToLow >= 0XDC00 && potentialToLow <= 0xDFFF); - - // Correct the split location, moving it 'before' the high surrogate. - if(flag) { - end = end - 1; - } - } - } - - // 1.3: take substring from start to the split point; determine SMP-aware length. - // This yields the SMP-aware divergence index, which gives the number of left-deletes. - let newCaret = from._kmwCodeUnitToCodePoint(end); - let deletedLeft = fromCaret - newCaret; - - // Step 2: Determine the other properties. - // Since the 'after' OutputTarget's caret indicates the end of any inserted text, we - // can easily calculate the rest. - let insertedLength = toCaret - newCaret; - let delta = to._kmwSubstr(newCaret, insertedLength); - - let undeletedRight = to._kmwLength() - toCaret; - let originalRight = from._kmwLength() - fromCaret; - let deletedRight = originalRight - undeletedRight; - - // May occur when reverting a suggestion that had been applied mid-word. - if(deletedRight < 0) { - // Restores deleteRight characters. - delta = delta + to._kmwSubstr(toCaret, -deletedRight); - deletedRight = 0; - } - - return new TextTransform(delta, deletedLeft, deletedRight); + return new TextTransform(insertedText, deletedLeft, deletedRight); } buildTranscriptionFrom(original: OutputTarget, keyEvent: KeyEvent, readonly: boolean, alternates?: Alternate[]): Transcription { diff --git a/common/web/keyboard-processor/src/text/stringDivergence.ts b/common/web/keyboard-processor/src/text/stringDivergence.ts new file mode 100644 index 0000000000..33213f7fdb --- /dev/null +++ b/common/web/keyboard-processor/src/text/stringDivergence.ts @@ -0,0 +1,68 @@ +/** + * Returns the index for the code point divergence point in code unit coordinates. + * @param str1 + * @param str2 + * @param commonRight If false or undefined, asserts a common prefix to the strings. If true, asserts a common suffix. + * @returns The code unit indices within each string for the start of the code point not common to both. + */ +export function searchStringDivergence(str1: string, str2: string, commonRight?: boolean): [number, number] { + let maxInterval = Math.min(str1.length, str2.length) - 1; + const commonLeft = !commonRight; + + let index: number; + let end: number; + + /** + * Index shift per loop iteration. + */ + let inc: number; + /** + * Difference in index for comparison between strings. + * Mostly matters when assuming a common right-hand side. + */ + let offset: number; + + if(commonLeft) { + index = 0; + end = maxInterval; + inc = 1; + offset = 0; + } else { + index = str1.length - 1; + end = index - maxInterval; + inc = -1; + offset = str2.length - str1.length; + } + + for(; commonLeft ? index <= end: index >= end; index += inc) { + if(str1.charAt(index) != str2.charAt(index + offset)) { + break; + } + } + + // `index` corresponds to the first char that is different _in the direction indicated by inc_. + + // if commonLeft, high surrogate; if commonRight, low surrogate. + const commonPotentialSurrogate = str1.charCodeAt(index - inc); + // Opposite surrogate type from the previous variable. + const divergentChar1 = str1.charCodeAt(index); + const divergentChar2 = str2.charCodeAt(index + offset); + + const isHigh = (charCode: number) => charCode >= 0xD800 && charCode <= 0xDBFF; + const isLow = (charCode: number) => charCode >= 0xDC00 && charCode <= 0xDFFF; + const commonChecker = commonLeft ? isHigh : isLow; + const divergentChecker = commonLeft ? isLow : isHigh; + + // If the last common char qualifies as a direction-appropriate SMP surrogate... + if(commonChecker(commonPotentialSurrogate)) { + // And one of the two divergent chars is a qualifying match - a surrogate + // of the opposite type... + if(divergentChecker(divergentChar1) || divergentChecker(divergentChar2)) { + // Our current index would split a surrogate pair; decrement the index to + // preserve the pair. + return [index - inc, index - inc + offset]; + } + } + + return [index, index + offset]; +} \ No newline at end of file diff --git a/common/web/keyboard-processor/tests/node/transcriptions.js b/common/web/keyboard-processor/tests/node/transcriptions.js index 3d8519ec12..5e02cc724e 100644 --- a/common/web/keyboard-processor/tests/node/transcriptions.js +++ b/common/web/keyboard-processor/tests/node/transcriptions.js @@ -1,20 +1,129 @@ import { assert } from 'chai'; -import { Mock } from '@keymanapp/keyboard-processor'; +import { Mock, searchStringDivergence } from '@keymanapp/keyboard-processor'; import { extendString } from '@keymanapp/web-utils'; extendString(); // Ensure KMW's string-extension functionality is available. String.kmwEnableSupplementaryPlane(false); +const toSupplementaryPairString = function(code){ + var H = Math.floor((code - 0x10000) / 0x400) + 0xD800; + var L = (code - 0x10000) % 0x400 + 0xDC00; + + return String.fromCharCode(H, L); +} +// A unicode-coding like alias for use in constructing SMP strings. +const u = toSupplementaryPairString; + +/** + * Returns the "Mathematical Sans-Serif Small" SMP encoding for + * a passed-in lowercase char between 'a' and 'z', inclusive. + * @param {*} char + * @returns + */ +const ss = (char) => { + const charCodeOffset = char.charCodeAt(0) - 'a'.charCodeAt(0); + return u(0x1d5ba + charCodeOffset); +} + +describe("String divergence calculations", function() { + describe("Common prefix", () => { + it("BMP text", () => { + const result1 = searchStringDivergence("apple", "applause", false); + assert.deepEqual(result1, [4, 4]); + + const result2 = searchStringDivergence("applesauce", "applause", false); + assert.deepEqual(result2, [4, 4]); + + const result3 = searchStringDivergence("applesauce", "applesauce", false); + assert.deepEqual(result3, [10, 10]); + }); + + it("SMP text", () => { + const smp_ify = (str) => str.split('').map(ss).join(''); + + const result1 = searchStringDivergence( + smp_ify('apple'), + smp_ify('applause'), + false + ); + + // 2 per SMP char; is in code-unit... units. + // Will avoid splitting code points, though. + assert.deepEqual(result1, [8, 8]); + + const result2 = searchStringDivergence( + smp_ify('applesauce'), + smp_ify('applause'), + false + ); + + assert.deepEqual(result2, [8, 8]); + + const result3 = searchStringDivergence( + smp_ify('applesauce'), + smp_ify('applesauce'), + false + ); + + assert.deepEqual(result3, [20, 20]); + }); + }); + + describe("Common suffix", () => { + it("BMP text", () => { + // att|endance + // transc|endance + const result1 = searchStringDivergence("attendance", "transcendance", true); + assert.deepEqual(result1, [2, 5]); + + // transcend|ance + // happenst|ance + const result2 = searchStringDivergence("transcendance", "happenstance", true); + assert.deepEqual(result2, [8, 7]); + + // And if the two are equal... + const result3 = searchStringDivergence("post-caret text", "post-caret text", true); + assert.deepEqual(result3, [-1, -1]); + }); + + it("SMP text", () => { + const smp_ify = (str) => str.split('').map(ss).join(''); + + // att|endance + // trans|endance + const result1 = searchStringDivergence( + smp_ify("attendance"), + smp_ify("transcendance"), + true + ); + + // 2 per SMP char; is in code-unit... units. + // Will avoid splitting code points; is odd b/c we get the index of the LAST char of the pair. + assert.deepEqual(result1, [5, 11]); + + // transcend|ance + // happenst|ance + const result2 = searchStringDivergence( + smp_ify("transcendance"), + smp_ify("happenstance"), + true + ); + assert.deepEqual(result2, [17, 15]); + + // And if the two are equal... + const result3 = searchStringDivergence( + smp_ify("post-caret text"), + smp_ify("post-caret text"), + true + ); + assert.deepEqual(result3, [-1, -1]); + }); + }) +}); + describe("Transcriptions and Transforms", function() { - var toSupplementaryPairString = function(code){ - var H = Math.floor((code - 0x10000) / 0x400) + 0xD800; - var L = (code - 0x10000) % 0x400 + 0xDC00; - - return String.fromCharCode(H, L); - } - // Built in-line via function. Looks functionally equivalent to "apple", but with SMP characters. let u = toSupplementaryPairString; let smpApple = u(0x1d5ba)+u(0x1d5c9)+u(0x1d5c9)+u(0x1d5c5)+u(0x1d5be); @@ -298,7 +407,7 @@ but not himself.`; // Sheev Palpatine, in the Star Wars prequels. } }); - it.only('from targets with existing selection', () => { + it('from targets with existing selection', () => { // | | const target = new Mock("testing testing one two three"); target.setSelection(8, 20) @@ -313,7 +422,7 @@ but not himself.`; // Sheev Palpatine, in the Star Wars prequels. }); }); - it.only('to targets with existing selection', () => { + it('to targets with existing selection', () => { // | | const target = new Mock("testing testing one two three"); target.setSelection(8, 20) diff --git a/web/src/app/browser/src/keymanEngine.ts b/web/src/app/browser/src/keymanEngine.ts index 5e11fa4809..6b95cb49a0 100644 --- a/web/src/app/browser/src/keymanEngine.ts +++ b/web/src/app/browser/src/keymanEngine.ts @@ -205,9 +205,6 @@ export default class KeymanEngine extends KeymanEngineBase Date: Wed, 7 Feb 2024 12:22:56 +0700 Subject: [PATCH 03/17] fix(web): restoreTo() tweaks --- common/web/keyboard-processor/src/text/outputTarget.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/common/web/keyboard-processor/src/text/outputTarget.ts b/common/web/keyboard-processor/src/text/outputTarget.ts index 1ef67b1f7b..2929a6c1fa 100644 --- a/common/web/keyboard-processor/src/text/outputTarget.ts +++ b/common/web/keyboard-processor/src/text/outputTarget.ts @@ -154,6 +154,13 @@ export default abstract class OutputTarget { * @param original An `OutputTarget` (usually a `Mock`). */ restoreTo(original: OutputTarget) { + this.clearSelection(); + // We currently do not restore selected text; the mechanism isn't supported at present for + // all output target types - especially in regard to re-selecting the text if restored. + // + // I believe this would mostly matter if/when reverting predictions based upon selected text. + // That pattern isn't well-supported yet, though. + // this.setTextBeforeCaret(original.getTextBeforeCaret()); this.setTextAfterCaret(original.getTextAfterCaret()); From f7d7e071f1965a13074080fa3c34c8f886b694d3 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 7 Feb 2024 15:13:15 +0700 Subject: [PATCH 04/17] fix(web): logic for divergence at start position --- .../src/text/stringDivergence.ts | 43 ++++++++++--------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/common/web/keyboard-processor/src/text/stringDivergence.ts b/common/web/keyboard-processor/src/text/stringDivergence.ts index 33213f7fdb..c2adcbe08f 100644 --- a/common/web/keyboard-processor/src/text/stringDivergence.ts +++ b/common/web/keyboard-processor/src/text/stringDivergence.ts @@ -9,6 +9,7 @@ export function searchStringDivergence(str1: string, str2: string, commonRight?: let maxInterval = Math.min(str1.length, str2.length) - 1; const commonLeft = !commonRight; + let start: number; let index: number; let end: number; @@ -23,12 +24,12 @@ export function searchStringDivergence(str1: string, str2: string, commonRight?: let offset: number; if(commonLeft) { - index = 0; + start = index = 0; end = maxInterval; inc = 1; offset = 0; } else { - index = str1.length - 1; + start = index = str1.length - 1; end = index - maxInterval; inc = -1; offset = str2.length - str1.length; @@ -41,26 +42,28 @@ export function searchStringDivergence(str1: string, str2: string, commonRight?: } // `index` corresponds to the first char that is different _in the direction indicated by inc_. + // If it's the start position, it can't split a (completed) surrogate pair. + if(index != start) { + // if commonLeft, high surrogate; if commonRight, low surrogate. + const commonPotentialSurrogate = str1.charCodeAt(index - inc); + // Opposite surrogate type from the previous variable. + const divergentChar1 = str1.charCodeAt(index); + const divergentChar2 = str2.charCodeAt(index + offset); - // if commonLeft, high surrogate; if commonRight, low surrogate. - const commonPotentialSurrogate = str1.charCodeAt(index - inc); - // Opposite surrogate type from the previous variable. - const divergentChar1 = str1.charCodeAt(index); - const divergentChar2 = str2.charCodeAt(index + offset); + const isHigh = (charCode: number) => charCode >= 0xD800 && charCode <= 0xDBFF; + const isLow = (charCode: number) => charCode >= 0xDC00 && charCode <= 0xDFFF; + const commonChecker = commonLeft ? isHigh : isLow; + const divergentChecker = commonLeft ? isLow : isHigh; - const isHigh = (charCode: number) => charCode >= 0xD800 && charCode <= 0xDBFF; - const isLow = (charCode: number) => charCode >= 0xDC00 && charCode <= 0xDFFF; - const commonChecker = commonLeft ? isHigh : isLow; - const divergentChecker = commonLeft ? isLow : isHigh; - - // If the last common char qualifies as a direction-appropriate SMP surrogate... - if(commonChecker(commonPotentialSurrogate)) { - // And one of the two divergent chars is a qualifying match - a surrogate - // of the opposite type... - if(divergentChecker(divergentChar1) || divergentChecker(divergentChar2)) { - // Our current index would split a surrogate pair; decrement the index to - // preserve the pair. - return [index - inc, index - inc + offset]; + // If the last common char qualifies as a direction-appropriate SMP surrogate... + if(commonChecker(commonPotentialSurrogate)) { + // And one of the two divergent chars is a qualifying match - a surrogate + // of the opposite type... + if(divergentChecker(divergentChar1) || divergentChecker(divergentChar2)) { + // Our current index would split a surrogate pair; decrement the index to + // preserve the pair. + return [index - inc, index - inc + offset]; + } } } From 3a1303fb6fc8351a5cc5c6acc93a4539c5c8b782 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 7 Feb 2024 15:19:35 +0700 Subject: [PATCH 05/17] change(web): PR change nomenclature --- .../keyboard-processor/src/text/outputTarget.ts | 4 ++-- .../src/text/stringDivergence.ts | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/common/web/keyboard-processor/src/text/outputTarget.ts b/common/web/keyboard-processor/src/text/outputTarget.ts index 2929a6c1fa..1a82bca32c 100644 --- a/common/web/keyboard-processor/src/text/outputTarget.ts +++ b/common/web/keyboard-processor/src/text/outputTarget.ts @@ -131,12 +131,12 @@ export default abstract class OutputTarget { const toRight = this.getTextAfterCaret(); const fromRight = original.getTextAfterCaret(); - const rightDivergence1 = searchStringDivergence(fromRight, toRight, true)[0]; + const rightDivergenceIndex = searchStringDivergence(fromRight, toRight, true)[0]; // Right insertions aren't supported, but right deletions will matter in some scenarios. // In particular, once we allow right-deletion for pred-text suggestions applied with the // caret mid-word.. - const deletedRight = fromRight.substring(0, rightDivergence1 + 1)._kmwLength(); + const deletedRight = fromRight.substring(0, rightDivergenceIndex + 1)._kmwLength(); return new TextTransform(insertedText, deletedLeft, deletedRight); } diff --git a/common/web/keyboard-processor/src/text/stringDivergence.ts b/common/web/keyboard-processor/src/text/stringDivergence.ts index c2adcbe08f..37bbaa57ce 100644 --- a/common/web/keyboard-processor/src/text/stringDivergence.ts +++ b/common/web/keyboard-processor/src/text/stringDivergence.ts @@ -2,12 +2,12 @@ * Returns the index for the code point divergence point in code unit coordinates. * @param str1 * @param str2 - * @param commonRight If false or undefined, asserts a common prefix to the strings. If true, asserts a common suffix. + * @param commonSuffix If false, asserts a common prefix to the strings. If true, asserts a common suffix. * @returns The code unit indices within each string for the start of the code point not common to both. */ -export function searchStringDivergence(str1: string, str2: string, commonRight?: boolean): [number, number] { +export function searchStringDivergence(str1: string, str2: string, commonSuffix: boolean): [number, number] { let maxInterval = Math.min(str1.length, str2.length) - 1; - const commonLeft = !commonRight; + const commonPrefix = !commonSuffix; let start: number; let index: number; @@ -23,7 +23,7 @@ export function searchStringDivergence(str1: string, str2: string, commonRight?: */ let offset: number; - if(commonLeft) { + if(commonPrefix) { start = index = 0; end = maxInterval; inc = 1; @@ -35,7 +35,7 @@ export function searchStringDivergence(str1: string, str2: string, commonRight?: offset = str2.length - str1.length; } - for(; commonLeft ? index <= end: index >= end; index += inc) { + for(; commonPrefix ? index <= end: index >= end; index += inc) { if(str1.charAt(index) != str2.charAt(index + offset)) { break; } @@ -52,8 +52,8 @@ export function searchStringDivergence(str1: string, str2: string, commonRight?: const isHigh = (charCode: number) => charCode >= 0xD800 && charCode <= 0xDBFF; const isLow = (charCode: number) => charCode >= 0xDC00 && charCode <= 0xDFFF; - const commonChecker = commonLeft ? isHigh : isLow; - const divergentChecker = commonLeft ? isLow : isHigh; + const commonChecker = commonPrefix ? isHigh : isLow; + const divergentChecker = commonPrefix ? isLow : isHigh; // If the last common char qualifies as a direction-appropriate SMP surrogate... if(commonChecker(commonPotentialSurrogate)) { From ce468303fb220b39627672fd0c7158006b77e966 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 7 Feb 2024 15:26:47 +0700 Subject: [PATCH 06/17] docs(web): a bit of documentation --- common/web/keyboard-processor/src/text/stringDivergence.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/common/web/keyboard-processor/src/text/stringDivergence.ts b/common/web/keyboard-processor/src/text/stringDivergence.ts index 37bbaa57ce..e0a3283352 100644 --- a/common/web/keyboard-processor/src/text/stringDivergence.ts +++ b/common/web/keyboard-processor/src/text/stringDivergence.ts @@ -35,12 +35,15 @@ export function searchStringDivergence(str1: string, str2: string, commonSuffix: offset = str2.length - str1.length; } + // Step 1: Find the index for the first code unit different between the strings. for(; commonPrefix ? index <= end: index >= end; index += inc) { if(str1.charAt(index) != str2.charAt(index + offset)) { break; } } + // Step 2: Ensure that we're not splitting a surrogate pair. + // `index` corresponds to the first char that is different _in the direction indicated by inc_. // If it's the start position, it can't split a (completed) surrogate pair. if(index != start) { From 7d4e3c70d065395bc32c5e8ea5d834a456eddb78 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 7 Feb 2024 15:27:14 +0700 Subject: [PATCH 07/17] change(web): replaces function for unit tests (just one instance) --- .../web/keyboard-processor/tests/node/transcriptions.js | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/common/web/keyboard-processor/tests/node/transcriptions.js b/common/web/keyboard-processor/tests/node/transcriptions.js index 5e02cc724e..60df8b2ddb 100644 --- a/common/web/keyboard-processor/tests/node/transcriptions.js +++ b/common/web/keyboard-processor/tests/node/transcriptions.js @@ -7,14 +7,8 @@ extendString(); // Ensure KMW's string-extension functionality is available. String.kmwEnableSupplementaryPlane(false); -const toSupplementaryPairString = function(code){ - var H = Math.floor((code - 0x10000) / 0x400) + 0xD800; - var L = (code - 0x10000) % 0x400 + 0xDC00; - - return String.fromCharCode(H, L); -} // A unicode-coding like alias for use in constructing SMP strings. -const u = toSupplementaryPairString; +const u = String.fromCodePoint; /** * Returns the "Mathematical Sans-Serif Small" SMP encoding for From a903bb91e7d7a47c98bf4aaa80e1754c671e0dfa Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 7 Feb 2024 15:36:21 +0700 Subject: [PATCH 08/17] chore(web): some more cleanup + docs --- .../src/text/stringDivergence.ts | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/common/web/keyboard-processor/src/text/stringDivergence.ts b/common/web/keyboard-processor/src/text/stringDivergence.ts index e0a3283352..cd665d9cb7 100644 --- a/common/web/keyboard-processor/src/text/stringDivergence.ts +++ b/common/web/keyboard-processor/src/text/stringDivergence.ts @@ -6,11 +6,21 @@ * @returns The code unit indices within each string for the start of the code point not common to both. */ export function searchStringDivergence(str1: string, str2: string, commonSuffix: boolean): [number, number] { - let maxInterval = Math.min(str1.length, str2.length) - 1; - const commonPrefix = !commonSuffix; + const maxInterval = Math.min(str1.length, str2.length) - 1; + /** + * The first valid index within the string. + */ let start: number; + + /** + * The current index within the string under consideration as the divergence point. + */ let index: number; + + /** + * The last valid index within the string to consider as the divergence point. + */ let end: number; /** @@ -23,20 +33,20 @@ export function searchStringDivergence(str1: string, str2: string, commonSuffix: */ let offset: number; - if(commonPrefix) { - start = index = 0; - end = maxInterval; - inc = 1; - offset = 0; - } else { + if(commonSuffix) { start = index = str1.length - 1; end = index - maxInterval; inc = -1; offset = str2.length - str1.length; + } else { + start = index = 0; + end = maxInterval; + inc = 1; + offset = 0; } // Step 1: Find the index for the first code unit different between the strings. - for(; commonPrefix ? index <= end: index >= end; index += inc) { + for(; commonSuffix ? index >= end: index <= end; index += inc) { if(str1.charAt(index) != str2.charAt(index + offset)) { break; } @@ -55,8 +65,8 @@ export function searchStringDivergence(str1: string, str2: string, commonSuffix: const isHigh = (charCode: number) => charCode >= 0xD800 && charCode <= 0xDBFF; const isLow = (charCode: number) => charCode >= 0xDC00 && charCode <= 0xDFFF; - const commonChecker = commonPrefix ? isHigh : isLow; - const divergentChecker = commonPrefix ? isLow : isHigh; + const commonChecker = commonSuffix ? isLow : isHigh; + const divergentChecker = commonSuffix ? isHigh : isLow; // If the last common char qualifies as a direction-appropriate SMP surrogate... if(commonChecker(commonPotentialSurrogate)) { From e3da931606f943ac1d432795bdc0c405b41b20f5 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 7 Feb 2024 15:40:36 +0700 Subject: [PATCH 09/17] change(web): clearer maxInterval handling --- .../web/keyboard-processor/src/text/stringDivergence.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/common/web/keyboard-processor/src/text/stringDivergence.ts b/common/web/keyboard-processor/src/text/stringDivergence.ts index cd665d9cb7..58936931ee 100644 --- a/common/web/keyboard-processor/src/text/stringDivergence.ts +++ b/common/web/keyboard-processor/src/text/stringDivergence.ts @@ -6,7 +6,10 @@ * @returns The code unit indices within each string for the start of the code point not common to both. */ export function searchStringDivergence(str1: string, str2: string, commonSuffix: boolean): [number, number] { - const maxInterval = Math.min(str1.length, str2.length) - 1; + /** + * The maximum number of iterations to consider; exceeding this would go past a string boundary. + */ + const maxInterval = Math.min(str1.length, str2.length); /** * The first valid index within the string. @@ -35,12 +38,12 @@ export function searchStringDivergence(str1: string, str2: string, commonSuffix: if(commonSuffix) { start = index = str1.length - 1; - end = index - maxInterval; + end = index - maxInterval + 1; // index - (maxInterval-1) inc = -1; offset = str2.length - str1.length; } else { start = index = 0; - end = maxInterval; + end = maxInterval - 1; inc = 1; offset = 0; } From b1adb857b3facb8b37a7c1c1cd73107da1813556 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 7 Feb 2024 15:41:56 +0700 Subject: [PATCH 10/17] fix(web): forgot to delete a ref to deleted name --- common/web/keyboard-processor/tests/node/transcriptions.js | 1 - 1 file changed, 1 deletion(-) diff --git a/common/web/keyboard-processor/tests/node/transcriptions.js b/common/web/keyboard-processor/tests/node/transcriptions.js index 60df8b2ddb..2ac7046ff0 100644 --- a/common/web/keyboard-processor/tests/node/transcriptions.js +++ b/common/web/keyboard-processor/tests/node/transcriptions.js @@ -119,7 +119,6 @@ describe("String divergence calculations", function() { describe("Transcriptions and Transforms", function() { // Built in-line via function. Looks functionally equivalent to "apple", but with SMP characters. - let u = toSupplementaryPairString; let smpApple = u(0x1d5ba)+u(0x1d5c9)+u(0x1d5c9)+u(0x1d5c5)+u(0x1d5be); it("does not store an alias for related OutputTargets", function() { From 1222b2b3e286fb969aaca1f423289b8a72948a71 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 7 Feb 2024 16:04:42 +0700 Subject: [PATCH 11/17] chore(web): minor simplification --- .../keyboard-processor/src/text/stringDivergence.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/common/web/keyboard-processor/src/text/stringDivergence.ts b/common/web/keyboard-processor/src/text/stringDivergence.ts index 58936931ee..8e385c1bff 100644 --- a/common/web/keyboard-processor/src/text/stringDivergence.ts +++ b/common/web/keyboard-processor/src/text/stringDivergence.ts @@ -22,7 +22,7 @@ export function searchStringDivergence(str1: string, str2: string, commonSuffix: let index: number; /** - * The last valid index within the string to consider as the divergence point. + * The index at which to terminate the search for a divergence point. */ let end: number; @@ -30,6 +30,7 @@ export function searchStringDivergence(str1: string, str2: string, commonSuffix: * Index shift per loop iteration. */ let inc: number; + /** * Difference in index for comparison between strings. * Mostly matters when assuming a common right-hand side. @@ -37,19 +38,19 @@ export function searchStringDivergence(str1: string, str2: string, commonSuffix: let offset: number; if(commonSuffix) { - start = index = str1.length - 1; - end = index - maxInterval + 1; // index - (maxInterval-1) + start = index = str1.length - 1; // e.g. str.length == 10 => start = 9. + end = index - maxInterval; // e.g. maxInterval 8, start 9 => iterate from 9 to 2, end at 1. inc = -1; offset = str2.length - str1.length; } else { start = index = 0; - end = maxInterval - 1; + end = maxInterval; // last valid index: - 1. e.g. maxInterval 8 => iterate from 0 to 7, end at 8. inc = 1; offset = 0; } // Step 1: Find the index for the first code unit different between the strings. - for(; commonSuffix ? index >= end: index <= end; index += inc) { + for(; index != end; index += inc) { if(str1.charAt(index) != str2.charAt(index + offset)) { break; } From 0f0784a0e79f2a87630aa5de79dd861cb23f300d Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 7 Feb 2024 16:24:42 +0700 Subject: [PATCH 12/17] change(web): simpler return value --- .../src/text/outputTarget.ts | 4 ++-- .../src/text/stringDivergence.ts | 8 +++---- .../tests/node/transcriptions.js | 24 +++++++++---------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/common/web/keyboard-processor/src/text/outputTarget.ts b/common/web/keyboard-processor/src/text/outputTarget.ts index 1a82bca32c..8399bbb834 100644 --- a/common/web/keyboard-processor/src/text/outputTarget.ts +++ b/common/web/keyboard-processor/src/text/outputTarget.ts @@ -124,14 +124,14 @@ export default abstract class OutputTarget { const toLeft = this.getTextBeforeCaret(); const fromLeft = original.getTextBeforeCaret(); - const leftDivergenceIndex = searchStringDivergence(fromLeft, toLeft, false)[0]; + const leftDivergenceIndex = searchStringDivergence(fromLeft, toLeft, false); const deletedLeft = fromLeft.substring(leftDivergenceIndex)._kmwLength(); // No need for our specialized variant here. const insertedText = toLeft.substring(leftDivergenceIndex); const toRight = this.getTextAfterCaret(); const fromRight = original.getTextAfterCaret(); - const rightDivergenceIndex = searchStringDivergence(fromRight, toRight, true)[0]; + const rightDivergenceIndex = searchStringDivergence(fromRight, toRight, true); // Right insertions aren't supported, but right deletions will matter in some scenarios. // In particular, once we allow right-deletion for pred-text suggestions applied with the diff --git a/common/web/keyboard-processor/src/text/stringDivergence.ts b/common/web/keyboard-processor/src/text/stringDivergence.ts index 8e385c1bff..7476bc2fd6 100644 --- a/common/web/keyboard-processor/src/text/stringDivergence.ts +++ b/common/web/keyboard-processor/src/text/stringDivergence.ts @@ -3,9 +3,9 @@ * @param str1 * @param str2 * @param commonSuffix If false, asserts a common prefix to the strings. If true, asserts a common suffix. - * @returns The code unit indices within each string for the start of the code point not common to both. + * @returns The code unit index within `str1` for the start of the code point not common to both. */ -export function searchStringDivergence(str1: string, str2: string, commonSuffix: boolean): [number, number] { +export function searchStringDivergence(str1: string, str2: string, commonSuffix: boolean): number { /** * The maximum number of iterations to consider; exceeding this would go past a string boundary. */ @@ -79,10 +79,10 @@ export function searchStringDivergence(str1: string, str2: string, commonSuffix: if(divergentChecker(divergentChar1) || divergentChecker(divergentChar2)) { // Our current index would split a surrogate pair; decrement the index to // preserve the pair. - return [index - inc, index - inc + offset]; + return index - inc; } } } - return [index, index + offset]; + return index; } \ No newline at end of file diff --git a/common/web/keyboard-processor/tests/node/transcriptions.js b/common/web/keyboard-processor/tests/node/transcriptions.js index 2ac7046ff0..cd074efb53 100644 --- a/common/web/keyboard-processor/tests/node/transcriptions.js +++ b/common/web/keyboard-processor/tests/node/transcriptions.js @@ -25,13 +25,13 @@ describe("String divergence calculations", function() { describe("Common prefix", () => { it("BMP text", () => { const result1 = searchStringDivergence("apple", "applause", false); - assert.deepEqual(result1, [4, 4]); + assert.equal(result1, 4); const result2 = searchStringDivergence("applesauce", "applause", false); - assert.deepEqual(result2, [4, 4]); + assert.equal(result2, 4); const result3 = searchStringDivergence("applesauce", "applesauce", false); - assert.deepEqual(result3, [10, 10]); + assert.equal(result3, 10); }); it("SMP text", () => { @@ -45,7 +45,7 @@ describe("String divergence calculations", function() { // 2 per SMP char; is in code-unit... units. // Will avoid splitting code points, though. - assert.deepEqual(result1, [8, 8]); + assert.equal(result1, 8); const result2 = searchStringDivergence( smp_ify('applesauce'), @@ -53,7 +53,7 @@ describe("String divergence calculations", function() { false ); - assert.deepEqual(result2, [8, 8]); + assert.equal(result2, 8); const result3 = searchStringDivergence( smp_ify('applesauce'), @@ -61,7 +61,7 @@ describe("String divergence calculations", function() { false ); - assert.deepEqual(result3, [20, 20]); + assert.equal(result3, 20); }); }); @@ -70,16 +70,16 @@ describe("String divergence calculations", function() { // att|endance // transc|endance const result1 = searchStringDivergence("attendance", "transcendance", true); - assert.deepEqual(result1, [2, 5]); + assert.equal(result1, 2); // transcend|ance // happenst|ance const result2 = searchStringDivergence("transcendance", "happenstance", true); - assert.deepEqual(result2, [8, 7]); + assert.equal(result2, 8); // And if the two are equal... const result3 = searchStringDivergence("post-caret text", "post-caret text", true); - assert.deepEqual(result3, [-1, -1]); + assert.equal(result3, -1); }); it("SMP text", () => { @@ -95,7 +95,7 @@ describe("String divergence calculations", function() { // 2 per SMP char; is in code-unit... units. // Will avoid splitting code points; is odd b/c we get the index of the LAST char of the pair. - assert.deepEqual(result1, [5, 11]); + assert.equal(result1, 5); // transcend|ance // happenst|ance @@ -104,7 +104,7 @@ describe("String divergence calculations", function() { smp_ify("happenstance"), true ); - assert.deepEqual(result2, [17, 15]); + assert.equal(result2, 17); // And if the two are equal... const result3 = searchStringDivergence( @@ -112,7 +112,7 @@ describe("String divergence calculations", function() { smp_ify("post-caret text"), true ); - assert.deepEqual(result3, [-1, -1]); + assert.equal(result3, -1); }); }) }); From c32691c2e7910e9ff2da71e6a4a0eef568ca6a7f Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 8 Feb 2024 08:19:40 +0700 Subject: [PATCH 13/17] feat(web): extra edge-case unit tests --- .../tests/node/transcriptions.js | 57 +++++++++++++++---- 1 file changed, 47 insertions(+), 10 deletions(-) diff --git a/common/web/keyboard-processor/tests/node/transcriptions.js b/common/web/keyboard-processor/tests/node/transcriptions.js index cd074efb53..98c47a21f1 100644 --- a/common/web/keyboard-processor/tests/node/transcriptions.js +++ b/common/web/keyboard-processor/tests/node/transcriptions.js @@ -29,9 +29,14 @@ describe("String divergence calculations", function() { const result2 = searchStringDivergence("applesauce", "applause", false); assert.equal(result2, 4); + }); - const result3 = searchStringDivergence("applesauce", "applesauce", false); - assert.equal(result3, 10); + it("BMP edge cases", () => { + const result1 = searchStringDivergence("applesauce", "applesauce", false); + assert.equal(result1, 10); + + const result2 = searchStringDivergence("applesauce", "banana bread", false); + assert.equal(result2, 0); }); it("SMP text", () => { @@ -54,15 +59,27 @@ describe("String divergence calculations", function() { ); assert.equal(result2, 8); + }); - const result3 = searchStringDivergence( + it("SMP edge cases", () => { + const smp_ify = (str) => str.split('').map(ss).join(''); + + const result1 = searchStringDivergence( smp_ify('applesauce'), smp_ify('applesauce'), false ); - assert.equal(result3, 20); - }); + assert.equal(result1, 20); + + const result2 = searchStringDivergence( + smp_ify('applesauce'), + smp_ify('banana bread'), + false + ); + + assert.equal(result2, 0); + }) }); describe("Common suffix", () => { @@ -77,11 +94,18 @@ describe("String divergence calculations", function() { const result2 = searchStringDivergence("transcendance", "happenstance", true); assert.equal(result2, 8); - // And if the two are equal... - const result3 = searchStringDivergence("post-caret text", "post-caret text", true); - assert.equal(result3, -1); }); + it("BMP edge cases", () => { + // If the two are equal... + const result1 = searchStringDivergence("post-caret text", "post-caret text", true); + assert.equal(result1, -1); + + // If the two are completely different... + const result2 = searchStringDivergence("post-caret text", "supercalifragilistic", true); + assert.equal(result2, "post-caret text".length-1); + }) + it("SMP text", () => { const smp_ify = (str) => str.split('').map(ss).join(''); @@ -106,14 +130,27 @@ describe("String divergence calculations", function() { ); assert.equal(result2, 17); - // And if the two are equal... + }); + + it("SMP edge cases", () => { + const smp_ify = (str) => str.split('').map(ss).join(''); + + // If the two are equal... const result3 = searchStringDivergence( smp_ify("post-caret text"), smp_ify("post-caret text"), true ); assert.equal(result3, -1); - }); + + // If the two are completely different... + const result2 = searchStringDivergence( + smp_ify("post-caret text"), + smp_ify("supercalifragilistic"), + true + ); + assert.equal(result2, smp_ify("post-caret text").length-1); + }) }) }); From f36aba7badbd046e949b214f47770acc626f0f71 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 8 Feb 2024 11:52:13 +0700 Subject: [PATCH 14/17] chore(web): explicit edge case condition --- common/web/keyboard-processor/src/text/stringDivergence.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/web/keyboard-processor/src/text/stringDivergence.ts b/common/web/keyboard-processor/src/text/stringDivergence.ts index 7476bc2fd6..f2de629b73 100644 --- a/common/web/keyboard-processor/src/text/stringDivergence.ts +++ b/common/web/keyboard-processor/src/text/stringDivergence.ts @@ -60,7 +60,7 @@ export function searchStringDivergence(str1: string, str2: string, commonSuffix: // `index` corresponds to the first char that is different _in the direction indicated by inc_. // If it's the start position, it can't split a (completed) surrogate pair. - if(index != start) { + if(index != start && index != end) { // if commonLeft, high surrogate; if commonRight, low surrogate. const commonPotentialSurrogate = str1.charCodeAt(index - inc); // Opposite surrogate type from the previous variable. From aefef2181db8a536d29345b8691ee185c1c3feda Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 8 Feb 2024 13:12:26 +0700 Subject: [PATCH 15/17] chore(web): further PR cleanup --- .../src/text/outputTarget.ts | 6 ++--- .../src/text/stringDivergence.ts | 21 ++++++++------- .../tests/node/transcriptions.js | 22 +++++++-------- common/web/utils/src/index.ts | 2 ++ common/web/utils/src/surrogates.ts | 27 +++++++++++++++++++ 5 files changed, 55 insertions(+), 23 deletions(-) create mode 100644 common/web/utils/src/surrogates.ts diff --git a/common/web/keyboard-processor/src/text/outputTarget.ts b/common/web/keyboard-processor/src/text/outputTarget.ts index 8399bbb834..ab9656b699 100644 --- a/common/web/keyboard-processor/src/text/outputTarget.ts +++ b/common/web/keyboard-processor/src/text/outputTarget.ts @@ -1,7 +1,7 @@ /// import { extendString } from "@keymanapp/web-utils"; -import { searchStringDivergence } from "./stringDivergence.js"; +import { findCommonSubstringEndIndex } from "./stringDivergence.js"; extendString(); @@ -124,14 +124,14 @@ export default abstract class OutputTarget { const toLeft = this.getTextBeforeCaret(); const fromLeft = original.getTextBeforeCaret(); - const leftDivergenceIndex = searchStringDivergence(fromLeft, toLeft, false); + const leftDivergenceIndex = findCommonSubstringEndIndex(fromLeft, toLeft, false); const deletedLeft = fromLeft.substring(leftDivergenceIndex)._kmwLength(); // No need for our specialized variant here. const insertedText = toLeft.substring(leftDivergenceIndex); const toRight = this.getTextAfterCaret(); const fromRight = original.getTextAfterCaret(); - const rightDivergenceIndex = searchStringDivergence(fromRight, toRight, true); + const rightDivergenceIndex = findCommonSubstringEndIndex(fromRight, toRight, true); // Right insertions aren't supported, but right deletions will matter in some scenarios. // In particular, once we allow right-deletion for pred-text suggestions applied with the diff --git a/common/web/keyboard-processor/src/text/stringDivergence.ts b/common/web/keyboard-processor/src/text/stringDivergence.ts index f2de629b73..79509d51a7 100644 --- a/common/web/keyboard-processor/src/text/stringDivergence.ts +++ b/common/web/keyboard-processor/src/text/stringDivergence.ts @@ -1,11 +1,15 @@ +// Future TODO: import from @keymanapp/common-types... once we no longer need to support ES5. +import { Uni_IsSurrogate1, Uni_IsSurrogate2 } from '@keymanapp/web-utils'; + /** - * Returns the index for the code point divergence point in code unit coordinates. + * Returns the index for the code point divergence point between two strings, as measured in code + * unit coordinates. * @param str1 * @param str2 * @param commonSuffix If false, asserts a common prefix to the strings. If true, asserts a common suffix. * @returns The code unit index within `str1` for the start of the code point not common to both. */ -export function searchStringDivergence(str1: string, str2: string, commonSuffix: boolean): number { +export function findCommonSubstringEndIndex(str1: string, str2: string, commonSuffix: boolean): number { /** * The maximum number of iterations to consider; exceeding this would go past a string boundary. */ @@ -67,16 +71,15 @@ export function searchStringDivergence(str1: string, str2: string, commonSuffix: const divergentChar1 = str1.charCodeAt(index); const divergentChar2 = str2.charCodeAt(index + offset); - const isHigh = (charCode: number) => charCode >= 0xD800 && charCode <= 0xDBFF; - const isLow = (charCode: number) => charCode >= 0xDC00 && charCode <= 0xDFFF; - const commonChecker = commonSuffix ? isLow : isHigh; - const divergentChecker = commonSuffix ? isHigh : isLow; + const commonSurrogateChecker = commonSuffix ? Uni_IsSurrogate2 : Uni_IsSurrogate1; + const divergentSurrogateChecker = commonSuffix ? Uni_IsSurrogate1 : Uni_IsSurrogate2; - // If the last common char qualifies as a direction-appropriate SMP surrogate... - if(commonChecker(commonPotentialSurrogate)) { + // If the last common character if of the direction-appropriate surrogate type (for + // comprising a potential split surrogate pair representing a non-BMP char)... + if(commonSurrogateChecker(commonPotentialSurrogate)) { // And one of the two divergent chars is a qualifying match - a surrogate // of the opposite type... - if(divergentChecker(divergentChar1) || divergentChecker(divergentChar2)) { + if(divergentSurrogateChecker(divergentChar1) || divergentSurrogateChecker(divergentChar2)) { // Our current index would split a surrogate pair; decrement the index to // preserve the pair. return index - inc; diff --git a/common/web/keyboard-processor/tests/node/transcriptions.js b/common/web/keyboard-processor/tests/node/transcriptions.js index 98c47a21f1..ef89ae1a07 100644 --- a/common/web/keyboard-processor/tests/node/transcriptions.js +++ b/common/web/keyboard-processor/tests/node/transcriptions.js @@ -7,11 +7,11 @@ extendString(); // Ensure KMW's string-extension functionality is available. String.kmwEnableSupplementaryPlane(false); -// A unicode-coding like alias for use in constructing SMP strings. +// A unicode-coding like alias for use in constructing non-BMP strings. const u = String.fromCodePoint; /** - * Returns the "Mathematical Sans-Serif Small" SMP encoding for + * Returns the "Mathematical Sans-Serif Small" non-BMP encoding for * a passed-in lowercase char between 'a' and 'z', inclusive. * @param {*} char * @returns @@ -39,7 +39,7 @@ describe("String divergence calculations", function() { assert.equal(result2, 0); }); - it("SMP text", () => { + it("non-BMP text", () => { const smp_ify = (str) => str.split('').map(ss).join(''); const result1 = searchStringDivergence( @@ -48,7 +48,7 @@ describe("String divergence calculations", function() { false ); - // 2 per SMP char; is in code-unit... units. + // 2 per non-BMP char; is in code-unit... units. // Will avoid splitting code points, though. assert.equal(result1, 8); @@ -61,7 +61,7 @@ describe("String divergence calculations", function() { assert.equal(result2, 8); }); - it("SMP edge cases", () => { + it("non-BMP edge cases", () => { const smp_ify = (str) => str.split('').map(ss).join(''); const result1 = searchStringDivergence( @@ -106,7 +106,7 @@ describe("String divergence calculations", function() { assert.equal(result2, "post-caret text".length-1); }) - it("SMP text", () => { + it("non-BMP text", () => { const smp_ify = (str) => str.split('').map(ss).join(''); // att|endance @@ -117,7 +117,7 @@ describe("String divergence calculations", function() { true ); - // 2 per SMP char; is in code-unit... units. + // 2 per non-BMP char; is in code-unit... units. // Will avoid splitting code points; is odd b/c we get the index of the LAST char of the pair. assert.equal(result1, 5); @@ -132,7 +132,7 @@ describe("String divergence calculations", function() { }); - it("SMP edge cases", () => { + it("non-BMP edge cases", () => { const smp_ify = (str) => str.split('').map(ss).join(''); // If the two are equal... @@ -155,7 +155,7 @@ describe("String divergence calculations", function() { }); describe("Transcriptions and Transforms", function() { - // Built in-line via function. Looks functionally equivalent to "apple", but with SMP characters. + // Built in-line via function. Looks functionally equivalent to "apple", but with non-BMP characters. let smpApple = u(0x1d5ba)+u(0x1d5c9)+u(0x1d5c9)+u(0x1d5c5)+u(0x1d5be); it("does not store an alias for related OutputTargets", function() { @@ -263,7 +263,7 @@ but not himself.`; // Sheev Palpatine, in the Star Wars prequels. assert.equal(transcription.transform.deleteRight, 1, "Incorrect count for right-of-caret deletions"); }); - it("handles deletions around the caret without text insertion (SMP text)", function() { + it("handles deletions around the caret without text insertion (non-BMP text)", function() { try { String.kmwEnableSupplementaryPlane(true); var target = new Mock(smpApple, 2); @@ -355,7 +355,7 @@ but not himself.`; // Sheev Palpatine, in the Star Wars prequels. assert.equal(transcription.transform.deleteRight, 3, "Incorrect count for right-of-caret deletions"); }); - it("handles deletions around the caret with text insertion (SMP text)", function() { + it("handles deletions around the caret with text insertion (non-BMP text)", function() { try { String.kmwEnableSupplementaryPlane(true); diff --git a/common/web/utils/src/index.ts b/common/web/utils/src/index.ts index fdd49e60f2..1d1e540a80 100644 --- a/common/web/utils/src/index.ts +++ b/common/web/utils/src/index.ts @@ -21,6 +21,8 @@ export { default as extendString } from "./kmwstring.js"; export { default as ManagedPromise } from "./managedPromise.js"; export { default as TimeoutPromise, timedPromise } from "./timeoutPromise.js"; +export { Uni_IsSurrogate1, Uni_IsSurrogate2 } from "./surrogates.js"; + // // Uncomment the following line and run the bundled output to verify successful // // esbuild bundling of this submodule: // console.log(Version.CURRENT.toString()); \ No newline at end of file diff --git a/common/web/utils/src/surrogates.ts b/common/web/utils/src/surrogates.ts new file mode 100644 index 0000000000..53c1a1e3fa --- /dev/null +++ b/common/web/utils/src/surrogates.ts @@ -0,0 +1,27 @@ +/* + * The definitions below are duplicated from common/web/types/util/util.ts; + * we can't downcompile the originals to ES5 when bundling with esbuild. + * `import type` stuff is fine, but not non-type `import` statements. + * + * TODO: Use those instead, once we're no longer building ES5 versions of Web. + */ + +export const Uni_LEAD_SURROGATE_START = 0xD800; +export const Uni_LEAD_SURROGATE_END = 0xDBFF; +export const Uni_TRAIL_SURROGATE_START = 0xDC00; +export const Uni_TRAIL_SURROGATE_END = 0xDFFF; + +/** + * @brief True if a lead surrogate + * \def Uni_IsSurrogate1 + */ +export function Uni_IsSurrogate1(ch : number) { + return ((ch) >= Uni_LEAD_SURROGATE_START && (ch) <= Uni_LEAD_SURROGATE_END); +} +/** + * @brief True if a trail surrogate + * \def Uni_IsSurrogate2 + */ +export function Uni_IsSurrogate2(ch : number) { + return ((ch) >= Uni_TRAIL_SURROGATE_START && (ch) <= Uni_TRAIL_SURROGATE_END); +} From 444aeec509b081d16fbe55f4764f26da1ebc4266 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 8 Feb 2024 15:40:39 +0700 Subject: [PATCH 16/17] chore(web): unit test patchup after method rename --- .../tests/node/transcriptions.js | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/common/web/keyboard-processor/tests/node/transcriptions.js b/common/web/keyboard-processor/tests/node/transcriptions.js index ef89ae1a07..6d6be7b540 100644 --- a/common/web/keyboard-processor/tests/node/transcriptions.js +++ b/common/web/keyboard-processor/tests/node/transcriptions.js @@ -1,6 +1,6 @@ import { assert } from 'chai'; -import { Mock, searchStringDivergence } from '@keymanapp/keyboard-processor'; +import { Mock, findCommonSubstringEndIndex } from '@keymanapp/keyboard-processor'; import { extendString } from '@keymanapp/web-utils'; extendString(); // Ensure KMW's string-extension functionality is available. @@ -24,25 +24,25 @@ const ss = (char) => { describe("String divergence calculations", function() { describe("Common prefix", () => { it("BMP text", () => { - const result1 = searchStringDivergence("apple", "applause", false); + const result1 = findCommonSubstringEndIndex("apple", "applause", false); assert.equal(result1, 4); - const result2 = searchStringDivergence("applesauce", "applause", false); + const result2 = findCommonSubstringEndIndex("applesauce", "applause", false); assert.equal(result2, 4); }); it("BMP edge cases", () => { - const result1 = searchStringDivergence("applesauce", "applesauce", false); + const result1 = findCommonSubstringEndIndex("applesauce", "applesauce", false); assert.equal(result1, 10); - const result2 = searchStringDivergence("applesauce", "banana bread", false); + const result2 = findCommonSubstringEndIndex("applesauce", "banana bread", false); assert.equal(result2, 0); }); it("non-BMP text", () => { const smp_ify = (str) => str.split('').map(ss).join(''); - const result1 = searchStringDivergence( + const result1 = findCommonSubstringEndIndex( smp_ify('apple'), smp_ify('applause'), false @@ -52,7 +52,7 @@ describe("String divergence calculations", function() { // Will avoid splitting code points, though. assert.equal(result1, 8); - const result2 = searchStringDivergence( + const result2 = findCommonSubstringEndIndex( smp_ify('applesauce'), smp_ify('applause'), false @@ -64,7 +64,7 @@ describe("String divergence calculations", function() { it("non-BMP edge cases", () => { const smp_ify = (str) => str.split('').map(ss).join(''); - const result1 = searchStringDivergence( + const result1 = findCommonSubstringEndIndex( smp_ify('applesauce'), smp_ify('applesauce'), false @@ -72,7 +72,7 @@ describe("String divergence calculations", function() { assert.equal(result1, 20); - const result2 = searchStringDivergence( + const result2 = findCommonSubstringEndIndex( smp_ify('applesauce'), smp_ify('banana bread'), false @@ -86,23 +86,23 @@ describe("String divergence calculations", function() { it("BMP text", () => { // att|endance // transc|endance - const result1 = searchStringDivergence("attendance", "transcendance", true); + const result1 = findCommonSubstringEndIndex("attendance", "transcendance", true); assert.equal(result1, 2); // transcend|ance // happenst|ance - const result2 = searchStringDivergence("transcendance", "happenstance", true); + const result2 = findCommonSubstringEndIndex("transcendance", "happenstance", true); assert.equal(result2, 8); }); it("BMP edge cases", () => { // If the two are equal... - const result1 = searchStringDivergence("post-caret text", "post-caret text", true); + const result1 = findCommonSubstringEndIndex("post-caret text", "post-caret text", true); assert.equal(result1, -1); // If the two are completely different... - const result2 = searchStringDivergence("post-caret text", "supercalifragilistic", true); + const result2 = findCommonSubstringEndIndex("post-caret text", "supercalifragilistic", true); assert.equal(result2, "post-caret text".length-1); }) @@ -111,7 +111,7 @@ describe("String divergence calculations", function() { // att|endance // trans|endance - const result1 = searchStringDivergence( + const result1 = findCommonSubstringEndIndex( smp_ify("attendance"), smp_ify("transcendance"), true @@ -123,7 +123,7 @@ describe("String divergence calculations", function() { // transcend|ance // happenst|ance - const result2 = searchStringDivergence( + const result2 = findCommonSubstringEndIndex( smp_ify("transcendance"), smp_ify("happenstance"), true @@ -136,7 +136,7 @@ describe("String divergence calculations", function() { const smp_ify = (str) => str.split('').map(ss).join(''); // If the two are equal... - const result3 = searchStringDivergence( + const result3 = findCommonSubstringEndIndex( smp_ify("post-caret text"), smp_ify("post-caret text"), true @@ -144,7 +144,7 @@ describe("String divergence calculations", function() { assert.equal(result3, -1); // If the two are completely different... - const result2 = searchStringDivergence( + const result2 = findCommonSubstringEndIndex( smp_ify("post-caret text"), smp_ify("supercalifragilistic"), true From 0e06b7a20bd5592e249457aadeda648855a30a1c Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 13 Feb 2024 11:05:38 +0700 Subject: [PATCH 17/17] docs(web): adds a bit of extra documentation re: method return value paradigm --- common/web/keyboard-processor/src/text/stringDivergence.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/web/keyboard-processor/src/text/stringDivergence.ts b/common/web/keyboard-processor/src/text/stringDivergence.ts index 79509d51a7..e3461e129a 100644 --- a/common/web/keyboard-processor/src/text/stringDivergence.ts +++ b/common/web/keyboard-processor/src/text/stringDivergence.ts @@ -8,6 +8,8 @@ import { Uni_IsSurrogate1, Uni_IsSurrogate2 } from '@keymanapp/web-utils'; * @param str2 * @param commonSuffix If false, asserts a common prefix to the strings. If true, asserts a common suffix. * @returns The code unit index within `str1` for the start of the code point not common to both. + * + * Follows the convention of (start, end) substring parameterizations having 'end' be exclusive. */ export function findCommonSubstringEndIndex(str1: string, str2: string, commonSuffix: boolean): number { /**