From a4a5ac2f608b1f72255fd4f94b2583494066a0a0 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 7 Jun 2024 11:56:18 +0700 Subject: [PATCH 1/3] fix(web): add limited Array.from polyfill for lm-worker use Fixes: #11502 Fixes: KEYMAN-WEB-K4 Cherry-pick-of: #11732 --- .../models/wordbreakers/src/default/index.ts | 2 +- common/web/lm-worker/build-polyfiller.js | 3 ++ .../web/lm-worker/src/polyfills/array.from.js | 47 +++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 common/web/lm-worker/src/polyfills/array.from.js diff --git a/common/models/wordbreakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts index 20e159ed19..ef50ab4b31 100644 --- a/common/models/wordbreakers/src/default/index.ts +++ b/common/models/wordbreakers/src/default/index.ts @@ -274,7 +274,7 @@ export class BreakerContext { * @param chunk a chunk of text. Starts and ends at word boundaries. */ function isNonSpace(chunk: string, options?: DefaultWordBreakerOptions): boolean { - return !Array.from(chunk).map((char) => property(char, options)).every(wb => ( + return !chunk.split('').map((char) => property(char, options)).every(wb => ( wb === WordBreakProperty.CR || wb === WordBreakProperty.LF || wb === WordBreakProperty.Newline || diff --git a/common/web/lm-worker/build-polyfiller.js b/common/web/lm-worker/build-polyfiller.js index 20f3ab587f..1978038836 100644 --- a/common/web/lm-worker/build-polyfiller.js +++ b/common/web/lm-worker/build-polyfiller.js @@ -144,6 +144,9 @@ let sourceFileSet = [ // Needed for Android / Chromium browser pre-41. loadPolyfill('../../../node_modules/string.prototype.codepointat/codepointat.js', 'src/polyfills/string.codepointat.js'), // Needed for Android / Chromium browser pre-45. + // Not used in this codebase, but used by some compiled model defaults. + loadPolyfill('src/polyfills/array.from.js', 'src/polyfills/array.from.js'), + // Needed for Android / Chromium browser pre-45. loadPolyfill('src/polyfills/array.fill.js', 'src/polyfills/array.fill.js'), // Needed for Android / Chromium browser pre-45. loadPolyfill('src/polyfills/array.findIndex.js', 'src/polyfills/array.findIndex.js'), diff --git a/common/web/lm-worker/src/polyfills/array.from.js b/common/web/lm-worker/src/polyfills/array.from.js new file mode 100644 index 0000000000..6234d4a90c --- /dev/null +++ b/common/web/lm-worker/src/polyfills/array.from.js @@ -0,0 +1,47 @@ +if(!Array.from) { + function isHighSurrogate(codeUnit) { + if(typeof codeUnit == 'string') { + codeUnit = codeUnit.charCodeAt(0); + } + + return codeUnit >= 0xD800 && codeUnit <= 0xDBFF; + } + + function isLowSurrogate(codeUnit) { + if(typeof codeUnit == 'string') { + codeUnit = codeUnit.charCodeAt(0); + } + + return codeUnit >= 0xDC00 && codeUnit <= 0xDFFF; + } + + Array.from = function (obj) { + if(Array.isArray(obj)) { + // Simple array clone + return obj.slice(); + } else if(typeof obj == 'string') { + // Array.from is surrogate-aware and will not split surrogate pairs. + // We can start with a full split and then remerge the pairs. + var simpleSplit = obj.split(); + + /** @type {string[]} */ + var finalSplit = []; + /** @type {number} */ + var i; + + for(i=0; i < simpleSplit.length; i++) { + // Do we have a surrogate pair? + var a = simpleSplit.shift(); + if(isHighSurrogate(a) && isLowSurrogate(simpleShift[0] || '')) { + // yes, so merge them before pushing. + a = a + simpleSplit.shift(); + } // else: 'no', so just push the current char to the array and continue + + finalSplit.push(a); + } + return finalSplit; + } else { + throw "Unexpected + nonpolyfilled use of Array.from encountered; aborting"; + } + } +} \ No newline at end of file From c8e36ace788d69d71350d4e7a8e4181f468f85b4 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 7 Jun 2024 14:04:41 +0700 Subject: [PATCH 2/3] fix(web): array.from polyfill scoping, fixes after extracted test --- .../web/lm-worker/src/polyfills/array.from.js | 91 ++++++++++--------- 1 file changed, 47 insertions(+), 44 deletions(-) diff --git a/common/web/lm-worker/src/polyfills/array.from.js b/common/web/lm-worker/src/polyfills/array.from.js index 6234d4a90c..7e87e66678 100644 --- a/common/web/lm-worker/src/polyfills/array.from.js +++ b/common/web/lm-worker/src/polyfills/array.from.js @@ -1,47 +1,50 @@ -if(!Array.from) { - function isHighSurrogate(codeUnit) { - if(typeof codeUnit == 'string') { - codeUnit = codeUnit.charCodeAt(0); - } - - return codeUnit >= 0xD800 && codeUnit <= 0xDBFF; - } - - function isLowSurrogate(codeUnit) { - if(typeof codeUnit == 'string') { - codeUnit = codeUnit.charCodeAt(0); - } - - return codeUnit >= 0xDC00 && codeUnit <= 0xDFFF; - } - - Array.from = function (obj) { - if(Array.isArray(obj)) { - // Simple array clone - return obj.slice(); - } else if(typeof obj == 'string') { - // Array.from is surrogate-aware and will not split surrogate pairs. - // We can start with a full split and then remerge the pairs. - var simpleSplit = obj.split(); - - /** @type {string[]} */ - var finalSplit = []; - /** @type {number} */ - var i; - - for(i=0; i < simpleSplit.length; i++) { - // Do we have a surrogate pair? - var a = simpleSplit.shift(); - if(isHighSurrogate(a) && isLowSurrogate(simpleShift[0] || '')) { - // yes, so merge them before pushing. - a = a + simpleSplit.shift(); - } // else: 'no', so just push the current char to the array and continue - - finalSplit.push(a); +(function() { + if(!Array.from) { + function isHighSurrogate(codeUnit) { + if(typeof codeUnit == 'string') { + codeUnit = codeUnit.charCodeAt(0); + } + + return codeUnit >= 0xD800 && codeUnit <= 0xDBFF; + } + + function isLowSurrogate(codeUnit) { + if(typeof codeUnit == 'string') { + codeUnit = codeUnit.charCodeAt(0); + } + + return codeUnit >= 0xDC00 && codeUnit <= 0xDFFF; + } + + Array.from = function (obj) { + if(Array.isArray(obj)) { + // Simple array clone + return obj.slice(); + } else if(typeof obj == 'string') { + // Array.from is surrogate-aware and will not split surrogate pairs. + // We can start with a full split and then remerge the pairs. + var simpleSplit = obj.split(''); + + /** @type {string[]} */ + var finalSplit = []; + /** @type {number} */ + var i; + + while(simpleSplit.length > 0) { + // Do we have a surrogate pair? + var a = simpleSplit.shift(); + if(isHighSurrogate(a) && (isLowSurrogate(simpleSplit[0] || ''))) { + // yes, so merge them before pushing. + a = a + simpleSplit.shift(); + console.log(a); + } // else: 'no', so just push the current char to the array and continue + + finalSplit.push(a); + } + return finalSplit; + } else { + throw "Unexpected + nonpolyfilled use of Array.from encountered; aborting"; } - return finalSplit; - } else { - throw "Unexpected + nonpolyfilled use of Array.from encountered; aborting"; } } -} \ No newline at end of file +}()); \ No newline at end of file From ec50747d0993f61953775b373150998005c95d01 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 7 Jun 2024 14:24:41 +0700 Subject: [PATCH 3/3] change(android): removes unneeded conditionals --- common/web/lm-worker/src/polyfills/array.from.js | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/common/web/lm-worker/src/polyfills/array.from.js b/common/web/lm-worker/src/polyfills/array.from.js index 7e87e66678..ae6ba84600 100644 --- a/common/web/lm-worker/src/polyfills/array.from.js +++ b/common/web/lm-worker/src/polyfills/array.from.js @@ -1,18 +1,12 @@ (function() { if(!Array.from) { function isHighSurrogate(codeUnit) { - if(typeof codeUnit == 'string') { - codeUnit = codeUnit.charCodeAt(0); - } - + codeUnit = codeUnit.charCodeAt(0); return codeUnit >= 0xD800 && codeUnit <= 0xDBFF; } function isLowSurrogate(codeUnit) { - if(typeof codeUnit == 'string') { - codeUnit = codeUnit.charCodeAt(0); - } - + codeUnit = codeUnit.charCodeAt(0); return codeUnit >= 0xDC00 && codeUnit <= 0xDFFF; }