diff --git a/common/models/wordbreakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts index 20e159ed19..ef50ab4b31 100644 --- a/common/models/wordbreakers/src/default/index.ts +++ b/common/models/wordbreakers/src/default/index.ts @@ -274,7 +274,7 @@ export class BreakerContext { * @param chunk a chunk of text. Starts and ends at word boundaries. */ function isNonSpace(chunk: string, options?: DefaultWordBreakerOptions): boolean { - return !Array.from(chunk).map((char) => property(char, options)).every(wb => ( + return !chunk.split('').map((char) => property(char, options)).every(wb => ( wb === WordBreakProperty.CR || wb === WordBreakProperty.LF || wb === WordBreakProperty.Newline || diff --git a/common/web/lm-worker/build-polyfiller.js b/common/web/lm-worker/build-polyfiller.js index 20f3ab587f..1978038836 100644 --- a/common/web/lm-worker/build-polyfiller.js +++ b/common/web/lm-worker/build-polyfiller.js @@ -144,6 +144,9 @@ let sourceFileSet = [ // Needed for Android / Chromium browser pre-41. loadPolyfill('../../../node_modules/string.prototype.codepointat/codepointat.js', 'src/polyfills/string.codepointat.js'), // Needed for Android / Chromium browser pre-45. + // Not used in this codebase, but used by some compiled model defaults. + loadPolyfill('src/polyfills/array.from.js', 'src/polyfills/array.from.js'), + // Needed for Android / Chromium browser pre-45. loadPolyfill('src/polyfills/array.fill.js', 'src/polyfills/array.fill.js'), // Needed for Android / Chromium browser pre-45. loadPolyfill('src/polyfills/array.findIndex.js', 'src/polyfills/array.findIndex.js'), diff --git a/common/web/lm-worker/src/polyfills/array.from.js b/common/web/lm-worker/src/polyfills/array.from.js new file mode 100644 index 0000000000..ae6ba84600 --- /dev/null +++ b/common/web/lm-worker/src/polyfills/array.from.js @@ -0,0 +1,44 @@ +(function() { + if(!Array.from) { + function isHighSurrogate(codeUnit) { + codeUnit = codeUnit.charCodeAt(0); + return codeUnit >= 0xD800 && codeUnit <= 0xDBFF; + } + + function isLowSurrogate(codeUnit) { + codeUnit = codeUnit.charCodeAt(0); + return codeUnit >= 0xDC00 && codeUnit <= 0xDFFF; + } + + Array.from = function (obj) { + if(Array.isArray(obj)) { + // Simple array clone + return obj.slice(); + } else if(typeof obj == 'string') { + // Array.from is surrogate-aware and will not split surrogate pairs. + // We can start with a full split and then remerge the pairs. + var simpleSplit = obj.split(''); + + /** @type {string[]} */ + var finalSplit = []; + /** @type {number} */ + var i; + + while(simpleSplit.length > 0) { + // Do we have a surrogate pair? + var a = simpleSplit.shift(); + if(isHighSurrogate(a) && (isLowSurrogate(simpleSplit[0] || ''))) { + // yes, so merge them before pushing. + a = a + simpleSplit.shift(); + console.log(a); + } // else: 'no', so just push the current char to the array and continue + + finalSplit.push(a); + } + return finalSplit; + } else { + throw "Unexpected + nonpolyfilled use of Array.from encountered; aborting"; + } + } + } +}()); \ No newline at end of file