From 23d6dda22c3aff859079e452ad383a25ccaa5bd0 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 18 Dec 2023 14:53:42 +0700 Subject: [PATCH 1/7] feat(web): es6 artifact for app/webview - required extra changes to do well --- common/models/templates/src/index.ts | 24 +-- common/models/templates/src/trie-model.ts | 220 +++++++++++----------- common/web/utils/package.json | 5 +- web/src/app/webview/build.sh | 17 +- 4 files changed, 135 insertions(+), 131 deletions(-) diff --git a/common/models/templates/src/index.ts b/common/models/templates/src/index.ts index f828066846..52650ba9bb 100644 --- a/common/models/templates/src/index.ts +++ b/common/models/templates/src/index.ts @@ -1,22 +1,8 @@ -import { +export { SENTINEL_CODE_UNIT, applyTransform, buildMergedTransform, isHighSurrogate, isLowSurrogate, isSentinel, transformToSuggestion, defaultApplyCasing } from "./common.js"; -import PriorityQueue, { Comparator } from "./priority-queue.js"; -import QuoteBehavior from "./quote-behavior.js"; -import { Tokenization, tokenize, getLastPreCaretToken, wordbreak } from "./tokenization.js"; -import TrieModel, { TrieModelOptions } from "./trie-model.js"; - -import { extendString } from "@keymanapp/web-utils"; - -// This package requires our string-extension functions. -extendString(); - -export { - SENTINEL_CODE_UNIT, applyTransform, buildMergedTransform, isHighSurrogate, isLowSurrogate, isSentinel, - transformToSuggestion, defaultApplyCasing, // "common.ts" - PriorityQueue, Comparator, // "priority-queue.ts" - QuoteBehavior, // "quote-behavior.ts", - Tokenization, tokenize, getLastPreCaretToken, wordbreak, // "tokenization.ts" - TrieModel, TrieModelOptions // "trie-model.ts" -}; +export { default as PriorityQueue, Comparator } from "./priority-queue.js"; +export { default as QuoteBehavior } from "./quote-behavior.js"; +export { Tokenization, tokenize, getLastPreCaretToken, wordbreak } from "./tokenization.js"; +export { default as TrieModel, TrieModelOptions } from "./trie-model.js"; \ No newline at end of file diff --git a/common/models/templates/src/trie-model.ts b/common/models/templates/src/trie-model.ts index 990ab61dc6..0a571517c7 100644 --- a/common/models/templates/src/trie-model.ts +++ b/common/models/templates/src/trie-model.ts @@ -33,6 +33,8 @@ import { applyTransform, isHighSurrogate, isSentinel, SENTINEL_CODE_UNIT, transf import { getLastPreCaretToken } from "./tokenization.js"; import PriorityQueue from "./priority-queue.js"; +extendString(); + /** * @file trie-model.ts * @@ -81,6 +83,114 @@ type TextWithProbability = { p: number; // real-number weight, from 0 to 1 } +class Traversal implements LexiconTraversal { + /** + * The lexical prefix corresponding to the current traversal state. + */ + prefix: String; + + /** + * The current traversal node. Serves as the 'root' of its own sub-Trie, + * and we cannot navigate back to its parent. + */ + root: Node; + + constructor(root: Node, prefix: string) { + this.root = root; + this.prefix = prefix; + } + + *children(): Generator<{char: string, traversal: () => LexiconTraversal}> { + let root = this.root; + + if(root.type == 'internal') { + for(let entry of root.values) { + let entryNode = root.children[entry]; + + // UTF-16 astral plane check. + if(isHighSurrogate(entry)) { + // First code unit of a UTF-16 code point. + // For now, we'll just assume the second always completes such a char. + // + // Note: Things get nasty here if this is only sometimes true; in the future, + // we should compile-time enforce that this assumption is always true if possible. + if(entryNode.type == 'internal') { + let internalNode = entryNode; + for(let lowSurrogate of internalNode.values) { + let prefix = this.prefix + entry + lowSurrogate; + yield { + char: entry + lowSurrogate, + traversal: function() { return new Traversal(internalNode.children[lowSurrogate], prefix) } + } + } + } else { + // Determine how much of the 'leaf' entry has no Trie nodes, emulate them. + let fullText = entryNode.entries[0].key; + entry = entry + fullText[this.prefix.length + 1]; // The other half of the non-BMP char. + let prefix = this.prefix + entry; + + yield { + char: entry, + traversal: function () {return new Traversal(entryNode, prefix)} + } + } + } else if(isSentinel(entry)) { + continue; + } else if(!entry) { + // Prevent any accidental 'null' or 'undefined' entries from having an effect. + continue; + } else { + let prefix = this.prefix + entry; + yield { + char: entry, + traversal: function() { return new Traversal(entryNode, prefix)} + } + } + } + + return; + } else { // type == 'leaf' + let prefix = this.prefix; + + let children = root.entries.filter(function(entry) { + return entry.key != prefix && prefix.length < entry.key.length; + }) + + for(let {key} of children) { + let nodeKey = key[prefix.length]; + + if(isHighSurrogate(nodeKey)) { + // Merge the other half of an SMP char in! + nodeKey = nodeKey + key[prefix.length+1]; + } + yield { + char: nodeKey, + traversal: function() { return new Traversal(root, prefix + nodeKey)} + } + }; + return; + } + } + + get entries(): string[] { + if(this.root.type == 'leaf') { + let prefix = this.prefix; + let matches = this.root.entries.filter(function(entry) { + return entry.key == prefix; + }); + + return matches.map(function(value) { return value.content }); + } else { + let matchingLeaf = this.root.children[SENTINEL_CODE_UNIT]; + if(matchingLeaf && matchingLeaf.type == 'leaf') { + return matchingLeaf.entries.map(function(value) { return value.content }); + } else { + return []; + } + } + } +} + /** * @class TrieModel * @@ -176,115 +286,7 @@ export default class TrieModel implements LexicalModel { } public traverseFromRoot(): LexiconTraversal { - return new TrieModel.Traversal(this._trie['root'], ''); - } - - private static Traversal = class implements LexiconTraversal { - /** - * The lexical prefix corresponding to the current traversal state. - */ - prefix: String; - - /** - * The current traversal node. Serves as the 'root' of its own sub-Trie, - * and we cannot navigate back to its parent. - */ - root: Node; - - constructor(root: Node, prefix: string) { - this.root = root; - this.prefix = prefix; - } - - *children(): Generator<{char: string, traversal: () => LexiconTraversal}> { - let root = this.root; - - if(root.type == 'internal') { - for(let entry of root.values) { - let entryNode = root.children[entry]; - - // UTF-16 astral plane check. - if(isHighSurrogate(entry)) { - // First code unit of a UTF-16 code point. - // For now, we'll just assume the second always completes such a char. - // - // Note: Things get nasty here if this is only sometimes true; in the future, - // we should compile-time enforce that this assumption is always true if possible. - if(entryNode.type == 'internal') { - let internalNode = entryNode; - for(let lowSurrogate of internalNode.values) { - let prefix = this.prefix + entry + lowSurrogate; - yield { - char: entry + lowSurrogate, - traversal: function() { return new TrieModel.Traversal(internalNode.children[lowSurrogate], prefix) } - } - } - } else { - // Determine how much of the 'leaf' entry has no Trie nodes, emulate them. - let fullText = entryNode.entries[0].key; - entry = entry + fullText[this.prefix.length + 1]; // The other half of the non-BMP char. - let prefix = this.prefix + entry; - - yield { - char: entry, - traversal: function () {return new TrieModel.Traversal(entryNode, prefix)} - } - } - } else if(isSentinel(entry)) { - continue; - } else if(!entry) { - // Prevent any accidental 'null' or 'undefined' entries from having an effect. - continue; - } else { - let prefix = this.prefix + entry; - yield { - char: entry, - traversal: function() { return new TrieModel.Traversal(entryNode, prefix)} - } - } - } - - return; - } else { // type == 'leaf' - let prefix = this.prefix; - - let children = root.entries.filter(function(entry) { - return entry.key != prefix && prefix.length < entry.key.length; - }) - - for(let {key} of children) { - let nodeKey = key[prefix.length]; - - if(isHighSurrogate(nodeKey)) { - // Merge the other half of an SMP char in! - nodeKey = nodeKey + key[prefix.length+1]; - } - yield { - char: nodeKey, - traversal: function() { return new TrieModel.Traversal(root, prefix + nodeKey)} - } - }; - return; - } - } - - get entries(): string[] { - if(this.root.type == 'leaf') { - let prefix = this.prefix; - let matches = this.root.entries.filter(function(entry) { - return entry.key == prefix; - }); - - return matches.map(function(value) { return value.content }); - } else { - let matchingLeaf = this.root.children[SENTINEL_CODE_UNIT]; - if(matchingLeaf && matchingLeaf.type == 'leaf') { - return matchingLeaf.entries.map(function(value) { return value.content }); - } else { - return []; - } - } - } + return new Traversal(this._trie['root'], ''); } }; diff --git a/common/web/utils/package.json b/common/web/utils/package.json index 1f73e92556..60bd928404 100644 --- a/common/web/utils/package.json +++ b/common/web/utils/package.json @@ -36,5 +36,8 @@ "type": "module", "paths": { "@keymanapp/keyman-version": "*" - } + }, + "sideEffects": [ + "./src/kmwstring.ts", "./build/obj/kmwstring.js" + ] } diff --git a/web/src/app/webview/build.sh b/web/src/app/webview/build.sh index 26eecaee55..20c93c9986 100755 --- a/web/src/app/webview/build.sh +++ b/web/src/app/webview/build.sh @@ -44,16 +44,29 @@ compile_and_copy() { compile $SUBPROJECT_NAME BUILD_ROOT="${KEYMAN_ROOT}/web/build/app/webview" + SRC_ROOT="${KEYMAN_ROOT}/web/src/app/webview/src" $BUNDLE_CMD "${BUILD_ROOT}/obj/debug-main.js" \ - --out "${BUILD_ROOT}/debug/keymanweb-webview.js" \ + --out "${BUILD_ROOT}/debug/keymanweb-webview.es5.js" \ --sourceRoot "@keymanapp/keyman/web/build/app/webview/debug" $BUNDLE_CMD "${BUILD_ROOT}/obj/release-main.js" \ + --out "${BUILD_ROOT}/release/keymanweb-webview.es5.js" \ + --profile "${BUILD_ROOT}/filesize-profile.es5.log" \ + --sourceRoot "@keymanapp/keyman/web/build/app/webview/release" \ + --minify + + $BUNDLE_CMD "${SRC_ROOT}/debug-main.js" \ + --out "${BUILD_ROOT}/debug/keymanweb-webview.js" \ + --sourceRoot "@keymanapp/keyman/web/build/app/webview/debug" \ + --target "es6" + + $BUNDLE_CMD "${SRC_ROOT}/release-main.js" \ --out "${BUILD_ROOT}/release/keymanweb-webview.js" \ --profile "${BUILD_ROOT}/filesize-profile.log" \ --sourceRoot "@keymanapp/keyman/web/build/app/webview/release" \ - --minify + --minify \ + --target "es6" mkdir -p "$KEYMAN_ROOT/web/build/app/resources/osk" cp -R "$KEYMAN_ROOT/web/src/resources/osk/." "$KEYMAN_ROOT/web/build/app/resources/osk/" From a2305c8f465aad966893b97021032090347eb1ea Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 19 Dec 2023 14:41:49 +0700 Subject: [PATCH 2/7] change(web): Android engine now includes both ES5 and ES6 KMW versions --- .../app/src/main/assets/keyboard.es5.html | 29 ++++++++++++ .../KMEA/app/src/main/assets/keyboard.html | 9 ---- .../java/com/keyman/engine/KMManager.java | 46 +++++++++++++++---- android/KMEA/build.sh | 2 + 4 files changed, 68 insertions(+), 18 deletions(-) create mode 100644 android/KMEA/app/src/main/assets/keyboard.es5.html diff --git a/android/KMEA/app/src/main/assets/keyboard.es5.html b/android/KMEA/app/src/main/assets/keyboard.es5.html new file mode 100644 index 0000000000..d9f1c938b0 --- /dev/null +++ b/android/KMEA/app/src/main/assets/keyboard.es5.html @@ -0,0 +1,29 @@ + + + + + + + Keyman + + + + + + + + + + + + diff --git a/android/KMEA/app/src/main/assets/keyboard.html b/android/KMEA/app/src/main/assets/keyboard.html index 599b3201bb..135081d9b7 100644 --- a/android/KMEA/app/src/main/assets/keyboard.html +++ b/android/KMEA/app/src/main/assets/keyboard.html @@ -7,15 +7,6 @@ Keyman - - - diff --git a/android/KMEA/app/src/main/java/com/keyman/engine/KMManager.java b/android/KMEA/app/src/main/java/com/keyman/engine/KMManager.java index 7f5f6fed2c..33fd7cffdd 100644 --- a/android/KMEA/app/src/main/java/com/keyman/engine/KMManager.java +++ b/android/KMEA/app/src/main/java/com/keyman/engine/KMManager.java @@ -288,8 +288,11 @@ public final class KMManager { // Keyman files protected static final String KMFilename_KeyboardHtml = "keyboard.html"; + protected static final String KMFilename_KeyboardHtml_Legacy = "keyboard.es5.html"; protected static final String KMFilename_JSEngine = "keymanweb-webview.js"; protected static final String KMFilename_JSEngine_Sourcemap = "keymanweb-webview.js.map"; + protected static final String KMFilename_JSLegacyEngine = "keymanweb-webview.es5.js"; + protected static final String KMFilename_JSLegacyEngine_Sourcemap = "keymanweb-webview.es5.js.map"; protected static final String KMFilename_JSSentry = "sentry.min.js"; protected static final String KMFilename_JSSentryInit = "keyman-sentry.js"; protected static final String KMFilename_AndroidHost = "android-host.js"; @@ -848,24 +851,45 @@ public final class KMManager { private static void copyAssets(Context context) { AssetManager assetManager = context.getAssets(); + + // Will build a temp WebView in order to check Chrome version internally. + boolean legacyMode = WebViewUtils.getEngineWebViewVersionStatus(context, null, null) != WebViewUtils.EngineWebViewVersionStatus.FULL; + try { // Copy KMW files - copyAsset(context, KMFilename_KeyboardHtml, "", true); - copyAsset(context, KMFilename_JSEngine, "", true); + if(legacyMode) { + // Replaces the standard ES6-friendly version of the host page with a legacy one that + // includes polyfill requests and that links the legacy, ES5-compatible version of KMW. + copyAssetWithRename(context, KMFilename_KeyboardHtml_Legacy, KMFilename_KeyboardHtml, "", true); + + copyAsset(context, KMFilename_JSLegacyEngine, "", true); + if (KMManager.isDebugMode()) { + copyAsset(context, KMFilename_JSLegacyEngine_Sourcemap, "", true); + } + } else { + copyAsset(context, KMFilename_KeyboardHtml, "", true); + + // For versions of Chrome with full ES6 support, we use the ES6 artifact. + copyAsset(context, KMFilename_JSEngine, "", true); + if (KMManager.isDebugMode()) { + copyAsset(context, KMFilename_JSEngine_Sourcemap, "", true); + } + } + // Is still built targeting ES5. copyAsset(context, KMFilename_JSSentry, "", true); copyAsset(context, KMFilename_JSSentryInit, "", true); copyAsset(context, KMFilename_AndroidHost, "", true); - if(KMManager.isDebugMode()) { - copyAsset(context, KMFilename_JSEngine_Sourcemap, "", true); - } copyAsset(context, KMFilename_KmwCss, "", true); copyAsset(context, KMFilename_KmwGlobeHintCss, "", true); copyAsset(context, KMFilename_Osk_Ttf_Font, "", true); // Copy default keyboard font copyAsset(context, KMDefault_KeyboardFont, "", true); - copyAsset(context, KMFilename_JSPolyfill, "", true); - copyAsset(context, KMFilename_JSPolyfill2, "", true); + + if(legacyMode) { + copyAsset(context, KMFilename_JSPolyfill, "", true); + copyAsset(context, KMFilename_JSPolyfill2, "", true); + } // Keyboard packages directory File packagesDir = new File(getPackagesDir()); @@ -961,6 +985,10 @@ public final class KMManager { } private static int copyAsset(Context context, String filename, String directory, boolean overwrite) { + return copyAssetWithRename(context, filename, filename, directory, overwrite); + } + + private static int copyAssetWithRename(Context context, String srcName, String destName, String directory, boolean overwrite) { int result; AssetManager assetManager = context.getAssets(); try { @@ -977,9 +1005,9 @@ public final class KMManager { dirPath = getResourceRoot(); } - File file = new File(dirPath, filename); + File file = new File(dirPath, destName); if (!file.exists() || overwrite) { - InputStream inputStream = assetManager.open(directory + filename); + InputStream inputStream = assetManager.open(directory + srcName); FileOutputStream outputStream = new FileOutputStream(file); FileUtils.copy(inputStream, outputStream); diff --git a/android/KMEA/build.sh b/android/KMEA/build.sh index ff78403c03..7ef77fff40 100755 --- a/android/KMEA/build.sh +++ b/android/KMEA/build.sh @@ -84,6 +84,8 @@ if builder_start_action build:engine; then echo "Copying Keyman Web artifacts" cp "$KEYMAN_WEB_ROOT/build/app/webview/$CONFIG/keymanweb-webview.js" "$ENGINE_ASSETS/keymanweb-webview.js" cp "$KEYMAN_WEB_ROOT/build/app/webview/$CONFIG/keymanweb-webview.js.map" "$ENGINE_ASSETS/keymanweb-webview.js.map" + cp "$KEYMAN_WEB_ROOT/build/app/webview/$CONFIG/keymanweb-webview.es5.js" "$ENGINE_ASSETS/keymanweb-webview.es5.js" + cp "$KEYMAN_WEB_ROOT/build/app/webview/$CONFIG/keymanweb-webview.es5.js.map" "$ENGINE_ASSETS/keymanweb-webview.es5.js.map" cp "$KEYMAN_WEB_ROOT/build/app/resources/osk/ajax-loader.gif" "$ENGINE_ASSETS/ajax-loader.gif" cp "$KEYMAN_WEB_ROOT/build/app/resources/osk/kmwosk.css" "$ENGINE_ASSETS/kmwosk.css" cp "$KEYMAN_WEB_ROOT/build/app/resources/osk/globe-hint.css" "$ENGINE_ASSETS/globe-hint.css" From 9c2e2b580bc212912d545468f20ecae99754a87d Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 19 Dec 2023 14:42:01 +0700 Subject: [PATCH 3/7] chore(android): oh right, .gitignore update --- android/.gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/android/.gitignore b/android/.gitignore index 9504f1f1b4..3d3618c78f 100644 --- a/android/.gitignore +++ b/android/.gitignore @@ -37,6 +37,8 @@ KMEA/**/assets/keymanandroid.js KMEA/**/assets/keyman.js.map KMEA/**/assets/keymanweb-webview.js KMEA/**/assets/keymanweb-webview.js.map +KMEA/**/assets/keymanweb-webview.es5.js +KMEA/**/assets/keymanweb-webview.es5.js.map KMEA/**/assets/sentry.min.js KMEA/**/assets/keyman-sentry.js KMEA/**/assets/es6-shim.min.js From f9df7638b7ff814f946010abd858f1e2ec52edf0 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 19 Dec 2023 14:42:32 +0700 Subject: [PATCH 4/7] fix(web): error that only appeared in ES6 mode (embedded-kbd specific) --- web/src/engine/osk/src/views/activator.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/web/src/engine/osk/src/views/activator.ts b/web/src/engine/osk/src/views/activator.ts index e2484ca6ba..d2b9ec33f5 100644 --- a/web/src/engine/osk/src/views/activator.ts +++ b/web/src/engine/osk/src/views/activator.ts @@ -34,6 +34,10 @@ export class StaticActivator extends Activator { return true; } + set enabled(value: boolean) { + // does nothing; it's static. + } + get activate(): boolean { return true; } From 639395e3b39a90850a3f139b9d1e9112db30b806 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 22 Dec 2023 13:28:32 +0700 Subject: [PATCH 5/7] fix(android): improper handling of map-polyfill link on last merge --- android/KMEA/app/src/main/assets/keyboard.es5.html | 1 + 1 file changed, 1 insertion(+) diff --git a/android/KMEA/app/src/main/assets/keyboard.es5.html b/android/KMEA/app/src/main/assets/keyboard.es5.html index d9f1c938b0..6384235456 100644 --- a/android/KMEA/app/src/main/assets/keyboard.es5.html +++ b/android/KMEA/app/src/main/assets/keyboard.es5.html @@ -16,6 +16,7 @@ --> + From d1a6cf3d4038d0e516247222d1e2e6e44587e72c Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 22 Dec 2023 14:01:12 +0700 Subject: [PATCH 6/7] chore(web): cleans lm-worker intermediate/, not just lib/ --- common/web/lm-worker/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/web/lm-worker/build.sh b/common/web/lm-worker/build.sh index 3732e41fd6..b4130a1244 100755 --- a/common/web/lm-worker/build.sh +++ b/common/web/lm-worker/build.sh @@ -105,6 +105,6 @@ function do_test() { } builder_run_action configure verify_npm_setup -builder_run_action clean rm -rf build/ +builder_run_action clean rm -rf build/ & rm -rf intermediate/ builder_run_action build do_build builder_run_action test do_test \ No newline at end of file From b3b11186ca0968da9401fae44ad3d9604ae20313 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 8 Jan 2024 13:55:25 +0700 Subject: [PATCH 7/7] chore(web): apply suggestion from code review Co-authored-by: Marc Durdin --- common/web/lm-worker/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/web/lm-worker/build.sh b/common/web/lm-worker/build.sh index b4130a1244..b7b5007161 100755 --- a/common/web/lm-worker/build.sh +++ b/common/web/lm-worker/build.sh @@ -105,6 +105,6 @@ function do_test() { } builder_run_action configure verify_npm_setup -builder_run_action clean rm -rf build/ & rm -rf intermediate/ +builder_run_action clean rm -rf build/ intermediate/ builder_run_action build do_build builder_run_action test do_test \ No newline at end of file