From 76807e478666ce7e3d8c16a9b8b6c78f0ccd5fce Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Tue, 19 May 2020 14:00:56 -0400 Subject: [PATCH 01/41] auto: increment master version to 14.0.78 --- HISTORY.md | 7 +++++++ VERSION.md | 2 +- common/core/web/input-processor/package-lock.json | 2 +- common/core/web/input-processor/package.json | 8 ++++---- common/core/web/keyboard-processor/package-lock.json | 2 +- common/core/web/keyboard-processor/package.json | 4 ++-- common/core/web/tools/recorder/package-lock.json | 2 +- common/core/web/tools/recorder/package.json | 8 ++++---- common/core/web/utils/package-lock.json | 2 +- common/core/web/utils/package.json | 2 +- common/lexical-model-types/package-lock.json | 2 +- common/lexical-model-types/package.json | 2 +- common/predictive-text/package-lock.json | 2 +- common/predictive-text/package.json | 4 ++-- lerna.json | 2 +- web/package-lock.json | 2 +- web/package.json | 12 ++++++------ 17 files changed, 36 insertions(+), 29 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index a40704ade2..cb7b7e4659 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,12 @@ # Keyman Version History +## 14.0.77 alpha 2020-05-19 + +* refactor(web/engine): Moves common utility functions into separate `web-utils` package (#3130) +* refactor(web/engine): renames DeviceSpec, moves to utils (#3132) +* fix(ios): Fixes keyboard metadata decoding, tweaks to project files (#3137) +* feat(windows): Use http: instead of file: for Configuration UI (#3127) + ## 14.0.76 alpha 2020-05-15 * fix(windows): use correct name for Sentry in C++ (#3129) diff --git a/VERSION.md b/VERSION.md index 5e0feefd1c..5206b08bb1 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -14.0.77 \ No newline at end of file +14.0.78 \ No newline at end of file diff --git a/common/core/web/input-processor/package-lock.json b/common/core/web/input-processor/package-lock.json index 5376fb80d0..03ce768cfa 100644 --- a/common/core/web/input-processor/package-lock.json +++ b/common/core/web/input-processor/package-lock.json @@ -1,6 +1,6 @@ { "name": "@keymanapp/input-processor", - "version": "14.0.77", + "version": "14.0.78", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/common/core/web/input-processor/package.json b/common/core/web/input-processor/package.json index c65c41f636..7690ef7658 100644 --- a/common/core/web/input-processor/package.json +++ b/common/core/web/input-processor/package.json @@ -1,6 +1,6 @@ { "name": "@keymanapp/input-processor", - "version": "14.0.77", + "version": "14.0.78", "description": "The core text and prediction processing engine for KeymanWeb", "repository": { "type": "git", @@ -28,9 +28,9 @@ "test": "bash ./unit_tests/test.sh" }, "dependencies": { - "@keymanapp/keyboard-processor": "^14.0.77", - "@keymanapp/lexical-model-layer": "^14.0.77", - "@keymanapp/lexical-model-types": "^14.0.77", + "@keymanapp/keyboard-processor": "^14.0.78", + "@keymanapp/lexical-model-layer": "^14.0.78", + "@keymanapp/lexical-model-types": "^14.0.78", "@keymanapp/web-utils": "^14.0.77", "@types/node": "^11.9.4", "eventemitter3": "^4.0.0", diff --git a/common/core/web/keyboard-processor/package-lock.json b/common/core/web/keyboard-processor/package-lock.json index 39dc169f25..7f1afcfd20 100644 --- a/common/core/web/keyboard-processor/package-lock.json +++ b/common/core/web/keyboard-processor/package-lock.json @@ -1,6 +1,6 @@ { "name": "@keymanapp/keyboard-processor", - "version": "14.0.77", + "version": "14.0.78", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/common/core/web/keyboard-processor/package.json b/common/core/web/keyboard-processor/package.json index 2d72cb0801..23db578ef6 100644 --- a/common/core/web/keyboard-processor/package.json +++ b/common/core/web/keyboard-processor/package.json @@ -1,6 +1,6 @@ { "name": "@keymanapp/keyboard-processor", - "version": "14.0.77", + "version": "14.0.78", "description": "Core module for Keyman keyboard support in KeymanWeb.", "repository": { "type": "git", @@ -30,8 +30,8 @@ "mocha": "mocha" }, "dependencies": { + "@keymanapp/lexical-model-types": "^14.0.78", "@keymanapp/web-utils": "^14.0.77", - "@keymanapp/lexical-model-types": "^14.0.77", "@types/node": "^11.9.4", "ts-node": "^8.0.2" } diff --git a/common/core/web/tools/recorder/package-lock.json b/common/core/web/tools/recorder/package-lock.json index 491ed7ad29..51287c0da7 100644 --- a/common/core/web/tools/recorder/package-lock.json +++ b/common/core/web/tools/recorder/package-lock.json @@ -1,6 +1,6 @@ { "name": "@keymanapp/recorder-core", - "version": "14.0.77", + "version": "14.0.78", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/common/core/web/tools/recorder/package.json b/common/core/web/tools/recorder/package.json index 96c43d9e89..cce5948b80 100644 --- a/common/core/web/tools/recorder/package.json +++ b/common/core/web/tools/recorder/package.json @@ -1,6 +1,6 @@ { "name": "@keymanapp/recorder-core", - "version": "14.0.77", + "version": "14.0.78", "description": "Core classes used to develop KeymanWeb test cases based on keystrokes", "main": "index.js", "scripts": { @@ -17,9 +17,9 @@ }, "homepage": "https://github.com/keymanapp/keyman#readme", "dependencies": { - "@keymanapp/keyboard-processor": "^14.0.77", - "@keymanapp/web-utils": "^14.0.77", - "@keymanapp/lexical-model-types": "^14.0.77" + "@keymanapp/keyboard-processor": "^14.0.78", + "@keymanapp/lexical-model-types": "^14.0.78", + "@keymanapp/web-utils": "^14.0.77" }, "devDependencies": { "typescript": "^3.7.2" diff --git a/common/core/web/utils/package-lock.json b/common/core/web/utils/package-lock.json index db9940f030..26b3c5f720 100644 --- a/common/core/web/utils/package-lock.json +++ b/common/core/web/utils/package-lock.json @@ -1,6 +1,6 @@ { "name": "@keymanapp/web-utils", - "version": "14.0.76", + "version": "14.0.78", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/common/core/web/utils/package.json b/common/core/web/utils/package.json index 5c809b9fea..530320674c 100644 --- a/common/core/web/utils/package.json +++ b/common/core/web/utils/package.json @@ -1,6 +1,6 @@ { "name": "@keymanapp/web-utils", - "version": "14.0.76", + "version": "14.0.78", "description": "Common utility functions used throughout other Keyman packages", "main": "index.js", "scripts": { diff --git a/common/lexical-model-types/package-lock.json b/common/lexical-model-types/package-lock.json index f2a9b4895e..558015421c 100644 --- a/common/lexical-model-types/package-lock.json +++ b/common/lexical-model-types/package-lock.json @@ -1,6 +1,6 @@ { "name": "@keymanapp/lexical-model-types", - "version": "14.0.77", + "version": "14.0.78", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/common/lexical-model-types/package.json b/common/lexical-model-types/package.json index aa577f3305..241f78bbed 100644 --- a/common/lexical-model-types/package.json +++ b/common/lexical-model-types/package.json @@ -1,6 +1,6 @@ { "name": "@keymanapp/lexical-model-types", - "version": "14.0.77", + "version": "14.0.78", "description": "Type definitions in used in the LMLayer (lexical model/predictive text) component of Keyman.", "types": "./index.d.ts", "scripts": { diff --git a/common/predictive-text/package-lock.json b/common/predictive-text/package-lock.json index dcf31ec62f..6e90be1cb8 100644 --- a/common/predictive-text/package-lock.json +++ b/common/predictive-text/package-lock.json @@ -1,6 +1,6 @@ { "name": "@keymanapp/lexical-model-layer", - "version": "14.0.77", + "version": "14.0.78", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/common/predictive-text/package.json b/common/predictive-text/package.json index f31c796aa1..5192580b71 100644 --- a/common/predictive-text/package.json +++ b/common/predictive-text/package.json @@ -1,6 +1,6 @@ { "name": "@keymanapp/lexical-model-layer", - "version": "14.0.77", + "version": "14.0.78", "description": "Keyman/Predictive Text integration layer", "main": "build/index.js", "scripts": { @@ -44,8 +44,8 @@ "typescript": "^3.7.2" }, "dependencies": { + "@keymanapp/lexical-model-types": "^14.0.78", "@keymanapp/web-utils": "^14.0.77", - "@keymanapp/lexical-model-types": "^14.0.77", "es6-shim": "^0.35.5", "string.prototype.codepointat": "^0.2.1", "string.prototype.startswith": "^0.2.0" diff --git a/lerna.json b/lerna.json index 961634f053..2981f9e5eb 100644 --- a/lerna.json +++ b/lerna.json @@ -8,5 +8,5 @@ "common/predictive-text", "web" ], - "version": "14.0.77" + "version": "14.0.78" } diff --git a/web/package-lock.json b/web/package-lock.json index 1ae91f7351..aa5301dfd8 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -1,6 +1,6 @@ { "name": "keyman", - "version": "14.0.77", + "version": "14.0.78", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/web/package.json b/web/package.json index 755a67ea93..bf89b75d2f 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "keyman", - "version": "14.0.77", + "version": "14.0.78", "description": "Facilitates text input in any language.", "repository": { "type": "git", @@ -48,11 +48,11 @@ "modernizr": "modernizr" }, "dependencies": { - "@keymanapp/input-processor": "^14.0.77", - "@keymanapp/keyboard-processor": "^14.0.77", - "@keymanapp/lexical-model-layer": "^14.0.77", - "@keymanapp/lexical-model-types": "^14.0.77", - "@keymanapp/recorder-core": "^14.0.77", + "@keymanapp/input-processor": "^14.0.78", + "@keymanapp/keyboard-processor": "^14.0.78", + "@keymanapp/lexical-model-layer": "^14.0.78", + "@keymanapp/lexical-model-types": "^14.0.78", + "@keymanapp/recorder-core": "^14.0.78", "@keymanapp/web-utils": "^14.0.77", "@types/node": "^11.9.4", "es6-shim": "^0.35.5", From cab9766fce380fbeace34d0d9e93ca75b5b7145e Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Wed, 13 May 2020 09:33:40 -0600 Subject: [PATCH 02/41] change(common/lmlayer): create package: @keymanapp/lexical-model-word-breakers --- common/lexical-model-word-breakers/README.md | 11 +++++ .../package-lock.json | 13 ++++++ .../lexical-model-word-breakers/package.json | 40 +++++++++++++++++++ lerna.json | 1 + 4 files changed, 65 insertions(+) create mode 100644 common/lexical-model-word-breakers/README.md create mode 100644 common/lexical-model-word-breakers/package-lock.json create mode 100644 common/lexical-model-word-breakers/package.json diff --git a/common/lexical-model-word-breakers/README.md b/common/lexical-model-word-breakers/README.md new file mode 100644 index 0000000000..5052cd34e6 --- /dev/null +++ b/common/lexical-model-word-breakers/README.md @@ -0,0 +1,11 @@ +# `@keymanapp/lexical-model-word-breakers` + +> TODO: description + +## Usage + +``` +const lexicalModelWordBreakers = require('@keymanapp/lexical-model-word-breakers'); + +// TODO: DEMONSTRATE API +``` diff --git a/common/lexical-model-word-breakers/package-lock.json b/common/lexical-model-word-breakers/package-lock.json new file mode 100644 index 0000000000..2082ed2212 --- /dev/null +++ b/common/lexical-model-word-breakers/package-lock.json @@ -0,0 +1,13 @@ +{ + "name": "@keymanapp/lexical-model-word-breakers", + "version": "14.0.75", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "typescript": { + "version": "3.9.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.2.tgz", + "integrity": "sha512-q2ktq4n/uLuNNShyayit+DTobV2ApPEo/6so68JaD5ojvc/6GClBipedB9zNWYxRSAlZXAe405Rlijzl6qDiSw==" + } + } +} diff --git a/common/lexical-model-word-breakers/package.json b/common/lexical-model-word-breakers/package.json new file mode 100644 index 0000000000..da989df52e --- /dev/null +++ b/common/lexical-model-word-breakers/package.json @@ -0,0 +1,40 @@ +{ + "name": "@keymanapp/lexical-model-word-breakers", + "version": "14.0.75", + "description": "Word breakers bundled in the LMLayer", + "keywords": [ + "lexical model", + "word breaker", + "lmlayer" + ], + "contributors": [ + "Eddie Antonio Santos " + ], + "homepage": "https://github.com/keymanapp/keyman", + "license": "MIT", + "main": "lib/index.js", + "directories": { + "lib": "lib", + "test": "tests" + }, + "files": [ + "lib" + ], + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/keymanapp/keyman.git" + }, + "scripts": { + "test": "echo \"Error: run tests from root\" && exit 1" + }, + "bugs": { + "url": "https://github.com/keymanapp/keyman/issues" + }, + "devDependencies": { + "typescript": "^3.2.1", + "@keymanapp/lexical-model-types": "^14.0.75" + } +} diff --git a/lerna.json b/lerna.json index 2981f9e5eb..e5b5b83b5a 100644 --- a/lerna.json +++ b/lerna.json @@ -6,6 +6,7 @@ "common/core/web/utils", "common/lexical-model-types", "common/predictive-text", + "common/lexical-model-word-breakers", "web" ], "version": "14.0.78" From 980d0c58f9248aaccff8a9a1f5b24cac4258bda8 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Wed, 13 May 2020 09:39:32 -0600 Subject: [PATCH 03/41] refactor(common/lmlayer): move one test over --- .../package-lock.json | 244 +++++++++++++++++- .../lexical-model-word-breakers/package.json | 12 +- .../test/test-ascii-word-breaker.js} | 0 3 files changed, 250 insertions(+), 6 deletions(-) rename common/{predictive-text/unit_tests/headless/ascii-word-breaker.js => lexical-model-word-breakers/test/test-ascii-word-breaker.js} (100%) diff --git a/common/lexical-model-word-breakers/package-lock.json b/common/lexical-model-word-breakers/package-lock.json index 2082ed2212..de0642f309 100644 --- a/common/lexical-model-word-breakers/package-lock.json +++ b/common/lexical-model-word-breakers/package-lock.json @@ -4,10 +4,252 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "assertion-error": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", + "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==", + "dev": true + }, + "balanced-match": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", + "dev": true + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "browser-stdout": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", + "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", + "dev": true + }, + "chai": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/chai/-/chai-4.2.0.tgz", + "integrity": "sha512-XQU3bhBukrOsQCuwZndwGcCVQHyZi53fQ6Ys1Fym7E4olpIqqZZhhoFJoaKVvV17lWQoXYwgWN2nF5crA8J2jw==", + "dev": true, + "requires": { + "assertion-error": "^1.1.0", + "check-error": "^1.0.2", + "deep-eql": "^3.0.1", + "get-func-name": "^2.0.0", + "pathval": "^1.1.0", + "type-detect": "^4.0.5" + } + }, + "check-error": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz", + "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=", + "dev": true + }, + "commander": { + "version": "2.15.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.1.tgz", + "integrity": "sha512-VlfT9F3V0v+jr4yxPc5gg9s62/fIVWsd2Bk2iD435um1NlGMYdVCq+MjcXnhYq2icNOizHr1kK+5TI6H0Hy0ag==", + "dev": true + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "dev": true + }, + "debug": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", + "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "dev": true, + "requires": { + "ms": "2.0.0" + } + }, + "deep-eql": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz", + "integrity": "sha512-+QeIQyN5ZuO+3Uk5DYh6/1eKO0m0YmJFGNmFHGACpf1ClL1nmlV/p4gNgbl2pJGxgXb4faqo6UE+M5ACEMyVcw==", + "dev": true, + "requires": { + "type-detect": "^4.0.0" + } + }, + "diff": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", + "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", + "dev": true + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "dev": true + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "dev": true + }, + "get-func-name": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", + "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=", + "dev": true + }, + "glob": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", + "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "growl": { + "version": "1.10.5", + "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", + "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", + "dev": true + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true + }, + "he": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", + "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=", + "dev": true + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "dev": true, + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dev": true, + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "minimist": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", + "dev": true + }, + "mkdirp": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "dev": true, + "requires": { + "minimist": "0.0.8" + } + }, + "mocha": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-5.2.0.tgz", + "integrity": "sha512-2IUgKDhc3J7Uug+FxMXuqIyYzH7gJjXECKe/w43IGgQHTSj3InJi+yAA7T24L9bQMRKiUEHxEX37G5JpVUGLcQ==", + "dev": true, + "requires": { + "browser-stdout": "1.3.1", + "commander": "2.15.1", + "debug": "3.1.0", + "diff": "3.5.0", + "escape-string-regexp": "1.0.5", + "glob": "7.1.2", + "growl": "1.10.5", + "he": "1.1.1", + "minimatch": "3.0.4", + "mkdirp": "0.5.1", + "supports-color": "5.4.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dev": true, + "requires": { + "wrappy": "1" + } + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "dev": true + }, + "pathval": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.0.tgz", + "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=", + "dev": true + }, + "supports-color": { + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz", + "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==", + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } + }, + "type-detect": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", + "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", + "dev": true + }, "typescript": { "version": "3.9.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.2.tgz", - "integrity": "sha512-q2ktq4n/uLuNNShyayit+DTobV2ApPEo/6so68JaD5ojvc/6GClBipedB9zNWYxRSAlZXAe405Rlijzl6qDiSw==" + "integrity": "sha512-q2ktq4n/uLuNNShyayit+DTobV2ApPEo/6so68JaD5ojvc/6GClBipedB9zNWYxRSAlZXAe405Rlijzl6qDiSw==", + "dev": true + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true } } } diff --git a/common/lexical-model-word-breakers/package.json b/common/lexical-model-word-breakers/package.json index da989df52e..2e17111b4e 100644 --- a/common/lexical-model-word-breakers/package.json +++ b/common/lexical-model-word-breakers/package.json @@ -8,14 +8,14 @@ "lmlayer" ], "contributors": [ - "Eddie Antonio Santos " + "Eddie Antonio Santos " ], "homepage": "https://github.com/keymanapp/keyman", "license": "MIT", "main": "lib/index.js", "directories": { "lib": "lib", - "test": "tests" + "test": "test" }, "files": [ "lib" @@ -28,13 +28,15 @@ "url": "git+https://github.com/keymanapp/keyman.git" }, "scripts": { - "test": "echo \"Error: run tests from root\" && exit 1" + "test": "mocha" }, "bugs": { "url": "https://github.com/keymanapp/keyman/issues" }, "devDependencies": { - "typescript": "^3.2.1", - "@keymanapp/lexical-model-types": "^14.0.75" + "@keymanapp/lexical-model-types": "^14.0.75", + "chai": "^4.2.0", + "mocha": "^5.2.0", + "typescript": "^3.2.1" } } diff --git a/common/predictive-text/unit_tests/headless/ascii-word-breaker.js b/common/lexical-model-word-breakers/test/test-ascii-word-breaker.js similarity index 100% rename from common/predictive-text/unit_tests/headless/ascii-word-breaker.js rename to common/lexical-model-word-breakers/test/test-ascii-word-breaker.js From b4efd9a6a1f84baf9d107b9c746b74434dbf2680 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Wed, 13 May 2020 09:58:15 -0600 Subject: [PATCH 04/41] refactor(common/lmlayer): move ASCII word breaker to new package --- common/lexical-model-word-breakers/.gitignore | 3 ++ .../lexical-model-word-breakers/package.json | 2 +- .../src/ascii-word-breaker.ts | 39 +++++++++++++++++++ .../lexical-model-word-breakers/src/index.ts | 2 + .../test/test-ascii-word-breaker.js | 5 +-- .../lexical-model-word-breakers/tsconfig.json | 18 +++++++++ .../word_breaking/ascii-word-breaker.ts | 39 ------------------- 7 files changed, 65 insertions(+), 43 deletions(-) create mode 100644 common/lexical-model-word-breakers/.gitignore create mode 100644 common/lexical-model-word-breakers/src/ascii-word-breaker.ts create mode 100644 common/lexical-model-word-breakers/src/index.ts create mode 100644 common/lexical-model-word-breakers/tsconfig.json delete mode 100644 common/predictive-text/worker/word_breaking/ascii-word-breaker.ts diff --git a/common/lexical-model-word-breakers/.gitignore b/common/lexical-model-word-breakers/.gitignore new file mode 100644 index 0000000000..75871e6dc2 --- /dev/null +++ b/common/lexical-model-word-breakers/.gitignore @@ -0,0 +1,3 @@ +*.js +*.js.map +*.d.ts diff --git a/common/lexical-model-word-breakers/package.json b/common/lexical-model-word-breakers/package.json index 2e17111b4e..57c357f026 100644 --- a/common/lexical-model-word-breakers/package.json +++ b/common/lexical-model-word-breakers/package.json @@ -14,7 +14,6 @@ "license": "MIT", "main": "lib/index.js", "directories": { - "lib": "lib", "test": "test" }, "files": [ @@ -28,6 +27,7 @@ "url": "git+https://github.com/keymanapp/keyman.git" }, "scripts": { + "build": "tsc", "test": "mocha" }, "bugs": { diff --git a/common/lexical-model-word-breakers/src/ascii-word-breaker.ts b/common/lexical-model-word-breakers/src/ascii-word-breaker.ts new file mode 100644 index 0000000000..7a933f0107 --- /dev/null +++ b/common/lexical-model-word-breakers/src/ascii-word-breaker.ts @@ -0,0 +1,39 @@ +/// + +/** + * Splits ASCII words. + * + * @param phrase + */ +export function ascii(phrase: string): Span[] { + let matchWord = /[A-Za-z0-9']+/g; + let words: Span[] = []; + let match: RegExpExecArray; + while ((match = matchWord.exec(phrase)) !== null) { + words.push(new RegExpDerivedSpan(match[0], match.index)); + } + + return words; +} + +/** + * A concrete span class that derives its properties from the result of + * RegExp.exec() array. + */ +class RegExpDerivedSpan implements Span { + readonly text: string; + readonly start: number; + + constructor(text: string, start: number) { + this.text = text; + this.start = start; + } + + get length(): number { + return this.text.length; + } + + get end(): number { + return this.start + this.text.length; + } +} diff --git a/common/lexical-model-word-breakers/src/index.ts b/common/lexical-model-word-breakers/src/index.ts new file mode 100644 index 0000000000..cd84d6d113 --- /dev/null +++ b/common/lexical-model-word-breakers/src/index.ts @@ -0,0 +1,2 @@ +import {ascii} from "./ascii-word-breaker"; +export {ascii}; diff --git a/common/lexical-model-word-breakers/test/test-ascii-word-breaker.js b/common/lexical-model-word-breakers/test/test-ascii-word-breaker.js index 7e6d129905..d529331eb0 100644 --- a/common/lexical-model-word-breakers/test/test-ascii-word-breaker.js +++ b/common/lexical-model-word-breakers/test/test-ascii-word-breaker.js @@ -1,6 +1,5 @@ var assert = require('chai').assert; - -var breakASCIIWords = require('../../build/intermediate').wordBreakers['ascii']; +var breakASCIIWords = require('../').ascii; describe('The ASCII word breaker', function () { it('should break simple English sentences', function () { @@ -8,4 +7,4 @@ describe('The ASCII word breaker', function () { let words = breaks.map(span => span.text); assert.deepEqual(words, ['Look', 'The', 'quick', 'brown', 'fox', 'jumps', 'over', 'the', 'lazy', 'dog']); }); -}); \ No newline at end of file +}); diff --git a/common/lexical-model-word-breakers/tsconfig.json b/common/lexical-model-word-breakers/tsconfig.json new file mode 100644 index 0000000000..358a3f8b91 --- /dev/null +++ b/common/lexical-model-word-breakers/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "module": "commonjs", + "declaration": true, + "sourceMap": true, + "outDir": "lib", + "allowJs": false, + "lib": ["es6"], + "target": "es5" + }, + "include": [ + "src/**/*" + ], + "exclude": [ + "node_modules", + "test/**/*.ts" + ] +} diff --git a/common/predictive-text/worker/word_breaking/ascii-word-breaker.ts b/common/predictive-text/worker/word_breaking/ascii-word-breaker.ts deleted file mode 100644 index c7c10726bb..0000000000 --- a/common/predictive-text/worker/word_breaking/ascii-word-breaker.ts +++ /dev/null @@ -1,39 +0,0 @@ -namespace wordBreakers { - /** - * A concrete span class that derives its properties from the result of - * RegExp.exec() array. - */ - class RegExpDerivedSpan implements Span { - readonly text: string; - readonly start: number; - - constructor(text: string, start: number) { - this.text = text; - this.start = start; - } - - get length(): number { - return this.text.length; - } - - get end(): number { - return this.start + this.text.length; - } - } - - /** - * Splits ASCII words. - * - * @param phrase - */ - export function ascii(phrase: string): Span[] { - let matchWord = /[A-Za-z0-9']+/g; - let words: Span[] = []; - let match: RegExpExecArray; - while ((match = matchWord.exec(phrase)) !== null) { - words.push(new RegExpDerivedSpan(match[0], match.index)); - } - - return words; - } -} \ No newline at end of file From 2a31420f6846d8a3e2535b7eb8d3986726e92a47 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Wed, 13 May 2020 10:04:31 -0600 Subject: [PATCH 05/41] refactor(common/lmlayer): move default word-breaker tests to package --- .../test/test-default-word-breaker.js} | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) rename common/{predictive-text/unit_tests/headless/default-word-breaker.js => lexical-model-word-breakers/test/test-default-word-breaker.js} (82%) diff --git a/common/predictive-text/unit_tests/headless/default-word-breaker.js b/common/lexical-model-word-breakers/test/test-default-word-breaker.js similarity index 82% rename from common/predictive-text/unit_tests/headless/default-word-breaker.js rename to common/lexical-model-word-breakers/test/test-default-word-breaker.js index bcf08efd83..9c9ac3bb58 100644 --- a/common/predictive-text/unit_tests/headless/default-word-breaker.js +++ b/common/lexical-model-word-breakers/test/test-default-word-breaker.js @@ -1,10 +1,10 @@ /** - * Smoke-test the default + * Smoke-test the default */ var assert = require('chai').assert; -var TrieModel = require('../../build/intermediate').models.TrieModel; +//var TrieModel = require('../../build/intermediate').models.TrieModel; -var breakWords = require('../../build/intermediate').wordBreakers['default']; +var breakWords = require('..').default; const SHY = '\u00AD'; describe('The default word breaker', function () { @@ -23,13 +23,13 @@ describe('The default word breaker', function () { // The following tests are performed with model integration as an internal // test for the wordbreaking API. - it('recognizes a word at end of complete lefthand context', function () { + it.skip('recognizes a word at end of complete lefthand context', function () { var model = new TrieModel(jsonFixture('tries/english-1000'), { wordBreaker: breakWords // wordBreakers['default'] when fully integrated. }); // Standard case - wordbreaking at the end of a word. - var context = { + var context = { left: 'The quick brown fox jumped', startOfBuffer: true, right: ' over the lazy dog.', endOfBuffer: true }; @@ -41,13 +41,13 @@ describe('The default word breaker', function () { // Same test as before, but we want to be sure the start/end of buffer flags // don't affect our results. - it('recognizes a word at end of incomplete lefthand context', function () { + it.skip('recognizes a word at end of incomplete lefthand context', function () { var model = new TrieModel(jsonFixture('tries/english-1000'), { wordBreaker: breakWords }); // Standard case - wordbreaking at the end of a word. - var context = { + var context = { left: 'The quick brown fox jumped', startOfBuffer: false, right: ' over the lazy dog.', endOfBuffer: false }; @@ -57,13 +57,13 @@ describe('The default word breaker', function () { assert.strictEqual(broken, 'jumped'); }); - it('returns text for a word in-progress', function() { + it.skip('returns text for a word in-progress', function() { var model = new TrieModel(jsonFixture('tries/english-1000'), { wordBreaker: breakWords }); // Standard case - midword (xylophone) call - var context = { + var context = { left: 'xyl', startOfBuffer: true, right: '', endOfBuffer: true }; @@ -72,14 +72,14 @@ describe('The default word breaker', function () { assert.strictEqual(broken, 'xyl'); }); - - it('returns empty string when called without word text', function() { + + it.skip('returns empty string when called without word text', function() { var model = new TrieModel(jsonFixture('tries/english-1000'), { wordBreaker: breakWords }); // Wordbreaking on a empty space => no word. - context = { + context = { left: 'The quick brown fox jumped ', startOfBuffer: true, right: 'over the lazy dog.', endOfBuffer: true }; @@ -89,13 +89,13 @@ describe('The default word breaker', function () { assert.strictEqual(broken, ''); }); - it('returns empty string when called with empty context', function() { + it.skip('returns empty string when called with empty context', function() { var model = new TrieModel(jsonFixture('tries/english-1000'), { wordBreaker: breakWords }); // Wordbreaking on a empty space => no word. - context = { + context = { left: '', startOfBuffer: true, right: '', endOfBuffer: true }; @@ -105,13 +105,13 @@ describe('The default word breaker', function () { assert.strictEqual(broken, ''); }); - it('returns empty string when called with nil context', function() { + it.skip('returns empty string when called with nil context', function() { var model = new TrieModel(jsonFixture('tries/english-1000'), { wordBreaker: breakWords }); // Wordbreaking on a empty space => no word. - context = { + context = { left: '', startOfBuffer: false, right: '', endOfBuffer: false }; @@ -128,13 +128,13 @@ describe('The default word breaker', function () { // A limitation of the current implementation; we should fix this before release. // Then again, when typing this is probably fine; just not when not typing. - context = { + context = { left: 'The quick brown fox jum', startOfBuffer: true, right: 'ped over the lazy dog.', endOfBuffer: true }; broken = model.wordbreak(context); - assert.strictEqual(broken, 'jumped'); // Current result: 'jum'. + assert.strictEqual(broken, 'jumped'); // Current result: 'jum'. }); }); From eb9bfd5ef067669c96a6cf7af8cc816f93fecc30 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Wed, 13 May 2020 10:05:16 -0600 Subject: [PATCH 06/41] chore(common/lmlayer): run build before tests and packaging --- common/lexical-model-word-breakers/package.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/lexical-model-word-breakers/package.json b/common/lexical-model-word-breakers/package.json index 57c357f026..50e394788a 100644 --- a/common/lexical-model-word-breakers/package.json +++ b/common/lexical-model-word-breakers/package.json @@ -28,6 +28,8 @@ }, "scripts": { "build": "tsc", + "prepublish": "npm run build", + "pretest": "npm run build", "test": "mocha" }, "bugs": { From 1f4de1ba93817b82b350fe40623f0759394f713f Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Wed, 13 May 2020 10:14:22 -0600 Subject: [PATCH 07/41] refactor(common/lmlayer): move default word-breaker to @keymanapp/lexical-model-word-breakers --- .../src/default}/data.ts | 4 - .../src/default/index.ts | 363 +++++++++++++++++ .../lexical-model-word-breakers/src/index.ts | 3 + .../default-word-breaking/index.ts | 378 ------------------ 4 files changed, 366 insertions(+), 382 deletions(-) rename common/{predictive-text/worker/word_breaking/default-word-breaking => lexical-model-word-breakers/src/default}/data.ts (99%) create mode 100644 common/lexical-model-word-breakers/src/default/index.ts delete mode 100644 common/predictive-text/worker/word_breaking/default-word-breaking/index.ts diff --git a/common/predictive-text/worker/word_breaking/default-word-breaking/data.ts b/common/lexical-model-word-breakers/src/default/data.ts similarity index 99% rename from common/predictive-text/worker/word_breaking/default-word-breaking/data.ts rename to common/lexical-model-word-breakers/src/default/data.ts index a0bb37c556..0f81b1a5c4 100644 --- a/common/predictive-text/worker/word_breaking/default-word-breaking/data.ts +++ b/common/lexical-model-word-breakers/src/default/data.ts @@ -1,5 +1,3 @@ -namespace wordBreakers { -export namespace data { // Automatically generated file. DO NOT MODIFY. /** * Valid values for a word break property. @@ -1292,5 +1290,3 @@ export const WORD_BREAK_PROPERTY: [number, number, WordBreakProperty][] = [ [/*start*/ 0xE0020, /*end*/ 0xE007F, WordBreakProperty.Extend], [/*start*/ 0xE0100, /*end*/ 0xE01EF, WordBreakProperty.Extend], ]; -} -} \ No newline at end of file diff --git a/common/lexical-model-word-breakers/src/default/index.ts b/common/lexical-model-word-breakers/src/default/index.ts new file mode 100644 index 0000000000..2eeabff507 --- /dev/null +++ b/common/lexical-model-word-breakers/src/default/index.ts @@ -0,0 +1,363 @@ +import {WordBreakProperty, WORD_BREAK_PROPERTY, I} from './data'; + +/** + * Word breaker based on Unicode Standard Annex #29, Section 4.1: + * Default Word Boundary Specification. + * + * @see http://unicode.org/reports/tr29/#Word_Boundaries + * @see https://github.com/eddieantonio/unicode-default-word-boundary/tree/v12.0.0 + */ +export function default_(text: string): Span[] { + let boundaries = findBoundaries(text); + if (boundaries.length == 0) { + return []; + } + + // All non-empty strings have at least TWO boundaries at the start and end of + // the string. + let spans = []; + for (let i = 0; i < boundaries.length - 1; i++) { + let start = boundaries[i]; + let end = boundaries[i + 1]; + let span = new LazySpan(text, start, end); + + if (isNonSpace(span.text)) { + spans.push(span); + // Preserve a sequence-final space if it exists. Needed to signal "end of word". + } else if (i == boundaries.length - 2) { // if "we just checked the final boundary"... + // We don't want to return the whitespace itself; the correct token is simply ''. + span = new LazySpan(text, end, end); + spans.push(span); + } + } + return spans; +} + +/** + * A span that does not cut out the substring until it absolutely has to! + */ +class LazySpan implements Span { + private _source: string; + readonly start: number; + readonly end: number; + constructor(source: string, start: number, end: number) { + this._source = source; + this.start = start; + this.end = end; + } + + get text(): string { + return this._source.substring(this.start, this.end); + } + + get length(): number { + return this.end - this.start; + } +} + +/** + * Returns true when the chunk does not solely consist of whitespace. + * + * @param chunk a chunk of text. Starts and ends at word boundaries. + */ +function isNonSpace(chunk: string): boolean { + return !Array.from(chunk).map(property).every(wb => ( + wb === WordBreakProperty.CR || + wb === WordBreakProperty.LF || + wb === WordBreakProperty.Newline || + wb === WordBreakProperty.WSegSpace + )); +} + +/** + * Yields a series of string indices where a word break should + * occur. That is, there should be a break BEFORE each string + * index yielded by this generator. + * + * @param text Text to find word boundaries in. + */ +function findBoundaries(text: string): number[] { + // WB1 and WB2: no boundaries if given an empty string. + if (text.length === 0) { + // There are no boundaries in an empty string! + return []; + } + + // This algorithm works by maintaining a sliding window of four SCALAR VALUES. + // + // - Scalar values? JavaScript strings are NOT actually a string of + // Unicode code points; some characters are made up of TWO + // JavaScript indices. e.g., + // "💩".length === 2; + // "💩"[0] === '\uD83D'; + // "💩"[1] === '\uDCA9'; + // + // These characters that are represented by TWO indices are + // called "surrogate pairs". Since we don't want to be in the + // "middle" of a character, make sure we're always advancing + // by scalar values, and NOT indices. That means, we sometimes + // need to advance by TWO indices, not just one. + // - Four values? Some rules look at what's to the left of + // left, and some look at what's to the right of right. So + // keep track of this! + + let boundaries = []; + + let rightPos: number; + let lookaheadPos = 0; // lookahead, one scalar value to the right of right. + // Before the start of the string is also the start of the string. + let lookbehind: WordBreakProperty; + let left = WordBreakProperty.sot; + let right = WordBreakProperty.sot; + let lookahead = wordbreakPropertyAt(0); + // Count RIs to make sure we're not splitting emoji flags: + let nConsecutiveRegionalIndicators = 0; + + do { + // Shift all positions, one scalar value to the right. + rightPos = lookaheadPos; + lookaheadPos = positionAfter(lookaheadPos); + // Shift all properties, one scalar value to the right. + [lookbehind, left, right, lookahead] = + [left, right, lookahead, wordbreakPropertyAt(lookaheadPos)]; + + // Break at the start and end of text, unless the text is empty. + // WB1: Break at start of text... + if (left === WordBreakProperty.sot) { + boundaries.push(rightPos); + continue; + } + // WB2: Break at the end of text... + if (right === WordBreakProperty.eot) { + boundaries.push(rightPos); + break; // Reached the end of the string. We're done! + } + // WB3: Do not break within CRLF: + if (left === WordBreakProperty.CR && right === WordBreakProperty.LF) + continue; + // WB3b: Otherwise, break after... + if (left === WordBreakProperty.Newline || + left === WordBreakProperty.CR || + left === WordBreakProperty.LF) { + boundaries.push(rightPos); + continue; + } + // WB3a: ...and before newlines + if (right === WordBreakProperty.Newline || + right === WordBreakProperty.CR || + right === WordBreakProperty.LF) { + boundaries.push(rightPos); + continue; + } + + // TODO: WB3c is not implemented, due to its complex, error-prone + // implementation, requiring a ginormous regexp, and the fact that + // the only thing it does is prevent big emoji sequences from being + // split up, like 🧚🏼‍♂️ + // https://www.unicode.org/Public/emoji/12.0/emoji-zwj-sequences.txt + + // WB3d: Keep horizontal whitespace together + if (left === WordBreakProperty.WSegSpace && right == WordBreakProperty.WSegSpace) + continue; + + // WB4: Ignore format and extend characters + // This is to keep grapheme clusters together! + // See: Section 6.2: https://unicode.org/reports/tr29/#Grapheme_Cluster_and_Format_Rules + // N.B.: The rule about "except after sot, CR, LF, and + // Newline" already been by WB1, WB2, WB3a, and WB3b above. + while (right === WordBreakProperty.Format || + right === WordBreakProperty.Extend || + right === WordBreakProperty.ZWJ) { + // Continue advancing in the string, as if these + // characters do not exist. DO NOT update left and + // lookbehind however! + [rightPos, lookaheadPos] = [lookaheadPos, positionAfter(lookaheadPos)]; + [right, lookahead] = [lookahead, wordbreakPropertyAt(lookaheadPos)]; + } + // In ignoring the characters in the previous loop, we could + // have fallen off the end of the string, so end the loop + // prematurely if that happens! + if (right === WordBreakProperty.eot) { + boundaries.push(rightPos); + break; + } + // WB4 (continued): Lookahead must ALSO ignore these format, + // extend, ZWJ characters! + while (lookahead === WordBreakProperty.Format || + lookahead === WordBreakProperty.Extend || + lookahead === WordBreakProperty.ZWJ) { + // Continue advancing in the string, as if these + // characters do not exist. DO NOT update left and right, + // however! + lookaheadPos = positionAfter(lookaheadPos); + lookahead = wordbreakPropertyAt(lookaheadPos); + } + + // WB5: Do not break between most letters. + if (isAHLetter(left) && isAHLetter(right)) + continue; + // Do not break across certain punctuation + // WB6: (Don't break before apostrophes in contractions) + if (isAHLetter(left) && isAHLetter(lookahead) && + (right === WordBreakProperty.MidLetter || isMidNumLetQ(right))) + continue; + // WB7: (Don't break after apostrophes in contractions) + if (isAHLetter(lookbehind) && isAHLetter(right) && + (left === WordBreakProperty.MidLetter || isMidNumLetQ(left))) + continue; + // WB7a + if (left === WordBreakProperty.Hebrew_Letter && right === WordBreakProperty.Single_Quote) + continue; + // WB7b + if (left === WordBreakProperty.Hebrew_Letter && right === WordBreakProperty.Double_Quote && + lookahead === WordBreakProperty.Hebrew_Letter) + continue; + // WB7c + if (lookbehind === WordBreakProperty.Hebrew_Letter && left === WordBreakProperty.Double_Quote && + right === WordBreakProperty.Hebrew_Letter) + continue; + // Do not break within sequences of digits, or digits adjacent to letters. + // e.g., "3a" or "A3" + // WB8 + if (left === WordBreakProperty.Numeric && right === WordBreakProperty.Numeric) + continue; + // WB9 + if (isAHLetter(left) && right === WordBreakProperty.Numeric) + continue; + // WB10 + if (left === WordBreakProperty.Numeric && isAHLetter(right)) + continue; + // Do not break within sequences, such as 3.2, 3,456.789 + // WB11 + if (lookbehind === WordBreakProperty.Numeric && right === WordBreakProperty.Numeric && + (left === WordBreakProperty.MidNum || isMidNumLetQ(left))) + continue; + // WB12 + if (left === WordBreakProperty.Numeric && lookahead === WordBreakProperty.Numeric && + (right === WordBreakProperty.MidNum || isMidNumLetQ(right))) + continue; + // WB13: Do not break between Katakana + if (left === WordBreakProperty.Katakana && right === WordBreakProperty.Katakana) + continue; + // Do not break from extenders (e.g., U+202F NARROW NO-BREAK SPACE) + // WB13a + if ((isAHLetter(left) || + left === WordBreakProperty.Numeric || + left === WordBreakProperty.Katakana || + left === WordBreakProperty.ExtendNumLet) && + right === WordBreakProperty.ExtendNumLet) + continue; + // WB13b + if ((isAHLetter(right) || + right === WordBreakProperty.Numeric || + right === WordBreakProperty.Katakana) && left === WordBreakProperty.ExtendNumLet) + continue; + + // WB15 & WB16: + // Do not break within emoji flag sequences. That is, do not break between + // regional indicator (RI) symbols if there is an odd number of RI + // characters before the break point. + if (right === WordBreakProperty.Regional_Indicator) { + // Emoji flags are actually composed of TWO scalar values, each being a + // "regional indicator". These indicators correspond to Latin letters. Put + // two of them together, and they spell out an ISO 3166-1-alpha-2 country + // code. Since these always come in pairs, NEVER split the pairs! So, if + // we happen to be inside the middle of an odd numbered of + // Regional_Indicators, DON'T SPLIT IT! + nConsecutiveRegionalIndicators += 1; + if ((nConsecutiveRegionalIndicators % 2) == 1) { + continue; + } + } else { + nConsecutiveRegionalIndicators = 0; + } + // WB999: Otherwise, break EVERYWHERE (including around ideographs) + boundaries.push(rightPos); + } while (rightPos < text.length); + + return boundaries; + + ///// Internal utility functions ///// + + /** + * Returns the position of the start of the next scalar value. This jumps + * over surrogate pairs. + * + * If asked for the character AFTER the end of the string, this always + * returns the length of the string. + */ + function positionAfter(pos: number): number { + if (pos >= text.length) { + return text.length; + } else if (isStartOfSurrogatePair(text[pos])) { + return pos + 2; + } + return pos + 1; + } + + /** + * Return the value of the Word_Break property at the given string index. + * @param pos position in the text. + */ + function wordbreakPropertyAt(pos: number) { + if (pos < 0) { + return WordBreakProperty.sot; // Always "start of string" before the string starts! + } else if (pos >= text.length) { + return WordBreakProperty.eot; // Always "end of string" after the string ends! + } else if (isStartOfSurrogatePair(text[pos])) { + // Surrogate pairs the next TWO items from the string! + return property(text[pos] + text[pos + 1]); + } + return property(text[pos]); + } + + // Word_Break rule macros + // See: https://unicode.org/reports/tr29/#WB_Rule_Macros + function isAHLetter(prop: WordBreakProperty): boolean { + return prop === WordBreakProperty.ALetter || + prop === WordBreakProperty.Hebrew_Letter; + } + + function isMidNumLetQ(prop: WordBreakProperty): boolean { + return prop === WordBreakProperty.MidNumLet || + prop === WordBreakProperty.Single_Quote; + } +} + +function isStartOfSurrogatePair(character: string) { + let codeUnit = character.charCodeAt(0); + return codeUnit >= 0xD800 && codeUnit <= 0xDBFF; +} + +/** + * Return the Word_Break property value for a character. + * Note that + * @param character a scalar value + */ +function property(character: string): WordBreakProperty { + // This MUST be a scalar value. + // TODO: remove dependence on character.codepointAt()? + let codepoint = character.codePointAt(0) as number; + return searchForProperty(codepoint, 0, WORD_BREAK_PROPERTY.length - 1); +} + +/** + * Binary search for the word break property of a given CODE POINT. + */ +function searchForProperty(codePoint: number, left: number, right: number): WordBreakProperty { + // All items that are not found in the array are assigned the 'Other' property. + if (right < left) { + return WordBreakProperty.Other; + } + + let midpoint = left + ~~((right - left) / 2); + let candidate = WORD_BREAK_PROPERTY[midpoint]; + if (codePoint < candidate[I.Start]) { + return searchForProperty(codePoint, left, midpoint - 1); + } else if (codePoint > candidate[I.End]) { + return searchForProperty(codePoint, midpoint + 1, right); + } else { + // We found it! + return candidate[I.Value]; + } +} diff --git a/common/lexical-model-word-breakers/src/index.ts b/common/lexical-model-word-breakers/src/index.ts index cd84d6d113..22c6cd74a9 100644 --- a/common/lexical-model-word-breakers/src/index.ts +++ b/common/lexical-model-word-breakers/src/index.ts @@ -1,2 +1,5 @@ import {ascii} from "./ascii-word-breaker"; +import {default_} from "./default"; + export {ascii}; +export {default_ as default}; diff --git a/common/predictive-text/worker/word_breaking/default-word-breaking/index.ts b/common/predictive-text/worker/word_breaking/default-word-breaking/index.ts deleted file mode 100644 index f35b22ac80..0000000000 --- a/common/predictive-text/worker/word_breaking/default-word-breaking/index.ts +++ /dev/null @@ -1,378 +0,0 @@ -// Include the word-breaking data here: -/// -namespace wordBreakers { - /** - * Word breaker based on Unicode Standard Annex #29, Section 4.1: - * Default Word Boundary Specification. - * - * @see http://unicode.org/reports/tr29/#Word_Boundaries - * @see https://github.com/eddieantonio/unicode-default-word-boundary/tree/v12.0.0 - */ - export function default_(text: string): Span[] { - let boundaries = findBoundaries(text); - if (boundaries.length == 0) { - return []; - } - - // All non-empty strings have at least TWO boundaries at the start and end of - // the string. - let spans = []; - for (let i = 0; i < boundaries.length - 1; i++) { - let start = boundaries[i]; - let end = boundaries[i + 1]; - let span = new LazySpan(text, start, end); - - if (isNonSpace(span.text)) { - spans.push(span); - // Preserve a sequence-final space if it exists. Needed to signal "end of word". - } else if (i == boundaries.length - 2) { // if "we just checked the final boundary"... - // We don't want to return the whitespace itself; the correct token is simply ''. - span = new LazySpan(text, end, end); - spans.push(span); - } - } - return spans; - } - - // Utilities // - import WordBreakProperty = wordBreakers.data.WordBreakProperty; - import WORD_BREAK_PROPERTY = wordBreakers.data.WORD_BREAK_PROPERTY; - import I = wordBreakers.data.I; - - /** - * A span that does not cut out the substring until it absolutely has to! - */ - class LazySpan implements Span { - private _source: string; - readonly start: number; - readonly end: number; - constructor(source: string, start: number, end: number) { - this._source = source; - this.start = start; - this.end = end; - } - - get text(): string { - return this._source.substring(this.start, this.end); - } - - get length(): number { - return this.end - this.start; - } - } - - /** - * Returns true when the chunk does not solely consist of whitespace. - * - * @param chunk a chunk of text. Starts and ends at word boundaries. - */ - function isNonSpace(chunk: string): boolean { - return !Array.from(chunk).map(property).every(wb => ( - wb === WordBreakProperty.CR || - wb === WordBreakProperty.LF || - wb === WordBreakProperty.Newline || - wb === WordBreakProperty.WSegSpace - )); - } - - /** - * Yields a series of string indices where a word break should - * occur. That is, there should be a break BEFORE each string - * index yielded by this generator. - * - * @param text Text to find word boundaries in. - */ - function findBoundaries(text: string): number[] { - // WB1 and WB2: no boundaries if given an empty string. - if (text.length === 0) { - // There are no boundaries in an empty string! - return []; - } - - // This algorithm works by maintaining a sliding window of four SCALAR VALUES. - // - // - Scalar values? JavaScript strings are NOT actually a string of - // Unicode code points; some characters are made up of TWO - // JavaScript indices. e.g., - // "💩".length === 2; - // "💩"[0] === '\uD83D'; - // "💩"[1] === '\uDCA9'; - // - // These characters that are represented by TWO indices are - // called "surrogate pairs". Since we don't want to be in the - // "middle" of a character, make sure we're always advancing - // by scalar values, and NOT indices. That means, we sometimes - // need to advance by TWO indices, not just one. - // - Four values? Some rules look at what's to the left of - // left, and some look at what's to the right of right. So - // keep track of this! - - let boundaries = []; - - let rightPos: number; - let lookaheadPos = 0; // lookahead, one scalar value to the right of right. - // Before the start of the string is also the start of the string. - let lookbehind: WordBreakProperty; - let left = WordBreakProperty.sot; - let right = WordBreakProperty.sot; - let lookahead = wordbreakPropertyAt(0); - // Count RIs to make sure we're not splitting emoji flags: - let nConsecutiveRegionalIndicators = 0; - - do { - // Shift all positions, one scalar value to the right. - rightPos = lookaheadPos; - lookaheadPos = positionAfter(lookaheadPos); - // Shift all properties, one scalar value to the right. - [lookbehind, left, right, lookahead] = - [left, right, lookahead, wordbreakPropertyAt(lookaheadPos)]; - - // Break at the start and end of text, unless the text is empty. - // WB1: Break at start of text... - if (left === WordBreakProperty.sot) { - boundaries.push(rightPos); - continue; - } - // WB2: Break at the end of text... - if (right === WordBreakProperty.eot) { - boundaries.push(rightPos); - break; // Reached the end of the string. We're done! - } - // WB3: Do not break within CRLF: - if (left === WordBreakProperty.CR && right === WordBreakProperty.LF) - continue; - // WB3b: Otherwise, break after... - if (left === WordBreakProperty.Newline || - left === WordBreakProperty.CR || - left === WordBreakProperty.LF) { - boundaries.push(rightPos); - continue; - } - // WB3a: ...and before newlines - if (right === WordBreakProperty.Newline || - right === WordBreakProperty.CR || - right === WordBreakProperty.LF) { - boundaries.push(rightPos); - continue; - } - - // TODO: WB3c is not implemented, due to its complex, error-prone - // implementation, requiring a ginormous regexp, and the fact that - // the only thing it does is prevent big emoji sequences from being - // split up, like 🧚🏼‍♂️ - // https://www.unicode.org/Public/emoji/12.0/emoji-zwj-sequences.txt - - // WB3d: Keep horizontal whitespace together - if (left === WordBreakProperty.WSegSpace && right == WordBreakProperty.WSegSpace) - continue; - - // WB4: Ignore format and extend characters - // This is to keep grapheme clusters together! - // See: Section 6.2: https://unicode.org/reports/tr29/#Grapheme_Cluster_and_Format_Rules - // N.B.: The rule about "except after sot, CR, LF, and - // Newline" already been by WB1, WB2, WB3a, and WB3b above. - while (right === WordBreakProperty.Format || - right === WordBreakProperty.Extend || - right === WordBreakProperty.ZWJ) { - // Continue advancing in the string, as if these - // characters do not exist. DO NOT update left and - // lookbehind however! - [rightPos, lookaheadPos] = [lookaheadPos, positionAfter(lookaheadPos)]; - [right, lookahead] = [lookahead, wordbreakPropertyAt(lookaheadPos)]; - } - // In ignoring the characters in the previous loop, we could - // have fallen off the end of the string, so end the loop - // prematurely if that happens! - if (right === WordBreakProperty.eot) { - boundaries.push(rightPos); - break; - } - // WB4 (continued): Lookahead must ALSO ignore these format, - // extend, ZWJ characters! - while (lookahead === WordBreakProperty.Format || - lookahead === WordBreakProperty.Extend || - lookahead === WordBreakProperty.ZWJ) { - // Continue advancing in the string, as if these - // characters do not exist. DO NOT update left and right, - // however! - lookaheadPos = positionAfter(lookaheadPos); - lookahead = wordbreakPropertyAt(lookaheadPos); - } - - // WB5: Do not break between most letters. - if (isAHLetter(left) && isAHLetter(right)) - continue; - // Do not break across certain punctuation - // WB6: (Don't break before apostrophes in contractions) - if (isAHLetter(left) && isAHLetter(lookahead) && - (right === WordBreakProperty.MidLetter || isMidNumLetQ(right))) - continue; - // WB7: (Don't break after apostrophes in contractions) - if (isAHLetter(lookbehind) && isAHLetter(right) && - (left === WordBreakProperty.MidLetter || isMidNumLetQ(left))) - continue; - // WB7a - if (left === WordBreakProperty.Hebrew_Letter && right === WordBreakProperty.Single_Quote) - continue; - // WB7b - if (left === WordBreakProperty.Hebrew_Letter && right === WordBreakProperty.Double_Quote && - lookahead === WordBreakProperty.Hebrew_Letter) - continue; - // WB7c - if (lookbehind === WordBreakProperty.Hebrew_Letter && left === WordBreakProperty.Double_Quote && - right === WordBreakProperty.Hebrew_Letter) - continue; - // Do not break within sequences of digits, or digits adjacent to letters. - // e.g., "3a" or "A3" - // WB8 - if (left === WordBreakProperty.Numeric && right === WordBreakProperty.Numeric) - continue; - // WB9 - if (isAHLetter(left) && right === WordBreakProperty.Numeric) - continue; - // WB10 - if (left === WordBreakProperty.Numeric && isAHLetter(right)) - continue; - // Do not break within sequences, such as 3.2, 3,456.789 - // WB11 - if (lookbehind === WordBreakProperty.Numeric && right === WordBreakProperty.Numeric && - (left === WordBreakProperty.MidNum || isMidNumLetQ(left))) - continue; - // WB12 - if (left === WordBreakProperty.Numeric && lookahead === WordBreakProperty.Numeric && - (right === WordBreakProperty.MidNum || isMidNumLetQ(right))) - continue; - // WB13: Do not break between Katakana - if (left === WordBreakProperty.Katakana && right === WordBreakProperty.Katakana) - continue; - // Do not break from extenders (e.g., U+202F NARROW NO-BREAK SPACE) - // WB13a - if ((isAHLetter(left) || - left === WordBreakProperty.Numeric || - left === WordBreakProperty.Katakana || - left === WordBreakProperty.ExtendNumLet) && - right === WordBreakProperty.ExtendNumLet) - continue; - // WB13b - if ((isAHLetter(right) || - right === WordBreakProperty.Numeric || - right === WordBreakProperty.Katakana) && left === WordBreakProperty.ExtendNumLet) - continue; - - // WB15 & WB16: - // Do not break within emoji flag sequences. That is, do not break between - // regional indicator (RI) symbols if there is an odd number of RI - // characters before the break point. - if (right === WordBreakProperty.Regional_Indicator) { - // Emoji flags are actually composed of TWO scalar values, each being a - // "regional indicator". These indicators correspond to Latin letters. Put - // two of them together, and they spell out an ISO 3166-1-alpha-2 country - // code. Since these always come in pairs, NEVER split the pairs! So, if - // we happen to be inside the middle of an odd numbered of - // Regional_Indicators, DON'T SPLIT IT! - nConsecutiveRegionalIndicators += 1; - if ((nConsecutiveRegionalIndicators % 2) == 1) { - continue; - } - } else { - nConsecutiveRegionalIndicators = 0; - } - // WB999: Otherwise, break EVERYWHERE (including around ideographs) - boundaries.push(rightPos); - } while (rightPos < text.length); - - return boundaries; - - ///// Internal utility functions ///// - - /** - * Returns the position of the start of the next scalar value. This jumps - * over surrogate pairs. - * - * If asked for the character AFTER the end of the string, this always - * returns the length of the string. - */ - function positionAfter(pos: number): number { - if (pos >= text.length) { - return text.length; - } else if (isStartOfSurrogatePair(text[pos])) { - return pos + 2; - } - return pos + 1; - } - - /** - * Return the value of the Word_Break property at the given string index. - * @param pos position in the text. - */ - function wordbreakPropertyAt(pos: number) { - if (pos < 0) { - return WordBreakProperty.sot; // Always "start of string" before the string starts! - } else if (pos >= text.length) { - return WordBreakProperty.eot; // Always "end of string" after the string ends! - } else if (isStartOfSurrogatePair(text[pos])) { - // Surrogate pairs the next TWO items from the string! - return property(text[pos] + text[pos + 1]); - } - return property(text[pos]); - } - - // Word_Break rule macros - // See: https://unicode.org/reports/tr29/#WB_Rule_Macros - function isAHLetter(prop: WordBreakProperty): boolean { - return prop === WordBreakProperty.ALetter || - prop === WordBreakProperty.Hebrew_Letter; - } - - function isMidNumLetQ(prop: WordBreakProperty): boolean { - return prop === WordBreakProperty.MidNumLet || - prop === WordBreakProperty.Single_Quote; - } - } - - function isStartOfSurrogatePair(character: string) { - let codeUnit = character.charCodeAt(0); - return codeUnit >= 0xD800 && codeUnit <= 0xDBFF; - } - - /** - * Return the Word_Break property value for a character. - * Note that - * @param character a scalar value - */ - function property(character: string): WordBreakProperty { - // This MUST be a scalar value. - // TODO: remove dependence on character.codepointAt()? - let codepoint = character.codePointAt(0) as number; - return searchForProperty(codepoint, 0, WORD_BREAK_PROPERTY.length - 1); - } - - /** - * Binary search for the word break property of a given CODE POINT. - */ - function searchForProperty(codePoint: number, left: number, right: number): WordBreakProperty { - // All items that are not found in the array are assigned the 'Other' property. - if (right < left) { - return WordBreakProperty.Other; - } - - let midpoint = left + ~~((right - left) / 2); - let candidate = WORD_BREAK_PROPERTY[midpoint]; - if (codePoint < candidate[I.Start]) { - return searchForProperty(codePoint, left, midpoint - 1); - } else if (codePoint > candidate[I.End]) { - return searchForProperty(codePoint, midpoint + 1, right); - } else { - // We found it! - return candidate[I.Value]; - } - } -} - -// We cannot export a member whose name is a reserved word when -// implementing a namespace, BUT we can manually make the -// assignment and **declare** it as part of the namespace. -wordBreakers['default'] = wordBreakers.default_; -declare namespace wordBreakers { - export { default_ as default }; -} From 87d3b00c2295f3229c1be22241676aaf18fbc8b6 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Wed, 13 May 2020 10:19:27 -0600 Subject: [PATCH 08/41] change(common/lmlayer): compile in strict mode --- .../lexical-model-word-breakers/src/ascii-word-breaker.ts | 2 +- common/lexical-model-word-breakers/tsconfig.json | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/common/lexical-model-word-breakers/src/ascii-word-breaker.ts b/common/lexical-model-word-breakers/src/ascii-word-breaker.ts index 7a933f0107..d81b490b9b 100644 --- a/common/lexical-model-word-breakers/src/ascii-word-breaker.ts +++ b/common/lexical-model-word-breakers/src/ascii-word-breaker.ts @@ -8,7 +8,7 @@ export function ascii(phrase: string): Span[] { let matchWord = /[A-Za-z0-9']+/g; let words: Span[] = []; - let match: RegExpExecArray; + let match: RegExpExecArray | null; while ((match = matchWord.exec(phrase)) !== null) { words.push(new RegExpDerivedSpan(match[0], match.index)); } diff --git a/common/lexical-model-word-breakers/tsconfig.json b/common/lexical-model-word-breakers/tsconfig.json index 358a3f8b91..94dba60b20 100644 --- a/common/lexical-model-word-breakers/tsconfig.json +++ b/common/lexical-model-word-breakers/tsconfig.json @@ -1,12 +1,13 @@ { "compilerOptions": { "module": "commonjs", + "lib": ["es6"], + "target": "es5", "declaration": true, "sourceMap": true, "outDir": "lib", - "allowJs": false, - "lib": ["es6"], - "target": "es5" + "strict": true, + "allowJs": false }, "include": [ "src/**/*" From 0b4193d9423b90e2a9a61f9f02a72084ce1da857 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Wed, 13 May 2020 10:27:38 -0600 Subject: [PATCH 09/41] test(common/lmlayer): use TypeScript for tests instead --- .../package-lock.json | 67 +++++++++++++++++++ .../lexical-model-word-breakers/package.json | 7 +- ...-breaker.js => test-ascii-word-breaker.ts} | 4 +- 3 files changed, 74 insertions(+), 4 deletions(-) rename common/lexical-model-word-breakers/test/{test-ascii-word-breaker.js => test-ascii-word-breaker.ts} (69%) diff --git a/common/lexical-model-word-breakers/package-lock.json b/common/lexical-model-word-breakers/package-lock.json index de0642f309..b50f15e5dc 100644 --- a/common/lexical-model-word-breakers/package-lock.json +++ b/common/lexical-model-word-breakers/package-lock.json @@ -4,6 +4,18 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@types/mocha": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-7.0.2.tgz", + "integrity": "sha512-ZvO2tAcjmMi8V/5Z3JsyofMe3hasRcaw88cto5etSVMwVQfeivGAlEYmaQgceUSVYFofVjT+ioHsATjdWcFt1w==", + "dev": true + }, + "arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true + }, "assertion-error": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", @@ -32,6 +44,12 @@ "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", "dev": true }, + "buffer-from": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", + "dev": true + }, "chai": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/chai/-/chai-4.2.0.tgz", @@ -154,6 +172,12 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "dev": true }, + "make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "dev": true + }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", @@ -224,6 +248,22 @@ "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=", "dev": true }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + }, + "source-map-support": { + "version": "0.5.19", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", + "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", + "dev": true, + "requires": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, "supports-color": { "version": "5.4.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz", @@ -233,6 +273,27 @@ "has-flag": "^3.0.0" } }, + "ts-node": { + "version": "8.10.1", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-8.10.1.tgz", + "integrity": "sha512-bdNz1L4ekHiJul6SHtZWs1ujEKERJnHs4HxN7rjTyyVOFf3HaJ6sLqe6aPG62XTzAB/63pKRh5jTSWL0D7bsvw==", + "dev": true, + "requires": { + "arg": "^4.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "source-map-support": "^0.5.17", + "yn": "3.1.1" + }, + "dependencies": { + "diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true + } + } + }, "type-detect": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", @@ -250,6 +311,12 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", "dev": true + }, + "yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true } } } diff --git a/common/lexical-model-word-breakers/package.json b/common/lexical-model-word-breakers/package.json index 50e394788a..cc924595ce 100644 --- a/common/lexical-model-word-breakers/package.json +++ b/common/lexical-model-word-breakers/package.json @@ -28,17 +28,18 @@ }, "scripts": { "build": "tsc", - "prepublish": "npm run build", - "pretest": "npm run build", - "test": "mocha" + "prepublish": "npm run build", + "test": "mocha -r ts-node/register test/**/*.ts" }, "bugs": { "url": "https://github.com/keymanapp/keyman/issues" }, "devDependencies": { "@keymanapp/lexical-model-types": "^14.0.75", + "@types/mocha": "^7.0.2", "chai": "^4.2.0", "mocha": "^5.2.0", + "ts-node": "^8.10.1", "typescript": "^3.2.1" } } diff --git a/common/lexical-model-word-breakers/test/test-ascii-word-breaker.js b/common/lexical-model-word-breakers/test/test-ascii-word-breaker.ts similarity index 69% rename from common/lexical-model-word-breakers/test/test-ascii-word-breaker.js rename to common/lexical-model-word-breakers/test/test-ascii-word-breaker.ts index d529331eb0..ce642f5a1c 100644 --- a/common/lexical-model-word-breakers/test/test-ascii-word-breaker.js +++ b/common/lexical-model-word-breakers/test/test-ascii-word-breaker.ts @@ -1,9 +1,11 @@ +/// + var assert = require('chai').assert; var breakASCIIWords = require('../').ascii; describe('The ASCII word breaker', function () { it('should break simple English sentences', function () { - let breaks = breakASCIIWords('Look! -- The quick brown fox jumps... over the lazy dog!'); + let breaks: Span[] = breakASCIIWords('Look! -- The quick brown fox jumps... over the lazy dog!'); let words = breaks.map(span => span.text); assert.deepEqual(words, ['Look', 'The', 'quick', 'brown', 'fox', 'jumps', 'over', 'the', 'lazy', 'dog']); }); From fbead6cd07e63dd75322ccd9409c6a2cf8913dc6 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Wed, 13 May 2020 10:29:24 -0600 Subject: [PATCH 10/41] test(common/lmlayer): use import syntax in tests --- common/lexical-model-word-breakers/package-lock.json | 6 ++++++ common/lexical-model-word-breakers/package.json | 1 + .../test/test-ascii-word-breaker.ts | 7 ++++--- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/common/lexical-model-word-breakers/package-lock.json b/common/lexical-model-word-breakers/package-lock.json index b50f15e5dc..78a82b4388 100644 --- a/common/lexical-model-word-breakers/package-lock.json +++ b/common/lexical-model-word-breakers/package-lock.json @@ -4,6 +4,12 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@types/chai": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.2.11.tgz", + "integrity": "sha512-t7uW6eFafjO+qJ3BIV2gGUyZs27egcNRkUdalkud+Qa3+kg//f129iuOFivHDXQ+vnU3fDXuwgv0cqMCbcE8sw==", + "dev": true + }, "@types/mocha": { "version": "7.0.2", "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-7.0.2.tgz", diff --git a/common/lexical-model-word-breakers/package.json b/common/lexical-model-word-breakers/package.json index cc924595ce..469b9b672c 100644 --- a/common/lexical-model-word-breakers/package.json +++ b/common/lexical-model-word-breakers/package.json @@ -36,6 +36,7 @@ }, "devDependencies": { "@keymanapp/lexical-model-types": "^14.0.75", + "@types/chai": "^4.2.11", "@types/mocha": "^7.0.2", "chai": "^4.2.0", "mocha": "^5.2.0", diff --git a/common/lexical-model-word-breakers/test/test-ascii-word-breaker.ts b/common/lexical-model-word-breakers/test/test-ascii-word-breaker.ts index ce642f5a1c..48287872ca 100644 --- a/common/lexical-model-word-breakers/test/test-ascii-word-breaker.ts +++ b/common/lexical-model-word-breakers/test/test-ascii-word-breaker.ts @@ -1,11 +1,12 @@ /// -var assert = require('chai').assert; -var breakASCIIWords = require('../').ascii; +import {assert} from 'chai'; +import {ascii as breakASCIIWords} from '../' + describe('The ASCII word breaker', function () { it('should break simple English sentences', function () { - let breaks: Span[] = breakASCIIWords('Look! -- The quick brown fox jumps... over the lazy dog!'); + let breaks = breakASCIIWords('Look! -- The quick brown fox jumps... over the lazy dog!'); let words = breaks.map(span => span.text); assert.deepEqual(words, ['Look', 'The', 'quick', 'brown', 'fox', 'jumps', 'over', 'the', 'lazy', 'dog']); }); From 5e7f1791a1620ba4de121bb6f675dcb7e38b8738 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Wed, 13 May 2020 10:35:17 -0600 Subject: [PATCH 11/41] test(common/lmlayer): factor out tests that require the TrieModel --- .../test/test-default-word-breaker.js | 118 ------------------ 1 file changed, 118 deletions(-) diff --git a/common/lexical-model-word-breakers/test/test-default-word-breaker.js b/common/lexical-model-word-breakers/test/test-default-word-breaker.js index 9c9ac3bb58..208f51eb0e 100644 --- a/common/lexical-model-word-breakers/test/test-default-word-breaker.js +++ b/common/lexical-model-word-breakers/test/test-default-word-breaker.js @@ -2,7 +2,6 @@ * Smoke-test the default */ var assert = require('chai').assert; -//var TrieModel = require('../../build/intermediate').models.TrieModel; var breakWords = require('..').default; const SHY = '\u00AD'; @@ -20,121 +19,4 @@ describe('The default word breaker', function () { "let's", 'eat', 'phở', '!', '🥣' ]); }); - - // The following tests are performed with model integration as an internal - // test for the wordbreaking API. - it.skip('recognizes a word at end of complete lefthand context', function () { - var model = new TrieModel(jsonFixture('tries/english-1000'), { - wordBreaker: breakWords // wordBreakers['default'] when fully integrated. - }); - - // Standard case - wordbreaking at the end of a word. - var context = { - left: 'The quick brown fox jumped', startOfBuffer: true, - right: ' over the lazy dog.', endOfBuffer: true - }; - - var broken = model.wordbreak(context); - - assert.strictEqual(broken, 'jumped'); - }); - - // Same test as before, but we want to be sure the start/end of buffer flags - // don't affect our results. - it.skip('recognizes a word at end of incomplete lefthand context', function () { - var model = new TrieModel(jsonFixture('tries/english-1000'), { - wordBreaker: breakWords - }); - - // Standard case - wordbreaking at the end of a word. - var context = { - left: 'The quick brown fox jumped', startOfBuffer: false, - right: ' over the lazy dog.', endOfBuffer: false - }; - - var broken = model.wordbreak(context); - - assert.strictEqual(broken, 'jumped'); - }); - - it.skip('returns text for a word in-progress', function() { - var model = new TrieModel(jsonFixture('tries/english-1000'), { - wordBreaker: breakWords - }); - - // Standard case - midword (xylophone) call - var context = { - left: 'xyl', startOfBuffer: true, - right: '', endOfBuffer: true - }; - - var broken = model.wordbreak(context); - - assert.strictEqual(broken, 'xyl'); - }); - - it.skip('returns empty string when called without word text', function() { - var model = new TrieModel(jsonFixture('tries/english-1000'), { - wordBreaker: breakWords - }); - - // Wordbreaking on a empty space => no word. - context = { - left: 'The quick brown fox jumped ', startOfBuffer: true, - right: 'over the lazy dog.', endOfBuffer: true - }; - - broken = model.wordbreak(context); - - assert.strictEqual(broken, ''); - }); - - it.skip('returns empty string when called with empty context', function() { - var model = new TrieModel(jsonFixture('tries/english-1000'), { - wordBreaker: breakWords - }); - - // Wordbreaking on a empty space => no word. - context = { - left: '', startOfBuffer: true, - right: '', endOfBuffer: true - }; - - broken = model.wordbreak(context); - - assert.strictEqual(broken, ''); - }); - - it.skip('returns empty string when called with nil context', function() { - var model = new TrieModel(jsonFixture('tries/english-1000'), { - wordBreaker: breakWords - }); - - // Wordbreaking on a empty space => no word. - context = { - left: '', startOfBuffer: false, - right: '', endOfBuffer: false - }; - - broken = model.wordbreak(context); - - assert.strictEqual(broken, ''); - }); - - it.skip('correctly breaks a word when the caret is placed within it', function() { - var model = new TrieModel(jsonFixture('tries/english-1000'), { - wordBreaker: breakWords - }); - - // A limitation of the current implementation; we should fix this before release. - // Then again, when typing this is probably fine; just not when not typing. - context = { - left: 'The quick brown fox jum', startOfBuffer: true, - right: 'ped over the lazy dog.', endOfBuffer: true - }; - - broken = model.wordbreak(context); - - assert.strictEqual(broken, 'jumped'); // Current result: 'jum'. - }); }); From 3435db7e09a37075e7d754b85b72c7b6f2257210 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Wed, 13 May 2020 10:36:01 -0600 Subject: [PATCH 12/41] test(common/lmlayer): convert default word breaker test to TypeScript --- ...-default-word-breaker.js => test-default-word-breaker.ts} | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) rename common/lexical-model-word-breakers/test/{test-default-word-breaker.js => test-default-word-breaker.ts} (88%) diff --git a/common/lexical-model-word-breakers/test/test-default-word-breaker.js b/common/lexical-model-word-breakers/test/test-default-word-breaker.ts similarity index 88% rename from common/lexical-model-word-breakers/test/test-default-word-breaker.js rename to common/lexical-model-word-breakers/test/test-default-word-breaker.ts index 208f51eb0e..0b66f0c976 100644 --- a/common/lexical-model-word-breakers/test/test-default-word-breaker.js +++ b/common/lexical-model-word-breakers/test/test-default-word-breaker.ts @@ -1,9 +1,10 @@ /** * Smoke-test the default */ -var assert = require('chai').assert; -var breakWords = require('..').default; +import {assert} from 'chai'; +import {default as breakWords} from '../' + const SHY = '\u00AD'; describe('The default word breaker', function () { From 6aabd10c7090e6163f8676faf45550dfdccad02a Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Wed, 13 May 2020 10:37:39 -0600 Subject: [PATCH 13/41] test(common/lmlayer): add test for the placeholder --- .../test/test-placeholder-word-breaker.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 common/lexical-model-word-breakers/test/test-placeholder-word-breaker.ts diff --git a/common/lexical-model-word-breakers/test/test-placeholder-word-breaker.ts b/common/lexical-model-word-breakers/test/test-placeholder-word-breaker.ts new file mode 100644 index 0000000000..75470106f7 --- /dev/null +++ b/common/lexical-model-word-breakers/test/test-placeholder-word-breaker.ts @@ -0,0 +1,13 @@ +/// + +import {assert} from 'chai'; +import {placeholder as breakWords} from '../' + + +describe('The placeholder word breaker', function () { + it('should break simple English sentences', function () { + let breaks = breakWords('Look! -- The quick brown fox jumps... over the lazy dog!'); + let words = breaks.map(span => span.text); + assert.deepEqual(words, ['Look', 'The', 'quick', 'brown', 'fox', 'jumps', 'over', 'the', 'lazy', 'dog']); + }); +}); From 73b16ae33eb62f08247fbcb1afe0fca032f3df97 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Wed, 13 May 2020 10:41:38 -0600 Subject: [PATCH 14/41] refactor(common/lmlayer): move placeholder word breaker to @keymanapp/lexical-model-word-breakers --- .../lexical-model-word-breakers/src/index.ts | 3 ++- .../src/placeholder.ts | 21 ++++++++++++++++ .../test/test-placeholder-word-breaker.ts | 4 ++-- .../word_breaking/placeholder-word-breaker.ts | 24 ------------------- 4 files changed, 25 insertions(+), 27 deletions(-) create mode 100644 common/lexical-model-word-breakers/src/placeholder.ts delete mode 100644 common/predictive-text/worker/word_breaking/placeholder-word-breaker.ts diff --git a/common/lexical-model-word-breakers/src/index.ts b/common/lexical-model-word-breakers/src/index.ts index 22c6cd74a9..824e9aab75 100644 --- a/common/lexical-model-word-breakers/src/index.ts +++ b/common/lexical-model-word-breakers/src/index.ts @@ -1,5 +1,6 @@ import {ascii} from "./ascii-word-breaker"; +import {placeholder} from './placeholder'; import {default_} from "./default"; -export {ascii}; +export {ascii, placeholder}; export {default_ as default}; diff --git a/common/lexical-model-word-breakers/src/placeholder.ts b/common/lexical-model-word-breakers/src/placeholder.ts new file mode 100644 index 0000000000..7924b196e4 --- /dev/null +++ b/common/lexical-model-word-breakers/src/placeholder.ts @@ -0,0 +1,21 @@ +/** + * A **VERY** dumb word breaker that simply splits at words. Do not use this + * word breaker! + * + * @param phrase The phrase in which to break words. + * @deprecated Use a word breaker tailored to your language instead! + */ +export function placeholder(phrase: string): Span[] { + let nextStart = 0; + return phrase.split(/\s+/).map(utterance => { + // XXX: The indices are NOT accurate to the original phrase! + let span = { + start: nextStart, + end: nextStart + utterance.length, + text: utterance, + length: utterance.length + }; + nextStart = span.end; + return span; + }); +} diff --git a/common/lexical-model-word-breakers/test/test-placeholder-word-breaker.ts b/common/lexical-model-word-breakers/test/test-placeholder-word-breaker.ts index 75470106f7..923d60b7cb 100644 --- a/common/lexical-model-word-breakers/test/test-placeholder-word-breaker.ts +++ b/common/lexical-model-word-breakers/test/test-placeholder-word-breaker.ts @@ -1,13 +1,13 @@ /// import {assert} from 'chai'; -import {placeholder as breakWords} from '../' +import {placeholder as breakWords} from '../src'; describe('The placeholder word breaker', function () { it('should break simple English sentences', function () { let breaks = breakWords('Look! -- The quick brown fox jumps... over the lazy dog!'); let words = breaks.map(span => span.text); - assert.deepEqual(words, ['Look', 'The', 'quick', 'brown', 'fox', 'jumps', 'over', 'the', 'lazy', 'dog']); + assert.deepEqual(words, ['Look!', '--', 'The', 'quick', 'brown', 'fox', 'jumps...', 'over', 'the', 'lazy', 'dog!']); }); }); diff --git a/common/predictive-text/worker/word_breaking/placeholder-word-breaker.ts b/common/predictive-text/worker/word_breaking/placeholder-word-breaker.ts deleted file mode 100644 index 1105b89fb9..0000000000 --- a/common/predictive-text/worker/word_breaking/placeholder-word-breaker.ts +++ /dev/null @@ -1,24 +0,0 @@ -namespace wordBreakers { - - /** - * A **VERY** dumb word breaker that simply splits at words. Do not use this - * word breaker! - * - * @param phrase The phrase in which to break words. - * @deprecated Use a word breaker tailored to your language instead! - */ - export function placeholder(phrase: string): Span[] { - let nextStart = 0; - return phrase.split(/\s+/).map(utterance => { - // XXX: The indices are NOT accurate to the original phrase! - let span = { - start: nextStart, - end: nextStart + utterance.length, - text: utterance, - length: utterance.length - }; - nextStart = span.end; - return span; - }); - } -} From 689c96ce2c4bab614c1323719b5a7ac2f1fd98c4 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Wed, 13 May 2020 10:42:32 -0600 Subject: [PATCH 15/41] test(common/lmlayer): import directly from src/ instead, to compile on test --- .../lexical-model-word-breakers/test/test-ascii-word-breaker.ts | 2 +- .../test/test-default-word-breaker.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/lexical-model-word-breakers/test/test-ascii-word-breaker.ts b/common/lexical-model-word-breakers/test/test-ascii-word-breaker.ts index 48287872ca..3725be863c 100644 --- a/common/lexical-model-word-breakers/test/test-ascii-word-breaker.ts +++ b/common/lexical-model-word-breakers/test/test-ascii-word-breaker.ts @@ -1,7 +1,7 @@ /// import {assert} from 'chai'; -import {ascii as breakASCIIWords} from '../' +import {ascii as breakASCIIWords} from '../src' describe('The ASCII word breaker', function () { diff --git a/common/lexical-model-word-breakers/test/test-default-word-breaker.ts b/common/lexical-model-word-breakers/test/test-default-word-breaker.ts index 0b66f0c976..7f3201945b 100644 --- a/common/lexical-model-word-breakers/test/test-default-word-breaker.ts +++ b/common/lexical-model-word-breakers/test/test-default-word-breaker.ts @@ -3,7 +3,7 @@ */ import {assert} from 'chai'; -import {default as breakWords} from '../' +import {default as breakWords} from '../src'; const SHY = '\u00AD'; From 23a74c992c3de12944d4d4bd6ec356e920a800e7 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Wed, 13 May 2020 10:44:03 -0600 Subject: [PATCH 16/41] refactor(common/lmlayer): drop the -word-breaker suffix --- .../src/{ascii-word-breaker.ts => ascii.ts} | 0 common/lexical-model-word-breakers/src/index.ts | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename common/lexical-model-word-breakers/src/{ascii-word-breaker.ts => ascii.ts} (100%) diff --git a/common/lexical-model-word-breakers/src/ascii-word-breaker.ts b/common/lexical-model-word-breakers/src/ascii.ts similarity index 100% rename from common/lexical-model-word-breakers/src/ascii-word-breaker.ts rename to common/lexical-model-word-breakers/src/ascii.ts diff --git a/common/lexical-model-word-breakers/src/index.ts b/common/lexical-model-word-breakers/src/index.ts index 824e9aab75..c3dd1420d2 100644 --- a/common/lexical-model-word-breakers/src/index.ts +++ b/common/lexical-model-word-breakers/src/index.ts @@ -1,4 +1,4 @@ -import {ascii} from "./ascii-word-breaker"; +import {ascii} from "./ascii"; import {placeholder} from './placeholder'; import {default_} from "./default"; From bde1a88abebe19fd5f73f1df4f2b9605e0e6fbec Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Wed, 13 May 2020 10:50:58 -0600 Subject: [PATCH 17/41] docs(common/lmlayer): update word-breakers README --- common/lexical-model-word-breakers/README.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/common/lexical-model-word-breakers/README.md b/common/lexical-model-word-breakers/README.md index 5052cd34e6..3b0716d4f0 100644 --- a/common/lexical-model-word-breakers/README.md +++ b/common/lexical-model-word-breakers/README.md @@ -1,11 +1,18 @@ # `@keymanapp/lexical-model-word-breakers` -> TODO: description +The word breakers bundled in Keyman's lexical model layer (LMLayer). + +Includes: + + - `default` — a word breaker based on the Unicode default word boundary specification + - _deprecated_ `ascii` — an example word breaker + - _deprecated_ `placeholder` — an example word breaker ## Usage ``` -const lexicalModelWordBreakers = require('@keymanapp/lexical-model-word-breakers'); +import {default as breakWords} from '@keymanapp/lexical-model-word-breakers'; -// TODO: DEMONSTRATE API +console.log(breakWords('Hello, World!').map(span => span.text)); +// prints: [ 'Hello', ',', 'World', '!' ] ``` From a1a087a3745b52cf9502ed0c1fcb20ebe9191223 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Wed, 13 May 2020 13:55:54 -0600 Subject: [PATCH 18/41] change(common/lmlayer): use typeRoots instead of triple-slash include --- common/lexical-model-word-breakers/src/ascii.ts | 2 -- .../test/test-ascii-word-breaker.ts | 2 -- .../test/test-placeholder-word-breaker.ts | 2 -- common/lexical-model-word-breakers/tsconfig.json | 6 +++++- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/common/lexical-model-word-breakers/src/ascii.ts b/common/lexical-model-word-breakers/src/ascii.ts index d81b490b9b..ce007c8d00 100644 --- a/common/lexical-model-word-breakers/src/ascii.ts +++ b/common/lexical-model-word-breakers/src/ascii.ts @@ -1,5 +1,3 @@ -/// - /** * Splits ASCII words. * diff --git a/common/lexical-model-word-breakers/test/test-ascii-word-breaker.ts b/common/lexical-model-word-breakers/test/test-ascii-word-breaker.ts index 3725be863c..2b77810864 100644 --- a/common/lexical-model-word-breakers/test/test-ascii-word-breaker.ts +++ b/common/lexical-model-word-breakers/test/test-ascii-word-breaker.ts @@ -1,5 +1,3 @@ -/// - import {assert} from 'chai'; import {ascii as breakASCIIWords} from '../src' diff --git a/common/lexical-model-word-breakers/test/test-placeholder-word-breaker.ts b/common/lexical-model-word-breakers/test/test-placeholder-word-breaker.ts index 923d60b7cb..1dba3cbffa 100644 --- a/common/lexical-model-word-breakers/test/test-placeholder-word-breaker.ts +++ b/common/lexical-model-word-breakers/test/test-placeholder-word-breaker.ts @@ -1,5 +1,3 @@ -/// - import {assert} from 'chai'; import {placeholder as breakWords} from '../src'; diff --git a/common/lexical-model-word-breakers/tsconfig.json b/common/lexical-model-word-breakers/tsconfig.json index 94dba60b20..6acf0c403c 100644 --- a/common/lexical-model-word-breakers/tsconfig.json +++ b/common/lexical-model-word-breakers/tsconfig.json @@ -7,7 +7,11 @@ "sourceMap": true, "outDir": "lib", "strict": true, - "allowJs": false + "allowJs": false, + "typeRoots": [ + "node_modules/@types", + "node_modules/@keymanapp" + ] }, "include": [ "src/**/*" From 73b5d520e24b788a7d7684cb0969c60098277206 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Tue, 19 May 2020 14:06:06 -0600 Subject: [PATCH 19/41] change(common/lmlayer): move to common/models/wordbreakers --- .../wordbreakers}/.gitignore | 0 .../wordbreakers}/README.md | 0 .../wordbreakers}/package-lock.json | 0 .../wordbreakers}/package.json | 9 ++++++--- .../wordbreakers}/src/ascii.ts | 0 .../wordbreakers}/src/default/data.ts | 0 .../wordbreakers}/src/default/index.ts | 0 .../wordbreakers}/src/index.ts | 0 .../wordbreakers}/src/placeholder.ts | 0 .../wordbreakers}/test/test-ascii-word-breaker.ts | 0 .../wordbreakers}/test/test-default-word-breaker.ts | 0 .../wordbreakers}/test/test-placeholder-word-breaker.ts | 0 .../wordbreakers}/tsconfig.json | 0 lerna.json | 2 +- 14 files changed, 7 insertions(+), 4 deletions(-) rename common/{lexical-model-word-breakers => models/wordbreakers}/.gitignore (100%) rename common/{lexical-model-word-breakers => models/wordbreakers}/README.md (100%) rename common/{lexical-model-word-breakers => models/wordbreakers}/package-lock.json (100%) rename common/{lexical-model-word-breakers => models/wordbreakers}/package.json (86%) rename common/{lexical-model-word-breakers => models/wordbreakers}/src/ascii.ts (100%) rename common/{lexical-model-word-breakers => models/wordbreakers}/src/default/data.ts (100%) rename common/{lexical-model-word-breakers => models/wordbreakers}/src/default/index.ts (100%) rename common/{lexical-model-word-breakers => models/wordbreakers}/src/index.ts (100%) rename common/{lexical-model-word-breakers => models/wordbreakers}/src/placeholder.ts (100%) rename common/{lexical-model-word-breakers => models/wordbreakers}/test/test-ascii-word-breaker.ts (100%) rename common/{lexical-model-word-breakers => models/wordbreakers}/test/test-default-word-breaker.ts (100%) rename common/{lexical-model-word-breakers => models/wordbreakers}/test/test-placeholder-word-breaker.ts (100%) rename common/{lexical-model-word-breakers => models/wordbreakers}/tsconfig.json (100%) diff --git a/common/lexical-model-word-breakers/.gitignore b/common/models/wordbreakers/.gitignore similarity index 100% rename from common/lexical-model-word-breakers/.gitignore rename to common/models/wordbreakers/.gitignore diff --git a/common/lexical-model-word-breakers/README.md b/common/models/wordbreakers/README.md similarity index 100% rename from common/lexical-model-word-breakers/README.md rename to common/models/wordbreakers/README.md diff --git a/common/lexical-model-word-breakers/package-lock.json b/common/models/wordbreakers/package-lock.json similarity index 100% rename from common/lexical-model-word-breakers/package-lock.json rename to common/models/wordbreakers/package-lock.json diff --git a/common/lexical-model-word-breakers/package.json b/common/models/wordbreakers/package.json similarity index 86% rename from common/lexical-model-word-breakers/package.json rename to common/models/wordbreakers/package.json index 469b9b672c..bda9215cb6 100644 --- a/common/lexical-model-word-breakers/package.json +++ b/common/models/wordbreakers/package.json @@ -1,10 +1,13 @@ { - "name": "@keymanapp/lexical-model-word-breakers", + "name": "@keymanapp/model-wordbreakers", "version": "14.0.75", - "description": "Word breakers bundled in the LMLayer", + "description": "Word breakers bundled in the LMlayer", "keywords": [ "lexical model", - "word breaker", + "models", + "word", + "breaker", + "breaking", "lmlayer" ], "contributors": [ diff --git a/common/lexical-model-word-breakers/src/ascii.ts b/common/models/wordbreakers/src/ascii.ts similarity index 100% rename from common/lexical-model-word-breakers/src/ascii.ts rename to common/models/wordbreakers/src/ascii.ts diff --git a/common/lexical-model-word-breakers/src/default/data.ts b/common/models/wordbreakers/src/default/data.ts similarity index 100% rename from common/lexical-model-word-breakers/src/default/data.ts rename to common/models/wordbreakers/src/default/data.ts diff --git a/common/lexical-model-word-breakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts similarity index 100% rename from common/lexical-model-word-breakers/src/default/index.ts rename to common/models/wordbreakers/src/default/index.ts diff --git a/common/lexical-model-word-breakers/src/index.ts b/common/models/wordbreakers/src/index.ts similarity index 100% rename from common/lexical-model-word-breakers/src/index.ts rename to common/models/wordbreakers/src/index.ts diff --git a/common/lexical-model-word-breakers/src/placeholder.ts b/common/models/wordbreakers/src/placeholder.ts similarity index 100% rename from common/lexical-model-word-breakers/src/placeholder.ts rename to common/models/wordbreakers/src/placeholder.ts diff --git a/common/lexical-model-word-breakers/test/test-ascii-word-breaker.ts b/common/models/wordbreakers/test/test-ascii-word-breaker.ts similarity index 100% rename from common/lexical-model-word-breakers/test/test-ascii-word-breaker.ts rename to common/models/wordbreakers/test/test-ascii-word-breaker.ts diff --git a/common/lexical-model-word-breakers/test/test-default-word-breaker.ts b/common/models/wordbreakers/test/test-default-word-breaker.ts similarity index 100% rename from common/lexical-model-word-breakers/test/test-default-word-breaker.ts rename to common/models/wordbreakers/test/test-default-word-breaker.ts diff --git a/common/lexical-model-word-breakers/test/test-placeholder-word-breaker.ts b/common/models/wordbreakers/test/test-placeholder-word-breaker.ts similarity index 100% rename from common/lexical-model-word-breakers/test/test-placeholder-word-breaker.ts rename to common/models/wordbreakers/test/test-placeholder-word-breaker.ts diff --git a/common/lexical-model-word-breakers/tsconfig.json b/common/models/wordbreakers/tsconfig.json similarity index 100% rename from common/lexical-model-word-breakers/tsconfig.json rename to common/models/wordbreakers/tsconfig.json diff --git a/lerna.json b/lerna.json index e5b5b83b5a..080660cc0c 100644 --- a/lerna.json +++ b/lerna.json @@ -6,7 +6,7 @@ "common/core/web/utils", "common/lexical-model-types", "common/predictive-text", - "common/lexical-model-word-breakers", + "common/models/*", "web" ], "version": "14.0.78" From f5aae5cbd547a1730926f6a93f23772a66fa0e12 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Tue, 19 May 2020 14:14:06 -0600 Subject: [PATCH 20/41] chore(common/lmlayer): let the worker depend on the word breakers --- common/models/wordbreakers/package.json | 10 +++++----- common/predictive-text/package.json | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/common/models/wordbreakers/package.json b/common/models/wordbreakers/package.json index bda9215cb6..b60f9b95dc 100644 --- a/common/models/wordbreakers/package.json +++ b/common/models/wordbreakers/package.json @@ -1,13 +1,13 @@ { - "name": "@keymanapp/model-wordbreakers", + "name": "@keymanapp/models-wordbreakers", "version": "14.0.75", "description": "Word breakers bundled in the LMlayer", "keywords": [ "lexical model", - "models", - "word", - "breaker", - "breaking", + "models", + "word", + "breaker", + "breaking", "lmlayer" ], "contributors": [ diff --git a/common/predictive-text/package.json b/common/predictive-text/package.json index 5192580b71..33515545c2 100644 --- a/common/predictive-text/package.json +++ b/common/predictive-text/package.json @@ -45,6 +45,7 @@ }, "dependencies": { "@keymanapp/lexical-model-types": "^14.0.78", + "@keymanapp/models-wordbreakers": "^14.0.75", "@keymanapp/web-utils": "^14.0.77", "es6-shim": "^0.35.5", "string.prototype.codepointat": "^0.2.1", From 49b93eb329cc5ebbc7f493d9bdde39ad4976cad5 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Tue, 19 May 2020 14:20:34 -0600 Subject: [PATCH 21/41] chore(common/lmlayer): disable tests before redoing some commits --- common/models/wordbreakers/test/test-default-word-breaker.ts | 2 +- .../models/wordbreakers/test/test-placeholder-word-breaker.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/models/wordbreakers/test/test-default-word-breaker.ts b/common/models/wordbreakers/test/test-default-word-breaker.ts index 7f3201945b..ed25c4bea4 100644 --- a/common/models/wordbreakers/test/test-default-word-breaker.ts +++ b/common/models/wordbreakers/test/test-default-word-breaker.ts @@ -7,7 +7,7 @@ import {default as breakWords} from '../src'; const SHY = '\u00AD'; -describe('The default word breaker', function () { +describe.skip('The default word breaker', function () { it('should break multilingual text', function () { let breaks = breakWords( `Добрый день! ᑕᐻ᙮ — after working on ka${SHY}wen${SHY}non:${SHY}nis, diff --git a/common/models/wordbreakers/test/test-placeholder-word-breaker.ts b/common/models/wordbreakers/test/test-placeholder-word-breaker.ts index 1dba3cbffa..dadd9cbd5f 100644 --- a/common/models/wordbreakers/test/test-placeholder-word-breaker.ts +++ b/common/models/wordbreakers/test/test-placeholder-word-breaker.ts @@ -2,7 +2,7 @@ import {assert} from 'chai'; import {placeholder as breakWords} from '../src'; -describe('The placeholder word breaker', function () { +describe.skip('The placeholder word breaker', function () { it('should break simple English sentences', function () { let breaks = breakWords('Look! -- The quick brown fox jumps... over the lazy dog!'); let words = breaks.map(span => span.text); From 430ba9d56140e417202fab3c2ebe0fd57457c1c0 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Tue, 19 May 2020 14:23:06 -0600 Subject: [PATCH 22/41] Revert "refactor(common/lmlayer): move ASCII word breaker to new package" This reverts commit b4efd9a6a1f84baf9d107b9c746b74434dbf2680. --- common/models/wordbreakers/package.json | 3 +- .../test/test-ascii-word-breaker.ts | 5 +-- .../word_breaking/ascii-word-breaker.ts | 39 +++++++++++++++++++ 3 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 common/predictive-text/worker/word_breaking/ascii-word-breaker.ts diff --git a/common/models/wordbreakers/package.json b/common/models/wordbreakers/package.json index b60f9b95dc..aa53d7656f 100644 --- a/common/models/wordbreakers/package.json +++ b/common/models/wordbreakers/package.json @@ -17,6 +17,7 @@ "license": "MIT", "main": "lib/index.js", "directories": { + "lib": "lib", "test": "test" }, "files": [ @@ -32,7 +33,7 @@ "scripts": { "build": "tsc", "prepublish": "npm run build", - "test": "mocha -r ts-node/register test/**/*.ts" + "test": "mocha" }, "bugs": { "url": "https://github.com/keymanapp/keyman/issues" diff --git a/common/models/wordbreakers/test/test-ascii-word-breaker.ts b/common/models/wordbreakers/test/test-ascii-word-breaker.ts index 2b77810864..f9f4fe390c 100644 --- a/common/models/wordbreakers/test/test-ascii-word-breaker.ts +++ b/common/models/wordbreakers/test/test-ascii-word-breaker.ts @@ -1,6 +1,5 @@ -import {assert} from 'chai'; -import {ascii as breakASCIIWords} from '../src' - +var assert = require('chai').assert; +var breakASCIIWords = require('../../build/intermediate').wordBreakers['ascii']; describe('The ASCII word breaker', function () { it('should break simple English sentences', function () { diff --git a/common/predictive-text/worker/word_breaking/ascii-word-breaker.ts b/common/predictive-text/worker/word_breaking/ascii-word-breaker.ts new file mode 100644 index 0000000000..c7c10726bb --- /dev/null +++ b/common/predictive-text/worker/word_breaking/ascii-word-breaker.ts @@ -0,0 +1,39 @@ +namespace wordBreakers { + /** + * A concrete span class that derives its properties from the result of + * RegExp.exec() array. + */ + class RegExpDerivedSpan implements Span { + readonly text: string; + readonly start: number; + + constructor(text: string, start: number) { + this.text = text; + this.start = start; + } + + get length(): number { + return this.text.length; + } + + get end(): number { + return this.start + this.text.length; + } + } + + /** + * Splits ASCII words. + * + * @param phrase + */ + export function ascii(phrase: string): Span[] { + let matchWord = /[A-Za-z0-9']+/g; + let words: Span[] = []; + let match: RegExpExecArray; + while ((match = matchWord.exec(phrase)) !== null) { + words.push(new RegExpDerivedSpan(match[0], match.index)); + } + + return words; + } +} \ No newline at end of file From 2ac6ca478752708aa0df5ffab5222691276336bc Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Tue, 19 May 2020 14:25:14 -0600 Subject: [PATCH 23/41] test(common/lmlayer): let word breaker work with module --- .../{test-ascii-word-breaker.ts => test-ascii-word-breaker.js} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename common/models/wordbreakers/test/{test-ascii-word-breaker.ts => test-ascii-word-breaker.js} (83%) diff --git a/common/models/wordbreakers/test/test-ascii-word-breaker.ts b/common/models/wordbreakers/test/test-ascii-word-breaker.js similarity index 83% rename from common/models/wordbreakers/test/test-ascii-word-breaker.ts rename to common/models/wordbreakers/test/test-ascii-word-breaker.js index f9f4fe390c..598780ad95 100644 --- a/common/models/wordbreakers/test/test-ascii-word-breaker.ts +++ b/common/models/wordbreakers/test/test-ascii-word-breaker.js @@ -1,5 +1,5 @@ var assert = require('chai').assert; -var breakASCIIWords = require('../../build/intermediate').wordBreakers['ascii']; +var breakASCIIWords = require('..').wordBreakers['ascii']; describe('The ASCII word breaker', function () { it('should break simple English sentences', function () { From 6b87ebbb371d291910ac2f9ecf2efba1efed4594 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Tue, 19 May 2020 14:28:43 -0600 Subject: [PATCH 24/41] chore(common/lmlayer): build word breakers as a namespace package --- common/models/wordbreakers/package.json | 1 + common/models/wordbreakers/tsconfig.json | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/common/models/wordbreakers/package.json b/common/models/wordbreakers/package.json index aa53d7656f..4ebce57169 100644 --- a/common/models/wordbreakers/package.json +++ b/common/models/wordbreakers/package.json @@ -33,6 +33,7 @@ "scripts": { "build": "tsc", "prepublish": "npm run build", + "pretest": "npm run build", "test": "mocha" }, "bugs": { diff --git a/common/models/wordbreakers/tsconfig.json b/common/models/wordbreakers/tsconfig.json index 6acf0c403c..5d9b97a291 100644 --- a/common/models/wordbreakers/tsconfig.json +++ b/common/models/wordbreakers/tsconfig.json @@ -1,11 +1,11 @@ { "compilerOptions": { - "module": "commonjs", + "module": "none", "lib": ["es6"], "target": "es5", "declaration": true, "sourceMap": true, - "outDir": "lib", + "outFile": "index.js", "strict": true, "allowJs": false, "typeRoots": [ From 6fc39df5d06f157cdf2b08169b2c58f71a448637 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Tue, 19 May 2020 14:32:19 -0600 Subject: [PATCH 25/41] Revert "refactor(common/lmlayer): move default word-breaker to @keymanapp/lexical-model-word-breakers" This reverts commit 1f4de1ba93817b82b350fe40623f0759394f713f. --- .../models/wordbreakers/src/default/index.ts | 363 ----------------- .../default-word-breaking}/data.ts | 4 + .../default-word-breaking/index.ts | 378 ++++++++++++++++++ 3 files changed, 382 insertions(+), 363 deletions(-) delete mode 100644 common/models/wordbreakers/src/default/index.ts rename common/{models/wordbreakers/src/default => predictive-text/worker/word_breaking/default-word-breaking}/data.ts (99%) create mode 100644 common/predictive-text/worker/word_breaking/default-word-breaking/index.ts diff --git a/common/models/wordbreakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts deleted file mode 100644 index 2eeabff507..0000000000 --- a/common/models/wordbreakers/src/default/index.ts +++ /dev/null @@ -1,363 +0,0 @@ -import {WordBreakProperty, WORD_BREAK_PROPERTY, I} from './data'; - -/** - * Word breaker based on Unicode Standard Annex #29, Section 4.1: - * Default Word Boundary Specification. - * - * @see http://unicode.org/reports/tr29/#Word_Boundaries - * @see https://github.com/eddieantonio/unicode-default-word-boundary/tree/v12.0.0 - */ -export function default_(text: string): Span[] { - let boundaries = findBoundaries(text); - if (boundaries.length == 0) { - return []; - } - - // All non-empty strings have at least TWO boundaries at the start and end of - // the string. - let spans = []; - for (let i = 0; i < boundaries.length - 1; i++) { - let start = boundaries[i]; - let end = boundaries[i + 1]; - let span = new LazySpan(text, start, end); - - if (isNonSpace(span.text)) { - spans.push(span); - // Preserve a sequence-final space if it exists. Needed to signal "end of word". - } else if (i == boundaries.length - 2) { // if "we just checked the final boundary"... - // We don't want to return the whitespace itself; the correct token is simply ''. - span = new LazySpan(text, end, end); - spans.push(span); - } - } - return spans; -} - -/** - * A span that does not cut out the substring until it absolutely has to! - */ -class LazySpan implements Span { - private _source: string; - readonly start: number; - readonly end: number; - constructor(source: string, start: number, end: number) { - this._source = source; - this.start = start; - this.end = end; - } - - get text(): string { - return this._source.substring(this.start, this.end); - } - - get length(): number { - return this.end - this.start; - } -} - -/** - * Returns true when the chunk does not solely consist of whitespace. - * - * @param chunk a chunk of text. Starts and ends at word boundaries. - */ -function isNonSpace(chunk: string): boolean { - return !Array.from(chunk).map(property).every(wb => ( - wb === WordBreakProperty.CR || - wb === WordBreakProperty.LF || - wb === WordBreakProperty.Newline || - wb === WordBreakProperty.WSegSpace - )); -} - -/** - * Yields a series of string indices where a word break should - * occur. That is, there should be a break BEFORE each string - * index yielded by this generator. - * - * @param text Text to find word boundaries in. - */ -function findBoundaries(text: string): number[] { - // WB1 and WB2: no boundaries if given an empty string. - if (text.length === 0) { - // There are no boundaries in an empty string! - return []; - } - - // This algorithm works by maintaining a sliding window of four SCALAR VALUES. - // - // - Scalar values? JavaScript strings are NOT actually a string of - // Unicode code points; some characters are made up of TWO - // JavaScript indices. e.g., - // "💩".length === 2; - // "💩"[0] === '\uD83D'; - // "💩"[1] === '\uDCA9'; - // - // These characters that are represented by TWO indices are - // called "surrogate pairs". Since we don't want to be in the - // "middle" of a character, make sure we're always advancing - // by scalar values, and NOT indices. That means, we sometimes - // need to advance by TWO indices, not just one. - // - Four values? Some rules look at what's to the left of - // left, and some look at what's to the right of right. So - // keep track of this! - - let boundaries = []; - - let rightPos: number; - let lookaheadPos = 0; // lookahead, one scalar value to the right of right. - // Before the start of the string is also the start of the string. - let lookbehind: WordBreakProperty; - let left = WordBreakProperty.sot; - let right = WordBreakProperty.sot; - let lookahead = wordbreakPropertyAt(0); - // Count RIs to make sure we're not splitting emoji flags: - let nConsecutiveRegionalIndicators = 0; - - do { - // Shift all positions, one scalar value to the right. - rightPos = lookaheadPos; - lookaheadPos = positionAfter(lookaheadPos); - // Shift all properties, one scalar value to the right. - [lookbehind, left, right, lookahead] = - [left, right, lookahead, wordbreakPropertyAt(lookaheadPos)]; - - // Break at the start and end of text, unless the text is empty. - // WB1: Break at start of text... - if (left === WordBreakProperty.sot) { - boundaries.push(rightPos); - continue; - } - // WB2: Break at the end of text... - if (right === WordBreakProperty.eot) { - boundaries.push(rightPos); - break; // Reached the end of the string. We're done! - } - // WB3: Do not break within CRLF: - if (left === WordBreakProperty.CR && right === WordBreakProperty.LF) - continue; - // WB3b: Otherwise, break after... - if (left === WordBreakProperty.Newline || - left === WordBreakProperty.CR || - left === WordBreakProperty.LF) { - boundaries.push(rightPos); - continue; - } - // WB3a: ...and before newlines - if (right === WordBreakProperty.Newline || - right === WordBreakProperty.CR || - right === WordBreakProperty.LF) { - boundaries.push(rightPos); - continue; - } - - // TODO: WB3c is not implemented, due to its complex, error-prone - // implementation, requiring a ginormous regexp, and the fact that - // the only thing it does is prevent big emoji sequences from being - // split up, like 🧚🏼‍♂️ - // https://www.unicode.org/Public/emoji/12.0/emoji-zwj-sequences.txt - - // WB3d: Keep horizontal whitespace together - if (left === WordBreakProperty.WSegSpace && right == WordBreakProperty.WSegSpace) - continue; - - // WB4: Ignore format and extend characters - // This is to keep grapheme clusters together! - // See: Section 6.2: https://unicode.org/reports/tr29/#Grapheme_Cluster_and_Format_Rules - // N.B.: The rule about "except after sot, CR, LF, and - // Newline" already been by WB1, WB2, WB3a, and WB3b above. - while (right === WordBreakProperty.Format || - right === WordBreakProperty.Extend || - right === WordBreakProperty.ZWJ) { - // Continue advancing in the string, as if these - // characters do not exist. DO NOT update left and - // lookbehind however! - [rightPos, lookaheadPos] = [lookaheadPos, positionAfter(lookaheadPos)]; - [right, lookahead] = [lookahead, wordbreakPropertyAt(lookaheadPos)]; - } - // In ignoring the characters in the previous loop, we could - // have fallen off the end of the string, so end the loop - // prematurely if that happens! - if (right === WordBreakProperty.eot) { - boundaries.push(rightPos); - break; - } - // WB4 (continued): Lookahead must ALSO ignore these format, - // extend, ZWJ characters! - while (lookahead === WordBreakProperty.Format || - lookahead === WordBreakProperty.Extend || - lookahead === WordBreakProperty.ZWJ) { - // Continue advancing in the string, as if these - // characters do not exist. DO NOT update left and right, - // however! - lookaheadPos = positionAfter(lookaheadPos); - lookahead = wordbreakPropertyAt(lookaheadPos); - } - - // WB5: Do not break between most letters. - if (isAHLetter(left) && isAHLetter(right)) - continue; - // Do not break across certain punctuation - // WB6: (Don't break before apostrophes in contractions) - if (isAHLetter(left) && isAHLetter(lookahead) && - (right === WordBreakProperty.MidLetter || isMidNumLetQ(right))) - continue; - // WB7: (Don't break after apostrophes in contractions) - if (isAHLetter(lookbehind) && isAHLetter(right) && - (left === WordBreakProperty.MidLetter || isMidNumLetQ(left))) - continue; - // WB7a - if (left === WordBreakProperty.Hebrew_Letter && right === WordBreakProperty.Single_Quote) - continue; - // WB7b - if (left === WordBreakProperty.Hebrew_Letter && right === WordBreakProperty.Double_Quote && - lookahead === WordBreakProperty.Hebrew_Letter) - continue; - // WB7c - if (lookbehind === WordBreakProperty.Hebrew_Letter && left === WordBreakProperty.Double_Quote && - right === WordBreakProperty.Hebrew_Letter) - continue; - // Do not break within sequences of digits, or digits adjacent to letters. - // e.g., "3a" or "A3" - // WB8 - if (left === WordBreakProperty.Numeric && right === WordBreakProperty.Numeric) - continue; - // WB9 - if (isAHLetter(left) && right === WordBreakProperty.Numeric) - continue; - // WB10 - if (left === WordBreakProperty.Numeric && isAHLetter(right)) - continue; - // Do not break within sequences, such as 3.2, 3,456.789 - // WB11 - if (lookbehind === WordBreakProperty.Numeric && right === WordBreakProperty.Numeric && - (left === WordBreakProperty.MidNum || isMidNumLetQ(left))) - continue; - // WB12 - if (left === WordBreakProperty.Numeric && lookahead === WordBreakProperty.Numeric && - (right === WordBreakProperty.MidNum || isMidNumLetQ(right))) - continue; - // WB13: Do not break between Katakana - if (left === WordBreakProperty.Katakana && right === WordBreakProperty.Katakana) - continue; - // Do not break from extenders (e.g., U+202F NARROW NO-BREAK SPACE) - // WB13a - if ((isAHLetter(left) || - left === WordBreakProperty.Numeric || - left === WordBreakProperty.Katakana || - left === WordBreakProperty.ExtendNumLet) && - right === WordBreakProperty.ExtendNumLet) - continue; - // WB13b - if ((isAHLetter(right) || - right === WordBreakProperty.Numeric || - right === WordBreakProperty.Katakana) && left === WordBreakProperty.ExtendNumLet) - continue; - - // WB15 & WB16: - // Do not break within emoji flag sequences. That is, do not break between - // regional indicator (RI) symbols if there is an odd number of RI - // characters before the break point. - if (right === WordBreakProperty.Regional_Indicator) { - // Emoji flags are actually composed of TWO scalar values, each being a - // "regional indicator". These indicators correspond to Latin letters. Put - // two of them together, and they spell out an ISO 3166-1-alpha-2 country - // code. Since these always come in pairs, NEVER split the pairs! So, if - // we happen to be inside the middle of an odd numbered of - // Regional_Indicators, DON'T SPLIT IT! - nConsecutiveRegionalIndicators += 1; - if ((nConsecutiveRegionalIndicators % 2) == 1) { - continue; - } - } else { - nConsecutiveRegionalIndicators = 0; - } - // WB999: Otherwise, break EVERYWHERE (including around ideographs) - boundaries.push(rightPos); - } while (rightPos < text.length); - - return boundaries; - - ///// Internal utility functions ///// - - /** - * Returns the position of the start of the next scalar value. This jumps - * over surrogate pairs. - * - * If asked for the character AFTER the end of the string, this always - * returns the length of the string. - */ - function positionAfter(pos: number): number { - if (pos >= text.length) { - return text.length; - } else if (isStartOfSurrogatePair(text[pos])) { - return pos + 2; - } - return pos + 1; - } - - /** - * Return the value of the Word_Break property at the given string index. - * @param pos position in the text. - */ - function wordbreakPropertyAt(pos: number) { - if (pos < 0) { - return WordBreakProperty.sot; // Always "start of string" before the string starts! - } else if (pos >= text.length) { - return WordBreakProperty.eot; // Always "end of string" after the string ends! - } else if (isStartOfSurrogatePair(text[pos])) { - // Surrogate pairs the next TWO items from the string! - return property(text[pos] + text[pos + 1]); - } - return property(text[pos]); - } - - // Word_Break rule macros - // See: https://unicode.org/reports/tr29/#WB_Rule_Macros - function isAHLetter(prop: WordBreakProperty): boolean { - return prop === WordBreakProperty.ALetter || - prop === WordBreakProperty.Hebrew_Letter; - } - - function isMidNumLetQ(prop: WordBreakProperty): boolean { - return prop === WordBreakProperty.MidNumLet || - prop === WordBreakProperty.Single_Quote; - } -} - -function isStartOfSurrogatePair(character: string) { - let codeUnit = character.charCodeAt(0); - return codeUnit >= 0xD800 && codeUnit <= 0xDBFF; -} - -/** - * Return the Word_Break property value for a character. - * Note that - * @param character a scalar value - */ -function property(character: string): WordBreakProperty { - // This MUST be a scalar value. - // TODO: remove dependence on character.codepointAt()? - let codepoint = character.codePointAt(0) as number; - return searchForProperty(codepoint, 0, WORD_BREAK_PROPERTY.length - 1); -} - -/** - * Binary search for the word break property of a given CODE POINT. - */ -function searchForProperty(codePoint: number, left: number, right: number): WordBreakProperty { - // All items that are not found in the array are assigned the 'Other' property. - if (right < left) { - return WordBreakProperty.Other; - } - - let midpoint = left + ~~((right - left) / 2); - let candidate = WORD_BREAK_PROPERTY[midpoint]; - if (codePoint < candidate[I.Start]) { - return searchForProperty(codePoint, left, midpoint - 1); - } else if (codePoint > candidate[I.End]) { - return searchForProperty(codePoint, midpoint + 1, right); - } else { - // We found it! - return candidate[I.Value]; - } -} diff --git a/common/models/wordbreakers/src/default/data.ts b/common/predictive-text/worker/word_breaking/default-word-breaking/data.ts similarity index 99% rename from common/models/wordbreakers/src/default/data.ts rename to common/predictive-text/worker/word_breaking/default-word-breaking/data.ts index 0f81b1a5c4..a0bb37c556 100644 --- a/common/models/wordbreakers/src/default/data.ts +++ b/common/predictive-text/worker/word_breaking/default-word-breaking/data.ts @@ -1,3 +1,5 @@ +namespace wordBreakers { +export namespace data { // Automatically generated file. DO NOT MODIFY. /** * Valid values for a word break property. @@ -1290,3 +1292,5 @@ export const WORD_BREAK_PROPERTY: [number, number, WordBreakProperty][] = [ [/*start*/ 0xE0020, /*end*/ 0xE007F, WordBreakProperty.Extend], [/*start*/ 0xE0100, /*end*/ 0xE01EF, WordBreakProperty.Extend], ]; +} +} \ No newline at end of file diff --git a/common/predictive-text/worker/word_breaking/default-word-breaking/index.ts b/common/predictive-text/worker/word_breaking/default-word-breaking/index.ts new file mode 100644 index 0000000000..f35b22ac80 --- /dev/null +++ b/common/predictive-text/worker/word_breaking/default-word-breaking/index.ts @@ -0,0 +1,378 @@ +// Include the word-breaking data here: +/// +namespace wordBreakers { + /** + * Word breaker based on Unicode Standard Annex #29, Section 4.1: + * Default Word Boundary Specification. + * + * @see http://unicode.org/reports/tr29/#Word_Boundaries + * @see https://github.com/eddieantonio/unicode-default-word-boundary/tree/v12.0.0 + */ + export function default_(text: string): Span[] { + let boundaries = findBoundaries(text); + if (boundaries.length == 0) { + return []; + } + + // All non-empty strings have at least TWO boundaries at the start and end of + // the string. + let spans = []; + for (let i = 0; i < boundaries.length - 1; i++) { + let start = boundaries[i]; + let end = boundaries[i + 1]; + let span = new LazySpan(text, start, end); + + if (isNonSpace(span.text)) { + spans.push(span); + // Preserve a sequence-final space if it exists. Needed to signal "end of word". + } else if (i == boundaries.length - 2) { // if "we just checked the final boundary"... + // We don't want to return the whitespace itself; the correct token is simply ''. + span = new LazySpan(text, end, end); + spans.push(span); + } + } + return spans; + } + + // Utilities // + import WordBreakProperty = wordBreakers.data.WordBreakProperty; + import WORD_BREAK_PROPERTY = wordBreakers.data.WORD_BREAK_PROPERTY; + import I = wordBreakers.data.I; + + /** + * A span that does not cut out the substring until it absolutely has to! + */ + class LazySpan implements Span { + private _source: string; + readonly start: number; + readonly end: number; + constructor(source: string, start: number, end: number) { + this._source = source; + this.start = start; + this.end = end; + } + + get text(): string { + return this._source.substring(this.start, this.end); + } + + get length(): number { + return this.end - this.start; + } + } + + /** + * Returns true when the chunk does not solely consist of whitespace. + * + * @param chunk a chunk of text. Starts and ends at word boundaries. + */ + function isNonSpace(chunk: string): boolean { + return !Array.from(chunk).map(property).every(wb => ( + wb === WordBreakProperty.CR || + wb === WordBreakProperty.LF || + wb === WordBreakProperty.Newline || + wb === WordBreakProperty.WSegSpace + )); + } + + /** + * Yields a series of string indices where a word break should + * occur. That is, there should be a break BEFORE each string + * index yielded by this generator. + * + * @param text Text to find word boundaries in. + */ + function findBoundaries(text: string): number[] { + // WB1 and WB2: no boundaries if given an empty string. + if (text.length === 0) { + // There are no boundaries in an empty string! + return []; + } + + // This algorithm works by maintaining a sliding window of four SCALAR VALUES. + // + // - Scalar values? JavaScript strings are NOT actually a string of + // Unicode code points; some characters are made up of TWO + // JavaScript indices. e.g., + // "💩".length === 2; + // "💩"[0] === '\uD83D'; + // "💩"[1] === '\uDCA9'; + // + // These characters that are represented by TWO indices are + // called "surrogate pairs". Since we don't want to be in the + // "middle" of a character, make sure we're always advancing + // by scalar values, and NOT indices. That means, we sometimes + // need to advance by TWO indices, not just one. + // - Four values? Some rules look at what's to the left of + // left, and some look at what's to the right of right. So + // keep track of this! + + let boundaries = []; + + let rightPos: number; + let lookaheadPos = 0; // lookahead, one scalar value to the right of right. + // Before the start of the string is also the start of the string. + let lookbehind: WordBreakProperty; + let left = WordBreakProperty.sot; + let right = WordBreakProperty.sot; + let lookahead = wordbreakPropertyAt(0); + // Count RIs to make sure we're not splitting emoji flags: + let nConsecutiveRegionalIndicators = 0; + + do { + // Shift all positions, one scalar value to the right. + rightPos = lookaheadPos; + lookaheadPos = positionAfter(lookaheadPos); + // Shift all properties, one scalar value to the right. + [lookbehind, left, right, lookahead] = + [left, right, lookahead, wordbreakPropertyAt(lookaheadPos)]; + + // Break at the start and end of text, unless the text is empty. + // WB1: Break at start of text... + if (left === WordBreakProperty.sot) { + boundaries.push(rightPos); + continue; + } + // WB2: Break at the end of text... + if (right === WordBreakProperty.eot) { + boundaries.push(rightPos); + break; // Reached the end of the string. We're done! + } + // WB3: Do not break within CRLF: + if (left === WordBreakProperty.CR && right === WordBreakProperty.LF) + continue; + // WB3b: Otherwise, break after... + if (left === WordBreakProperty.Newline || + left === WordBreakProperty.CR || + left === WordBreakProperty.LF) { + boundaries.push(rightPos); + continue; + } + // WB3a: ...and before newlines + if (right === WordBreakProperty.Newline || + right === WordBreakProperty.CR || + right === WordBreakProperty.LF) { + boundaries.push(rightPos); + continue; + } + + // TODO: WB3c is not implemented, due to its complex, error-prone + // implementation, requiring a ginormous regexp, and the fact that + // the only thing it does is prevent big emoji sequences from being + // split up, like 🧚🏼‍♂️ + // https://www.unicode.org/Public/emoji/12.0/emoji-zwj-sequences.txt + + // WB3d: Keep horizontal whitespace together + if (left === WordBreakProperty.WSegSpace && right == WordBreakProperty.WSegSpace) + continue; + + // WB4: Ignore format and extend characters + // This is to keep grapheme clusters together! + // See: Section 6.2: https://unicode.org/reports/tr29/#Grapheme_Cluster_and_Format_Rules + // N.B.: The rule about "except after sot, CR, LF, and + // Newline" already been by WB1, WB2, WB3a, and WB3b above. + while (right === WordBreakProperty.Format || + right === WordBreakProperty.Extend || + right === WordBreakProperty.ZWJ) { + // Continue advancing in the string, as if these + // characters do not exist. DO NOT update left and + // lookbehind however! + [rightPos, lookaheadPos] = [lookaheadPos, positionAfter(lookaheadPos)]; + [right, lookahead] = [lookahead, wordbreakPropertyAt(lookaheadPos)]; + } + // In ignoring the characters in the previous loop, we could + // have fallen off the end of the string, so end the loop + // prematurely if that happens! + if (right === WordBreakProperty.eot) { + boundaries.push(rightPos); + break; + } + // WB4 (continued): Lookahead must ALSO ignore these format, + // extend, ZWJ characters! + while (lookahead === WordBreakProperty.Format || + lookahead === WordBreakProperty.Extend || + lookahead === WordBreakProperty.ZWJ) { + // Continue advancing in the string, as if these + // characters do not exist. DO NOT update left and right, + // however! + lookaheadPos = positionAfter(lookaheadPos); + lookahead = wordbreakPropertyAt(lookaheadPos); + } + + // WB5: Do not break between most letters. + if (isAHLetter(left) && isAHLetter(right)) + continue; + // Do not break across certain punctuation + // WB6: (Don't break before apostrophes in contractions) + if (isAHLetter(left) && isAHLetter(lookahead) && + (right === WordBreakProperty.MidLetter || isMidNumLetQ(right))) + continue; + // WB7: (Don't break after apostrophes in contractions) + if (isAHLetter(lookbehind) && isAHLetter(right) && + (left === WordBreakProperty.MidLetter || isMidNumLetQ(left))) + continue; + // WB7a + if (left === WordBreakProperty.Hebrew_Letter && right === WordBreakProperty.Single_Quote) + continue; + // WB7b + if (left === WordBreakProperty.Hebrew_Letter && right === WordBreakProperty.Double_Quote && + lookahead === WordBreakProperty.Hebrew_Letter) + continue; + // WB7c + if (lookbehind === WordBreakProperty.Hebrew_Letter && left === WordBreakProperty.Double_Quote && + right === WordBreakProperty.Hebrew_Letter) + continue; + // Do not break within sequences of digits, or digits adjacent to letters. + // e.g., "3a" or "A3" + // WB8 + if (left === WordBreakProperty.Numeric && right === WordBreakProperty.Numeric) + continue; + // WB9 + if (isAHLetter(left) && right === WordBreakProperty.Numeric) + continue; + // WB10 + if (left === WordBreakProperty.Numeric && isAHLetter(right)) + continue; + // Do not break within sequences, such as 3.2, 3,456.789 + // WB11 + if (lookbehind === WordBreakProperty.Numeric && right === WordBreakProperty.Numeric && + (left === WordBreakProperty.MidNum || isMidNumLetQ(left))) + continue; + // WB12 + if (left === WordBreakProperty.Numeric && lookahead === WordBreakProperty.Numeric && + (right === WordBreakProperty.MidNum || isMidNumLetQ(right))) + continue; + // WB13: Do not break between Katakana + if (left === WordBreakProperty.Katakana && right === WordBreakProperty.Katakana) + continue; + // Do not break from extenders (e.g., U+202F NARROW NO-BREAK SPACE) + // WB13a + if ((isAHLetter(left) || + left === WordBreakProperty.Numeric || + left === WordBreakProperty.Katakana || + left === WordBreakProperty.ExtendNumLet) && + right === WordBreakProperty.ExtendNumLet) + continue; + // WB13b + if ((isAHLetter(right) || + right === WordBreakProperty.Numeric || + right === WordBreakProperty.Katakana) && left === WordBreakProperty.ExtendNumLet) + continue; + + // WB15 & WB16: + // Do not break within emoji flag sequences. That is, do not break between + // regional indicator (RI) symbols if there is an odd number of RI + // characters before the break point. + if (right === WordBreakProperty.Regional_Indicator) { + // Emoji flags are actually composed of TWO scalar values, each being a + // "regional indicator". These indicators correspond to Latin letters. Put + // two of them together, and they spell out an ISO 3166-1-alpha-2 country + // code. Since these always come in pairs, NEVER split the pairs! So, if + // we happen to be inside the middle of an odd numbered of + // Regional_Indicators, DON'T SPLIT IT! + nConsecutiveRegionalIndicators += 1; + if ((nConsecutiveRegionalIndicators % 2) == 1) { + continue; + } + } else { + nConsecutiveRegionalIndicators = 0; + } + // WB999: Otherwise, break EVERYWHERE (including around ideographs) + boundaries.push(rightPos); + } while (rightPos < text.length); + + return boundaries; + + ///// Internal utility functions ///// + + /** + * Returns the position of the start of the next scalar value. This jumps + * over surrogate pairs. + * + * If asked for the character AFTER the end of the string, this always + * returns the length of the string. + */ + function positionAfter(pos: number): number { + if (pos >= text.length) { + return text.length; + } else if (isStartOfSurrogatePair(text[pos])) { + return pos + 2; + } + return pos + 1; + } + + /** + * Return the value of the Word_Break property at the given string index. + * @param pos position in the text. + */ + function wordbreakPropertyAt(pos: number) { + if (pos < 0) { + return WordBreakProperty.sot; // Always "start of string" before the string starts! + } else if (pos >= text.length) { + return WordBreakProperty.eot; // Always "end of string" after the string ends! + } else if (isStartOfSurrogatePair(text[pos])) { + // Surrogate pairs the next TWO items from the string! + return property(text[pos] + text[pos + 1]); + } + return property(text[pos]); + } + + // Word_Break rule macros + // See: https://unicode.org/reports/tr29/#WB_Rule_Macros + function isAHLetter(prop: WordBreakProperty): boolean { + return prop === WordBreakProperty.ALetter || + prop === WordBreakProperty.Hebrew_Letter; + } + + function isMidNumLetQ(prop: WordBreakProperty): boolean { + return prop === WordBreakProperty.MidNumLet || + prop === WordBreakProperty.Single_Quote; + } + } + + function isStartOfSurrogatePair(character: string) { + let codeUnit = character.charCodeAt(0); + return codeUnit >= 0xD800 && codeUnit <= 0xDBFF; + } + + /** + * Return the Word_Break property value for a character. + * Note that + * @param character a scalar value + */ + function property(character: string): WordBreakProperty { + // This MUST be a scalar value. + // TODO: remove dependence on character.codepointAt()? + let codepoint = character.codePointAt(0) as number; + return searchForProperty(codepoint, 0, WORD_BREAK_PROPERTY.length - 1); + } + + /** + * Binary search for the word break property of a given CODE POINT. + */ + function searchForProperty(codePoint: number, left: number, right: number): WordBreakProperty { + // All items that are not found in the array are assigned the 'Other' property. + if (right < left) { + return WordBreakProperty.Other; + } + + let midpoint = left + ~~((right - left) / 2); + let candidate = WORD_BREAK_PROPERTY[midpoint]; + if (codePoint < candidate[I.Start]) { + return searchForProperty(codePoint, left, midpoint - 1); + } else if (codePoint > candidate[I.End]) { + return searchForProperty(codePoint, midpoint + 1, right); + } else { + // We found it! + return candidate[I.Value]; + } + } +} + +// We cannot export a member whose name is a reserved word when +// implementing a namespace, BUT we can manually make the +// assignment and **declare** it as part of the namespace. +wordBreakers['default'] = wordBreakers.default_; +declare namespace wordBreakers { + export { default_ as default }; +} From 374f0dfd8eb275eefd8fd23ded66ff047d645b4f Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Tue, 19 May 2020 14:34:11 -0600 Subject: [PATCH 26/41] Revert "refactor(common/lmlayer): move placeholder word breaker to @keymanapp/lexical-model-word-breakers" This reverts commit 73b16ae33eb62f08247fbcb1afe0fca032f3df97. --- common/models/wordbreakers/src/index.ts | 5 +--- common/models/wordbreakers/src/placeholder.ts | 21 ---------------- .../test/test-placeholder-word-breaker.ts | 4 ++-- .../word_breaking/placeholder-word-breaker.ts | 24 +++++++++++++++++++ 4 files changed, 27 insertions(+), 27 deletions(-) delete mode 100644 common/models/wordbreakers/src/placeholder.ts create mode 100644 common/predictive-text/worker/word_breaking/placeholder-word-breaker.ts diff --git a/common/models/wordbreakers/src/index.ts b/common/models/wordbreakers/src/index.ts index c3dd1420d2..ea1cc87eea 100644 --- a/common/models/wordbreakers/src/index.ts +++ b/common/models/wordbreakers/src/index.ts @@ -1,6 +1,3 @@ import {ascii} from "./ascii"; -import {placeholder} from './placeholder'; -import {default_} from "./default"; -export {ascii, placeholder}; -export {default_ as default}; +export {ascii}; diff --git a/common/models/wordbreakers/src/placeholder.ts b/common/models/wordbreakers/src/placeholder.ts deleted file mode 100644 index 7924b196e4..0000000000 --- a/common/models/wordbreakers/src/placeholder.ts +++ /dev/null @@ -1,21 +0,0 @@ -/** - * A **VERY** dumb word breaker that simply splits at words. Do not use this - * word breaker! - * - * @param phrase The phrase in which to break words. - * @deprecated Use a word breaker tailored to your language instead! - */ -export function placeholder(phrase: string): Span[] { - let nextStart = 0; - return phrase.split(/\s+/).map(utterance => { - // XXX: The indices are NOT accurate to the original phrase! - let span = { - start: nextStart, - end: nextStart + utterance.length, - text: utterance, - length: utterance.length - }; - nextStart = span.end; - return span; - }); -} diff --git a/common/models/wordbreakers/test/test-placeholder-word-breaker.ts b/common/models/wordbreakers/test/test-placeholder-word-breaker.ts index dadd9cbd5f..7a90b0b34d 100644 --- a/common/models/wordbreakers/test/test-placeholder-word-breaker.ts +++ b/common/models/wordbreakers/test/test-placeholder-word-breaker.ts @@ -1,11 +1,11 @@ import {assert} from 'chai'; -import {placeholder as breakWords} from '../src'; +import {placeholder as breakWords} from '../' describe.skip('The placeholder word breaker', function () { it('should break simple English sentences', function () { let breaks = breakWords('Look! -- The quick brown fox jumps... over the lazy dog!'); let words = breaks.map(span => span.text); - assert.deepEqual(words, ['Look!', '--', 'The', 'quick', 'brown', 'fox', 'jumps...', 'over', 'the', 'lazy', 'dog!']); + assert.deepEqual(words, ['Look', 'The', 'quick', 'brown', 'fox', 'jumps', 'over', 'the', 'lazy', 'dog']); }); }); diff --git a/common/predictive-text/worker/word_breaking/placeholder-word-breaker.ts b/common/predictive-text/worker/word_breaking/placeholder-word-breaker.ts new file mode 100644 index 0000000000..1105b89fb9 --- /dev/null +++ b/common/predictive-text/worker/word_breaking/placeholder-word-breaker.ts @@ -0,0 +1,24 @@ +namespace wordBreakers { + + /** + * A **VERY** dumb word breaker that simply splits at words. Do not use this + * word breaker! + * + * @param phrase The phrase in which to break words. + * @deprecated Use a word breaker tailored to your language instead! + */ + export function placeholder(phrase: string): Span[] { + let nextStart = 0; + return phrase.split(/\s+/).map(utterance => { + // XXX: The indices are NOT accurate to the original phrase! + let span = { + start: nextStart, + end: nextStart + utterance.length, + text: utterance, + length: utterance.length + }; + nextStart = span.end; + return span; + }); + } +} From 863b83883b2ab15a9631579c673765b9ce9afbbc Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Tue, 19 May 2020 14:36:03 -0600 Subject: [PATCH 27/41] chore(common/lmlayer): get ASCII word breaker compiling --- common/models/wordbreakers/src/ascii.ts | 62 +++++++++++++------------ common/models/wordbreakers/src/index.ts | 4 +- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/common/models/wordbreakers/src/ascii.ts b/common/models/wordbreakers/src/ascii.ts index ce007c8d00..81be11f395 100644 --- a/common/models/wordbreakers/src/ascii.ts +++ b/common/models/wordbreakers/src/ascii.ts @@ -1,37 +1,39 @@ -/** - * Splits ASCII words. - * - * @param phrase - */ -export function ascii(phrase: string): Span[] { - let matchWord = /[A-Za-z0-9']+/g; - let words: Span[] = []; - let match: RegExpExecArray | null; - while ((match = matchWord.exec(phrase)) !== null) { - words.push(new RegExpDerivedSpan(match[0], match.index)); +namespace wordBreakers { + /** + * Splits ASCII words. + * + * @param phrase + */ + export function ascii(phrase: string): Span[] { + let matchWord = /[A-Za-z0-9']+/g; + let words: Span[] = []; + let match: RegExpExecArray | null; + while ((match = matchWord.exec(phrase)) !== null) { + words.push(new RegExpDerivedSpan(match[0], match.index)); + } + + return words; } - return words; -} + /** + * A concrete span class that derives its properties from the result of + * RegExp.exec() array. + */ + class RegExpDerivedSpan implements Span { + readonly text: string; + readonly start: number; -/** - * A concrete span class that derives its properties from the result of - * RegExp.exec() array. - */ -class RegExpDerivedSpan implements Span { - readonly text: string; - readonly start: number; + constructor(text: string, start: number) { + this.text = text; + this.start = start; + } - constructor(text: string, start: number) { - this.text = text; - this.start = start; - } + get length(): number { + return this.text.length; + } - get length(): number { - return this.text.length; - } - - get end(): number { - return this.start + this.text.length; + get end(): number { + return this.start + this.text.length; + } } } diff --git a/common/models/wordbreakers/src/index.ts b/common/models/wordbreakers/src/index.ts index ea1cc87eea..d86a893ada 100644 --- a/common/models/wordbreakers/src/index.ts +++ b/common/models/wordbreakers/src/index.ts @@ -1,3 +1 @@ -import {ascii} from "./ascii"; - -export {ascii}; +/// From 7b1f86b39943d71dccd0ba4b8bcd18f98b04dcca Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Tue, 19 May 2020 14:46:33 -0600 Subject: [PATCH 28/41] fix(common/lmlayer): export wordBreakers namespace --- common/models/wordbreakers/src/index.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/common/models/wordbreakers/src/index.ts b/common/models/wordbreakers/src/index.ts index d86a893ada..244e75c2b8 100644 --- a/common/models/wordbreakers/src/index.ts +++ b/common/models/wordbreakers/src/index.ts @@ -1 +1,7 @@ /// + +// Let the word breakers be available both in the browser and in Node. +declare var module: unknown; +if (typeof module !== "undefined" && (module as any).exports) { + (module as any).exports['wordBreakers'] = wordBreakers; +} From 1bd4f6e94d8d68b5a1db6fec97dd6ac30d71242a Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Tue, 19 May 2020 14:48:21 -0600 Subject: [PATCH 29/41] fix(common/lmlayer): forgot to remove ascii-word-breaker --- .../word_breaking/ascii-word-breaker.ts | 39 ------------------- 1 file changed, 39 deletions(-) delete mode 100644 common/predictive-text/worker/word_breaking/ascii-word-breaker.ts diff --git a/common/predictive-text/worker/word_breaking/ascii-word-breaker.ts b/common/predictive-text/worker/word_breaking/ascii-word-breaker.ts deleted file mode 100644 index c7c10726bb..0000000000 --- a/common/predictive-text/worker/word_breaking/ascii-word-breaker.ts +++ /dev/null @@ -1,39 +0,0 @@ -namespace wordBreakers { - /** - * A concrete span class that derives its properties from the result of - * RegExp.exec() array. - */ - class RegExpDerivedSpan implements Span { - readonly text: string; - readonly start: number; - - constructor(text: string, start: number) { - this.text = text; - this.start = start; - } - - get length(): number { - return this.text.length; - } - - get end(): number { - return this.start + this.text.length; - } - } - - /** - * Splits ASCII words. - * - * @param phrase - */ - export function ascii(phrase: string): Span[] { - let matchWord = /[A-Za-z0-9']+/g; - let words: Span[] = []; - let match: RegExpExecArray; - while ((match = matchWord.exec(phrase)) !== null) { - words.push(new RegExpDerivedSpan(match[0], match.index)); - } - - return words; - } -} \ No newline at end of file From e98ae7d7a185a1bc283778fb7aa415e82fcb0b14 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Tue, 19 May 2020 14:57:55 -0600 Subject: [PATCH 30/41] fix(common/lmlayer): add @types/node to @keymanapp/models-wordbreaker to prevent failure --- common/models/wordbreakers/package-lock.json | 7 ++++++- common/models/wordbreakers/package.json | 3 +++ common/models/wordbreakers/src/index.ts | 1 - 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/common/models/wordbreakers/package-lock.json b/common/models/wordbreakers/package-lock.json index 78a82b4388..8e636b7edf 100644 --- a/common/models/wordbreakers/package-lock.json +++ b/common/models/wordbreakers/package-lock.json @@ -1,5 +1,5 @@ { - "name": "@keymanapp/lexical-model-word-breakers", + "name": "@keymanapp/models-wordbreakers", "version": "14.0.75", "lockfileVersion": 1, "requires": true, @@ -16,6 +16,11 @@ "integrity": "sha512-ZvO2tAcjmMi8V/5Z3JsyofMe3hasRcaw88cto5etSVMwVQfeivGAlEYmaQgceUSVYFofVjT+ioHsATjdWcFt1w==", "dev": true }, + "@types/node": { + "version": "14.0.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.0.3.tgz", + "integrity": "sha512-a8TR2N5VEJCL9HEJrAfwv3UI1bZq50HydowDDVV6pfnY7ZwG5Pjii+nSDhrDtGW3XKMoVKOgG8zS/Kv5j399uA==" + }, "arg": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", diff --git a/common/models/wordbreakers/package.json b/common/models/wordbreakers/package.json index 4ebce57169..fae4d76ae5 100644 --- a/common/models/wordbreakers/package.json +++ b/common/models/wordbreakers/package.json @@ -47,5 +47,8 @@ "mocha": "^5.2.0", "ts-node": "^8.10.1", "typescript": "^3.2.1" + }, + "dependencies": { + "@types/node": "^14.0.3" } } diff --git a/common/models/wordbreakers/src/index.ts b/common/models/wordbreakers/src/index.ts index 244e75c2b8..2bb4a506fb 100644 --- a/common/models/wordbreakers/src/index.ts +++ b/common/models/wordbreakers/src/index.ts @@ -1,7 +1,6 @@ /// // Let the word breakers be available both in the browser and in Node. -declare var module: unknown; if (typeof module !== "undefined" && (module as any).exports) { (module as any).exports['wordBreakers'] = wordBreakers; } From 349adaf9f21ee13efaf54857b6846b2729041c1f Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Tue, 19 May 2020 14:58:31 -0600 Subject: [PATCH 31/41] change(common/lmlayer): include word breakers from separate package --- common/predictive-text/worker/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/predictive-text/worker/index.ts b/common/predictive-text/worker/index.ts index de7b70699f..3529cf4005 100644 --- a/common/predictive-text/worker/index.ts +++ b/common/predictive-text/worker/index.ts @@ -31,7 +31,7 @@ /// /// -/// +/// /// /** From af343138cfbb1c679e631684c36f8e4353720dda Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Tue, 19 May 2020 15:00:43 -0600 Subject: [PATCH 32/41] test(common/lmlayer): placeholder word breaker --- ...der-word-breaker.ts => test-placeholder-word-breaker.js} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename common/models/wordbreakers/test/{test-placeholder-word-breaker.ts => test-placeholder-word-breaker.js} (67%) diff --git a/common/models/wordbreakers/test/test-placeholder-word-breaker.ts b/common/models/wordbreakers/test/test-placeholder-word-breaker.js similarity index 67% rename from common/models/wordbreakers/test/test-placeholder-word-breaker.ts rename to common/models/wordbreakers/test/test-placeholder-word-breaker.js index 7a90b0b34d..ae7e82cfca 100644 --- a/common/models/wordbreakers/test/test-placeholder-word-breaker.ts +++ b/common/models/wordbreakers/test/test-placeholder-word-breaker.js @@ -1,8 +1,8 @@ -import {assert} from 'chai'; -import {placeholder as breakWords} from '../' +const assert = require('chai').assert; +const breakWords = require('..').wordBreakers['placeholder']; -describe.skip('The placeholder word breaker', function () { +describe('The placeholder word breaker', function () { it('should break simple English sentences', function () { let breaks = breakWords('Look! -- The quick brown fox jumps... over the lazy dog!'); let words = breaks.map(span => span.text); From 6af00a3358a0e2daf8b1b3aed5ce48e62c7610fd Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Tue, 19 May 2020 15:03:43 -0600 Subject: [PATCH 33/41] test(common/lmlayer): correct expected output of placeholder word breaker --- .../models/wordbreakers/test/test-placeholder-word-breaker.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/models/wordbreakers/test/test-placeholder-word-breaker.js b/common/models/wordbreakers/test/test-placeholder-word-breaker.js index ae7e82cfca..b23fd5d6bb 100644 --- a/common/models/wordbreakers/test/test-placeholder-word-breaker.js +++ b/common/models/wordbreakers/test/test-placeholder-word-breaker.js @@ -6,6 +6,6 @@ describe('The placeholder word breaker', function () { it('should break simple English sentences', function () { let breaks = breakWords('Look! -- The quick brown fox jumps... over the lazy dog!'); let words = breaks.map(span => span.text); - assert.deepEqual(words, ['Look', 'The', 'quick', 'brown', 'fox', 'jumps', 'over', 'the', 'lazy', 'dog']); + assert.deepEqual(words, ['Look!', '--', 'The', 'quick', 'brown', 'fox', 'jumps...', 'over', 'the', 'lazy', 'dog!']); }); }); From 0baf04ae060c5e1c75a617de897f9bfe569dcf69 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Tue, 19 May 2020 15:04:07 -0600 Subject: [PATCH 34/41] refactor(common/lmlayer): move placeholder wordbreaker to models-wordbreakers --- common/models/wordbreakers/src/index.ts | 1 + .../wordbreakers/src/placeholder.ts} | 0 2 files changed, 1 insertion(+) rename common/{predictive-text/worker/word_breaking/placeholder-word-breaker.ts => models/wordbreakers/src/placeholder.ts} (100%) diff --git a/common/models/wordbreakers/src/index.ts b/common/models/wordbreakers/src/index.ts index 2bb4a506fb..1e31e002cc 100644 --- a/common/models/wordbreakers/src/index.ts +++ b/common/models/wordbreakers/src/index.ts @@ -1,4 +1,5 @@ /// +/// // Let the word breakers be available both in the browser and in Node. if (typeof module !== "undefined" && (module as any).exports) { diff --git a/common/predictive-text/worker/word_breaking/placeholder-word-breaker.ts b/common/models/wordbreakers/src/placeholder.ts similarity index 100% rename from common/predictive-text/worker/word_breaking/placeholder-word-breaker.ts rename to common/models/wordbreakers/src/placeholder.ts From 940639abf9527c5cd3aba3f43a07202eba348382 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Tue, 19 May 2020 15:05:57 -0600 Subject: [PATCH 35/41] test(common/lmlayer): adjust default word breaker tests to work with namespaces --- ...default-word-breaker.ts => test-default-word-breaker.js} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename common/models/wordbreakers/test/{test-default-word-breaker.ts => test-default-word-breaker.js} (78%) diff --git a/common/models/wordbreakers/test/test-default-word-breaker.ts b/common/models/wordbreakers/test/test-default-word-breaker.js similarity index 78% rename from common/models/wordbreakers/test/test-default-word-breaker.ts rename to common/models/wordbreakers/test/test-default-word-breaker.js index ed25c4bea4..6cf2362695 100644 --- a/common/models/wordbreakers/test/test-default-word-breaker.ts +++ b/common/models/wordbreakers/test/test-default-word-breaker.js @@ -2,12 +2,12 @@ * Smoke-test the default */ -import {assert} from 'chai'; -import {default as breakWords} from '../src'; +const assert = require('chai').assert; +const breakWords = require('..').wordBreakers['default']; const SHY = '\u00AD'; -describe.skip('The default word breaker', function () { +describe('The default word breaker', function () { it('should break multilingual text', function () { let breaks = breakWords( `Добрый день! ᑕᐻ᙮ — after working on ka${SHY}wen${SHY}non:${SHY}nis, From a4dae628178dae0e28c5e6b7cf500523d58b5422 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Tue, 19 May 2020 15:07:06 -0600 Subject: [PATCH 36/41] refactor(common/lmlayer): move default word breaker to models-wordbreakers package --- .../wordbreakers/src/default}/data.ts | 0 .../wordbreakers/src/default}/index.ts | 0 common/models/wordbreakers/src/index.ts | 1 + 3 files changed, 1 insertion(+) rename common/{predictive-text/worker/word_breaking/default-word-breaking => models/wordbreakers/src/default}/data.ts (100%) rename common/{predictive-text/worker/word_breaking/default-word-breaking => models/wordbreakers/src/default}/index.ts (100%) diff --git a/common/predictive-text/worker/word_breaking/default-word-breaking/data.ts b/common/models/wordbreakers/src/default/data.ts similarity index 100% rename from common/predictive-text/worker/word_breaking/default-word-breaking/data.ts rename to common/models/wordbreakers/src/default/data.ts diff --git a/common/predictive-text/worker/word_breaking/default-word-breaking/index.ts b/common/models/wordbreakers/src/default/index.ts similarity index 100% rename from common/predictive-text/worker/word_breaking/default-word-breaking/index.ts rename to common/models/wordbreakers/src/default/index.ts diff --git a/common/models/wordbreakers/src/index.ts b/common/models/wordbreakers/src/index.ts index 1e31e002cc..2140ae6bf4 100644 --- a/common/models/wordbreakers/src/index.ts +++ b/common/models/wordbreakers/src/index.ts @@ -1,4 +1,5 @@ /// +/// /// // Let the word breakers be available both in the browser and in Node. From c9cf6058335dfca145eedc09468b85e7de668f41 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Tue, 19 May 2020 15:10:41 -0600 Subject: [PATCH 37/41] fix(common/lmlayer): remove unused reference for placeholder word breaker --- common/predictive-text/worker/models/trie-model.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/common/predictive-text/worker/models/trie-model.ts b/common/predictive-text/worker/models/trie-model.ts index 896c3c1d45..1a2a1e80ac 100644 --- a/common/predictive-text/worker/models/trie-model.ts +++ b/common/predictive-text/worker/models/trie-model.ts @@ -22,7 +22,6 @@ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -/// /// /** From e84ffb86af23c2a117cb7c14c790c7dd197c3795 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Tue, 19 May 2020 15:17:46 -0600 Subject: [PATCH 38/41] docs(common/lmlayer): correct package name in README --- common/models/wordbreakers/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/common/models/wordbreakers/README.md b/common/models/wordbreakers/README.md index 3b0716d4f0..533085d45f 100644 --- a/common/models/wordbreakers/README.md +++ b/common/models/wordbreakers/README.md @@ -1,4 +1,4 @@ -# `@keymanapp/lexical-model-word-breakers` +# `@keymanapp/models-wordbreakers` The word breakers bundled in Keyman's lexical model layer (LMLayer). @@ -11,7 +11,8 @@ Includes: ## Usage ``` -import {default as breakWords} from '@keymanapp/lexical-model-word-breakers'; +import {wordBreakers} from '@keymanapp/models-wordbreakers'; +const breakWords = wordBreakers['default']; console.log(breakWords('Hello, World!').map(span => span.text)); // prints: [ 'Hello', ',', 'World', '!' ] From 6909ebff59e813506786ba050d69c7269df7b267 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Tue, 19 May 2020 15:21:54 -0600 Subject: [PATCH 39/41] fix(common/lmlayer): restore 'default-word-breaker.js' tests --- .../headless/default-word-breaker.js | 140 ++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 common/predictive-text/unit_tests/headless/default-word-breaker.js diff --git a/common/predictive-text/unit_tests/headless/default-word-breaker.js b/common/predictive-text/unit_tests/headless/default-word-breaker.js new file mode 100644 index 0000000000..bcf08efd83 --- /dev/null +++ b/common/predictive-text/unit_tests/headless/default-word-breaker.js @@ -0,0 +1,140 @@ +/** + * Smoke-test the default + */ +var assert = require('chai').assert; +var TrieModel = require('../../build/intermediate').models.TrieModel; + +var breakWords = require('../../build/intermediate').wordBreakers['default']; +const SHY = '\u00AD'; + +describe('The default word breaker', function () { + it('should break multilingual text', function () { + let breaks = breakWords( + `Добрый день! ᑕᐻ᙮ — after working on ka${SHY}wen${SHY}non:${SHY}nis, + let's eat phở! 🥣` + ); + let words = breaks.map(span => span.text); + assert.deepEqual(words, [ + 'Добрый', 'день', '!', 'ᑕᐻ', '᙮', '—', 'after', + 'working', 'on', `ka${SHY}wen${SHY}non:${SHY}nis`, ',', + "let's", 'eat', 'phở', '!', '🥣' + ]); + }); + + // The following tests are performed with model integration as an internal + // test for the wordbreaking API. + it('recognizes a word at end of complete lefthand context', function () { + var model = new TrieModel(jsonFixture('tries/english-1000'), { + wordBreaker: breakWords // wordBreakers['default'] when fully integrated. + }); + + // Standard case - wordbreaking at the end of a word. + var context = { + left: 'The quick brown fox jumped', startOfBuffer: true, + right: ' over the lazy dog.', endOfBuffer: true + }; + + var broken = model.wordbreak(context); + + assert.strictEqual(broken, 'jumped'); + }); + + // Same test as before, but we want to be sure the start/end of buffer flags + // don't affect our results. + it('recognizes a word at end of incomplete lefthand context', function () { + var model = new TrieModel(jsonFixture('tries/english-1000'), { + wordBreaker: breakWords + }); + + // Standard case - wordbreaking at the end of a word. + var context = { + left: 'The quick brown fox jumped', startOfBuffer: false, + right: ' over the lazy dog.', endOfBuffer: false + }; + + var broken = model.wordbreak(context); + + assert.strictEqual(broken, 'jumped'); + }); + + it('returns text for a word in-progress', function() { + var model = new TrieModel(jsonFixture('tries/english-1000'), { + wordBreaker: breakWords + }); + + // Standard case - midword (xylophone) call + var context = { + left: 'xyl', startOfBuffer: true, + right: '', endOfBuffer: true + }; + + var broken = model.wordbreak(context); + + assert.strictEqual(broken, 'xyl'); + }); + + it('returns empty string when called without word text', function() { + var model = new TrieModel(jsonFixture('tries/english-1000'), { + wordBreaker: breakWords + }); + + // Wordbreaking on a empty space => no word. + context = { + left: 'The quick brown fox jumped ', startOfBuffer: true, + right: 'over the lazy dog.', endOfBuffer: true + }; + + broken = model.wordbreak(context); + + assert.strictEqual(broken, ''); + }); + + it('returns empty string when called with empty context', function() { + var model = new TrieModel(jsonFixture('tries/english-1000'), { + wordBreaker: breakWords + }); + + // Wordbreaking on a empty space => no word. + context = { + left: '', startOfBuffer: true, + right: '', endOfBuffer: true + }; + + broken = model.wordbreak(context); + + assert.strictEqual(broken, ''); + }); + + it('returns empty string when called with nil context', function() { + var model = new TrieModel(jsonFixture('tries/english-1000'), { + wordBreaker: breakWords + }); + + // Wordbreaking on a empty space => no word. + context = { + left: '', startOfBuffer: false, + right: '', endOfBuffer: false + }; + + broken = model.wordbreak(context); + + assert.strictEqual(broken, ''); + }); + + it.skip('correctly breaks a word when the caret is placed within it', function() { + var model = new TrieModel(jsonFixture('tries/english-1000'), { + wordBreaker: breakWords + }); + + // A limitation of the current implementation; we should fix this before release. + // Then again, when typing this is probably fine; just not when not typing. + context = { + left: 'The quick brown fox jum', startOfBuffer: true, + right: 'ped over the lazy dog.', endOfBuffer: true + }; + + broken = model.wordbreak(context); + + assert.strictEqual(broken, 'jumped'); // Current result: 'jum'. + }); +}); From 44ce9e8afcbba4f709e9fee9e17ed0204f1ff9dc Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Tue, 19 May 2020 15:26:16 -0600 Subject: [PATCH 40/41] test(common/lmlayer): let file only contain Trie/default WB integration tests --- .../headless/default-word-breaker.js | 40 +++++++------------ 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/common/predictive-text/unit_tests/headless/default-word-breaker.js b/common/predictive-text/unit_tests/headless/default-word-breaker.js index bcf08efd83..613336458e 100644 --- a/common/predictive-text/unit_tests/headless/default-word-breaker.js +++ b/common/predictive-text/unit_tests/headless/default-word-breaker.js @@ -1,5 +1,5 @@ /** - * Smoke-test the default + * Smoke-test the default word breaker. */ var assert = require('chai').assert; var TrieModel = require('../../build/intermediate').models.TrieModel; @@ -7,29 +7,19 @@ var TrieModel = require('../../build/intermediate').models.TrieModel; var breakWords = require('../../build/intermediate').wordBreakers['default']; const SHY = '\u00AD'; +// The following are **integration tests** testing the interaction between the +// Trie model "template" and the default word breaker. +// +// They exercise whether the default word breaker produces the correct word +// breaks, and whether the Trie model can consume the word breaks properly. describe('The default word breaker', function () { - it('should break multilingual text', function () { - let breaks = breakWords( - `Добрый день! ᑕᐻ᙮ — after working on ka${SHY}wen${SHY}non:${SHY}nis, - let's eat phở! 🥣` - ); - let words = breaks.map(span => span.text); - assert.deepEqual(words, [ - 'Добрый', 'день', '!', 'ᑕᐻ', '᙮', '—', 'after', - 'working', 'on', `ka${SHY}wen${SHY}non:${SHY}nis`, ',', - "let's", 'eat', 'phở', '!', '🥣' - ]); - }); - - // The following tests are performed with model integration as an internal - // test for the wordbreaking API. it('recognizes a word at end of complete lefthand context', function () { var model = new TrieModel(jsonFixture('tries/english-1000'), { wordBreaker: breakWords // wordBreakers['default'] when fully integrated. }); // Standard case - wordbreaking at the end of a word. - var context = { + var context = { left: 'The quick brown fox jumped', startOfBuffer: true, right: ' over the lazy dog.', endOfBuffer: true }; @@ -47,7 +37,7 @@ describe('The default word breaker', function () { }); // Standard case - wordbreaking at the end of a word. - var context = { + var context = { left: 'The quick brown fox jumped', startOfBuffer: false, right: ' over the lazy dog.', endOfBuffer: false }; @@ -63,7 +53,7 @@ describe('The default word breaker', function () { }); // Standard case - midword (xylophone) call - var context = { + var context = { left: 'xyl', startOfBuffer: true, right: '', endOfBuffer: true }; @@ -72,14 +62,14 @@ describe('The default word breaker', function () { assert.strictEqual(broken, 'xyl'); }); - + it('returns empty string when called without word text', function() { var model = new TrieModel(jsonFixture('tries/english-1000'), { wordBreaker: breakWords }); // Wordbreaking on a empty space => no word. - context = { + context = { left: 'The quick brown fox jumped ', startOfBuffer: true, right: 'over the lazy dog.', endOfBuffer: true }; @@ -95,7 +85,7 @@ describe('The default word breaker', function () { }); // Wordbreaking on a empty space => no word. - context = { + context = { left: '', startOfBuffer: true, right: '', endOfBuffer: true }; @@ -111,7 +101,7 @@ describe('The default word breaker', function () { }); // Wordbreaking on a empty space => no word. - context = { + context = { left: '', startOfBuffer: false, right: '', endOfBuffer: false }; @@ -128,13 +118,13 @@ describe('The default word breaker', function () { // A limitation of the current implementation; we should fix this before release. // Then again, when typing this is probably fine; just not when not typing. - context = { + context = { left: 'The quick brown fox jum', startOfBuffer: true, right: 'ped over the lazy dog.', endOfBuffer: true }; broken = model.wordbreak(context); - assert.strictEqual(broken, 'jumped'); // Current result: 'jum'. + assert.strictEqual(broken, 'jumped'); // Current result: 'jum'. }); }); From 3cc8ee2399151c8bffb11776a0ae431b9c754f03 Mon Sep 17 00:00:00 2001 From: Eddie Antonio Santos Date: Tue, 19 May 2020 15:30:43 -0600 Subject: [PATCH 41/41] docs(common/models): Apply suggestions from code review Co-authored-by: Marc Durdin --- common/models/wordbreakers/README.md | 2 +- common/models/wordbreakers/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/models/wordbreakers/README.md b/common/models/wordbreakers/README.md index 533085d45f..29ba5d51c7 100644 --- a/common/models/wordbreakers/README.md +++ b/common/models/wordbreakers/README.md @@ -1,6 +1,6 @@ # `@keymanapp/models-wordbreakers` -The word breakers bundled in Keyman's lexical model layer (LMLayer). +The word breakers bundled in Keyman's modeling layer (common/models). Includes: diff --git a/common/models/wordbreakers/package.json b/common/models/wordbreakers/package.json index fae4d76ae5..143c08c773 100644 --- a/common/models/wordbreakers/package.json +++ b/common/models/wordbreakers/package.json @@ -1,7 +1,7 @@ { "name": "@keymanapp/models-wordbreakers", "version": "14.0.75", - "description": "Word breakers bundled in the LMlayer", + "description": "Word breakers bundled in the modeling layer", "keywords": [ "lexical model", "models",