From ad8c30c6c07b7f5272c1b1e55976a9b146fbbf06 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 5 Sep 2022 10:26:31 +0700 Subject: [PATCH 01/59] fix(web): pred-text context tracking when wordbreak not caused by whitespace --- .../src/correction/context-tracker.ts | 78 +++++++++++-------- common/web/lm-worker/src/model-compositor.ts | 36 ++------- common/web/lm-worker/src/transformUtils.ts | 25 ++++++ 3 files changed, 78 insertions(+), 61 deletions(-) create mode 100644 common/web/lm-worker/src/transformUtils.ts diff --git a/common/web/lm-worker/src/correction/context-tracker.ts b/common/web/lm-worker/src/correction/context-tracker.ts index 52e89bedc8..1a94723546 100644 --- a/common/web/lm-worker/src/correction/context-tracker.ts +++ b/common/web/lm-worker/src/correction/context-tracker.ts @@ -89,7 +89,7 @@ namespace correction { if(token.replacementText) { copy.replacementText = token.replacementText; } - + return copy; }); this.searchSpace = obj.searchSpace; @@ -139,8 +139,10 @@ namespace correction { // Track the Transform that resulted in the whitespace 'token'. // Will be needed for phrase-level correction/prediction. - whitespaceToken.transformDistributions = [transformDistribution]; - + if(transformDistribution) { + whitespaceToken.transformDistributions = [transformDistribution]; + } + whitespaceToken.raw = null; this.tokens.push(whitespaceToken); } @@ -149,19 +151,19 @@ namespace correction { * Used for 14.0's backspace workaround, which flattens all previous Distribution * entries because of limitations with direct use of backspace transforms. * @param tokenText - * @param transformId + * @param transformId */ replaceTailForBackspace(tokenText: USVString, transformId: number) { this.tokens.pop(); // It's a backspace transform; time for special handling! // - // For now, with 14.0, we simply compress all remaining Transforms for the token into - // multiple single-char transforms. Probabalistically modeling BKSP is quite complex, + // For now, with 14.0, we simply compress all remaining Transforms for the token into + // multiple single-char transforms. Probabalistically modeling BKSP is quite complex, // so we simplify by assuming everything remaining after a BKSP is 'true' and 'intended' text. // // Note that we cannot just use a single, monolithic transform at this point b/c - // of our current edit-distance optimization strategy; diagonalization is currently... + // of our current edit-distance optimization strategy; diagonalization is currently... // not very compatible with that. let backspacedTokenContext: Distribution[] = textToCharTransforms(tokenText, transformId).map(function(transform) { return [{sample: transform, p: 1.0}]; @@ -175,7 +177,7 @@ namespace correction { updateTail(transformDistribution: Distribution, tokenText?: USVString) { let editedToken = this.tail; - + // Preserve existing text if new text isn't specified. tokenText = tokenText || (tokenText === '' ? '' : editedToken.raw); @@ -191,7 +193,7 @@ namespace correction { toRawTokenization() { let sequence: USVString[] = []; - + for(let token of this.tokens) { // Hide any tokens representing wordbreaks. (Thinking ahead to phrase-level possibilities) if(token.currentText !== null) { @@ -281,7 +283,7 @@ namespace correction { /** * Returns items contained within the circular array, ordered from 'oldest' to 'newest' - * the same order in which the items will be dequeued. - * @param index + * @param index */ item(index: number) { if(index >= this.count) { @@ -294,7 +296,7 @@ namespace correction { } export class ContextTracker extends CircularArray { - static attemptMatchContext(tokenizedContext: USVString[], + static attemptMatchContext(tokenizedContext: USVString[], matchState: TrackedContextState, transformDistribution?: Distribution,): TrackedContextState { // Map the previous tokenized state to an edit-distance friendly version. @@ -335,7 +337,7 @@ namespace correction { } // Can happen for the first text input after backspace deletes a wordbreaking character, - // thus the new input continues a previous word while dropping the empty word after + // thus the new input continues a previous word while dropping the empty word after // that prior wordbreaking character. // // We can't handle it reliably from this match state, but a previous entry (without the empty token) @@ -353,7 +355,7 @@ namespace correction { // If we've made it here... success! We have a context match! let state: TrackedContextState; - + if(pushedTail) { // On suggestion acceptance, we should update the previous final token. // We do it first so that the acceptance is replicated in the new TrackedContextState @@ -376,7 +378,9 @@ namespace correction { if(primaryInput && primaryInput.insert == "" && primaryInput.deleteLeft == 0 && !primaryInput.deleteRight) { primaryInput = null; } - const isBackspace = primaryInput && primaryInput.insert == "" && primaryInput.deleteLeft > 0 && !primaryInput.deleteRight; + + const isWhitespace = primaryInput && TransformUtils.isWhitespace(primaryInput); + const isBackspace = primaryInput && TransformUtils.isBackspace(primaryInput); const finalToken = tokenizedContext[tokenizedContext.length-1]; /* Assumption: This is an adequate check for its two sub-branches. @@ -388,7 +392,7 @@ namespace correction { * - Assumption: one keystroke may only cause a single token to be appended to the context * - That is, no "reasonable" keystroke would emit a Transform adding two separate word tokens * - For languages using whitespace to word-break, said keystroke would have to include said whitespace to break the assumption. - */ + */ // If there is/was more than one context token available... if(editPath.length > 1) { @@ -399,17 +403,29 @@ namespace correction { // We're adding an additional context token. if(pushedTail) { - // ASSUMPTION: any transform that triggers this case is a pure-whitespace Transform, as we - // need a word-break before beginning a new word's context. - // Worth note: when invalid, the lm-layer already has problems in other aspects too. - state.pushWhitespaceToTail(transformDistribution); + const tokenizedTail = tokenizedContext[tokenizedContext.length - 1]; + /* + * Common-case: most transforms that trigger this case are from pure-whitespace Transforms. MOST. + * + * Less-common, but noteworthy: some wordbreaks may occur without whitespace. Example: + * `"o` => ['"', 'o']. Make sure to double-check against `tokenizedContext`! + */ + let pushedToken = new TrackedContextToken(); + pushedToken.raw = tokenizedTail; - let emptyToken = new TrackedContextToken(); - emptyToken.raw = ''; - // Continuing the earlier assumption, that 'pure-whitespace Transform' does not emit any initial characters - // for the new word (token), so the input keystrokes do not correspond to the new text token. - emptyToken.transformDistributions = []; - state.pushTail(emptyToken); + if(isWhitespace) { + state.pushWhitespaceToTail(transformDistribution); + // Continuing the earlier assumption, that 'pure-whitespace Transform' does not emit any initial characters + // for the new word (token), so the input keystrokes do not correspond to the new text token. + pushedToken.transformDistributions = []; + } else { + state.pushWhitespaceToTail(); + // Assumption: Since we only allow one-transform-at-a-time changes between states, we shouldn't be missing + // any metadata used to construct the new context state token. + pushedToken.transformDistributions = [transformDistribution]; + } + + state.pushTail(pushedToken); } else { // We're editing the final context token. // TODO: Assumption: we didn't 'miss' any inputs somehow. // As is, may be prone to fragility should the lm-layer's tracked context 'desync' from its host's. @@ -483,13 +499,13 @@ namespace correction { * Compares the current, post-input context against the most recently-seen contexts from previous prediction calls, returning * the most information-rich `TrackedContextState` possible. If a match is found, the state will be annotated with the * input information provided to previous prediction calls and persisted correction-search calculations for re-use. - * - * @param model - * @param context - * @param mainTransform - * @param transformDistribution + * + * @param model + * @param context + * @param mainTransform + * @param transformDistribution */ - analyzeState(model: LexicalModel, + analyzeState(model: LexicalModel, context: Context, transformDistribution?: Distribution): TrackedContextState { if(!model.traverseFromRoot) { diff --git a/common/web/lm-worker/src/model-compositor.ts b/common/web/lm-worker/src/model-compositor.ts index 329460776d..84687c867d 100644 --- a/common/web/lm-worker/src/model-compositor.ts +++ b/common/web/lm-worker/src/model-compositor.ts @@ -16,30 +16,6 @@ class ModelCompositor { this.punctuation = ModelCompositor.determinePunctuationFromModel(lexicalModel); } - protected isWhitespace(transform: Transform): boolean { - // Matches prefixed text + any instance of a character with Unicode general property Z* or the following: CR, LF, and Tab. - let whitespaceRemover = /.*[\u0009\u000A\u000D\u0020\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b\u2028\u2029\u202f\u205f\u3000]/i; - - // Filter out null-inserts; their high probability can cause issues. - if(transform.insert == '') { // Can actually register as 'whitespace'. - return false; - } - - let insert = transform.insert; - - insert = insert.replace(whitespaceRemover, ''); - - return insert == ''; - } - - protected isBackspace(transform: Transform): boolean { - return transform.insert == "" && transform.deleteLeft > 0; - } - - protected isEmpty(transform: Transform): boolean { - return transform.insert == '' && transform.deleteLeft == 0; - } - private predictFromCorrections(corrections: ProbabilityMass[], context: Context): Distribution { let returnedPredictions: Distribution = []; @@ -98,8 +74,8 @@ class ModelCompositor { })[0].sample; // Only allow new-word suggestions if space was the most likely keypress. - let allowSpace = this.isWhitespace(inputTransform); - let allowBksp = this.isBackspace(inputTransform); + let allowSpace = TransformUtils.isWhitespace(inputTransform); + let allowBksp = TransformUtils.isBackspace(inputTransform); let postContext = models.applyTransform(inputTransform, context); let keepOptionText = this.wordbreak(postContext); @@ -146,9 +122,9 @@ class ModelCompositor { } else { contextState = this.contextTracker.analyzeState(this.lexicalModel, postContext, - !this.isEmpty(inputTransform) ? - transformDistribution: - null + !TransformUtils.isEmpty(inputTransform) ? + transformDistribution: + null ); // TODO: Should we filter backspaces & whitespaces out of the transform distribution? @@ -164,7 +140,7 @@ class ModelCompositor { // Detect if we're starting a new context state. let contextTokens = contextState.tokens; if(contextTokens.length == 0 || contextTokens[contextTokens.length - 1].isNew) { - if(this.isEmpty(inputTransform) || this.isWhitespace(inputTransform)) { + if(TransformUtils.isEmpty(inputTransform) || TransformUtils.isWhitespace(inputTransform)) { newEmptyToken = true; prefixTransform = inputTransform; context = postContext; // Ensure the whitespace token is preapplied! diff --git a/common/web/lm-worker/src/transformUtils.ts b/common/web/lm-worker/src/transformUtils.ts new file mode 100644 index 0000000000..8512d4e9a2 --- /dev/null +++ b/common/web/lm-worker/src/transformUtils.ts @@ -0,0 +1,25 @@ +class TransformUtils { + static isWhitespace(transform: Transform): boolean { + // Matches prefixed text + any instance of a character with Unicode general property Z* or the following: CR, LF, and Tab. + let whitespaceRemover = /.*[\u0009\u000A\u000D\u0020\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b\u2028\u2029\u202f\u205f\u3000]/i; + + // Filter out null-inserts; their high probability can cause issues. + if(transform.insert == '') { // Can actually register as 'whitespace'. + return false; + } + + let insert = transform.insert; + + insert = insert.replace(whitespaceRemover, ''); + + return insert == ''; + } + + static isBackspace(transform: Transform): boolean { + return transform.insert == "" && transform.deleteLeft > 0 && !transform.deleteRight; + } + + static isEmpty(transform: Transform): boolean { + return transform.insert == '' && transform.deleteLeft == 0; + } +} \ No newline at end of file From be562b30225410afa77d65d1fbed58f6257f4784 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 5 Sep 2022 10:37:50 +0700 Subject: [PATCH 02/59] fix(web): missed method references --- common/web/lm-worker/src/model-compositor.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/web/lm-worker/src/model-compositor.ts b/common/web/lm-worker/src/model-compositor.ts index 84687c867d..657a7b1ec1 100644 --- a/common/web/lm-worker/src/model-compositor.ts +++ b/common/web/lm-worker/src/model-compositor.ts @@ -100,18 +100,18 @@ class ModelCompositor { predictionRoots = [{sample: inputTransform, p: 1.0}]; prefixTransform = inputTransform; } else { - predictionRoots = transformDistribution.map(function(alt) { + predictionRoots = transformDistribution.map((alt) => { let transform = alt.sample; // Filter out special keys unless they're expected. - if(this.isWhitespace(transform) && !allowSpace) { + if(TransformUtils.isWhitespace(transform) && !allowSpace) { return null; - } else if(this.isBackspace(transform) && !allowBksp) { + } else if(TransformUtils.isBackspace(transform) && !allowBksp) { return null; } return alt; - }, this); + }); } // Remove `null` entries. From b0184ec98bcf1106119afab36b6b7d09b9db70e1 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 5 Sep 2022 11:08:26 +0700 Subject: [PATCH 03/59] fix(web): adds unit test targeting issue, handler for edge case --- .../headless/edit-distance/context-tracker.js | 26 ++++++++++++++++--- .../src/correction/context-tracker.ts | 2 +- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js b/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js index 6a4148b669..5960a47658 100644 --- a/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js +++ b/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js @@ -45,7 +45,7 @@ describe('ContextTracker', function() { assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); }); - it("properly matches and aligns when a 'wordbreak' is added'", function() { + it("properly matches and aligns when a 'wordbreak' is added", function() { let existingContext = ["an", "apple", "a", "day", "keeps", "the", "doctor"]; let transform = { insert: ' ', @@ -56,7 +56,7 @@ describe('ContextTracker', function() { let rawTokens = ["an", null, "apple", null, "a", null, "day", null, "keeps", null, "the", null, "doctor", null, ""]; let existingState = ContextTracker.modelContextState(existingContext); - let state = ContextTracker.attemptMatchContext(newContext, existingState, null, toWrapperDistribution(transform)); + let state = ContextTracker.attemptMatchContext(newContext, existingState, toWrapperDistribution(transform)); assert.isNotNull(state); assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); @@ -65,6 +65,26 @@ describe('ContextTracker', function() { assert.isEmpty(state.tokens[state.tokens.length - 1].transformDistributions); }); + it("properly matches and aligns when an implied 'wordbreak' occurs \"'\"", function() { + let existingContext = ["'"]; + let transform = { + insert: 'a', + deleteLeft: 0 + } + let newContext = Array.from(existingContext); + newContext.push('a'); // The incoming transform should produce a new token WITH TEXT. + let rawTokens = ["'", null, "a"]; + + let existingState = ContextTracker.modelContextState(existingContext); + let state = ContextTracker.attemptMatchContext(newContext, existingState, toWrapperDistribution(transform)); + assert.isNotNull(state); + assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); + + // The 'wordbreak' transform + assert.isEmpty(state.tokens[state.tokens.length - 2].transformDistributions); + assert.isNotEmpty(state.tokens[state.tokens.length - 1].transformDistributions); + }); + it("properly matches and aligns when lead token is removed AND a 'wordbreak' is added'", function() { let existingContext = ["an", "apple", "a", "day", "keeps", "the", "doctor"]; let transform = { @@ -77,7 +97,7 @@ describe('ContextTracker', function() { let rawTokens = ["apple", null, "a", null, "day", null, "keeps", null, "the", null, "doctor", null, ""]; let existingState = ContextTracker.modelContextState(existingContext); - let state = ContextTracker.attemptMatchContext(newContext, existingState, null, toWrapperDistribution(transform)); + let state = ContextTracker.attemptMatchContext(newContext, existingState, toWrapperDistribution(transform)); assert.isNotNull(state); assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); diff --git a/common/web/lm-worker/src/correction/context-tracker.ts b/common/web/lm-worker/src/correction/context-tracker.ts index 1a94723546..0f50657f0e 100644 --- a/common/web/lm-worker/src/correction/context-tracker.ts +++ b/common/web/lm-worker/src/correction/context-tracker.ts @@ -413,7 +413,7 @@ namespace correction { let pushedToken = new TrackedContextToken(); pushedToken.raw = tokenizedTail; - if(isWhitespace) { + if(isWhitespace || !primaryInput) { state.pushWhitespaceToTail(transformDistribution); // Continuing the earlier assumption, that 'pure-whitespace Transform' does not emit any initial characters // for the new word (token), so the input keystrokes do not correspond to the new text token. From 0a237f800fbbca7aa78483ae0f3371a71534b52f Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 5 Sep 2022 11:31:57 +0700 Subject: [PATCH 04/59] chore(web): test name tweak --- .../unit_tests/headless/edit-distance/context-tracker.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js b/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js index 5960a47658..fa09ca20de 100644 --- a/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js +++ b/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js @@ -65,7 +65,7 @@ describe('ContextTracker', function() { assert.isEmpty(state.tokens[state.tokens.length - 1].transformDistributions); }); - it("properly matches and aligns when an implied 'wordbreak' occurs \"'\"", function() { + it("properly matches and aligns when an implied 'wordbreak' occurs (as when following \"'\")", function() { let existingContext = ["'"]; let transform = { insert: 'a', From d33a9f05bf999cbc476bb600f50eada8ee934395 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 5 Sep 2022 11:41:42 +0700 Subject: [PATCH 05/59] docs(web): fixes missed doc update --- common/web/lm-worker/src/correction/context-tracker.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/common/web/lm-worker/src/correction/context-tracker.ts b/common/web/lm-worker/src/correction/context-tracker.ts index 0f50657f0e..bc187fa03f 100644 --- a/common/web/lm-worker/src/correction/context-tracker.ts +++ b/common/web/lm-worker/src/correction/context-tracker.ts @@ -502,7 +502,6 @@ namespace correction { * * @param model * @param context - * @param mainTransform * @param transformDistribution */ analyzeState(model: LexicalModel, From 57ce30e0685fb6e6c8c3d06207045d4960e2131d Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 5 Sep 2022 12:17:17 +0700 Subject: [PATCH 06/59] fix(web): post-suggestion-apply error --- common/web/lm-worker/src/correction/context-tracker.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/web/lm-worker/src/correction/context-tracker.ts b/common/web/lm-worker/src/correction/context-tracker.ts index bc187fa03f..99842b8299 100644 --- a/common/web/lm-worker/src/correction/context-tracker.ts +++ b/common/web/lm-worker/src/correction/context-tracker.ts @@ -414,7 +414,7 @@ namespace correction { pushedToken.raw = tokenizedTail; if(isWhitespace || !primaryInput) { - state.pushWhitespaceToTail(transformDistribution); + state.pushWhitespaceToTail(transformDistribution ?? []); // Continuing the earlier assumption, that 'pure-whitespace Transform' does not emit any initial characters // for the new word (token), so the input keystrokes do not correspond to the new text token. pushedToken.transformDistributions = []; @@ -422,7 +422,7 @@ namespace correction { state.pushWhitespaceToTail(); // Assumption: Since we only allow one-transform-at-a-time changes between states, we shouldn't be missing // any metadata used to construct the new context state token. - pushedToken.transformDistributions = [transformDistribution]; + pushedToken.transformDistributions = transformDistribution ? [transformDistribution] : []; } state.pushTail(pushedToken); From b47596a803cc9c65c329d85a33c8c1d3736f74e0 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 5 Sep 2022 12:35:11 +0700 Subject: [PATCH 07/59] fix(common/models): context token .isNew maintenance --- common/web/lm-worker/src/correction/context-tracker.ts | 5 ++++- common/web/lm-worker/src/model-compositor.ts | 9 ++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/common/web/lm-worker/src/correction/context-tracker.ts b/common/web/lm-worker/src/correction/context-tracker.ts index 99842b8299..08ca11b136 100644 --- a/common/web/lm-worker/src/correction/context-tracker.ts +++ b/common/web/lm-worker/src/correction/context-tracker.ts @@ -27,13 +27,14 @@ namespace correction { export class TrackedContextToken { raw: string; replacementText: string; + newFlag: boolean = false; transformDistributions: Distribution[] = []; replacements: TrackedContextSuggestion[]; activeReplacementId: number = -1; get isNew(): boolean { - return this.transformDistributions.length == 0; + return this.newFlag; } get currentText(): string { @@ -126,6 +127,7 @@ namespace correction { } else { this.searchSpace = []; } + token.newFlag = true; this.tokens.push(token); let state = this; @@ -189,6 +191,7 @@ namespace correction { } // Replace old token's raw-text with new token's raw-text. editedToken.raw = tokenText; + editedToken.newFlag = false; } toRawTokenization() { diff --git a/common/web/lm-worker/src/model-compositor.ts b/common/web/lm-worker/src/model-compositor.ts index 657a7b1ec1..6c40d36e98 100644 --- a/common/web/lm-worker/src/model-compositor.ts +++ b/common/web/lm-worker/src/model-compositor.ts @@ -136,12 +136,15 @@ class ModelCompositor { // The 'eventual' logic will be significantly more complex, though still manageable. let searchSpace = contextState.searchSpace[0]; - let newEmptyToken = false; + let newToken = false; // Detect if we're starting a new context state. let contextTokens = contextState.tokens; if(contextTokens.length == 0 || contextTokens[contextTokens.length - 1].isNew) { + // Always note if we have a new token (so that we don't try to delete existing context) + newToken = true; + // If the new token is due to whitespace, or if we had a context-reset trigger this (thus, no input...) + // (Lingering question: do we need the .isEmpty check here? Track `prefixTransform` and find out.) if(TransformUtils.isEmpty(inputTransform) || TransformUtils.isWhitespace(inputTransform)) { - newEmptyToken = true; prefixTransform = inputTransform; context = postContext; // Ensure the whitespace token is preapplied! } @@ -170,7 +173,7 @@ class ModelCompositor { let deleteLeft = 0; // remove actual token string. If new token, there should be nothing to delete. - if(!newEmptyToken) { + if(!newToken) { // If this is triggered from a backspace, make sure to use its results // and also include its left-deletions! It's the one post-input context case. if(allowBksp) { From 26c5150977cf60de364b6d2ef8aa893668d00aeb Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 5 Sep 2022 14:02:55 +0700 Subject: [PATCH 08/59] fix(web): context-tracker newFlag management for new contexts --- .../lm-worker/src/correction/context-tracker.ts | 17 +++++++++++++++-- common/web/lm-worker/src/model-compositor.ts | 2 +- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/common/web/lm-worker/src/correction/context-tracker.ts b/common/web/lm-worker/src/correction/context-tracker.ts index 08ca11b136..fba8826168 100644 --- a/common/web/lm-worker/src/correction/context-tracker.ts +++ b/common/web/lm-worker/src/correction/context-tracker.ts @@ -461,7 +461,9 @@ namespace correction { return state; } - static modelContextState(tokenizedContext: USVString[], lexicalModel: LexicalModel): TrackedContextState { + static modelContextState(tokenizedContext: USVString[], + transformDistribution: Distribution, + lexicalModel: LexicalModel): TrackedContextState { let baseTokens = tokenizedContext.map(function(entry) { let token = new TrackedContextToken(); token.raw = entry; @@ -495,6 +497,17 @@ namespace correction { state.pushTail(token); } + for(let i = 0; i < state.tokens.length - 1; i++) { + state.tokens[i].newFlag = false; + } + + const finalToken = state.tokens[state.tokens.length - 1]; + const baseTransform = (transformDistribution && transformDistribution.length > 0) ? transformDistribution[0] : null; + + if(!baseTransform || baseTransform.sample.insert != finalToken.raw) { + finalToken.newFlag = false; + } + return state; } @@ -537,7 +550,7 @@ namespace correction { // // Assumption: as a caret needs to move to context before any actual transform distributions occur, // this state is only reached on caret moves; thus, transformDistribution is actually just a single null transform. - let state = ContextTracker.modelContextState(tokenizedContext.left, model); + let state = ContextTracker.modelContextState(tokenizedContext.left, transformDistribution, model); state.taggedContext = context; this.enqueue(state); return state; diff --git a/common/web/lm-worker/src/model-compositor.ts b/common/web/lm-worker/src/model-compositor.ts index 6c40d36e98..8709ab0c6b 100644 --- a/common/web/lm-worker/src/model-compositor.ts +++ b/common/web/lm-worker/src/model-compositor.ts @@ -638,7 +638,7 @@ class ModelCompositor { // than before. if(this.contextTracker) { let tokenizedContext = models.tokenize(this.lexicalModel.wordbreaker || wordBreakers.default, context); - let contextState = correction.ContextTracker.modelContextState(tokenizedContext.left, this.lexicalModel); + let contextState = correction.ContextTracker.modelContextState(tokenizedContext.left, null, this.lexicalModel); this.contextTracker.enqueue(contextState); } } From 9634c7635340a93babd6bee323941d97d797a54d Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 6 Sep 2022 08:57:56 +0700 Subject: [PATCH 09/59] chore(common/models): suggested tweak from review --- common/web/lm-worker/src/transformUtils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/web/lm-worker/src/transformUtils.ts b/common/web/lm-worker/src/transformUtils.ts index 8512d4e9a2..3bc66f33ce 100644 --- a/common/web/lm-worker/src/transformUtils.ts +++ b/common/web/lm-worker/src/transformUtils.ts @@ -16,10 +16,10 @@ class TransformUtils { } static isBackspace(transform: Transform): boolean { - return transform.insert == "" && transform.deleteLeft > 0 && !transform.deleteRight; + return transform.insert == "" && transform.deleteLeft > 0 && transform.deleteRight == 0; } static isEmpty(transform: Transform): boolean { - return transform.insert == '' && transform.deleteLeft == 0; + return transform.insert == '' && transform.deleteLeft == 0 && transform.deleteRight == 0; } } \ No newline at end of file From 9be8839685ae8d65fbae37ff5eb9ab836f863804 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 6 Sep 2022 10:47:51 +0700 Subject: [PATCH 10/59] fix(common/models): undefined != 0 --- common/web/lm-worker/src/transformUtils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/web/lm-worker/src/transformUtils.ts b/common/web/lm-worker/src/transformUtils.ts index 3bc66f33ce..b91951722f 100644 --- a/common/web/lm-worker/src/transformUtils.ts +++ b/common/web/lm-worker/src/transformUtils.ts @@ -16,10 +16,10 @@ class TransformUtils { } static isBackspace(transform: Transform): boolean { - return transform.insert == "" && transform.deleteLeft > 0 && transform.deleteRight == 0; + return transform.insert == "" && transform.deleteLeft > 0 && !transform.deleteRight; } static isEmpty(transform: Transform): boolean { - return transform.insert == '' && transform.deleteLeft == 0 && transform.deleteRight == 0; + return transform.insert == '' && transform.deleteLeft == 0 && !transform.deleteRight; } } \ No newline at end of file From f96e3493af6f0ae8d928f18f4d5eeb0be9da1538 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 7 Sep 2022 11:13:22 +1000 Subject: [PATCH 11/59] fix(developer): hide key-sizes when in desktop layout in touch layout editor Fixes #7028. Note that the desktop layout is not currently used by KeymanWeb. The designer has a number of additional issues, as the .keyman-touch-layout format is not well suited to describing a fixed hardware layout, but fixing this is outside the scope of this issue. --- developer/src/tike/xml/layoutbuilder/builder.css | 5 +++++ developer/src/tike/xml/layoutbuilder/builder.js | 5 +---- developer/src/tike/xml/layoutbuilder/constants.js | 3 ++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/developer/src/tike/xml/layoutbuilder/builder.css b/developer/src/tike/xml/layoutbuilder/builder.css index 60daff8f3c..6e40f5b5e1 100644 --- a/developer/src/tike/xml/layoutbuilder/builder.css +++ b/developer/src/tike/xml/layoutbuilder/builder.css @@ -613,6 +613,11 @@ body:not(.text-controls-in-toolbar) input#inpSubKeyCap { color: white; } +#kbd.desktop .key-size { + /* the layout is fixed on desktop so key size is not useful */ + display: none; +} + /* Position flick keys relative to the flick grid, by hand */ #flick .key { diff --git a/developer/src/tike/xml/layoutbuilder/builder.js b/developer/src/tike/xml/layoutbuilder/builder.js index 561c4d6d0a..a432f8cd97 100644 --- a/developer/src/tike/xml/layoutbuilder/builder.js +++ b/developer/src/tike/xml/layoutbuilder/builder.js @@ -12,10 +12,7 @@ $(function() { this.getPresentation = function () { - var platform = $('#selPlatformPresentation').val(); - //if(platform == 'tablet') return 'tablet-ipad'; - //if(platform == 'phone') return 'phone-iphone5'; - return platform; + return $('#selPlatformPresentation').val(); } this.saveSelection = function() { diff --git a/developer/src/tike/xml/layoutbuilder/constants.js b/developer/src/tike/xml/layoutbuilder/constants.js index 593c7e4ca0..807a8bd30a 100644 --- a/developer/src/tike/xml/layoutbuilder/constants.js +++ b/developer/src/tike/xml/layoutbuilder/constants.js @@ -604,7 +604,8 @@ $(function() { "tablet-ipad-landscape": { "x": 829, "y": 299, "name": "iPad (landscape)" }, // 829x622 = iPad tablet box size; (97,101)-(926,723) "tablet-ipad-portrait": { "x": 605, "y": 300, "name": "iPad (portrait)" }, // 605x806 = iPad tablet box size; (98,94)-(703,900) "phone-iphone5-landscape": { "x": 731, "y": 196, "name": "iPhone 5 (landscape)" }, // 731x412 = iPhone box size; (144,39)-(875,451) - "phone-iphone5-portrait": { "x": 526, "y": 266, "name": "iPhone 5 (portrait)"} // 528x936 = iPhone box size; (90,204)-(618,1040) + "phone-iphone5-portrait": { "x": 526, "y": 266, "name": "iPhone 5 (portrait)"}, // 528x936 = iPhone box size; (90,204)-(618,1040) + "desktop": { "x": 640, "y": 300, "name": "Desktop" }, }; this.keyMargin = 15; From fb0cc5086cd5f0abbf6d0cac4d166cd691fbca88 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 7 Sep 2022 14:57:53 +0700 Subject: [PATCH 12/59] fix(common/models): backspacing shouldn't make 'new' tokens --- .../web/lm-worker/src/correction/context-tracker.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/common/web/lm-worker/src/correction/context-tracker.ts b/common/web/lm-worker/src/correction/context-tracker.ts index fba8826168..b670f1dd4a 100644 --- a/common/web/lm-worker/src/correction/context-tracker.ts +++ b/common/web/lm-worker/src/correction/context-tracker.ts @@ -127,7 +127,6 @@ namespace correction { } else { this.searchSpace = []; } - token.newFlag = true; this.tokens.push(token); let state = this; @@ -428,6 +427,7 @@ namespace correction { pushedToken.transformDistributions = transformDistribution ? [transformDistribution] : []; } + pushedToken.newFlag = true; state.pushTail(pushedToken); } else { // We're editing the final context token. // TODO: Assumption: we didn't 'miss' any inputs somehow. @@ -448,6 +448,7 @@ namespace correction { let token = new TrackedContextToken(); token.raw = tokenizedContext[0]; token.transformDistributions = [transformDistribution]; + token.newFlag = true; state.pushTail(token); } else { // Edit the lone context token. // Consider backspace entry for this case? @@ -497,15 +498,11 @@ namespace correction { state.pushTail(token); } - for(let i = 0; i < state.tokens.length - 1; i++) { - state.tokens[i].newFlag = false; - } - const finalToken = state.tokens[state.tokens.length - 1]; const baseTransform = (transformDistribution && transformDistribution.length > 0) ? transformDistribution[0] : null; - if(!baseTransform || baseTransform.sample.insert != finalToken.raw) { - finalToken.newFlag = false; + if(baseTransform && baseTransform.sample.insert == finalToken.raw) { + finalToken.newFlag = true; } return state; From 52cb9aa19d944f53f15838dace9c156d81984ce7 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 8 Sep 2022 10:48:12 +0700 Subject: [PATCH 13/59] change(common/models): drops .isNew, replaces with tokenized context contrast logic --- .../src/correction/context-tracker.ts | 15 --- common/web/lm-worker/src/model-compositor.ts | 104 ++++++++++++------ 2 files changed, 71 insertions(+), 48 deletions(-) diff --git a/common/web/lm-worker/src/correction/context-tracker.ts b/common/web/lm-worker/src/correction/context-tracker.ts index b670f1dd4a..f6104ef921 100644 --- a/common/web/lm-worker/src/correction/context-tracker.ts +++ b/common/web/lm-worker/src/correction/context-tracker.ts @@ -27,16 +27,11 @@ namespace correction { export class TrackedContextToken { raw: string; replacementText: string; - newFlag: boolean = false; transformDistributions: Distribution[] = []; replacements: TrackedContextSuggestion[]; activeReplacementId: number = -1; - get isNew(): boolean { - return this.newFlag; - } - get currentText(): string { if(this.replacementText === undefined || this.replacementText === null) { return this.raw; @@ -190,7 +185,6 @@ namespace correction { } // Replace old token's raw-text with new token's raw-text. editedToken.raw = tokenText; - editedToken.newFlag = false; } toRawTokenization() { @@ -427,7 +421,6 @@ namespace correction { pushedToken.transformDistributions = transformDistribution ? [transformDistribution] : []; } - pushedToken.newFlag = true; state.pushTail(pushedToken); } else { // We're editing the final context token. // TODO: Assumption: we didn't 'miss' any inputs somehow. @@ -448,7 +441,6 @@ namespace correction { let token = new TrackedContextToken(); token.raw = tokenizedContext[0]; token.transformDistributions = [transformDistribution]; - token.newFlag = true; state.pushTail(token); } else { // Edit the lone context token. // Consider backspace entry for this case? @@ -498,13 +490,6 @@ namespace correction { state.pushTail(token); } - const finalToken = state.tokens[state.tokens.length - 1]; - const baseTransform = (transformDistribution && transformDistribution.length > 0) ? transformDistribution[0] : null; - - if(baseTransform && baseTransform.sample.insert == finalToken.raw) { - finalToken.newFlag = true; - } - return state; } diff --git a/common/web/lm-worker/src/model-compositor.ts b/common/web/lm-worker/src/model-compositor.ts index 8709ab0c6b..9a7da82a80 100644 --- a/common/web/lm-worker/src/model-compositor.ts +++ b/common/web/lm-worker/src/model-compositor.ts @@ -85,7 +85,7 @@ class ModelCompositor { // Used to restore whitespaces if operations would remove them. let prefixTransform: Transform; - let contextState: correction.TrackedContextState = null; + let postContextState: correction.TrackedContextState = null; // Section 1: determining 'prediction roots'. if(!this.contextTracker) { @@ -120,12 +120,15 @@ class ModelCompositor { // Running in bulk over all suggestions, duplicate entries may be possible. rawPredictions = this.predictFromCorrections(predictionRoots, context); } else { - contextState = this.contextTracker.analyzeState(this.lexicalModel, - postContext, - !TransformUtils.isEmpty(inputTransform) ? - transformDistribution: - null - ); + // Token replacement benefits greatly from knowledge of the prior context state. + let contextState = this.contextTracker.analyzeState(this.lexicalModel, context, null); + // Corrections and predictions are based upon the post-context state, though. + postContextState = this.contextTracker.analyzeState(this.lexicalModel, + postContext, + !TransformUtils.isEmpty(inputTransform) ? + transformDistribution: + null + ); // TODO: Should we filter backspaces & whitespaces out of the transform distribution? // Ideally, the answer (in the future) will be no, but leaving it in right now may pose an issue. @@ -134,20 +137,68 @@ class ModelCompositor { // let's just note that right now, there will only ever be one. // // The 'eventual' logic will be significantly more complex, though still manageable. - let searchSpace = contextState.searchSpace[0]; + let searchSpace = postContextState.searchSpace[0]; - let newToken = false; - // Detect if we're starting a new context state. - let contextTokens = contextState.tokens; - if(contextTokens.length == 0 || contextTokens[contextTokens.length - 1].isNew) { - // Always note if we have a new token (so that we don't try to delete existing context) - newToken = true; - // If the new token is due to whitespace, or if we had a context-reset trigger this (thus, no input...) - // (Lingering question: do we need the .isEmpty check here? Track `prefixTransform` and find out.) - if(TransformUtils.isEmpty(inputTransform) || TransformUtils.isWhitespace(inputTransform)) { + // No matter the prediction, once we know the root of the prediction, we'll always 'replace' the + // same amount of text. We can handle this before the big 'prediction root' loop. + let deleteLeft = 0; + + // The amount of text to 'replace' depends upon whatever sort of context change occurs + // from the received input. + let postContextLength = postContextState.tokens.length; + let contextLengthDelta = postContextState.tokens.length - contextState.tokens.length; + // If the context now has more tokens, the token we'll be 'predicting' didn't originally exist. + if(postContextLength == 0 || contextLengthDelta > 0) { + // As the word/token being corrected/predicted didn't originally exist, there's no + // part of it to 'replace'. + deleteLeft = 0; + + // If the new token is due to whitespace or due to a different input type that would + // likely imply a tokenization boundary... + if(TransformUtils.isWhitespace(inputTransform)) { + /* TODO: consider/implement: the second half of the comment above. + * For example: on input of a `'`, predict new words instead of replacing the `'`. + * (since after a letter, the `'` will be ignored, anyway) + * + * Idea: if the model's most likely prediction (with no root) would make a new + * token if appended to the current token, that's probably a good case. + * Keeps the check simple & quick. + * + * Might need a mixed mode, though: ';' is close enough that `l` is a reasonable + * fat-finger guess. So yeah, we're not addressing this idea right now. + * - so... consider multiple context behavior angles when building prediction roots? + * + * May need something similar to help handle contractions during their construction, + * but that'd be within `ContextTracker`. + * can' => [`can`, `'`] + * can't => [`can't`] (WB6, 7 of https://unicode.org/reports/tr29/#Word_Boundary_Rules) + * + * (Would also helps WB7b+c for Hebrew text) + */ + + // Infer 'new word' mode, even if we received new text when reaching + // this position. That new text didn't exist before, so still - nothing + // to 'replace'. prefixTransform = inputTransform; - context = postContext; // Ensure the whitespace token is preapplied! + context = postContext; // As far as predictions are concerned, the post-context state + // should not be replaced. Predictions are to be rooted on + // text "up for correction" - so we want a null root for this + // branch. + contextState = postContextState; } + // If the tokenized context length is shorter... sounds like a backspace (or similar). + } else if (contextLengthDelta < 0) { + /* Ooh, we've dropped context here. Almost certainly from a backspace. + * Even if we drop multiple tokens... well, we know exactly how many chars + * were actually deleted - `inputTransform.deleteLeft`. + * Since we replace a word being corrected/predicted, we take length of the remaining + * context's tail token in addition to however far was deleted to reach that state. + */ + deleteLeft = this.wordbreak(postContext).kmwLength() + inputTransform.deleteLeft; + } else { + // Suggestions are applied to the pre-input context, so get the token's original length. + // We're on the same token, so just delete its text for the replacement op. + deleteLeft = this.wordbreak(context).kmwLength(); } // TODO: whitespace, backspace filtering. Do it here. @@ -171,19 +222,6 @@ class ModelCompositor { finalInput = inputTransform; // A fallback measure. Greatly matters for empty contexts. } - let deleteLeft = 0; - // remove actual token string. If new token, there should be nothing to delete. - if(!newToken) { - // If this is triggered from a backspace, make sure to use its results - // and also include its left-deletions! It's the one post-input context case. - if(allowBksp) { - deleteLeft = this.wordbreak(postContext).kmwLength() + inputTransform.deleteLeft; - } else { - // Normal case - use the pre-input context. - deleteLeft = this.wordbreak(context).kmwLength(); - } - } - // Replace the existing context with the correction. let correctionTransform: Transform = { insert: correction, // insert correction string @@ -390,8 +428,8 @@ class ModelCompositor { // Store the suggestions on the final token of the current context state (if it exists). // Or, once phrase-level suggestions are possible, on whichever token serves as each prediction's root. - if(contextState) { - contextState.tail.replacements = suggestions.map(function(suggestion) { + if(postContextState) { + postContextState.tail.replacements = suggestions.map(function(suggestion) { return { suggestion: suggestion, tokenWidth: 1 From 3aed0bf39556a2f7431b1c17bc2655efa7bc152c Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 8 Sep 2022 10:48:23 +0700 Subject: [PATCH 14/59] feat(common/models): also, unit tests --- .../headless/worker-model-compositor.js | 43 ++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/common/predictive-text/unit_tests/headless/worker-model-compositor.js b/common/predictive-text/unit_tests/headless/worker-model-compositor.js index f7e9173cee..dc8a36e9dc 100644 --- a/common/predictive-text/unit_tests/headless/worker-model-compositor.js +++ b/common/predictive-text/unit_tests/headless/worker-model-compositor.js @@ -66,12 +66,53 @@ describe('ModelCompositor', function() { // Suggestions always delete the full root of the suggestion. // // After a backspace, that means the text 'the' - 3 chars. - // Char 4 is for the original backspace, as suggstions are built + // Char 4 is for the original backspace, as suggestions are built // based on the context state BEFORE the triggering input - // here, a backspace. assert.equal(suggestion.transform.deleteLeft, 4); }); }); + + it('properly handles suggestions for the first letter after a ` `', function() { + let compositor = new ModelCompositor(plainModel); + let context = { + left: 'the', startOfBuffer: true, endOfBuffer: true, + }; + + let inputTransform = { + insert: ' ', + deleteLeft: 0 + }; + + let suggestions = compositor.predict(inputTransform, context); + suggestions.forEach(function(suggestion) { + // After a space, predictions are based on a new, zero-length root. + // With nothing to replace, .deleteLeft should be zero. + assert.equal(suggestion.transform.deleteLeft, 0); + }); + }); + + it('properly handles suggestions for the first letter after a `\'`', function() { + let compositor = new ModelCompositor(plainModel); + let context = { + left: "the '", startOfBuffer: true, endOfBuffer: true, + }; + + // This results in a new word boundary (between the `'` and the `a`). + // Basically, an implied (but nonexistent) ` `. + let inputTransform = { + insert: "a", + deleteLeft: 0 + }; + + let suggestions = compositor.predict(inputTransform, context); + suggestions.forEach(function(suggestion) { + // Suggestions always delete the full root of the suggestion. + // Which, here, didn't exist before the input. Nothing to + // replace => nothing for the suggestion to delete. + assert.equal(suggestion.transform.deleteLeft, 0); + }); + }); }); describe('applySuggestionCasing', function() { From 75e0b27bf379f80813855d6fb35f12bb26f64c2a Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 8 Sep 2022 10:54:01 +0700 Subject: [PATCH 15/59] change(web): pushWhitespaceToTail tweak --- common/web/lm-worker/src/correction/context-tracker.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/web/lm-worker/src/correction/context-tracker.ts b/common/web/lm-worker/src/correction/context-tracker.ts index f6104ef921..528edc2ce8 100644 --- a/common/web/lm-worker/src/correction/context-tracker.ts +++ b/common/web/lm-worker/src/correction/context-tracker.ts @@ -137,6 +137,8 @@ namespace correction { // Will be needed for phrase-level correction/prediction. if(transformDistribution) { whitespaceToken.transformDistributions = [transformDistribution]; + } else { + whitespaceToken.transformDistributions = []; } whitespaceToken.raw = null; From 6c50de9e9a220f28c49050dae44e6f3e01ef7a18 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 8 Sep 2022 10:56:32 +0700 Subject: [PATCH 16/59] change(web): conciser version of last commit --- common/web/lm-worker/src/correction/context-tracker.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/common/web/lm-worker/src/correction/context-tracker.ts b/common/web/lm-worker/src/correction/context-tracker.ts index 528edc2ce8..e17e201d78 100644 --- a/common/web/lm-worker/src/correction/context-tracker.ts +++ b/common/web/lm-worker/src/correction/context-tracker.ts @@ -135,11 +135,7 @@ namespace correction { // Track the Transform that resulted in the whitespace 'token'. // Will be needed for phrase-level correction/prediction. - if(transformDistribution) { - whitespaceToken.transformDistributions = [transformDistribution]; - } else { - whitespaceToken.transformDistributions = []; - } + whitespaceToken.transformDistributions = transformDistribution ? [transformDistribution] : []; whitespaceToken.raw = null; this.tokens.push(whitespaceToken); From 9b29339594309048634226ef78d969164dc4b695 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 8 Sep 2022 13:13:58 +0700 Subject: [PATCH 17/59] fix(common/models): blocks full-text corrections --- .../src/correction/distance-modeler.ts | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/common/web/lm-worker/src/correction/distance-modeler.ts b/common/web/lm-worker/src/correction/distance-modeler.ts index 7b4456e4d8..47c7ad88eb 100644 --- a/common/web/lm-worker/src/correction/distance-modeler.ts +++ b/common/web/lm-worker/src/correction/distance-modeler.ts @@ -204,6 +204,15 @@ namespace correction { // TODO: might should also track diagonalWidth. return inputString + models.SENTINEL_CODE_UNIT + matchString; } + + get isFullReplacement(): boolean { + // If the known edit-distance cost is equal to the input length, this means + // that literally every input has been full-on replaced. Thus, this is + // likely not a good 'root' to use for predictions. + // + // Logic exception: 0 cost, 0 length != a "replacement". + return this.knownCost && this.knownCost == this.priorInput.length; + } } class SearchSpaceTier { @@ -590,6 +599,14 @@ namespace correction { // Build batches of same-cost entries. while(preprocessedQueue.count > 0) { let entry = preprocessedQueue.dequeue(); + + // Is the entry a reasonable result? + if(entry.isFullReplacement) { + // If the entry's 'match' fully replaces the input string, we consider it + // unreasonable and ignore it. + continue; + } + let batch = batcher.checkAndAdd(entry); if(batch) { @@ -626,6 +643,13 @@ namespace correction { if(newResult.type == 'none') { break; } else if(newResult.type == 'complete') { + // Is the entry a reasonable result? + if(newResult.finalNode.isFullReplacement) { + // If the entry's 'match' fully replaces the input string, we consider it + // unreasonable and ignore it. Also, if we've reached this point... + // we can(?) assume that everything thereafter is as well. + break; + } batch = batcher.checkAndAdd(newResult.finalNode); } From 666b63940407f2a945263dc9090ff25ecc2435f1 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 9 Sep 2022 15:35:12 +0700 Subject: [PATCH 18/59] fix(common/models): adds 'soft bound' for reasonable 100% fat-finger corrections --- common/web/lm-worker/src/model-compositor.ts | 42 +++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/common/web/lm-worker/src/model-compositor.ts b/common/web/lm-worker/src/model-compositor.ts index 329460776d..0010371e63 100644 --- a/common/web/lm-worker/src/model-compositor.ts +++ b/common/web/lm-worker/src/model-compositor.ts @@ -171,6 +171,16 @@ class ModelCompositor { } } + // Is the token under construction newly-constructed / is there no pre-existing root? + // If so, we want to strongly avoid overcorrection, even for 'nearby' keys. + // (Strong lexical frequency differences can easily cause overcorrection when only + // one key's available.) + // + // NOTE: we only want this applied word-initially, when any corrections 'correct' + // 100% of the word. Things are generally fine once it's not "all or nothing." + let tailToken = contextTokens[contextTokens.length - 1]; + const isTokenStart = tailToken.transformDistributions.length <= 1; + // TODO: whitespace, backspace filtering. Do it here. // Whitespace is probably fine, actually. Less sure about backspace. @@ -212,9 +222,39 @@ class ModelCompositor { id: inputTransform.id // The correction should always be based on the most recent external transform/transcription ID. } + let rootCost = match.totalCost; + + /* If we're dealing with the FIRST keystroke of a new sequence, we'll **dramatically** boost + * the exponent to ensure only VERY nearby corrections have a chance of winning, and only if + * there are significantly more likely words. We only need this to allow very minor fat-finger + * adjustments for 100% keystroke-sequence corrections in order to prevent finickiness on + * key borders. + * + * Technically, the probabilities this produces won't be normalized as-is... but there's no + * true NEED to do so for it, even if it'd be 'nice to have'. Consistently tracking when + * to apply it could become tricky, so it's simpler to leave out. + * + * Worst-case, it's possible to temporarily add normalization if a code deep-dive + * is needed in the future. + */ + if(isTokenStart) { + /* Suppose a key distribution: most likely with p=0.5,, second-most with 0.4 - a pretty + * ambiguous case that would only arise very near the center of the boundary between two keys. + * Raising (0.5/0.4)^16 ~= 35.53. + * That seems 'within reason' for correction very near boundaries. + * + * So, with the second-most-likely key being that close in probability, its best suggestion + * must be ~ 35.5x more likely than that of the truly-most-likely key to "win". So, it's not + * a HARD cutoff, but more of a 'soft' one. Keeping the principles in mind documented above, + * it's possible to tweak this to a more harsh or lenient setting if desired, rather than + * being totally "all or nothing" on which key is taken for highly-ambiguous keypresses. + */ + rootCost *= 16; + } + return { sample: correctionTransform, - p: Math.exp(-match.totalCost) + p: Math.exp(-rootCost) }; }, this); From fbb3375ed99d12e6dbbd7ecc3fd8b52424f6f5e1 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 12 Sep 2022 12:02:09 +1000 Subject: [PATCH 19/59] fix(developer): show more useful error if out of space during Setup Fixes #7261. --- common/windows/delphi/setup/SFX.pas | 45 ++++++++++++++++++----------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/common/windows/delphi/setup/SFX.pas b/common/windows/delphi/setup/SFX.pas index 13491802cb..5af3972261 100644 --- a/common/windows/delphi/setup/SFX.pas +++ b/common/windows/delphi/setup/SFX.pas @@ -82,27 +82,38 @@ var fs: TFileStream; ms: TMemoryStream; begin - fs := TFileStream.Create(ParamStr(0), fmOpenRead or fmShareDenyWrite); - ms := TMemoryStream.Create; try - if not FindFirstHeader(fs) then - Exit(False); - - fs.Seek(StartOfFile, TSeekOrigin.soBeginning); - - ms.CopyFrom(fs, fs.Size - StartOfFile); - ms.Position := 0; - - with TZipFile.Create do + fs := TFileStream.Create(ParamStr(0), fmOpenRead or fmShareDenyWrite); + ms := TMemoryStream.Create; try - Open(ms, zmRead); - ExtractAll(ExtPath); + if not FindFirstHeader(fs) then + Exit(False); + + fs.Seek(StartOfFile, TSeekOrigin.soBeginning); + + ms.CopyFrom(fs, fs.Size - StartOfFile); + ms.Position := 0; + + with TZipFile.Create do + try + Open(ms, zmRead); + ExtractAll(ExtPath); + finally + Free; + end; finally - Free; + fs.Free; + ms.Free; + end; + except + on E:Exception do + begin + raise Exception.Create( + 'Failed to extract setup archive. '+ + 'You may have run out of disk space or there may be a '+ + 'problem with the source files.'#13#10#13#10+ + 'The error received was: '+E.Message); end; - finally - fs.Free; - ms.Free; end; Result := True; end; From 6632d597d524e83a71001869582af698d09a8af4 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 12 Sep 2022 10:25:24 +0700 Subject: [PATCH 20/59] feat(common/models): adds unit test, minor branch polish --- .../headless/worker-model-compositor.js | 39 +++++++++++++++++++ common/web/lm-worker/src/model-compositor.ts | 23 +++++++++-- 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/common/predictive-text/unit_tests/headless/worker-model-compositor.js b/common/predictive-text/unit_tests/headless/worker-model-compositor.js index f7e9173cee..464ed7b203 100644 --- a/common/predictive-text/unit_tests/headless/worker-model-compositor.js +++ b/common/predictive-text/unit_tests/headless/worker-model-compositor.js @@ -50,6 +50,45 @@ describe('ModelCompositor', function() { }); }); + it('strongly avoids corrections for single-character roots', function() { + let compositor = new ModelCompositor(plainModel); + let context = { + left: '', startOfBuffer: true, endOfBuffer: true, + }; + + // The 'weights' involved imply that we have an edge-case fat finger on the bottom of + // the 'q' key, slightly in its favor. + let inputDistribution = [ + {sample: {insert: 'q', deleteLeft: 0}, p: 0.5}, // 'quite' (679) and 'question' (644) are included! + {sample: {insert: 'a', deleteLeft: 0}, p: 0.4} // but at lower weight than 'and' (998). + ]; + + compositor.predict({insert: '', deleteLeft: 0}, context); // Initialize context tracking first! + let suggestions = compositor.predict(inputDistribution, context); + + // remove the keep suggestion; we're not testing that here. + suggestions = suggestions.filter((suggestion) => suggestion.tag != 'keep'); + + suggestions.sort((a, b) => b.p - a.p); + + // There are only 4 suggestions in this limited test model that begin with 'q'. + // We expect more than that, since 'a' is indicated to be very close by. + assert.isAbove(suggestions.length, 4, "fat-finger style corrections needed for test comparisons are missing"); + + // Note: 'and' is (currently) modeled by the text-fixture model to have 9.3x the base probability + // that the worst 'q'-rooted suggestion ('quality') does. Without single-character correction + // avoidance logic, this test _will_ fail. + // + // In case a tweak to test parameters is desired, note that 'and' beats rank #3 - 'questions' - + // at 3.36x base. At the time of writing this test, upping 'a's probability to 0.45 will block + // 'quality' while the top three 'q's (ending with 'questions') remain in place. + let qRange = suggestions.slice(0, 4); + assert.isUndefined(qRange.find((suggestion) => suggestion.transform.insert.charAt(0) != 'q')); + + let aRange = suggestions.slice(4); + assert.isUndefined(aRange.find((suggestion) => suggestion.transform.insert.charAt(0) == 'q')); + }); + it('properly handles suggestions after a backspace', function() { let compositor = new ModelCompositor(plainModel); let context = { diff --git a/common/web/lm-worker/src/model-compositor.ts b/common/web/lm-worker/src/model-compositor.ts index 0010371e63..d048d8b246 100644 --- a/common/web/lm-worker/src/model-compositor.ts +++ b/common/web/lm-worker/src/model-compositor.ts @@ -6,6 +6,23 @@ class ModelCompositor { private static readonly MAX_SUGGESTIONS = 12; readonly punctuation: LexicalModelPunctuation; + /** + * Controls the strength of anti-corrective measures for single-character scenarios. + * The base key probability will be raised to this power for this specific case. + * + * Current selection's motivation: (0.5 / 0.4) ^ 16 ~= 35.5. + * - if the most likely has p=0.5 and second-most has p=0.4 - a highly-inaccurate key + * stroke - the net effect will apply a factor of 35.5 to the lexical probability of + * the best key's prediction roots, favoring it in this manner. + * - less extreme edge cases will have a significantly stronger factor, acting as a + * "soft threshold". + * - truly ambiguous, "coin flip" cases will have a lower factor and thus favor the + * more likely words from the pair. + * - Our OSK key-element borders aren't visible to the user, so the 'spot' where + * behavior changes might feel arbitrary to users if we used a hard threshold instead. + */ + private static readonly SINGLE_CHAR_KEY_PROB_EXPONENT = 16; + private SUGGESTION_ID_SEED = 0; constructor(lexicalModel: LexicalModel) { @@ -238,9 +255,9 @@ class ModelCompositor { * is needed in the future. */ if(isTokenStart) { - /* Suppose a key distribution: most likely with p=0.5,, second-most with 0.4 - a pretty + /* Suppose a key distribution: most likely with p=0.5, second-most with 0.4 - a pretty * ambiguous case that would only arise very near the center of the boundary between two keys. - * Raising (0.5/0.4)^16 ~= 35.53. + * Raising (0.5/0.4)^16 ~= 35.53. (At time of writing, SINGLE_CHAR_KEY_PROB_EXPONENT = 16.) * That seems 'within reason' for correction very near boundaries. * * So, with the second-most-likely key being that close in probability, its best suggestion @@ -249,7 +266,7 @@ class ModelCompositor { * it's possible to tweak this to a more harsh or lenient setting if desired, rather than * being totally "all or nothing" on which key is taken for highly-ambiguous keypresses. */ - rootCost *= 16; + rootCost *= ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT; // note the `Math.exp` below. } return { From 6e000d10743803a53a37f5287324c6cda3d52d05 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 8 Sep 2022 14:50:37 +0700 Subject: [PATCH 21/59] change(web): better touch-distrib weighting --- .../src/text/inputProcessor.ts | 3 ++- .../src/keyboards/activeLayout.ts | 26 ++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/common/web/input-processor/src/text/inputProcessor.ts b/common/web/input-processor/src/text/inputProcessor.ts index 9471c244fe..b45098e538 100644 --- a/common/web/input-processor/src/text/inputProcessor.ts +++ b/common/web/input-processor/src/text/inputProcessor.ts @@ -12,7 +12,7 @@ namespace com.keyman.text { /** * Indicates the device (platform) to be used for non-keystroke events, - * such as those sent to `begin postkeystroke` and `begin newcontext` + * such as those sent to `begin postkeystroke` and `begin newcontext` * entry points. */ private contextDevice: utils.DeviceSpec; @@ -273,6 +273,7 @@ namespace com.keyman.text { let totalMass = 0; // Tracks sum of non-error probabilities. for(let pair of keyDistribution) { if(pair.p < KEYSTROKE_EPSILON) { + totalMass += pair.p; break; } else if(timer && timer() >= TIMEOUT_THRESHOLD) { // Note: it's always possible that the thread _executing_ our JS diff --git a/common/web/keyboard-processor/src/keyboards/activeLayout.ts b/common/web/keyboard-processor/src/keyboards/activeLayout.ts index b1be473bc6..432b051f73 100644 --- a/common/web/keyboard-processor/src/keyboards/activeLayout.ts +++ b/common/web/keyboard-processor/src/keyboards/activeLayout.ts @@ -511,17 +511,41 @@ namespace com.keyman.keyboards { let keyProbs: {[keyId: string]: number} = {}; let totalMass = 0; + let bestKey = null; + let bestProb = Number.MIN_VALUE; // Should we wish to allow multiple different transforms for distance -> probability, use a function parameter in place // of the formula in the loop below. for(let key in keyDists) { - totalMass += keyProbs[key] = 1 / (keyDists[key] + 1e-6); // Prevent div-by-0 errors. + keyProbs[key] = 1 / (Math.pow(keyDists[key], 2) + 1e-6); // Prevent div-by-0 errors. + totalMass += keyProbs[key]; + + if(keyProbs[key] > bestProb) { + bestProb = keyProbs[key]; + bestKey = key; + } } for(let key in keyProbs) { keyProbs[key] /= totalMass; } + // To help ensure the highest probability key gets priority, we'll square-root its probability, + // then renormalize. (p <= 1) Has the largest effect when near the edge of the best key. + const originalBestProb = keyProbs[bestKey]; + const finalBestProb = Math.sqrt(keyProbs[bestKey]); + + const normDelta = finalBestProb - originalBestProb; // will be positive. + const renormalizer = 1 / (1 + normDelta); // as we're increasing the sum-total probability mass. + + for(let key in keyProbs) { + if(key == bestKey) { + keyProbs[key] = finalBestProb * renormalizer; // override with the adjusted value, renorm'd. + } else { + keyProbs[key] *= renormalizer; + } + } + return keyProbs; } From c05df0e323b4e1fc523d1b023cf8af40004832d8 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 9 Sep 2022 15:34:00 +0700 Subject: [PATCH 22/59] fix(web): fixes final-key-of-row dist mapping, drops arbitrary prob adjustment for best key --- .../src/keyboards/activeLayout.ts | 30 ++----------------- 1 file changed, 2 insertions(+), 28 deletions(-) diff --git a/common/web/keyboard-processor/src/keyboards/activeLayout.ts b/common/web/keyboard-processor/src/keyboards/activeLayout.ts index 432b051f73..9e8c84896f 100644 --- a/common/web/keyboard-processor/src/keyboards/activeLayout.ts +++ b/common/web/keyboard-processor/src/keyboards/activeLayout.ts @@ -337,14 +337,12 @@ namespace com.keyman.keyboards { // Allow for right OSK margin (15 layout units) let rightMargin = ActiveKey.DEFAULT_RIGHT_MARGIN/totalWidth; - totalPercent += rightMargin; // If a single key, and padding is negative, add padding to right align the key if(keys.length == 1 && parseInt(keys[0]['pad'],10) < 0) { keyPercent=parseInt(keys[0]['width'],10)/totalWidth; keys[0]['widthpc']=keyPercent; - totalPercent += keyPercent; - keys[0]['padpc']=1-totalPercent; + keys[0]['padpc']=1-(totalPercent + keyPercent + rightMargin); // compute center's default x-coord (used in headless modes) setProportions(keys[0] as ActiveKey, padPercent, keyPercent, totalPercent); @@ -352,8 +350,7 @@ namespace com.keyman.keyboards { let j=keys.length-1; padPercent=parseInt(keys[j]['pad'],10)/totalWidth; keys[j]['padpc']=padPercent; - totalPercent += padPercent; - keys[j]['widthpc'] = keyPercent = 1-totalPercent; + keys[j]['widthpc'] = keyPercent = 1-(totalPercent + padPercent + rightMargin); // compute center's default x-coord (used in headless modes) setProportions(keys[j] as ActiveKey, padPercent, keyPercent, totalPercent); @@ -511,41 +508,18 @@ namespace com.keyman.keyboards { let keyProbs: {[keyId: string]: number} = {}; let totalMass = 0; - let bestKey = null; - let bestProb = Number.MIN_VALUE; // Should we wish to allow multiple different transforms for distance -> probability, use a function parameter in place // of the formula in the loop below. for(let key in keyDists) { keyProbs[key] = 1 / (Math.pow(keyDists[key], 2) + 1e-6); // Prevent div-by-0 errors. totalMass += keyProbs[key]; - - if(keyProbs[key] > bestProb) { - bestProb = keyProbs[key]; - bestKey = key; - } } for(let key in keyProbs) { keyProbs[key] /= totalMass; } - // To help ensure the highest probability key gets priority, we'll square-root its probability, - // then renormalize. (p <= 1) Has the largest effect when near the edge of the best key. - const originalBestProb = keyProbs[bestKey]; - const finalBestProb = Math.sqrt(keyProbs[bestKey]); - - const normDelta = finalBestProb - originalBestProb; // will be positive. - const renormalizer = 1 / (1 + normDelta); // as we're increasing the sum-total probability mass. - - for(let key in keyProbs) { - if(key == bestKey) { - keyProbs[key] = finalBestProb * renormalizer; // override with the adjusted value, renorm'd. - } else { - keyProbs[key] *= renormalizer; - } - } - return keyProbs; } From b8dd9baf5b9863c0bac2fac468e9d9d437038fde Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 9 Sep 2022 15:51:42 +0700 Subject: [PATCH 23/59] chore(web): minor cleanup --- common/web/keyboard-processor/src/keyboards/activeLayout.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/common/web/keyboard-processor/src/keyboards/activeLayout.ts b/common/web/keyboard-processor/src/keyboards/activeLayout.ts index 9e8c84896f..cd4baa0a27 100644 --- a/common/web/keyboard-processor/src/keyboards/activeLayout.ts +++ b/common/web/keyboard-processor/src/keyboards/activeLayout.ts @@ -512,8 +512,7 @@ namespace com.keyman.keyboards { // Should we wish to allow multiple different transforms for distance -> probability, use a function parameter in place // of the formula in the loop below. for(let key in keyDists) { - keyProbs[key] = 1 / (Math.pow(keyDists[key], 2) + 1e-6); // Prevent div-by-0 errors. - totalMass += keyProbs[key]; + totalMass += keyProbs[key] = 1 / (Math.pow(keyDists[key], 2) + 1e-6); // Prevent div-by-0 errors. } for(let key in keyProbs) { From d91132bc0fdf1bc4dfad24d3dd487c73b7b55e3e Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 12 Sep 2022 10:39:41 +0700 Subject: [PATCH 24/59] fix(web): filters more non-output keys from fat-finger distrib --- .../web/keyboard-processor/src/keyboards/activeLayout.ts | 9 +++++++++ common/web/keyboard-processor/src/text/codes.ts | 5 ++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/common/web/keyboard-processor/src/keyboards/activeLayout.ts b/common/web/keyboard-processor/src/keyboards/activeLayout.ts index cd4baa0a27..93a00f84c9 100644 --- a/common/web/keyboard-processor/src/keyboards/activeLayout.ts +++ b/common/web/keyboard-processor/src/keyboards/activeLayout.ts @@ -37,6 +37,7 @@ namespace com.keyman.keyboards { layer: string; displayLayer: string; nextlayer: string; + sp?: ButtonClass; private baseKeyEvent: text.KeyEvent; isMnemonic: boolean = false; @@ -73,6 +74,12 @@ namespace com.keyman.keyboards { return this.id; } + public get isPadding(): boolean { + // Does not include 9 (class: blank) as that may be an intentional 'catch' for misplaced + // keystrokes. + return this['sp'] == 10; // Button class: hidden. + } + /** * A unique identifier based on both the key ID & the 'desktop layer' to be used for the key. * @@ -547,6 +554,8 @@ namespace com.keyman.keyboards { // Results in a more optimized distribution. if(text.Codes.isKnownOSKModifierKey(key.baseKeyID)) { return; + } else if(key.isPadding) { // to the user, blank / padding keys do not exist. + return; } } // These represent the within-key distance of the touch from the key's center. diff --git a/common/web/keyboard-processor/src/text/codes.ts b/common/web/keyboard-processor/src/text/codes.ts index 915ae7feed..48fde0d728 100644 --- a/common/web/keyboard-processor/src/text/codes.ts +++ b/common/web/keyboard-processor/src/text/codes.ts @@ -81,10 +81,13 @@ namespace com.keyman.text { case 'K_SHIFT': case 'K_LOPT': case 'K_ROPT': - case 'K_NUMLOCK': // Often used for numeric layers. + case 'K_NUMLOCK': // Often used for numeric layers. case 'K_CAPS': return true; default: + if(Codes.keyCodes[keyID] >= 50000) { // A few are used by `sil_euro_latin`. + return true; // is a 'K_' key defined for layer shifting or 'control' use. + } // Refer to text/codes.ts - these are Keyman-custom "keycodes" used for // layer shifting keys. To be safe, we currently let K_TABBACK and // K_TABFWD through, though we might be able to drop them too. From 5cd48a578493f2c2c15236e1a9accc844b9a1482 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 12 Sep 2022 11:05:27 +0700 Subject: [PATCH 25/59] fix(web): new key prop needs enumerable --- common/web/keyboard-processor/src/keyboards/activeLayout.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/common/web/keyboard-processor/src/keyboards/activeLayout.ts b/common/web/keyboard-processor/src/keyboards/activeLayout.ts index 93a00f84c9..7f0f2f935f 100644 --- a/common/web/keyboard-processor/src/keyboards/activeLayout.ts +++ b/common/web/keyboard-processor/src/keyboards/activeLayout.ts @@ -74,6 +74,7 @@ namespace com.keyman.keyboards { return this.id; } + @Enumerable public get isPadding(): boolean { // Does not include 9 (class: blank) as that may be an intentional 'catch' for misplaced // keystrokes. From 415201bf15d7cdb429df44a8f86bf2984ba82399 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 12 Sep 2022 15:03:48 +0700 Subject: [PATCH 26/59] fix(common/models): isWhitespace, adds related unit tests --- .../unit_tests/headless/transform-utils.js | 58 +++++++++++++++++++ common/web/lm-worker/src/transformUtils.ts | 10 +--- 2 files changed, 61 insertions(+), 7 deletions(-) create mode 100644 common/predictive-text/unit_tests/headless/transform-utils.js diff --git a/common/predictive-text/unit_tests/headless/transform-utils.js b/common/predictive-text/unit_tests/headless/transform-utils.js new file mode 100644 index 0000000000..b00a24a123 --- /dev/null +++ b/common/predictive-text/unit_tests/headless/transform-utils.js @@ -0,0 +1,58 @@ +var assert = require('chai').assert; + +let TransformUtils = require('../../../web/lm-worker/build/intermediate.js').TransformUtils; + +describe('TransformUtils', function () { + describe('isWhitespace', function () { + it("should not match a string containing standard alphabetic characters", function () { + let testTransforms = [{ + insert: "a ", + deleteLeft: 0 + }, { + insert: " a", + deleteLeft: 0 + }, { + insert: "ab", + deleteLeft: 0 + }]; + + testTransforms.forEach((transform) => assert.isFalse(TransformUtils.isWhitespace(transform), `failed with: '${transform.insert}'`)); + }); + + it("should match a simple ' ' transform", function() { + transform = { + insert: " ", + deleteLeft: 0 + }; + + assert.isTrue(TransformUtils.isWhitespace(transform)); + }); + + it("should match a simple ' ' transform with delete-left", function() { + transform = { + insert: " ", + deleteLeft: 1 + }; + + assert.isTrue(TransformUtils.isWhitespace(transform)); + }); + + it("should match a transform consisting of multiple characters of only whitespace", function() { + transform = { + insert: " \n\r\u00a0\t\u2000 ", + deleteLeft: 0 + }; + + assert.isTrue(TransformUtils.isWhitespace(transform)); + }); + + it("stress tests", function() { + transform = { + insert: " \n\r\u00a0\ta\u2000 ", // the 'a' should cause failure. + deleteLeft: 0 + }; + + assert.isFalse(TransformUtils.isWhitespace(transform)); + }); + }); +}); diff --git a/common/web/lm-worker/src/transformUtils.ts b/common/web/lm-worker/src/transformUtils.ts index b91951722f..8c82f16b3b 100644 --- a/common/web/lm-worker/src/transformUtils.ts +++ b/common/web/lm-worker/src/transformUtils.ts @@ -1,18 +1,14 @@ class TransformUtils { static isWhitespace(transform: Transform): boolean { // Matches prefixed text + any instance of a character with Unicode general property Z* or the following: CR, LF, and Tab. - let whitespaceRemover = /.*[\u0009\u000A\u000D\u0020\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b\u2028\u2029\u202f\u205f\u3000]/i; + const whitespaceRemover = /^[\u0009\u000A\u000D\u0020\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b\u2028\u2029\u202f\u205f\u3000]+$/i; // Filter out null-inserts; their high probability can cause issues. - if(transform.insert == '') { // Can actually register as 'whitespace'. + if(transform.insert == '') { return false; } - let insert = transform.insert; - - insert = insert.replace(whitespaceRemover, ''); - - return insert == ''; + return transform.insert.match(whitespaceRemover) != null; } static isBackspace(transform: Transform): boolean { From 665b149b096735a361ae4dc79f6678198abd872c Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 12 Sep 2022 15:04:17 +0700 Subject: [PATCH 27/59] fix(common/models): needed export for unit tests --- common/web/lm-worker/src/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/web/lm-worker/src/index.ts b/common/web/lm-worker/src/index.ts index 7cc3bcbde1..e3c16fa2c8 100644 --- a/common/web/lm-worker/src/index.ts +++ b/common/web/lm-worker/src/index.ts @@ -32,6 +32,7 @@ /// /// /// +/// /** * Encapsulates all the state required for the LMLayer's worker thread. @@ -407,6 +408,7 @@ if (typeof module !== 'undefined' && typeof module.exports !== 'undefined') { module.exports['wordBreakers'] = wordBreakers; /// XXX: export the ModelCompositor for testing. module.exports['ModelCompositor'] = ModelCompositor; + module.exports['TransformUtils'] = TransformUtils; } else if (typeof self !== 'undefined' && 'postMessage' in self && 'importScripts' in self) { // Automatically install if we're in a Web Worker. LMLayerWorker.install(self as any); // really, 'as typeof globalThis', but we're currently getting TS errors from use of that. From 5bad267e11f4d76078ef9e394aa7331c90b053f2 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 7 Sep 2022 09:52:37 +0700 Subject: [PATCH 28/59] refactor(common/models): default wordbreaker state tracking --- .../models/wordbreakers/src/default/index.ts | 235 +++++++++++++----- 1 file changed, 166 insertions(+), 69 deletions(-) diff --git a/common/models/wordbreakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts index 5f14165c13..415d0b0046 100644 --- a/common/models/wordbreakers/src/default/index.ts +++ b/common/models/wordbreakers/src/default/index.ts @@ -61,6 +61,121 @@ namespace wordBreakers { } } + /** + * Provides a useful presentation for wordbreaker's context for use in word-breaking rules. + * + * @see https://unicode.org/reports/tr29/#Word_Boundary_Rules + */ + export class BreakerContext { + private readonly text: string; + + /** + * Represents the property of character immediately preceding `left`'s character. + */ + readonly lookbehind: WordBreakProperty = WordBreakProperty.sot; + + /** + * Represents the property of the character immediately preceding the potential word boundary. + */ + readonly left: WordBreakProperty = WordBreakProperty.sot; + + /** + * Represents the property of the character immediately following the potential word boundary. + */ + readonly right: WordBreakProperty = WordBreakProperty.sot; + + /** + * Represents the property of the character immediately following `right`'s character. + */ + readonly lookahead: WordBreakProperty; + + /** + * Initializes the word-breaking context at the start of the word-breaker's boundary-detection + * algorithm. + * @param text The text to be word-broken + * @param lookaheadPos The position corresponding to `lookahead`. + */ + constructor(text: string, lookaheadPos: number); + /** + * Used internally by the boundary-detection algorithm during context-shifting operations. + * @param text + * @param lookbehind + * @param left + * @param right + * @param lookahead + */ + constructor(text: string, + lookbehind: WordBreakProperty, + left: WordBreakProperty, + right: WordBreakProperty, + lookahead: WordBreakProperty); + constructor(text: string, + prop1: WordBreakProperty | number, + prop2?: WordBreakProperty, + prop3?: WordBreakProperty, + prop4?: WordBreakProperty) { + this.text = text; + + if(arguments.length == 2) { + this.lookahead = this.wordbreakPropertyAt(prop1);// prop1; + } else /*if(arguments.length == 5)*/ { + this.lookbehind = prop1 as WordBreakProperty; + this.left = prop2 as WordBreakProperty; + this.right = prop3 as WordBreakProperty; + this.lookahead = prop4 as WordBreakProperty; + } + } + + /** + * The general use-case when shifting boundary-check position if WB4 is not active. + * @param lookahead The WordBreakProperty for the character to become `lookahead`. + * @returns + */ + public next(lookaheadPos: number): BreakerContext { + let newLookahead = this.wordbreakPropertyAt(lookaheadPos); + return new BreakerContext(this.text, this.left, this.right, this.lookahead, newLookahead); + } + + /** + * Used for WB4: when ignoring characters before an intervening linebreak, we + * replace `right` with the current `lookahead`, without affecting `lookbehind` + * or `left`. A new `lookahead` is then needed. + * @param lookahead + * @returns + */ + public ignoringRight(lookaheadPos: number) { + let newLookahead = this.wordbreakPropertyAt(lookaheadPos); + return new BreakerContext(this.text, this.lookbehind, this.left, this.lookahead, newLookahead); + } + + /** + * Used for WB4: when ignoring characters after an intervening linebreak, it's + * `lookahead` that gets replaced without shifting the other tracked properties. + * @param lookahead + * @returns + */ + public ignoringLookahead(lookaheadPos: number) { + let newLookahead = this.wordbreakPropertyAt(lookaheadPos); + return new BreakerContext(this.text, this.lookbehind, this.left, this.right, newLookahead); + } + + /** + * Return the value of the Word_Break property at the given string index. + * @param pos position in the text. + */ + private wordbreakPropertyAt(pos: number) { + if (pos < 0) { + return WordBreakProperty.sot; // Always "start of string" before the string starts! + } else if (pos >= this.text.length) { + return WordBreakProperty.eot; // Always "end of string" after the string ends! + } else if (isStartOfSurrogatePair(this.text[pos])) { + // Surrogate pairs the next TWO items from the string! + return property(this.text[pos] + this.text[pos + 1]); + } + return property(this.text[pos]); + } + } + /** * Returns true when the chunk does not solely consist of whitespace. * @@ -112,10 +227,7 @@ namespace wordBreakers { let rightPos: number; let lookaheadPos = 0; // lookahead, one scalar value to the right of right. // Before the start of the string is also the start of the string. - let lookbehind: WordBreakProperty; - let left = WordBreakProperty.sot; - let right = WordBreakProperty.sot; - let lookahead = wordbreakPropertyAt(0); + let state = new BreakerContext(text, lookaheadPos); // Count RIs to make sure we're not splitting emoji flags: let nConsecutiveRegionalIndicators = 0; @@ -124,34 +236,33 @@ namespace wordBreakers { rightPos = lookaheadPos; lookaheadPos = positionAfter(lookaheadPos); // Shift all properties, one scalar value to the right. - [lookbehind, left, right, lookahead] = - [left, right, lookahead, wordbreakPropertyAt(lookaheadPos)]; + state = state.next(lookaheadPos); // Break at the start and end of text, unless the text is empty. // WB1: Break at start of text... - if (left === WordBreakProperty.sot) { + if (state.left === WordBreakProperty.sot) { boundaries.push(rightPos); continue; } // WB2: Break at the end of text... - if (right === WordBreakProperty.eot) { + if (state.right === WordBreakProperty.eot) { boundaries.push(rightPos); break; // Reached the end of the string. We're done! } // WB3: Do not break within CRLF: - if (left === WordBreakProperty.CR && right === WordBreakProperty.LF) + if (state.left === WordBreakProperty.CR && state.right === WordBreakProperty.LF) continue; // WB3b: Otherwise, break after... - if (left === WordBreakProperty.Newline || - left === WordBreakProperty.CR || - left === WordBreakProperty.LF) { + if (state.left === WordBreakProperty.Newline || + state.left === WordBreakProperty.CR || + state.left === WordBreakProperty.LF) { boundaries.push(rightPos); continue; } // WB3a: ...and before newlines - if (right === WordBreakProperty.Newline || - right === WordBreakProperty.CR || - right === WordBreakProperty.LF) { + if (state.right === WordBreakProperty.Newline || + state.right === WordBreakProperty.CR || + state.right === WordBreakProperty.LF) { boundaries.push(rightPos); continue; } @@ -163,7 +274,7 @@ namespace wordBreakers { // https://www.unicode.org/Public/emoji/12.0/emoji-zwj-sequences.txt // WB3d: Keep horizontal whitespace together - if (left === WordBreakProperty.WSegSpace && right == WordBreakProperty.WSegSpace) + if (state.left === WordBreakProperty.WSegSpace && state.right == WordBreakProperty.WSegSpace) continue; // WB4: Ignore format and extend characters @@ -171,99 +282,99 @@ namespace wordBreakers { // See: Section 6.2: https://unicode.org/reports/tr29/#Grapheme_Cluster_and_Format_Rules // N.B.: The rule about "except after sot, CR, LF, and // Newline" already been by WB1, WB2, WB3a, and WB3b above. - while (right === WordBreakProperty.Format || - right === WordBreakProperty.Extend || - right === WordBreakProperty.ZWJ) { + while (state.right === WordBreakProperty.Format || + state.right === WordBreakProperty.Extend || + state.right === WordBreakProperty.ZWJ) { // Continue advancing in the string, as if these // characters do not exist. DO NOT update left and // lookbehind however! [rightPos, lookaheadPos] = [lookaheadPos, positionAfter(lookaheadPos)]; - [right, lookahead] = [lookahead, wordbreakPropertyAt(lookaheadPos)]; + state = state.ignoringRight(lookaheadPos); } // In ignoring the characters in the previous loop, we could // have fallen off the end of the string, so end the loop // prematurely if that happens! - if (right === WordBreakProperty.eot) { + if (state.right === WordBreakProperty.eot) { boundaries.push(rightPos); break; } // WB4 (continued): Lookahead must ALSO ignore these format, // extend, ZWJ characters! - while (lookahead === WordBreakProperty.Format || - lookahead === WordBreakProperty.Extend || - lookahead === WordBreakProperty.ZWJ) { + while (state.lookahead === WordBreakProperty.Format || + state.lookahead === WordBreakProperty.Extend || + state.lookahead === WordBreakProperty.ZWJ) { // Continue advancing in the string, as if these // characters do not exist. DO NOT update left and right, // however! lookaheadPos = positionAfter(lookaheadPos); - lookahead = wordbreakPropertyAt(lookaheadPos); + state = state.ignoringLookahead(lookaheadPos); } // WB5: Do not break between most letters. - if (isAHLetter(left) && isAHLetter(right)) + if (isAHLetter(state.left) && isAHLetter(state.right)) continue; // Do not break across certain punctuation // WB6: (Don't break before apostrophes in contractions) - if (isAHLetter(left) && isAHLetter(lookahead) && - (right === WordBreakProperty.MidLetter || isMidNumLetQ(right))) + if (isAHLetter(state.left) && isAHLetter(state.lookahead) && + (state.right === WordBreakProperty.MidLetter || isMidNumLetQ(state.right))) continue; // WB7: (Don't break after apostrophes in contractions) - if (isAHLetter(lookbehind) && isAHLetter(right) && - (left === WordBreakProperty.MidLetter || isMidNumLetQ(left))) + if (isAHLetter(state.lookbehind) && isAHLetter(state.right) && + (state.left === WordBreakProperty.MidLetter || isMidNumLetQ(state.left))) continue; // WB7a - if (left === WordBreakProperty.Hebrew_Letter && right === WordBreakProperty.Single_Quote) + if (state.left === WordBreakProperty.Hebrew_Letter && state.right === WordBreakProperty.Single_Quote) continue; // WB7b - if (left === WordBreakProperty.Hebrew_Letter && right === WordBreakProperty.Double_Quote && - lookahead === WordBreakProperty.Hebrew_Letter) + if (state.left === WordBreakProperty.Hebrew_Letter && state.right === WordBreakProperty.Double_Quote && + state.lookahead === WordBreakProperty.Hebrew_Letter) continue; // WB7c - if (lookbehind === WordBreakProperty.Hebrew_Letter && left === WordBreakProperty.Double_Quote && - right === WordBreakProperty.Hebrew_Letter) + if (state.lookbehind === WordBreakProperty.Hebrew_Letter && state.left === WordBreakProperty.Double_Quote && + state.right === WordBreakProperty.Hebrew_Letter) continue; // Do not break within sequences of digits, or digits adjacent to letters. // e.g., "3a" or "A3" // WB8 - if (left === WordBreakProperty.Numeric && right === WordBreakProperty.Numeric) + if (state.left === WordBreakProperty.Numeric && state.right === WordBreakProperty.Numeric) continue; // WB9 - if (isAHLetter(left) && right === WordBreakProperty.Numeric) + if (isAHLetter(state.left) && state.right === WordBreakProperty.Numeric) continue; // WB10 - if (left === WordBreakProperty.Numeric && isAHLetter(right)) + if (state.left === WordBreakProperty.Numeric && isAHLetter(state.right)) continue; // Do not break within sequences, such as 3.2, 3,456.789 // WB11 - if (lookbehind === WordBreakProperty.Numeric && right === WordBreakProperty.Numeric && - (left === WordBreakProperty.MidNum || isMidNumLetQ(left))) + if (state.lookbehind === WordBreakProperty.Numeric && state.right === WordBreakProperty.Numeric && + (state.left === WordBreakProperty.MidNum || isMidNumLetQ(state.left))) continue; // WB12 - if (left === WordBreakProperty.Numeric && lookahead === WordBreakProperty.Numeric && - (right === WordBreakProperty.MidNum || isMidNumLetQ(right))) + if (state.left === WordBreakProperty.Numeric && state.lookahead === WordBreakProperty.Numeric && + (state.right === WordBreakProperty.MidNum || isMidNumLetQ(state.right))) continue; // WB13: Do not break between Katakana - if (left === WordBreakProperty.Katakana && right === WordBreakProperty.Katakana) + if (state.left === WordBreakProperty.Katakana && state.right === WordBreakProperty.Katakana) continue; // Do not break from extenders (e.g., U+202F NARROW NO-BREAK SPACE) // WB13a - if ((isAHLetter(left) || - left === WordBreakProperty.Numeric || - left === WordBreakProperty.Katakana || - left === WordBreakProperty.ExtendNumLet) && - right === WordBreakProperty.ExtendNumLet) + if ((isAHLetter(state.left) || + state.left === WordBreakProperty.Numeric || + state.left === WordBreakProperty.Katakana || + state.left === WordBreakProperty.ExtendNumLet) && + state.right === WordBreakProperty.ExtendNumLet) continue; // WB13b - if ((isAHLetter(right) || - right === WordBreakProperty.Numeric || - right === WordBreakProperty.Katakana) && left === WordBreakProperty.ExtendNumLet) + if ((isAHLetter(state.right) || + state.right === WordBreakProperty.Numeric || + state.right === WordBreakProperty.Katakana) && state.left === WordBreakProperty.ExtendNumLet) continue; // WB15 & WB16: // Do not break within emoji flag sequences. That is, do not break between // regional indicator (RI) symbols if there is an odd number of RI // characters before the break point. - if (right === WordBreakProperty.Regional_Indicator) { + if (state.right === WordBreakProperty.Regional_Indicator) { // Emoji flags are actually composed of TWO scalar values, each being a // "regional indicator". These indicators correspond to Latin letters. Put // two of them together, and they spell out an ISO 3166-1-alpha-2 country @@ -301,22 +412,6 @@ namespace wordBreakers { return pos + 1; } - /** - * Return the value of the Word_Break property at the given string index. - * @param pos position in the text. - */ - function wordbreakPropertyAt(pos: number) { - if (pos < 0) { - return WordBreakProperty.sot; // Always "start of string" before the string starts! - } else if (pos >= text.length) { - return WordBreakProperty.eot; // Always "end of string" after the string ends! - } else if (isStartOfSurrogatePair(text[pos])) { - // Surrogate pairs the next TWO items from the string! - return property(text[pos] + text[pos + 1]); - } - return property(text[pos]); - } - // Word_Break rule macros // See: https://unicode.org/reports/tr29/#WB_Rule_Macros function isAHLetter(prop: WordBreakProperty): boolean { @@ -340,7 +435,7 @@ namespace wordBreakers { * Note that * @param character a scalar value */ - function property(character: string): WordBreakProperty { + export function property(character: string): WordBreakProperty { // This MUST be a scalar value. // TODO: remove dependence on character.codepointAt()? let codepoint = character.codePointAt(0) as number; @@ -383,6 +478,8 @@ namespace wordBreakers { // implementing a namespace, BUT we can manually make the // assignment and **declare** it as part of the namespace. wordBreakers['default'] = wordBreakers.default_; +wordBreakers['unicodeProperty'] = wordBreakers.property; declare namespace wordBreakers { export { default_ as default }; + export { property as unicodeProperty }; } From 991a5bc271500334b1e48444a12ffce10527c98e Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 7 Sep 2022 10:33:50 +0700 Subject: [PATCH 29/59] feat(common/models): wordbreaker context .match method --- .../models/wordbreakers/src/default/index.ts | 136 +++++++++++------- 1 file changed, 81 insertions(+), 55 deletions(-) diff --git a/common/models/wordbreakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts index 415d0b0046..4f86efc507 100644 --- a/common/models/wordbreakers/src/default/index.ts +++ b/common/models/wordbreakers/src/default/index.ts @@ -87,7 +87,7 @@ namespace wordBreakers { /** * Represents the property of the character immediately following `right`'s character. */ - readonly lookahead: WordBreakProperty; + readonly lookahead: WordBreakProperty; // Always initialized by constructor. /** * Initializes the word-breaking context at the start of the word-breaker's boundary-detection @@ -174,6 +174,25 @@ namespace wordBreakers { } return property(this.text[pos]); } + + /** + * Returns `true` if and only if each member of the context has a property included within + * its corresponding set (when specified). Any set may be replaced with null to disable + * a check against its corresponding property. + * @param lookbehindSet + * @param leftSet + * @param rightSet + * @param lookaheadSet + */ + public match(lookbehindSet: WordBreakProperty[] | null, + leftSet: WordBreakProperty[] | null, + rightSet: WordBreakProperty[] | null, + lookaheadSet: WordBreakProperty[] | null) : boolean { + let result: boolean = lookbehindSet?.includes(this.lookbehind) ?? true; + result = result && (leftSet?.includes(this.left) ?? true); + result = result && (rightSet?.includes(this.right) ?? true); + return result && (lookaheadSet?.includes(this.lookahead) ?? true); + } } /** @@ -240,29 +259,28 @@ namespace wordBreakers { // Break at the start and end of text, unless the text is empty. // WB1: Break at start of text... - if (state.left === WordBreakProperty.sot) { + if (state.match(null, [WordBreakProperty.sot], null, null)) { boundaries.push(rightPos); continue; } // WB2: Break at the end of text... - if (state.right === WordBreakProperty.eot) { + if (state.match(null, null, [WordBreakProperty.eot], null)) { boundaries.push(rightPos); break; // Reached the end of the string. We're done! } // WB3: Do not break within CRLF: - if (state.left === WordBreakProperty.CR && state.right === WordBreakProperty.LF) + if (state.match(null, [WordBreakProperty.CR], [WordBreakProperty.LF], null)) { continue; + } + // WB3b: Otherwise, break after... - if (state.left === WordBreakProperty.Newline || - state.left === WordBreakProperty.CR || - state.left === WordBreakProperty.LF) { + const NEWLINE_SET = [WordBreakProperty.Newline, WordBreakProperty.CR, WordBreakProperty.LF]; + if(state.match(null, NEWLINE_SET, null, null)) { boundaries.push(rightPos); continue; } // WB3a: ...and before newlines - if (state.right === WordBreakProperty.Newline || - state.right === WordBreakProperty.CR || - state.right === WordBreakProperty.LF) { + if (state.match(null, null, NEWLINE_SET, null)) { boundaries.push(rightPos); continue; } @@ -274,17 +292,17 @@ namespace wordBreakers { // https://www.unicode.org/Public/emoji/12.0/emoji-zwj-sequences.txt // WB3d: Keep horizontal whitespace together - if (state.left === WordBreakProperty.WSegSpace && state.right == WordBreakProperty.WSegSpace) + if (state.match(null, [WordBreakProperty.WSegSpace], [WordBreakProperty.WSegSpace], null)) { continue; + } // WB4: Ignore format and extend characters // This is to keep grapheme clusters together! // See: Section 6.2: https://unicode.org/reports/tr29/#Grapheme_Cluster_and_Format_Rules // N.B.: The rule about "except after sot, CR, LF, and // Newline" already been by WB1, WB2, WB3a, and WB3b above. - while (state.right === WordBreakProperty.Format || - state.right === WordBreakProperty.Extend || - state.right === WordBreakProperty.ZWJ) { + const SET_WB4_IGNORE = [WordBreakProperty.Format, WordBreakProperty.Extend, WordBreakProperty.ZWJ]; + while (state.match(null, null, SET_WB4_IGNORE, null)) { // Continue advancing in the string, as if these // characters do not exist. DO NOT update left and // lookbehind however! @@ -300,9 +318,7 @@ namespace wordBreakers { } // WB4 (continued): Lookahead must ALSO ignore these format, // extend, ZWJ characters! - while (state.lookahead === WordBreakProperty.Format || - state.lookahead === WordBreakProperty.Extend || - state.lookahead === WordBreakProperty.ZWJ) { + while (state.match(null, null, null, SET_WB4_IGNORE)) { // Continue advancing in the string, as if these // characters do not exist. DO NOT update left and right, // however! @@ -310,65 +326,87 @@ namespace wordBreakers { state = state.ignoringLookahead(lookaheadPos); } + // See: https://unicode.org/reports/tr29/#WB_Rule_Macros + const SET_AHLETTER = [WordBreakProperty.ALetter, WordBreakProperty.Hebrew_Letter]; + const SET_MIDNUMLETQ = [WordBreakProperty.MidNumLet, WordBreakProperty.Single_Quote]; + // WB5: Do not break between most letters. - if (isAHLetter(state.left) && isAHLetter(state.right)) + // if (isAHLetter(state.left) && isAHLetter(state.right)) + if(state.match(null, SET_AHLETTER, SET_AHLETTER, null)) { continue; + } // Do not break across certain punctuation // WB6: (Don't break before apostrophes in contractions) - if (isAHLetter(state.left) && isAHLetter(state.lookahead) && - (state.right === WordBreakProperty.MidLetter || isMidNumLetQ(state.right))) + const SET_ALL_MIDLETTER = [WordBreakProperty.MidLetter, ...SET_MIDNUMLETQ]; + if(state.match(null, SET_AHLETTER, SET_ALL_MIDLETTER, SET_AHLETTER)) { continue; + } // WB7: (Don't break after apostrophes in contractions) - if (isAHLetter(state.lookbehind) && isAHLetter(state.right) && - (state.left === WordBreakProperty.MidLetter || isMidNumLetQ(state.left))) + if(state.match(SET_AHLETTER, SET_ALL_MIDLETTER, SET_AHLETTER, null)) { continue; + } + // WB7a - if (state.left === WordBreakProperty.Hebrew_Letter && state.right === WordBreakProperty.Single_Quote) + if(state.match(null, [WordBreakProperty.Hebrew_Letter], [WordBreakProperty.Single_Quote], null)) { continue; + } // WB7b - if (state.left === WordBreakProperty.Hebrew_Letter && state.right === WordBreakProperty.Double_Quote && - state.lookahead === WordBreakProperty.Hebrew_Letter) + if(state.match(null, + [WordBreakProperty.Hebrew_Letter], + [WordBreakProperty.Double_Quote], + [WordBreakProperty.Hebrew_Letter])) { continue; + } // WB7c - if (state.lookbehind === WordBreakProperty.Hebrew_Letter && state.left === WordBreakProperty.Double_Quote && - state.right === WordBreakProperty.Hebrew_Letter) + if(state.match([WordBreakProperty.Hebrew_Letter], + [WordBreakProperty.Double_Quote], + [WordBreakProperty.Hebrew_Letter], + null)) { continue; + } // Do not break within sequences of digits, or digits adjacent to letters. // e.g., "3a" or "A3" // WB8 - if (state.left === WordBreakProperty.Numeric && state.right === WordBreakProperty.Numeric) + if(state.match(null, [WordBreakProperty.Numeric], [WordBreakProperty.Numeric], null)) { continue; + } // WB9 - if (isAHLetter(state.left) && state.right === WordBreakProperty.Numeric) + if(state.match(null, SET_AHLETTER, [WordBreakProperty.Numeric], null)) { continue; + } // WB10 - if (state.left === WordBreakProperty.Numeric && isAHLetter(state.right)) + if(state.match(null, [WordBreakProperty.Numeric], SET_AHLETTER, null)) { continue; + } // Do not break within sequences, such as 3.2, 3,456.789 // WB11 - if (state.lookbehind === WordBreakProperty.Numeric && state.right === WordBreakProperty.Numeric && - (state.left === WordBreakProperty.MidNum || isMidNumLetQ(state.left))) + const SET_ALL_MIDNUM = [WordBreakProperty.MidNum, ...SET_MIDNUMLETQ]; + if(state.match([WordBreakProperty.Numeric], SET_ALL_MIDNUM, [WordBreakProperty.Numeric], null)) { continue; + } // WB12 - if (state.left === WordBreakProperty.Numeric && state.lookahead === WordBreakProperty.Numeric && - (state.right === WordBreakProperty.MidNum || isMidNumLetQ(state.right))) + if(state.match(null, [WordBreakProperty.Numeric], SET_ALL_MIDNUM, [WordBreakProperty.Numeric])) { continue; + } // WB13: Do not break between Katakana - if (state.left === WordBreakProperty.Katakana && state.right === WordBreakProperty.Katakana) + if(state.match(null, [WordBreakProperty.Katakana], [WordBreakProperty.Katakana], null)) { continue; + } // Do not break from extenders (e.g., U+202F NARROW NO-BREAK SPACE) // WB13a - if ((isAHLetter(state.left) || - state.left === WordBreakProperty.Numeric || - state.left === WordBreakProperty.Katakana || - state.left === WordBreakProperty.ExtendNumLet) && - state.right === WordBreakProperty.ExtendNumLet) + const SET_NUM_KAT_LET = [WordBreakProperty.Katakana, + WordBreakProperty.Numeric, + ...SET_AHLETTER]; + if(state.match(null, SET_NUM_KAT_LET, [WordBreakProperty.ExtendNumLet], null)) { continue; + } + if(state.match(null, [WordBreakProperty.ExtendNumLet], [WordBreakProperty.ExtendNumLet], null)) { + continue; + } // WB13b - if ((isAHLetter(state.right) || - state.right === WordBreakProperty.Numeric || - state.right === WordBreakProperty.Katakana) && state.left === WordBreakProperty.ExtendNumLet) + if(state.match(null, [WordBreakProperty.ExtendNumLet], SET_NUM_KAT_LET, null)) { continue; + } // WB15 & WB16: // Do not break within emoji flag sequences. That is, do not break between @@ -411,18 +449,6 @@ namespace wordBreakers { } return pos + 1; } - - // Word_Break rule macros - // See: https://unicode.org/reports/tr29/#WB_Rule_Macros - function isAHLetter(prop: WordBreakProperty): boolean { - return prop === WordBreakProperty.ALetter || - prop === WordBreakProperty.Hebrew_Letter; - } - - function isMidNumLetQ(prop: WordBreakProperty): boolean { - return prop === WordBreakProperty.MidNumLet || - prop === WordBreakProperty.Single_Quote; - } } function isStartOfSurrogatePair(character: string) { From 51caf533a874643822683e247ea5e9f8217287bc Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 7 Sep 2022 11:52:32 +0700 Subject: [PATCH 30/59] feat(common/models): interface for custom breaker rules --- .../models/wordbreakers/src/default/data.ts | 2 +- .../models/wordbreakers/src/default/index.ts | 48 +++++++++++++++++-- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/common/models/wordbreakers/src/default/data.ts b/common/models/wordbreakers/src/default/data.ts index ffb6b8e79a..7401fd667e 100644 --- a/common/models/wordbreakers/src/default/data.ts +++ b/common/models/wordbreakers/src/default/data.ts @@ -4,7 +4,7 @@ export namespace data { /** * Valid values for a word break property. */ -export const enum WordBreakProperty { +export const enum WordBreakProperty { // Scary bit: this does not exist as an object at run-time! Other, LF, Newline, diff --git a/common/models/wordbreakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts index 4f86efc507..9e4d4a27ee 100644 --- a/common/models/wordbreakers/src/default/index.ts +++ b/common/models/wordbreakers/src/default/index.ts @@ -61,13 +61,35 @@ namespace wordBreakers { } } + /** + * An abstraction supporting custom wordbreaker boundary rules. While this doesn't provide + * support for more complex rules like WB4, WB15, or WB16, this is sufficient for all other + * default word-breaking rules. Thus, this should cover the majority of cases requiring + * custom handling for specific languages. + * + * @see https://unicode.org/reports/tr29/#WB_Rule_Macros + */ + export interface WordbreakerRule { + /** + * Indicates whether or not the rule applies in the specified context. + * @param context + */ + match(context: BreakerContext): boolean; + + /** + * Indicates whether or not the rule indicates a word boundary at the context's site when it matches. + */ + breakIfMatch: boolean; + } + /** * Provides a useful presentation for wordbreaker's context for use in word-breaking rules. * * @see https://unicode.org/reports/tr29/#Word_Boundary_Rules */ export class BreakerContext { - private readonly text: string; + // Referenced by this object in order to facilitate `lookahead` maintenance. + private readonly text: string; /** * Represents the property of character immediately preceding `left`'s character. @@ -163,7 +185,7 @@ namespace wordBreakers { * Return the value of the Word_Break property at the given string index. * @param pos position in the text. */ - private wordbreakPropertyAt(pos: number) { + private wordbreakPropertyAt(pos: number) { if (pos < 0) { return WordBreakProperty.sot; // Always "start of string" before the string starts! } else if (pos >= this.text.length) { @@ -216,13 +238,17 @@ namespace wordBreakers { * * @param text Text to find word boundaries in. */ - function findBoundaries(text: string): number[] { + function findBoundaries(text: string, options?: WordbreakerRule[]): number[] { // WB1 and WB2: no boundaries if given an empty string. if (text.length === 0) { // There are no boundaries in an empty string! return []; } + if(!options) { + options = []; + } + // This algorithm works by maintaining a sliding window of four SCALAR VALUES. // // - Scalar values? JavaScript strings are NOT actually a string of @@ -330,6 +356,22 @@ namespace wordBreakers { const SET_AHLETTER = [WordBreakProperty.ALetter, WordBreakProperty.Hebrew_Letter]; const SET_MIDNUMLETQ = [WordBreakProperty.MidNumLet, WordBreakProperty.Single_Quote]; + // Start: Custom rules + let customMatch: boolean = false; + for(const rule of options) { + customMatch = rule.match(state); + if(customMatch) { + if(rule.breakIfMatch) { + boundaries.push(rightPos); + } + break; // as customMatch == true here, this will trigger the `continue` that follows. + } + } + if(customMatch) { + continue; + } + // End: Custom rules + // WB5: Do not break between most letters. // if (isAHLetter(state.left) && isAHLetter(state.right)) if(state.match(null, SET_AHLETTER, SET_AHLETTER, null)) { From 075147ba30c945c70d68bc985990bc23b2f978d6 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Mon, 12 Sep 2022 14:04:02 -0400 Subject: [PATCH 31/59] auto: increment master version to 16.0.62 --- HISTORY.md | 8 ++++++++ VERSION.md | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 48a2da1054..3431ee2692 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,13 @@ # Keyman Version History +## 16.0.61 alpha 2022-09-12 + +* docs(windows): add steps for using testhost debugging (#7263) +* fix(developer): compiler mismatch on currentLine (#7190) +* fix(developer): suppress repeated warnings about unreachable code (#7219) +* chore: try disabling concurrency for browserstack tests (#7258) +* chore(web): disable browserstack on non-web-specific builds (#7260) + ## 16.0.60 alpha 2022-09-10 * fix(web): enhanced timer for prediction algorithm (#7037) diff --git a/VERSION.md b/VERSION.md index d6db1906b4..f4737a994e 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.61 \ No newline at end of file +16.0.62 \ No newline at end of file From 8e076fe3fdd94598af53fb12e9aca425d480cbc4 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 13 Sep 2022 10:18:24 +0700 Subject: [PATCH 32/59] feat(common/models): actual wordbreaker customization --- .../models/wordbreakers/src/default/data.ts | 26 + .../models/wordbreakers/src/default/index.ts | 152 ++++-- .../test/test-default-word-breaker.js | 476 ++++++++++++------ 3 files changed, 457 insertions(+), 197 deletions(-) diff --git a/common/models/wordbreakers/src/default/data.ts b/common/models/wordbreakers/src/default/data.ts index 7401fd667e..16a4d9beb3 100644 --- a/common/models/wordbreakers/src/default/data.ts +++ b/common/models/wordbreakers/src/default/data.ts @@ -28,6 +28,32 @@ export const enum WordBreakProperty { // Scary bit: this does not exist as an o eot }; +// Not currently built by the auto-generator tool, but it easily could be. +// If and when we import the data.ts rebuilder, we can add this in. +export const propertyMap = [ + "Other", + "LF", + "Newline", + "CR", + "WSegSpace", + "Double_Quote", + "Single_Quote", + "MidNum", + "MidNumLet", + "Numeric", + "MidLetter", + "ALetter", + "ExtendNumLet", + "Format", + "Extend", + "Hebrew_Letter", + "ZWJ", + "Katakana", + "Regional_Indicator", + "sot", + "eot" +]; + /** * Constants for indexing values in WORD_BREAK_PROPERTY. */ diff --git a/common/models/wordbreakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts index 9e4d4a27ee..cfe251fbf2 100644 --- a/common/models/wordbreakers/src/default/index.ts +++ b/common/models/wordbreakers/src/default/index.ts @@ -1,6 +1,34 @@ // Include the word-breaking data here: /// namespace wordBreakers { + /** + * A set of options used to customize and extend the behavior of the default + * Unicode wordbreaker. + */ + export interface DefaultWordBreakerOptions { + /** + * Allows addition of custom wordbreaking rules, which will be applied + * after WB1-WB4 and before all other default wordbreaking rules. + * + * @see `WordbreakerRule` + */ + rules?: WordbreakerRule[]; + + /** + * Allows assignment of characters to different word-breaking properties than + * their standard word-breaking assignment, including to custom properties + * specified within `customProperties`. + * @param char + */ + propertyMapping?(char: string): string; + + /** + * Allows definition of extra word-breaking properties for use with custom + * rules. + */ + customProperties?: [string]; + } + /** * Word breaker based on Unicode Standard Annex #29, Section 4.1: * Default Word Boundary Specification. @@ -8,8 +36,8 @@ namespace wordBreakers { * @see http://unicode.org/reports/tr29/#Word_Boundaries * @see https://github.com/eddieantonio/unicode-default-word-boundary/tree/v12.0.0 */ - export function default_(text: string): Span[] { - let boundaries = findBoundaries(text); + export function default_(text: string, options?: DefaultWordBreakerOptions): Span[] { + let boundaries = findBoundaries(text, options); if (boundaries.length == 0) { return []; } @@ -22,7 +50,7 @@ namespace wordBreakers { let end = boundaries[i + 1]; let span = new LazySpan(text, start, end); - if (isNonSpace(span.text)) { + if (isNonSpace(span.text, options)) { spans.push(span); // Preserve a sequence-final space if it exists. Needed to signal "end of word". } else if (i == boundaries.length - 2) { // if "we just checked the final boundary"... @@ -64,8 +92,7 @@ namespace wordBreakers { /** * An abstraction supporting custom wordbreaker boundary rules. While this doesn't provide * support for more complex rules like WB4, WB15, or WB16, this is sufficient for all other - * default word-breaking rules. Thus, this should cover the majority of cases requiring - * custom handling for specific languages. + * default word-breaking rules and can be used to define custom rules of similar structure. * * @see https://unicode.org/reports/tr29/#WB_Rule_Macros */ @@ -90,6 +117,7 @@ namespace wordBreakers { export class BreakerContext { // Referenced by this object in order to facilitate `lookahead` maintenance. private readonly text: string; + readonly options?: DefaultWordBreakerOptions; /** * Represents the property of character immediately preceding `left`'s character. @@ -117,7 +145,7 @@ namespace wordBreakers { * @param text The text to be word-broken * @param lookaheadPos The position corresponding to `lookahead`. */ - constructor(text: string, lookaheadPos: number); + constructor(text: string, options: DefaultWordBreakerOptions | undefined, lookaheadPos: number); /** * Used internally by the boundary-detection algorithm during context-shifting operations. * @param text @@ -127,20 +155,23 @@ namespace wordBreakers { * @param lookahead */ constructor(text: string, + options: DefaultWordBreakerOptions | undefined, lookbehind: WordBreakProperty, left: WordBreakProperty, right: WordBreakProperty, lookahead: WordBreakProperty); constructor(text: string, + options: DefaultWordBreakerOptions | undefined, prop1: WordBreakProperty | number, prop2?: WordBreakProperty, prop3?: WordBreakProperty, prop4?: WordBreakProperty) { this.text = text; + this.options = options; - if(arguments.length == 2) { + if(arguments.length == 3) { this.lookahead = this.wordbreakPropertyAt(prop1);// prop1; - } else /*if(arguments.length == 5)*/ { + } else /*if(arguments.length == 6)*/ { this.lookbehind = prop1 as WordBreakProperty; this.left = prop2 as WordBreakProperty; this.right = prop3 as WordBreakProperty; @@ -155,7 +186,7 @@ namespace wordBreakers { */ public next(lookaheadPos: number): BreakerContext { let newLookahead = this.wordbreakPropertyAt(lookaheadPos); - return new BreakerContext(this.text, this.left, this.right, this.lookahead, newLookahead); + return new BreakerContext(this.text, this.options, this.left, this.right, this.lookahead, newLookahead); } /** @@ -167,7 +198,7 @@ namespace wordBreakers { */ public ignoringRight(lookaheadPos: number) { let newLookahead = this.wordbreakPropertyAt(lookaheadPos); - return new BreakerContext(this.text, this.lookbehind, this.left, this.lookahead, newLookahead); + return new BreakerContext(this.text, this.options, this.lookbehind, this.left, this.lookahead, newLookahead); } /** @@ -178,7 +209,7 @@ namespace wordBreakers { */ public ignoringLookahead(lookaheadPos: number) { let newLookahead = this.wordbreakPropertyAt(lookaheadPos); - return new BreakerContext(this.text, this.lookbehind, this.left, this.right, newLookahead); + return new BreakerContext(this.text, this.options, this.lookbehind, this.left, this.right, newLookahead); } /** @@ -194,7 +225,7 @@ namespace wordBreakers { // Surrogate pairs the next TWO items from the string! return property(this.text[pos] + this.text[pos + 1]); } - return property(this.text[pos]); + return property(this.text[pos], this.options); } /** @@ -215,15 +246,38 @@ namespace wordBreakers { result = result && (rightSet?.includes(this.right) ?? true); return result && (lookaheadSet?.includes(this.lookahead) ?? true); } - } + + /** + * Returns `true` if and only if each member of the context has a property included within + * its corresponding set (when specified). Any set may be replaced with null to disable + * a check against its corresponding property. + * + * Names should match those found at https://unicode.org/reports/tr29/#Word_Boundary_Rules + * or defined in the word-breaker customization options; matching is case-insensitive. + * @param lookbehindSet + * @param leftSet + * @param rightSet + * @param lookaheadSet + */ + public propertyMatch(lookbehindSet: string[] | null, + leftSet: string[] | null, + rightSet: string[] | null, + lookaheadSet: string[] | null) : boolean { + const propMapper = (name: string) => propertyVal(name, this.options); + return this.match(lookbehindSet?.map(propMapper) as WordBreakProperty[] | null, + leftSet?.map(propMapper) as WordBreakProperty[] | null, + rightSet?.map(propMapper) as WordBreakProperty[] | null, + lookaheadSet?.map(propMapper) as WordBreakProperty[] | null); + } + } /** * Returns true when the chunk does not solely consist of whitespace. * * @param chunk a chunk of text. Starts and ends at word boundaries. */ - function isNonSpace(chunk: string): boolean { - return !Array.from(chunk).map(property).every(wb => ( + function isNonSpace(chunk: string, options?: DefaultWordBreakerOptions): boolean { + return !Array.from(chunk).map((char) => property(char, options)).every(wb => ( wb === WordBreakProperty.CR || wb === WordBreakProperty.LF || wb === WordBreakProperty.Newline || @@ -238,15 +292,15 @@ namespace wordBreakers { * * @param text Text to find word boundaries in. */ - function findBoundaries(text: string, options?: WordbreakerRule[]): number[] { + function findBoundaries(text: string, options?: DefaultWordBreakerOptions): number[] { // WB1 and WB2: no boundaries if given an empty string. if (text.length === 0) { // There are no boundaries in an empty string! return []; } - if(!options) { - options = []; + if(options && !options.rules) { + options.rules = []; } // This algorithm works by maintaining a sliding window of four SCALAR VALUES. @@ -272,7 +326,7 @@ namespace wordBreakers { let rightPos: number; let lookaheadPos = 0; // lookahead, one scalar value to the right of right. // Before the start of the string is also the start of the string. - let state = new BreakerContext(text, lookaheadPos); + let state = new BreakerContext(text, options, lookaheadPos); // Count RIs to make sure we're not splitting emoji flags: let nConsecutiveRegionalIndicators = 0; @@ -356,21 +410,22 @@ namespace wordBreakers { const SET_AHLETTER = [WordBreakProperty.ALetter, WordBreakProperty.Hebrew_Letter]; const SET_MIDNUMLETQ = [WordBreakProperty.MidNumLet, WordBreakProperty.Single_Quote]; - // Start: Custom rules - let customMatch: boolean = false; - for(const rule of options) { - customMatch = rule.match(state); - if(customMatch) { - if(rule.breakIfMatch) { - boundaries.push(rightPos); + // Custom rules may override the base ruleset aside from the first few fundamental ones. + if(options?.rules) { + let customMatch: boolean = false; + for(const rule of options.rules) { + customMatch = rule.match(state); + if(customMatch) { + if(rule.breakIfMatch) { + boundaries.push(rightPos); + } + break; // as customMatch == true here, this will trigger the `continue` that follows. } - break; // as customMatch == true here, this will trigger the `continue` that follows. + } + if(customMatch) { + continue; } } - if(customMatch) { - continue; - } - // End: Custom rules // WB5: Do not break between most letters. // if (isAHLetter(state.left) && isAHLetter(state.right)) @@ -503,13 +558,43 @@ namespace wordBreakers { * Note that * @param character a scalar value */ - export function property(character: string): WordBreakProperty { + function property(character: string, options?: DefaultWordBreakerOptions): WordBreakProperty { + // If there is a customized mapping for the character, prioritize that. + if(options?.propertyMapping) { + let propName = options.propertyMapping(character); + if(propName) { + return propertyVal(propName, options); + } + } + // This MUST be a scalar value. // TODO: remove dependence on character.codepointAt()? let codepoint = character.codePointAt(0) as number; + return searchForProperty(codepoint, 0, WORD_BREAK_PROPERTY.length - 1); } + function propertyVal(propName: string, options?: DefaultWordBreakerOptions) { + const matcher = (name: string) => name.toLowerCase() == propName.toLowerCase() + + const customIndex = options?.customProperties?.findIndex(matcher) ?? -1; + return customIndex != -1 ? -customIndex - 1 : data.propertyMap.findIndex(matcher); + } + + // /** + // * Provides the word-breaking property name for the specified character based on the property + // * values used by https://unicode.org/reports/tr29/#Word_Boundary_Rules. + // * @param character + // * @returns + // */ + // export function unicodeProperty(character: string): string { + // // Since we use a const enum for property names, the TS compiler optimizes it away + // // and does not provide a reverse lookup for us. So, we do that here. + // const enumVal = property(character); + + // return data.propertyMap[enumVal]; + // } + /** * Binary search for the word break property of a given CODE POINT. * @@ -546,8 +631,7 @@ namespace wordBreakers { // implementing a namespace, BUT we can manually make the // assignment and **declare** it as part of the namespace. wordBreakers['default'] = wordBreakers.default_; -wordBreakers['unicodeProperty'] = wordBreakers.property; +// wordBreakers['unicodeProperty'] = wordBreakers.unicodeProperty; declare namespace wordBreakers { export { default_ as default }; - export { property as unicodeProperty }; } diff --git a/common/models/wordbreakers/test/test-default-word-breaker.js b/common/models/wordbreakers/test/test-default-word-breaker.js index b4db8cb1a3..85129b836c 100644 --- a/common/models/wordbreakers/test/test-default-word-breaker.js +++ b/common/models/wordbreakers/test/test-default-word-breaker.js @@ -8,184 +8,334 @@ const breakWords = require('../build').wordBreakers['default']; const SHY = '\u00AD'; // Other, Format. The "Soft HYphen" - usually invisible unless needed for word-wrapping. describe('The default word breaker', function () { - it('should break multilingual text', function () { - let breaks = breakWords( - `Добрый день! ᑕᐻ᙮ — after working on ka${SHY}wen${SHY}non:${SHY}nis, - let's eat phở! 🥣` - ); - let words = breaks.map(span => span.text); - assert.deepEqual(words, [ - 'Добрый', 'день', '!', 'ᑕᐻ', '᙮', '—', 'after', - 'working', 'on', `ka${SHY}wen${SHY}non:${SHY}nis`, ',', - "let's", 'eat', 'phở', '!', '🥣' - ]); + describe('default configuration', function() { + it('should break multilingual text', function () { + let breaks = breakWords( + `Добрый день! ᑕᐻ᙮ — after working on ka${SHY}wen${SHY}non:${SHY}nis, + let's eat phở! 🥣` + ); + let words = breaks.map(span => span.text); + assert.deepEqual(words, [ + 'Добрый', 'день', '!', 'ᑕᐻ', '᙮', '—', 'after', + 'working', 'on', `ka${SHY}wen${SHY}non:${SHY}nis`, ',', + "let's", 'eat', 'phở', '!', '🥣' + ]); + }); + + it('handles heavily-punctuated English text', function() { + // This test case brought to you by http://unicode.org/reports/tr29/#Word_Boundaries, Figure 1. + let breaks = breakWords( + `The quick ("brown") fox can't jump 32.3 feet, right?` + ); + let words = breaks.map(span => span.text); + assert.deepEqual(words, [ + 'The', 'quick', '(', '"', 'brown', '"', ')', 'fox', "can't", + 'jump', '32.3', 'feet', ',', 'right', '?' + ]); + }); + + // The way these two tests are written is a bit much on the "white-box" style, + // but they do decently cover the boundary rules mentioned. + it('Does not split empty contexts (WB1 + WB2)', function() { + let breaks = breakWords(''); + let words = breaks.map(span => span.text); + assert.deepEqual(words, []); + }); + + it('Does split at context boundaries (WB1 + WB2)', function() { + let breaks = breakWords('a'); + let words = breaks.map(span => span.text); + assert.deepEqual(words, ['a']); + }); + + // WB3, WB3a, WB3b are all handled internally, within the top-level function. + + // iff, as in "if and only if" + it('ignores the zero-width joiner iff appropriate (WB4)', function() { + const zwj = '\u200d'; + + let breaks = breakWords(`a${zwj}b\n${zwj}c${zwj}\nd`); + let words = breaks.map(span => span.text); + + // Does NOT ignore the zwj immediately after a newline - the notable exception + // (the reason for "iff", not "if"). + assert.deepEqual(words, [`a${zwj}b`, `${zwj}`, `c${zwj}`, `d`]); + }) + + it('ignores extend characters iff appropriate (WB4)', function() { + const comboGrave = '\u0300'; // The 'combining grave accent', as used in NFD. + + let breaks = breakWords(`a${comboGrave}e\n${comboGrave}i${comboGrave}\no`); + let words = breaks.map(span => span.text); + + // Does NOT ignore the zwj immediately after a newline - the notable exception + // (the reason for "iff", not "if"). + assert.deepEqual(words, [`a${comboGrave}e`, `${comboGrave}`, `i${comboGrave}`, `o`]); + }); + + it('ignores format characters iff appropriate (WB4)', function() { + // Re-uses `const SHY` from above. + let breaks = breakWords(`a${SHY}e\n${SHY}i${SHY}\no`); + let words = breaks.map(span => span.text); + + // Does NOT ignore the zwj immediately after a newline - the notable exception + // (the reason for "iff", not "if"). + assert.deepEqual(words, [`a${SHY}e`, `${SHY}`, `i${SHY}`, `o`]); + }); + + it('does not break between most alphabetic characters (WB5)', function() { + let breaks = breakWords(`aəσאБ лאʈγX`); // a mix of latin, hebrew, greek, cyrillic, and IPA chars + // for both "words". + let words = breaks.map(span => span.text); + + assert.deepEqual(words, [`aəσאБ`, `лאʈγX`]); + }); + + it('does not break letters across specific punctuation patterns (WB6, WB7)', function() { + // `'`: MidNumLetQ (from Single_Quote) + // '.': MidNumLet + // ':': MidLetter + let breaks = breakWords(`don't b.r.e.a.k t:h:e:s:e`); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, [`don't`, `b.r.e.a.k`, `t:h:e:s:e`]); + + let breaks2 = breakWords(`.drop: :the' 'extras.`); + let words2 = breaks2.map(span => span.text); + + assert.deepEqual(words2, [`.`, `drop`, `:`, `:`, `the`, `'`, `'`, `extras`, `.`]); + + // ',': MidNum (is NOT included by rule!) + let breaks3 = breakWords('do br,eak that'); + let words3 = breaks3.map(span => span.text); + + assert.deepEqual(words3, ['do', 'br', ',', 'eak', 'that']); + }); + + it('treats Hebrew properly (WB7a-c)', function() { + const aleph = 'א'; + const bet = 'ב'; + + // As Hebrew is RTL... this is probably the clearest way for us LTR people to + // clearly see what's going on without ordering mechanics messing up the render. + let breaks = breakWords(`${aleph}' ${aleph}" ${aleph}"${bet}`); + let words = breaks.map(span => span.text); + + // A lingering double-quote isn't cool, but one in the middle's fine. + // Lingering single-quote is fine regardless. + assert.deepEqual(words, [`${aleph}'`, `${aleph}`, `"`, `${aleph}"${bet}`]); + }); + + it(`doesn't break within digit + digit/letter sequences (WB8-10)`, function() { + let breaks = breakWords('a1b2c3 hunter2 ab12cd34 1234567890'); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, ['a1b2c3', 'hunter2', 'ab12cd34', '1234567890']); + }); + + it('does not break within formatted number sequences (WB11-12)', function() { + // Note: `'` fits "MidNumLetQ", part of the two rules! + let breaks = breakWords(`1.2.3 3,458.01 3.45'8,01`); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, [`1.2.3`, `3,458.01`, `3.45'8,01`]); + + let breaks2 = breakWords(`.1' ,3.`); + let words2 = breaks2.map(span => span.text); + + assert.deepEqual(words2, [`.`, `1`, `'`, `,`, `3`, `.`]); + }); + + it('does not break between Katakana (WB13)', function() { + const kataSmA = '\u30a2'; //ァ + const kataA = '\u30a2'; //ア + const kataSound = '\u309b'; // ゛ + + let breaks = breakWords(`${kataSound}${kataA} ${kataSmA}${kataSound}b ${kataA}${kataSound}${kataSmA}`); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, [ + `${kataSound}${kataA}`, + `${kataSmA}${kataSound}`, + 'b', + `${kataA}${kataSound}${kataSmA}` + ]); + }); + + it('does not break form extenders (WB13a-b)', function() { + // The `_` (underscore) fits the ExtendNumLet class this rule focuses on. + const kataA = '\u30a2'; //ア + + let breaks = breakWords(`${kataA}_a__0_b_${kataA} _${kataA} 1_ _c_ ____`); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, [ + `${kataA}_a__0_b_${kataA}`, + `_${kataA}`, + `1_`, + `_c_`, + `____` + ]); + }); + + it('handles emoji flag sequences properly (WB15-16)', function() { + // For clarity on what's being tested... + let CA_FLAG = '\u{1f1e8}\u{1f1e6}' // '🇨🇦' (canadian flag emoji); should not be broken. + let KH_FLAG = '\u{1f1f0}\u{1f1ed}' // '🇰🇭' (khmer flag emoji); same + let X_FLAG_PIECE = '\u{1f1fd}' // '🇽' (half of a flag emoji; '🇽🇽' doesn't match a flag) + let breaks = breakWords(`${CA_FLAG}${KH_FLAG}${X_FLAG_PIECE}${X_FLAG_PIECE}`); + let words = breaks.map(span => span.text); + + // Note that the emoji may not render well within VSCode, but they show up nicely on GitHub. + assert.deepEqual(words, ['🇨🇦', '🇰🇭', '🇽🇽']); + }); + + it('breaks hyphenated words by default', function() { + let breaks = breakWords('Smith-Jones'); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, ['Smith', '-', 'Jones']); + }); }); - it('handles heavily-punctuated English text', function() { - // This test case brought to you by http://unicode.org/reports/tr29/#Word_Boundaries, Figure 1. - let breaks = breakWords( - `The quick ("brown") fox can't jump 32.3 feet, right?` - ); - let words = breaks.map(span => span.text); - assert.deepEqual(words, [ - 'The', 'quick', '(', '"', 'brown', '"', ')', 'fox', "can't", - 'jump', '32.3', 'feet', ',', 'right', '?' - ]); - }); + describe('customization', function() { + // Refer to https://unicode.org/reports/tr29/#Word_Boundary_Rules, third bullet point. + it('custom prop, rule: do not break on letter-adjacent hyphens', function() { + let customization = { + rules: [{ + match: (context) => { + if(context.propertyMatch(null, ["ALetter"], ["Hyphen"], ["ALetter"])) { + return true; + } else if(context.propertyMatch(["ALetter"], ["Hyphen"], ["ALetter"], null)) { + return true; + } else { + return false; + } + }, + breakIfMatch: false + }], + propertyMapping: (char) => { + const validHyphenCodes = [ + '\u002d', '\u2010', '\u058a', '\u30a0' + ]; + if(validHyphenCodes.includes(char)) { + return "Hyphen"; + } - // The way these two tests are written is a bit much on the "white-box" style, - // but they do decently cover the boundary rules mentioned. - it('Does not split empty contexts (WB1 + WB2)', function() { - let breaks = breakWords(''); - let words = breaks.map(span => span.text); - assert.deepEqual(words, []); - }); + return null; + }, + customProperties: ["Hyphen"] + } - it('Does split at context boundaries (WB1 + WB2)', function() { - let breaks = breakWords('a'); - let words = breaks.map(span => span.text); - assert.deepEqual(words, ['a']); - }); + let breaks = breakWords('Smith-Jones', customization); + let words = breaks.map(span => span.text); - // WB3, WB3a, WB3b are all handled internally, within the top-level function. + assert.deepEqual(words, ['Smith-Jones']); + }); - // iff, as in "if and only if" - it('ignores the zero-width joiner iff appropriate (WB4)', function() { - const zwj = '\u200d'; + it('mid-word hyphen via reassignment to MidLetter', function() { + let customization = { + propertyMapping: (char) => { + const validHyphenCodes = [ + '\u002d', '\u2010', '\u058a', '\u30a0' + ]; + if(validHyphenCodes.includes(char)) { + return "MidLetter"; + } - let breaks = breakWords(`a${zwj}b\n${zwj}c${zwj}\nd`); - let words = breaks.map(span => span.text); + return null; + } + } - // Does NOT ignore the zwj immediately after a newline - the notable exception - // (the reason for "iff", not "if"). - assert.deepEqual(words, [`a${zwj}b`, `${zwj}`, `c${zwj}`, `d`]); - }) + let breaks = breakWords('Smith-Jones', customization); + let words = breaks.map(span => span.text); - it('ignores extend characters iff appropriate (WB4)', function() { - const comboGrave = '\u0300'; // The 'combining grave accent', as used in NFD. + assert.deepEqual(words, ['Smith-Jones']); + }); - let breaks = breakWords(`a${comboGrave}e\n${comboGrave}i${comboGrave}\no`); - let words = breaks.map(span => span.text); + // Useful for some regional minority languages that prefer word-breaking spaces. + it('character reassignment: Khmer letters as ALetter', function() { + let customization = { + propertyMapping: (char) => { + if(char >= '\u1780' && char <= '\u17b3') { + return "ALetter"; + } else { + // The other Khmer characters already have useful word-breaking + // property assignments. + return null; + } + } + } - // Does NOT ignore the zwj immediately after a newline - the notable exception - // (the reason for "iff", not "if"). - assert.deepEqual(words, [`a${comboGrave}e`, `${comboGrave}`, `i${comboGrave}`, `o`]); - }); + let breaks = breakWords('ស្រុក ខ្មែរ', customization); + let words = breaks.map(span => span.text); - it('ignores format characters iff appropriate (WB4)', function() { - // Re-uses `const SHY` from above. - let breaks = breakWords(`a${SHY}e\n${SHY}i${SHY}\no`); - let words = breaks.map(span => span.text); + assert.deepEqual(words, ['ស្រុក', 'ខ្មែរ']); + }); - // Does NOT ignore the zwj immediately after a newline - the notable exception - // (the reason for "iff", not "if"). - assert.deepEqual(words, [`a${SHY}e`, `${SHY}`, `i${SHY}`, `o`]); - }); + // See: suggested language-specific WB5a from the spec's notes. + it("french/italian apostrophe / vowel boundaries", function() { + let customization = { + rules: [ + // WB5, but with differentiated consonants (ALetter) and vowels (AVowel) + { + match: (context) => { + if(context.propertyMatch(null, ["ALetter", "AVowel"], ["ALetter", "AVowel"], null)) { + return true; + } else { + return false; + } + }, + breakIfMatch: false + }, + // Proposed WB5a + { + match: (context) => { + if(context.propertyMatch(null, ["Single_Quote"], ["AVowel"], null)) { + return true; + } else { + return false; + } + }, + breakIfMatch: true + }, + // WB6, 7 + { + match: (context) => { + if(context.propertyMatch(null, + ["ALetter", "AVowel"], + ["MidLetter", "MidNumLet", "Single_Quote"], + ["ALetter", "AVowel"])) { + return true; + } else if(context.propertyMatch(["ALetter", "AVowel"], + ["MidLetter", "MidNumLet", "Single_Quote"], + ["ALetter", "AVowel"], + null)) { + return true; + } else { + return false; + } + }, + breakIfMatch: false + } + // Similar extensions to WB9, 10, 13a, and 13b would also be needed for robustness. + // And I kind of left the Hebrew_Letter out of the WB5, 6, and 7 rewrites. + ], + propertyMapping: (char) => { + const vowels = ['a', 'e', 'i', 'o', 'u']; + if(vowels.includes(char)) { + return "AVowel"; + } - it('does not break between most alphabetic characters (WB5)', function() { - let breaks = breakWords(`aəσאБ лאʈγX`); // a mix of latin, hebrew, greek, cyrillic, and IPA chars - // for both "words". - let words = breaks.map(span => span.text); + return null; + }, + customProperties: ["AVowel"] + } - assert.deepEqual(words, [`aəσאБ`, `лאʈγX`]); - }); + let breaks = breakWords("l'objectif aujourd'hui", customization); + let words = breaks.map(span => span.text); - it('does not break letters across specific punctuation patterns (WB6, WB7)', function() { - // `'`: MidNumLetQ (from Single_Quote) - // '.': MidNumLet - // ':': MidLetter - let breaks = breakWords(`don't b.r.e.a.k t:h:e:s:e`); - let words = breaks.map(span => span.text); - - assert.deepEqual(words, [`don't`, `b.r.e.a.k`, `t:h:e:s:e`]); - - let breaks2 = breakWords(`.drop: :the' 'extras.`); - let words2 = breaks2.map(span => span.text); - - assert.deepEqual(words2, [`.`, `drop`, `:`, `:`, `the`, `'`, `'`, `extras`, `.`]); - - // ',': MidNum (is NOT included by rule!) - let breaks3 = breakWords('do br,eak that'); - let words3 = breaks3.map(span => span.text); - - assert.deepEqual(words3, ['do', 'br', ',', 'eak', 'that']); - }); - - it('treats Hebrew properly (WB7a-c)', function() { - const aleph = 'א'; - const bet = 'ב'; - - // As Hebrew is RTL... this is probably the clearest way for us LTR people to - // clearly see what's going on without ordering mechanics messing up the render. - let breaks = breakWords(`${aleph}' ${aleph}" ${aleph}"${bet}`); - let words = breaks.map(span => span.text); - - // A lingering double-quote isn't cool, but one in the middle's fine. - // Lingering single-quote is fine regardless. - assert.deepEqual(words, [`${aleph}'`, `${aleph}`, `"`, `${aleph}"${bet}`]); - }); - - it(`doesn't break within digit + digit/letter sequences (WB8-10)`, function() { - let breaks = breakWords('a1b2c3 hunter2 ab12cd34 1234567890'); - let words = breaks.map(span => span.text); - - assert.deepEqual(words, ['a1b2c3', 'hunter2', 'ab12cd34', '1234567890']); - }); - - it('does not break within formatted number sequences (WB11-12)', function() { - // Note: `'` fits "MidNumLetQ", part of the two rules! - let breaks = breakWords(`1.2.3 3,458.01 3.45'8,01`); - let words = breaks.map(span => span.text); - - assert.deepEqual(words, [`1.2.3`, `3,458.01`, `3.45'8,01`]); - - let breaks2 = breakWords(`.1' ,3.`); - let words2 = breaks2.map(span => span.text); - - assert.deepEqual(words2, [`.`, `1`, `'`, `,`, `3`, `.`]); - }); - - it('does not break between Katakana (WB13)', function() { - const kataSmA = '\u30a2'; //ァ - const kataA = '\u30a2'; //ア - const kataSound = '\u309b'; // ゛ - - let breaks = breakWords(`${kataSound}${kataA} ${kataSmA}${kataSound}b ${kataA}${kataSound}${kataSmA}`); - let words = breaks.map(span => span.text); - - assert.deepEqual(words, [ - `${kataSound}${kataA}`, - `${kataSmA}${kataSound}`, - 'b', - `${kataA}${kataSound}${kataSmA}` - ]); - }); - - it('does not break form extenders (WB13a-b)', function() { - // The `_` (underscore) fits the ExtendNumLet class this rule focuses on. - const kataA = '\u30a2'; //ア - - let breaks = breakWords(`${kataA}_a__0_b_${kataA} _${kataA} 1_ _c_ ____`); - let words = breaks.map(span => span.text); - - assert.deepEqual(words, [ - `${kataA}_a__0_b_${kataA}`, - `_${kataA}`, - `1_`, - `_c_`, - `____` - ]); - }); - - it('handles emoji flag sequences properly (WB15-16)', function() { - // For clarity on what's being tested... - let CA_FLAG = '\u{1f1e8}\u{1f1e6}' // '🇨🇦' (canadian flag emoji); should not be broken. - let KH_FLAG = '\u{1f1f0}\u{1f1ed}' // '🇰🇭' (khmer flag emoji); same - let X_FLAG_PIECE = '\u{1f1fd}' // '🇽' (half of a flag emoji; '🇽🇽' doesn't match a flag) - let breaks = breakWords(`${CA_FLAG}${KH_FLAG}${X_FLAG_PIECE}${X_FLAG_PIECE}`); - let words = breaks.map(span => span.text); - - // Note that the emoji may not render well within VSCode, but they show up nicely on GitHub. - assert.deepEqual(words, ['🇨🇦', '🇰🇭', '🇽🇽']); + assert.deepEqual(words, ["l'", "objectif", "aujourd'hui"]); + }); }); }); From 261e25202615add0198a8ed3c78f5f1fc06bb869 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 13 Sep 2022 10:38:34 +0700 Subject: [PATCH 33/59] docs(common/models): adds minor doc re 'sot', 'eot' --- common/models/wordbreakers/src/default/index.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/common/models/wordbreakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts index cfe251fbf2..228e19a62e 100644 --- a/common/models/wordbreakers/src/default/index.ts +++ b/common/models/wordbreakers/src/default/index.ts @@ -254,6 +254,9 @@ namespace wordBreakers { * * Names should match those found at https://unicode.org/reports/tr29/#Word_Boundary_Rules * or defined in the word-breaker customization options; matching is case-insensitive. + * Also includes two extra properties: + * - `sot` - start of text + * - `eot` - end of text * @param lookbehindSet * @param leftSet * @param rightSet From 94face23d0b27735d51873a62391897ccf12dadf Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 13 Sep 2022 11:06:54 +0700 Subject: [PATCH 34/59] chore(common/models): minor cleanup --- common/models/wordbreakers/src/default/index.ts | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/common/models/wordbreakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts index 228e19a62e..13081e7bc2 100644 --- a/common/models/wordbreakers/src/default/index.ts +++ b/common/models/wordbreakers/src/default/index.ts @@ -584,20 +584,6 @@ namespace wordBreakers { return customIndex != -1 ? -customIndex - 1 : data.propertyMap.findIndex(matcher); } - // /** - // * Provides the word-breaking property name for the specified character based on the property - // * values used by https://unicode.org/reports/tr29/#Word_Boundary_Rules. - // * @param character - // * @returns - // */ - // export function unicodeProperty(character: string): string { - // // Since we use a const enum for property names, the TS compiler optimizes it away - // // and does not provide a reverse lookup for us. So, we do that here. - // const enumVal = property(character); - - // return data.propertyMap[enumVal]; - // } - /** * Binary search for the word break property of a given CODE POINT. * @@ -634,7 +620,6 @@ namespace wordBreakers { // implementing a namespace, BUT we can manually make the // assignment and **declare** it as part of the namespace. wordBreakers['default'] = wordBreakers.default_; -// wordBreakers['unicodeProperty'] = wordBreakers.unicodeProperty; declare namespace wordBreakers { export { default_ as default }; } From e6403c3092580a79fba4c741499720caa13ad6a5 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Tue, 13 Sep 2022 11:20:37 +0200 Subject: [PATCH 35/59] chore(linux): Update debian changelog (cherry picked from commit 6c31e9ed837c90c0b976744b345ae138b6378166) --- linux/debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/linux/debian/changelog b/linux/debian/changelog index a6a93469c5..ab682e3d73 100644 --- a/linux/debian/changelog +++ b/linux/debian/changelog @@ -1,3 +1,10 @@ +keyman (15.0.270-1) unstable; urgency=medium + + * New upstream release. + * Re-release to Debian + + -- Eberhard Beilharz Tue, 13 Sep 2022 11:20:25 +0200 + keyman (15.0.269-1) unstable; urgency=medium * New upstream release. From 34b8e136b015737b6117c8de22f5ae2955037e21 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Tue, 13 Sep 2022 16:01:10 +0200 Subject: [PATCH 36/59] fix(linux): Fix ignored error Previously we ignored an error that showed in the build logs because the eval line got executed before calling reconf.sh which created the version file. I think despite this error things worked, but it didn't look nice to have and ignore a build error in the build log. Fixes #7275. --- linux/keyman-config/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/linux/keyman-config/Makefile b/linux/keyman-config/Makefile index ee01e2bfc9..cb0fe8ab51 100644 --- a/linux/keyman-config/Makefile +++ b/linux/keyman-config/Makefile @@ -55,8 +55,10 @@ deb: dist man: ./build-help.sh --man --no-reconf -version: +version_reconf: cd .. && ./scripts/reconf.sh keyman-config + +version: version_reconf $(eval VERSION := $(shell python3 -c "from keyman_config import __releaseversion__; print(__releaseversion__)")) # i18n From 2e66c99a832fedd5de656e5be8d091f5270e7f7c Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Tue, 13 Sep 2022 14:01:39 -0400 Subject: [PATCH 37/59] auto: increment master version to 16.0.63 --- HISTORY.md | 5 +++++ VERSION.md | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 3431ee2692..d7897516bf 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,10 @@ # Keyman Version History +## 16.0.62 alpha 2022-09-13 + +* fix(developer): hide key-sizes when in desktop layout in touch layout editor (#7225) +* fix(developer): show more useful error if out of space during Setup (#7267) + ## 16.0.61 alpha 2022-09-12 * docs(windows): add steps for using testhost debugging (#7263) diff --git a/VERSION.md b/VERSION.md index f4737a994e..923642153a 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.62 \ No newline at end of file +16.0.63 \ No newline at end of file From 62aee6d11bcb63052005d6d39bd453f400b9e2a4 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 14 Sep 2022 10:28:30 +1000 Subject: [PATCH 38/59] chore: improve auto labeling --- .github/multi-labeler.yml | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/.github/multi-labeler.yml b/.github/multi-labeler.yml index 1278db2a28..97fd20f4f5 100644 --- a/.github/multi-labeler.yml +++ b/.github/multi-labeler.yml @@ -11,6 +11,8 @@ version: v1 # labels: + # conventional commit / semantic PR styles + - label: 'feat' matcher: title: '^feat(\(|:)' @@ -20,26 +22,38 @@ labels: - label: 'chore' matcher: title: '^chore(\(|:)' - # "change", - label: 'docs' matcher: title: '^docs(\(|:)' - # "style", - label: 'refactor' matcher: title: '^refactor(\(|:)' - # "test", - label: 'auto' matcher: title: '^auto(\(|:)' - # Below are the scopes that we look for in the PR title + # additional meta flags + + # note, this does not pick up chained PRs automatically + - label: 'stable' + matcher: + baseBranch: '^stable-\\d+\\.' + + - label: 'cherry-pick' + matcher: + title: '(🍒|:cherries:)' + + # Scopes that we look for in the PR title + - label: 'android/' matcher: title: '\(.*android.*\):' - label: 'common/' matcher: title: '\(.*common.*\):' + - label: 'core/' + matcher: + title: '\(.*core.*\):' - label: 'developer/' matcher: title: '\(.*developer.*\):' @@ -62,6 +76,8 @@ labels: matcher: title: '\(.*windows.*\):' - - label: 'cherry-pick' + # epics -- we will add/remove these as we work on new epics each release + + - label: 'epic-ldml' matcher: - title: '(🍒|:cherries:)' + branch: '.*epic-ldml.*' # anywhere in the branch name, e.g. feat/epic-ldml/developer/... or feat/developer/foo-epic-ldml From 747d3776713992957e234668b7d406b48647aed6 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Wed, 14 Sep 2022 09:52:27 +0700 Subject: [PATCH 39/59] fix(android/engine): Switch keyboard if uninstalling current one --- .../main/java/com/tavultesoft/kmea/KeyboardPickerActivity.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardPickerActivity.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardPickerActivity.java index 50a478dd30..ec32425899 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardPickerActivity.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardPickerActivity.java @@ -433,7 +433,7 @@ public final class KeyboardPickerActivity extends BaseActivity { if(adapter != null) { adapter.notifyDataSetChanged(); } - if (position == curKbPos && listView != null) { + if (position == curKbPos) { switchKeyboard(0,false); } else if(listView != null) { // A bit of a hack, since LanguageSettingsActivity calls this method too. curKbPos = KeyboardController.getInstance().getKeyboardIndex(KMKeyboard.currentKeyboard()); From 0845210f2000dc99ebfe10edf099359c268dc062 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 14 Sep 2022 09:57:29 +0700 Subject: [PATCH 40/59] fix(common/models): max prediction wait check --- common/web/lm-worker/src/correction/distance-modeler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/web/lm-worker/src/correction/distance-modeler.ts b/common/web/lm-worker/src/correction/distance-modeler.ts index b0cd2179a9..1d23c14b1e 100644 --- a/common/web/lm-worker/src/correction/distance-modeler.ts +++ b/common/web/lm-worker/src/correction/distance-modeler.ts @@ -653,7 +653,7 @@ namespace correction { shouldTimeout(): boolean { const now = Date.now(); - if(this.start - now > this.maxTrueTime) { + if(now - this.start > this.maxTrueTime) { return true; } From 6fbe0892441fbf5eb3888b7b7f8246f0e94fc3d6 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 14 Sep 2022 10:13:29 +0700 Subject: [PATCH 41/59] change(web): tweaks max wait time threshold to mitigate lag --- common/web/lm-worker/src/correction/distance-modeler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/web/lm-worker/src/correction/distance-modeler.ts b/common/web/lm-worker/src/correction/distance-modeler.ts index 1d23c14b1e..d29b9048a0 100644 --- a/common/web/lm-worker/src/correction/distance-modeler.ts +++ b/common/web/lm-worker/src/correction/distance-modeler.ts @@ -710,7 +710,7 @@ namespace correction { let batcher = new BatchingAssistant(); - const timer = new ExecutionTimer(maxTime*3, maxTime); + const timer = new ExecutionTimer(maxTime*1.5, maxTime); // Stage 1 - if we already have extracted results, build a queue just for them and iterate over it first. let returnedValues = Object.values(this.returnedValues); From be947eb76abf17c61f64e194781655da564f028b Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 14 Sep 2022 11:58:10 +1000 Subject: [PATCH 42/59] chore: improve label match for stable --- .github/multi-labeler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/multi-labeler.yml b/.github/multi-labeler.yml index 97fd20f4f5..76b2cbb647 100644 --- a/.github/multi-labeler.yml +++ b/.github/multi-labeler.yml @@ -37,7 +37,7 @@ labels: # note, this does not pick up chained PRs automatically - label: 'stable' matcher: - baseBranch: '^stable-\\d+\\.' + baseBranch: '^stable-\\d+\\.\\d+' - label: 'cherry-pick' matcher: From 68819c6b6c18559ab1dfb0b7258aa8fe8f2fae90 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 14 Sep 2022 11:59:46 +1000 Subject: [PATCH 43/59] chore: improve label match for stable --- .github/multi-labeler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/multi-labeler.yml b/.github/multi-labeler.yml index 76b2cbb647..f5f509cd3d 100644 --- a/.github/multi-labeler.yml +++ b/.github/multi-labeler.yml @@ -37,7 +37,7 @@ labels: # note, this does not pick up chained PRs automatically - label: 'stable' matcher: - baseBranch: '^stable-\\d+\\.\\d+' + baseBranch: '^stable-.+' - label: 'cherry-pick' matcher: From 59ad38eea3b0f407df55d7c897bbd5f137f53c6c Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 14 Sep 2022 13:49:18 +1000 Subject: [PATCH 44/59] chore: upgrade multi-labeler --- .github/workflows/labeler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 1c8704da25..6faa005e70 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -12,7 +12,7 @@ jobs: repo-token: "${{ secrets.GITHUB_TOKEN }}" - name: Update labels based on PR title id: labeler - uses: fuxingloh/multi-labeler@8afa186ed03230c98fe24ebf9fe35093072ad46e # v1.4.0 + uses: fuxingloh/multi-labeler@fb9bc28b2d65e406ffd208384c5095793c3fd59a # v1.8.0 with: github-token: ${{secrets.GITHUB_TOKEN}} config-path: .github/multi-labeler.yml From ec865c911d35747a5a4940a447e812c4da37440f Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 14 Sep 2022 14:08:01 +1000 Subject: [PATCH 45/59] chore: add feature- branches to labeler --- .github/multi-labeler.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/multi-labeler.yml b/.github/multi-labeler.yml index f5f509cd3d..ffab4f5572 100644 --- a/.github/multi-labeler.yml +++ b/.github/multi-labeler.yml @@ -11,7 +11,9 @@ version: v1 # labels: + # # conventional commit / semantic PR styles + # - label: 'feat' matcher: @@ -32,18 +34,28 @@ labels: matcher: title: '^auto(\(|:)' + # # additional meta flags + # - # note, this does not pick up chained PRs automatically + # stable-targeted patches; note, this does not pick up chained PRs automatically - label: 'stable' matcher: baseBranch: '^stable-.+' + # PRs marked as cherry-picks by title - label: 'cherry-pick' matcher: title: '(🍒|:cherries:)' + # long-lived feature branches + - label: 'feature-branch' + matcher: + branch: '^feature-.+' + + # # Scopes that we look for in the PR title + # - label: 'android/' matcher: @@ -76,7 +88,9 @@ labels: matcher: title: '\(.*windows.*\):' + # # epics -- we will add/remove these as we work on new epics each release + # - label: 'epic-ldml' matcher: From a390c2fb2ffce809ed4f8cd5289bff172a2b062b Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Wed, 14 Sep 2022 13:32:14 +0700 Subject: [PATCH 46/59] chore(common/models): Apply suggestions from code review Co-authored-by: Marc Durdin --- common/web/lm-worker/src/transformUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/web/lm-worker/src/transformUtils.ts b/common/web/lm-worker/src/transformUtils.ts index 8c82f16b3b..b5aa598b65 100644 --- a/common/web/lm-worker/src/transformUtils.ts +++ b/common/web/lm-worker/src/transformUtils.ts @@ -1,6 +1,6 @@ class TransformUtils { static isWhitespace(transform: Transform): boolean { - // Matches prefixed text + any instance of a character with Unicode general property Z* or the following: CR, LF, and Tab. + // Matches a string that is entirely one or more characters with Unicode general property Z* or the following: CR, LF, and Tab. const whitespaceRemover = /^[\u0009\u000A\u000D\u0020\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b\u2028\u2029\u202f\u205f\u3000]+$/i; // Filter out null-inserts; their high probability can cause issues. From fee6acabed3ae666aa43b33a5c4d37ccf4c36582 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Wed, 14 Sep 2022 14:01:46 -0400 Subject: [PATCH 47/59] auto: increment master version to 16.0.64 --- HISTORY.md | 5 +++++ VERSION.md | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index d7897516bf..b4c028256e 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,10 @@ # Keyman Version History +## 16.0.63 alpha 2022-09-14 + +* chore(linux): Update debian changelog (#7281) +* fix(linux): Fix ignored error (#7284) + ## 16.0.62 alpha 2022-09-13 * fix(developer): hide key-sizes when in desktop layout in touch layout editor (#7225) diff --git a/VERSION.md b/VERSION.md index 923642153a..f6b5bf34d8 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.63 \ No newline at end of file +16.0.64 \ No newline at end of file From 917fce0750ae35026d1dd6c1c0f24eb7e35e4bcd Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 15 Sep 2022 08:20:36 +0700 Subject: [PATCH 48/59] chore(common/models): final requested tweak --- common/web/lm-worker/src/transformUtils.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/common/web/lm-worker/src/transformUtils.ts b/common/web/lm-worker/src/transformUtils.ts index b5aa598b65..cbb2c151fe 100644 --- a/common/web/lm-worker/src/transformUtils.ts +++ b/common/web/lm-worker/src/transformUtils.ts @@ -2,12 +2,6 @@ class TransformUtils { static isWhitespace(transform: Transform): boolean { // Matches a string that is entirely one or more characters with Unicode general property Z* or the following: CR, LF, and Tab. const whitespaceRemover = /^[\u0009\u000A\u000D\u0020\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b\u2028\u2029\u202f\u205f\u3000]+$/i; - - // Filter out null-inserts; their high probability can cause issues. - if(transform.insert == '') { - return false; - } - return transform.insert.match(whitespaceRemover) != null; } From 26fb6da3abdd1df7c6a9f36ed9aaa0797468da9d Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 15 Sep 2022 13:23:21 +0700 Subject: [PATCH 49/59] feat(common/models): adds tokenization tests for wordbreaks near the caret --- .../templates/test/test-tokenization.js | 76 +++++++++++++++++++ .../models/wordbreakers/src/default/index.ts | 2 +- 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/common/models/templates/test/test-tokenization.js b/common/models/templates/test/test-tokenization.js index 4775064b38..a7ae0835e0 100644 --- a/common/models/templates/test/test-tokenization.js +++ b/common/models/templates/test/test-tokenization.js @@ -289,6 +289,82 @@ describe('Tokenization functions', function() { assert.deepEqual(tokenization, expectedResult); }); + + let midLetterNonbreaker = (text) => { + let customization = { + rules: [{ + match: (context) => { + if(context.propertyMatch(null, ["ALetter"], ["MidLetter"], ["eot"])) { + return true; + } else { + return false; + } + }, + breakIfMatch: false + }], + propertyMapping: (char) => { + let hyphens = ['\u002d', '\u2010', '\u058a', '\u30a0']; + if(hyphens.includes(char)) { + return "MidLetter"; + } else { + return null; + } + } + }; + + return wordBreakers.default(text, customization); + } + + it('treats caret as `eot` for pre-caret text', function() { + let context = { + left: "don-", // We use a hyphen here b/c single-quote is hardcoded. + right: " worry", + endOfBuffer: true, + startOfBuffer: true + }; + + let tokenization = models.tokenize(wordBreakers.default, context); + + assert.deepEqual(tokenization, { + left: ["don", "-"], + right: ["worry"], + caretSplitsToken: false + }); + + tokenization = models.tokenize(midLetterNonbreaker, context); + + assert.deepEqual(tokenization, { + left: ["don-"], + right: ["worry"], + caretSplitsToken: false + }); + }); + + it('handles mid-contraction tokenization', function() { + let context = { + left: "don:", + right: "t worry", + endOfBuffer: true, + startOfBuffer: true + }; + + let tokenization = models.tokenize(wordBreakers.default, context); + + assert.deepEqual(tokenization, { + left: ["don", ":"], // This particular case feels like a possible issue. + right: ["t", "worry"], // It'd be a three-way split token, as "don:t" would + // be a single token were it not for the caret in the middle. + caretSplitsToken: false + }) + + tokenization = models.tokenize(midLetterNonbreaker, context); + + assert.deepEqual(tokenization, { + left: ["don:"], + right: ["t", "worry"], + caretSplitsToken: true + }); + }); }); describe('getLastPreCaretToken', function() { diff --git a/common/models/wordbreakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts index 13081e7bc2..715b722932 100644 --- a/common/models/wordbreakers/src/default/index.ts +++ b/common/models/wordbreakers/src/default/index.ts @@ -26,7 +26,7 @@ namespace wordBreakers { * Allows definition of extra word-breaking properties for use with custom * rules. */ - customProperties?: [string]; + customProperties?: string[]; } /** From af6c6db49ac8e3eacfdbad8d80e760fb064e0354 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Thu, 15 Sep 2022 18:53:44 +0200 Subject: [PATCH 50/59] chore(linux): Remove unused IBusLookupTable --- linux/ibus-keyman/src/engine.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/linux/ibus-keyman/src/engine.c b/linux/ibus-keyman/src/engine.c index bfc267486c..b0c03bb439 100644 --- a/linux/ibus-keyman/src/engine.c +++ b/linux/ibus-keyman/src/engine.c @@ -70,7 +70,6 @@ struct _IBusKeymanEngine { gboolean lalt_pressed; gboolean ralt_pressed; gboolean emitting_keystroke; - IBusLookupTable *table; IBusProperty *status_prop; IBusPropList *prop_list; #ifdef GDK_WINDOWING_X11 @@ -262,8 +261,6 @@ ibus_keyman_engine_init(IBusKeymanEngine *keyman) { g_object_ref_sink(keyman->prop_list); ibus_prop_list_append(keyman->prop_list, keyman->status_prop); - keyman->table = ibus_lookup_table_new(9, 0, TRUE, TRUE); - g_object_ref_sink(keyman->table); keyman->state = NULL; #ifdef GDK_WINDOWING_X11 keyman->xdisplay = NULL; @@ -469,11 +466,6 @@ ibus_keyman_engine_destroy (IBusKeymanEngine *keyman) keyman->status_prop = NULL; } - if (keyman->table) { - g_debug("DAR: unref keyman->table"); - g_object_unref (keyman->table); - keyman->table = NULL; - } if (keyman->state) { km_kbp_state_dispose(keyman->state); keyman->state = NULL; From 6b847e3ec7c19a1575b69fb5b9ba3a64c3b708b9 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Thu, 15 Sep 2022 14:02:06 -0400 Subject: [PATCH 51/59] auto: increment master version to 16.0.65 --- HISTORY.md | 6 ++++++ VERSION.md | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index b4c028256e..ec8afd279d 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,11 @@ # Keyman Version History +## 16.0.64 alpha 2022-09-15 + +* fix(android/engine): Switch keyboard if uninstalling current one (#7291) +* fix(common/models): fixes quote-adjacent pred-text suggestions (#7205) +* fix(common/models): max prediction wait check (#7290) + ## 16.0.63 alpha 2022-09-14 * chore(linux): Update debian changelog (#7281) diff --git a/VERSION.md b/VERSION.md index f6b5bf34d8..c3bbc393bb 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.64 \ No newline at end of file +16.0.65 \ No newline at end of file From 11afaf03f80bf3db795c927c9472939428d825d7 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Fri, 16 Sep 2022 11:39:42 +0700 Subject: [PATCH 52/59] test(android): Add final keyboard to test K_ENTER --- .../app/src/main/assets/final.kmp | Bin 0 -> 31745 bytes .../tests/keyboardHarness/MainActivity.java | 16 ++++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 android/Tests/KeyboardHarness/app/src/main/assets/final.kmp diff --git a/android/Tests/KeyboardHarness/app/src/main/assets/final.kmp b/android/Tests/KeyboardHarness/app/src/main/assets/final.kmp new file mode 100644 index 0000000000000000000000000000000000000000..80d024623b2672ec2d9c80eb0efc1a42f9c599be GIT binary patch literal 31745 zcmZ6yV|Z;#(=Hs_wr$%sS8Ut%ifw1bwrz98wrxB4vfuqY-#PF3F~{hheOGnW-8HVB zv#N)JG%yGX000C8K&mQ(3flPLwF(db06r)H05|{ufR&9sorSHbOTwnyx&T7(HSr-X zjh_F3nCov6?Z2Q!jV^y?AcTvk%F87?SEQFZX0I~zw@U3+Z}BC=yiBfd+0BN{%os-& zv5=oSw%^AZccuE>cb>i2>8+Es*p{hg(*%3mMi{c|rnMPn9y00#EgO985H%KDhmGan zoEj&9wSy~^mVlna@FTl5gkZx0JxX8;{rDA$2zxmMM(*dLb`fKYQ(#JXKAyQal?( zFpZ6^fs=J%r$gAzWnuL)y)uYWcg40zC^!{%J0IOSgTwld=u#MwrW5Kv@4~c}R}84* zE@uIc967Z?1*FzHIo6_2seJW=itBLLtk%tfc1BdXX5{tubM7)7P4i5R39!tZI&4BP z&8r%>&YeYXG8(DhkpbF*n7Ugt`v*m4y!H(oX2<19enQy?^Luj&=ZBte(nRDebGmJ<63JJI_JJxU3Z5&EM`t~F016AMM zGzt(R^}2(wR($Rv2VZZa>cgnKyGoWZe|SSwDraJQp8;{44f-NzmF=jaBlNS$T_)9X zyC{MOwY5Azw}o^31XuW9k(X2a>(SvS@b?Q`D&3$qSC$~Et8sDBp|ng)K+BSFN;=z^k}B!PS^NbZIxAVy-)*3 zoou|{9Yd$PY{;&?#Bc#MQ$mU*WYH@nqT?Tm&7gOF!`o76o6%J<`Y1;b3Q=0ocb#@Q7T(Y$!!wMQqt!6yc)q~hMgg}`R1~U~#2!JieNUNH7vt$i^{W+N9=?dq zLhs_WEw4!z-u2S5QJ~Ch&po%aA_Qfnm^8F=IesWltC`tQskfYw?QXb@qwcZ{?3&av zAlxSQ*2Fl`P*?P+#m0`rnO?zRPSt~hm5GMb^R6gP<}tdBF^QJDR;0xOIw!n9)893I z!Azrlf+sA#;i~+15@J$HfhU0yACLK><#j^E?Z3iO$<2yx3VYlV6Cyui9KhOp3CrM$ zIk{iVzJYL{=FFH3q0tW0oY#Q^hqbBcxX6(i=Fb=>1v?phKL=pI0Vg*HKA^uwUy>}P zWgBi7_rEUWd#!b}_KugO)rtkhY*ZplLaFE}j|H}#lio|{_!nnU{%(xt}S?^BCKgiDW-f!&l} zhBRZdsNgupHFd4em}|uBxGVVzsI=*Z8qKzE@w3~wem#Zc`v&~qVTm%UY?u!Q0Dy%C z008+DmZlcA2G(>|HtvUMKISSe7$bi3DnxA4sLY7*CZ^5KlSLDMt!ksQlOCmT+T ze1eaWkEflZ_jP=M-Qtzgd46O|=rhl;kB5-A`%5#O80T6@{Z6$u{GV~^e(}EMl|541 z>`U8))2@%Ud4Go^>WV2Juc6{J4I~)M?4tN{WcLZ-Kw;N$|Lw9 zS~i@3-@PYSi<}0XzY;cS*jEqlInukZKD+_Jr)G3L+K$D$*w=^MXgRei zyooQ6mnT2WI)2wkPLuFKp26R~4FC^8-+=W6mKl%1`iGLPY79A1+Ecm6b+WjlJhr=I z-l*RXVS}3~6%0)TIF+#vP6XZ`Vz|*83j$KwcLz4X%T>jAdlBfNxQs8*cHv{H`qX0Ke=~0ynAR+QA=c*g$+Kt;{)$ z=_5DJqIE;!qdhlk%h_%C5zlJ4E~t$3INpt12&!H#Xamthq#re0>s>CR8Qc7A7jD+J zUVCmcPmz5N#!s7l=-JyfH=zHER=T z1y>2YSdYey!da#Jz|QVM&0bOesi3#qnl~nB zEacw(R%hLfd%5t_WRMe3x^Sj69gk?|Z!&y37-P8Ezd>e^W^GC@T>?i4@>BlDKOLx6 zsyI#&TSjf-?8CF2ke`7zG528i6fH=0aHi1Fre3R`X*Jz_2%N$;z}yqvQ`TYp5Jz!{ zT2Qy>vJYK*^+116mqBhns@Ph2|4~KXsmC*c`y2kB0o)MopJ<4^k@x>g=qSFs|HB-J z_dgR}v3QH|Ok@T7K^w!xzXNv@WFOM?rvj%7A5`})|IFy}V*1aFv@bF4DNfq^q+&lX zFRZJdMf2@s^mo6A7Qw-*q6nBCd|@ zuJ@##?YBH`gZlAhR^H*0VUX z;rVU(i6ZwRv*k=^I4AAFtH zE-}mSol}oo^3L5SK(kP@!THJg_z|k%bVxchnzUG`pAwB?jjD}u4@R?|itnV(vNa0W zBB^0)5p<|JWbN9{!6zlU1KVq6x_`7=NpH|W#=2{yV7^0#e08F1zAI!-+9Tzh;calU z#Is*4ZMpZD$deRTpUcxMZFMd)q{l}awXie6Dj^vN5+Wx->5F-KocN`?^?lyz^vywygTC^MwXL+)hmpRh zWBv@IF~*Q=J0uS>xe^Quv>poK7z%))Z&R&|Z)S~J5ZTf?=g2RUWSwgv&N>>cHcwNFAIXYmy=W1O7k31v zNXtxkF~lVwDR3oUy-Az@L`phOYHZgL?$v*G1}w#HJNcJT4J&#gBS$%4*)hl@HtlcLGj*pz5AZrMFh z&o3@}i*RM;C)QYBVXZFcZHLss)d|ND;}=I(gp~o8_AlyV($}cPQOBzUC=Q?v#MF=M z+3xNBt)b7h0$~lr24)rb*axN`st;Qmi|^oyYz^TGwFJZs%nkmIdI3}ktO^pjTX>oS z@P>Ne~h>yU7OR37x)SD1bG3uf~1Zk z1IP{J6_K~~y9>19_vPZzu!_TQ8<;I(f0Cbhr}$kxb}a8J-HPPOP?seVlL+y$Tq??ns4;xssSs8|>VesD6%tzl39Igt^^BQ@)twPcmJ@?O2G{3oa+BhfTR=BZo z=`BTE9*1aJ)6-KcuBQB^}hx^VRK1Awk_sI-#^A z=i5XNyV4tOQ#i2fCu^WrT>(Em>}McGPD+pUTgXXAXepkNfz&J{vGMQcHUl0S*>7EI z%%YYOU+(74XI=RVpG_@1+>T|+ji`TQ+( zrRVHTzV7Fuwemgwm}Bsran3*TwOswJc2U#z1%0t?|A~9y*U-}x3MK&aECKZ>0l(LP z$XW40pmjMv+r|v0QsQ=mTi>kfVtZ-NfStvBP^3d{1=;f17yT9Ud9D48`jo2k4f%xM z`XznAuJg_KWY_+M_q43zx6D24vLGs%!GqFJzwC8P}RC=R*># zury+}r4FVDa+t=_*+jLX31xnFJ4B7XxhM({ED>1k5ZSa-f0 zQe2X<@iml7ZnWO)Nsf$O(%I9Xyh>A}<{dxYa9=_^#q3SKx$g@ez4GkhG?6n^&EXP$ zRux@kv%Ra^;<<0JZo9@~bq#?wBVP7EB>$M1ws_;cMwd*@BF6?GE=`;{exz$M5^G)r z4(;O*ewCFqed8rD`3IoP1Dk*qHOJytN?|dLq^w3lhGXr*1&G9??(c<8__&NufHIAr zTNOar>K9!ZC8)%t?eB$3FbP>!0ZZMVp5_k>E-vHeW{sc(Y~>iT)H;#fu_$(2J{bzo z!BU|?3&9P*fifWHq8ptn6n>T|@8AP>~fFhLNN)xOQ?mZq) z@C(}en52d21uf&64&Js*blK^o0F86r>yz}9LnQ^#g~fYc={OIuQi2rU#bY3Lr2N@i z`A9L|qYxpVkTs<7coch_@bp!xJR!DTD&YxzJY~9{Y@hPM?i1Sl{zZiD0X`pIlkS@V zDDQ9)TrF&fCJ(f~Em-5%8J{#Ua8*UOWFVpmCpWZmGFfyZGo>UqGBfRANl|N(oz!bl zd5~W6T~RCLkaR{;jpS`T3Vq@`V*FkYDdH+=xQ?VD`ACBj4|Uyy$wgRApsnQu48JI;dTWhJF*NNiv3p{o|Z7jfuKOLO(a} zbxp2YJR7jHN)G+xHBhOPhKee@G!_@NqgxZ{qS@9cW6^m=Rq@q$f@RlOhIlaxvCMGP zOwnTkpC0{YaubzF9_?hK$SG%D+Os+YO-l3EG&||aya?g{mS5-iFv-)t~_@7Utw~!I{-?!KIE-Qb6#f}uNH@;oa z4!*s5LigU-|7KzBIiIwF*;NY4GYzLWheS5Nm3H`*qU(%nVQZPopgUYVNi*2Oh~0`= zpx0@6XNfLdX~&9y$e5+ znas#)9uyxOr(m78k?Qe5Q)(g){d&*lrGCDI`L40$ZGS|1|LlRhJ6STic=iTAsL{&# zdAgR6FT~pqG2bOx_Eut@rEilEBwJ$a73Qi+zNI(YzdIPe+8^1y%SaVksv$Sq3C$e% z^wAnz&O<$|#OA6{aqc||LJ+EJw%d*&epTb|`ux!ve}pd5o`yX1+`aTZ-SQ?5?JPd} z9O02~Tj<}`Zo^`a+{Ord-e;C%Ejzw}w{wji7=E~4ol*Y)F-WRS8hyM-{;>FkI<+_X zoLMIT3P~|FQuR{rB=<1=E%LJq%B(LJJGY1EcD5n%SF|t2Rhya7TIlMl z0EL*8?nNf3s_>0310aM-NgW%AKoFe*!S5q_Fv<`qdHI$WC8naL0dDT`B%YRjd4eL4f}S*-_#V z{;NRzzhhqiqaEB$`Ne++5lxt9U?C0NNrDf;gbC7nk5ANK=@6`G#AGQ>{-6QaGzTYs z`ZI0u(+NPLJI>@f>5m{WEi&E@7pnP2|M4kYjsoww|Mz{ny(a>`sVG;`%m2^-yz~zr z_U`s>e`y26)qhyikes6U*BV%k|2R$E$@U*hr+SqB^&-Pv@IRIshW>|;LAw9C!~0(z z-~VwQ4*n{}L*ND?@@J8(cJ2Pl$J#%QslAvEqpy4TGs83j;WNYk;W~ipKT8DaBJ=MY z&}XFos-`K``;TpcUH?Bw=m)fWvt!%gNA4apgh21&&~mkm0^xR!_mHuU5~Q(iH(Jx0 z9{lGpj2$GaJAW}pwfvAEAV9^SmjDQb(GT`(MPbP3+7NDE`(&ttCK-N@|vLa4A zMNHO8WZ+8JCqx&?zC`!(w>O_2QoP@bi}G`Rn0S2HJ28tTmO)}MB%4)+L1Zy#v!$lN zpJLc%t2KkrVqjNGE`!u!Xfjr8dxd)_I19GjT2ts4^JW9724Hd4n%y*0^cbtBouDIF z8LO>+$w|JU3gjc&Y6dP=}BbwqOwqaR{33eS>TT0P}^s*e(lz~u}ocjXsOMGUt+ zx+|=cR_VNbyx%b$FkDxTC#}@E`@Vpk99rRZnPS6CBr#-+5y@Z~qGj_gU?vwMTH|6Jra-myRmz z9qs@8A$IsFbv4zs!gBsB>(kE%>L;`7XkuV&V?t-{Y~!OiAveH)(0NQ9VsASy0+Giy z3F=uI1cKCM@0=iOxVzG~6($x2zsLTVnGz-fVWOy5%N5&<#u#(j|>hprLxq#5RFP+t}oM-hZ0C0*v6=sm^|4D z5CFzq+=!(nz8Y*;(76~L+dIfG_3M{(fo0)xk0w_WLkdkz!)fnZH-LhksD+{s2!#})g(4A+0DvGQ1W-f)4@4k^ zfzCoB2$X7t65ywEB5T1jh$wNoEY5{)$xm6iTV6TuyxVVI-rpV?8kG%lQ`x-CUQd7R zrX>e$Y*;O!*r0%bfGk}&U<3sO9uQl%2#JBd0vQ*#KtS?w&lp5~rf-{mxsbf;T*ri0 zh_rz`GkZnRRwqrbJ4(R_co9G*4tLftD89{Y_jI=6A9mirj_-*!HD^|xWaPkKXE zpP{rMXD)c;wA^C2(xle((@Bv+Vt}oHlpc6o*fz-epY!3%8OeW6K+KAKjlkZAf+O=g z-tiT#&LL6*;V0K{*4SsyW8d+;k4T&>;unSUrf|!0b8|y((qDDsjr($}`@#Xa>JkhZ z4^T&00qvWBv50!MpqRO%X(?TD8YyT!;Vs~5X=%Otke*P(UvGApnJ0YQkrv_eXP{IQ zV8}y_*qd==BP&dp!X-|(#PufOOm{%IS10d?c$u=Z+cA@^iOIfCz1iwgq^b}IaZFI@ zV~lOC?CNUM<0RKGdDqgjp1dEC1Z|dof4tt8^)2rD`zI&LN+H297yx2|tMekENkCnL zjg*#~h>nQYM~{Sa3mP|`{C3CGWDyDOIK&A8`6N!m4Ds#IUVJKyO$Y{)!RUK^cB3=Q z-3LRQ1Bo5%??kr6f?85<3Y8B|;AVj5)XxSSTusykL;wWygsTaU40neJ0-}>B+A_#2a^_mXIDl`ao_QbJ#huAW=br!5YuyL)alsPlA#&ZBUik7DCwq0J}r_BHrf*~LdO4hS)1#~M;tVi!eU z7cB#iOQAE&;mcNhA)2`1evt#sCs3*lSE`7bJ)k_0HV>;XBsynEV1=!#`@&9;0|<=e zmdu|EsGqjtHvvK|jH0RD1NMP#v_=tmgIbHt*%z<@#Raka6()KM83B#=b7uBV_^*dQ zB_%oeSGVy6Q+=oyUp(N7vC!Z_2&R4s0Hkw7$XPH@q0Tn%@`Iw>C$K|3bAu?W3zOyZ z9-Xz#r_-t5F&f;)vhr=ucAJu=3vbNZtvVm63J}n+fe}hJBs(Q*J*?5A{at&*p-EhQ z+mUuq!ORKM&`5&4_q5+QHC1<`eTCKE8SW1IDN>t50|EAZ{v#vFM!_oBtkl<>ITVo| zC1_U5>J1kYJQTQC29c6zV5l&Bkmz<8&6HGlj6tB$YR}7D>}m8vGVZmU&>6j(v6J4M z5f0s+pr23kiR%H=*PX@{@!m-Yg5i{I+<4PnAx5fDnFjll8MRJ!YTmTBp$ZoRlo_;K zS|hH3xYY6dgpZz^{hRiRGm6INXro@PJ~lPUz?0R=XDa&X(yOuKSY`NMYez<}53P1m zLW)<(tgP_X^kQHs3F!-q<6`r5)MwyV!e6^hMA5u0+D$zn2eK_gY|z3g1*ZJ6p3n8q zTuNn}JdE?`Z1 zR27$tY*XL>wYT$o4DDI!_@%H30q5}caV0~~SO%~yptGpRH{La{J@ue;7*E^~oH;Eh zSzv`)xZ6#5zdnu>BvuGesWgOdWaJhdsR}Qeep%qY%$$`wzG%_W>UP$}Nvw!R_Ijyr ze9f2pSi`gfos!&9_>ki;a{wt%q&YA6RER%vMhgKomMd&g#ogEn_DeL0WIG;&Gec>0 zHasqAoX&=c>SEVCSNu(VKBu23RHj2dQkuCpfiA60CBEohf)>l|lIAGA<`#_!2Zp}24}v1 zj&;S`k^PZvdOU57^Lwi)&mnKR{x3CB@h+BQq&5sn-i1R@bGV=oS%@&5E6*1OmVAaa zL;4*Ba-wLI%3m3hi5kZuGTN)Q``|8+VF!x3)(ZzNwp!zC>m(7xM4hg- zClyu$DBR7!5r0-{Vkd5viQMH3k`l{xD%1UI}cs=CT@v!OYF3dML0(QWVQGo*GN0 z58yF$0-X^#C79tyX8L)AI%1OU>22E&?s*iJFcW~Q|?9XS~!i1 zGED^JSN_pfe`QznBXwN$d0E~1Nyf?wZAsbER zGqg_fxmGb!`~QKY!;UGywV+6|dC>8Bc6RQGSguerB#`O2pE@cz)AD0sOuY~JKrl7T z!yJH*8nhW6w-DU^CJb2lq7W@g8P%gpyAjo!p^K*nxdb-@z6#>*v5w(ulIClkMJ52^ju8rd<0_fUb6sN$)iq%xokl>(YAuFhpn3h>zeNAzGGkeqzJC zF7oQES#3J%NoM)-ndF$RX`^4FMi(Hv-iU4_er06y2nl(*j=kw2lC9b}rl*zK{~3n( zu-dL6escz?{n=)9WHLSruZSK?fChbJpp{&Z+widNk{ms}8HDzANU>05;kjC|Keb%$ zh8K8!$bw_sqR=P9SuXzPgAwlvTP!qn%?|N&gFnoToaixBe_I1j0cygc(S~4@)BJ*y z_wnxlQB?^^jsC}gGTy?;?8Z^QdC+%M!WH+Reejh659%y1YZe59J{i@zdh4lI$8!Fl zpHeh#D?28+=9<22*bNN6m=CdQK<5p)`PQ^6n8C+T! zrXJ6W|DIr$&s{J`EEhRllhfnkFI;ZludkBl2|EkR6gI+$B56=I|N6rESFHmNEe8v* zD%Gq^f6wnGkqz;c!JNFjJnmq%3>P6Xgx|XQow|hAr48q^s!Qh9pISlV| zZ|zuq^3!rt-bTBrdCP@`X|c9}fSGg0Bg^bXW%ps~tvtz^WNY(Za9N`Yu4d2iX=kU?i1Wjo-=e@N{X+3#}inUO#QyIIfn9uxc0dG|f29UQ4QYH9? zP@svR{$n7~qJm~56Tj*Q(f$}Ge0K4S{*Fc1qN6X}^{^aHYgtskNQ zs8TLne6JRa^q4bD8n2+DT6;RioLOh?!RP9c;c2hVHUYouqFBf85_l2uH`jlK0lSs1 zfU9l}daE1EhKFnPfHee}jFyu9v3-L(%_&WrTvsOC1!bD=(bpDMz9xa1{EWtXozZgE{Hb*{#Kf z6ah+dA1D50w$aj?;UUh-f|oK8`=@RgZpA2&J>qQ^_~~!j9z?)*{^+9_fB52VxCSkk z(Gyq11Lm=yK_hJgX0LtP@{Kz$-~%>8EiT3kLjw<5MpoBqv}cR-Jr#c3tj8PwS#}nF zgriQY6%_0!nsM54$!Pc6IwX4O00~evX4_Jz@AKD97NQBAtlnVN@4Q%hL=r*m!Fe>q z5I?G z+ExwK(I->W%YB&9o1;4VFXpA)`5C`lPtI}I+$bxBGZD_mt|yFk;w?tmwyUy>O0klTwOfNY zer930ZC{J}hGn6$EMHi)XD73Vxt@jAqVhGA&62ZnDlh`Y^#cJA9-z?q& z;oYz^))e&uaB5pv95Hjk@wgKNocg{#ATrv%@WbgPLv zq1k)RBOHiKQq?8Hg9W2za7(Dgqlv^$@y*?vudF{FJpDXPWm|IO-$F2H=I~U1f0~=105tP67N_Tb4 zRF+0&9N^G^TK2&QIg_Y`&)8g_2;lH9U;eX0iGlrC6~1{0W-}TO57czJ&x8U>ls0<)khVh@{fWlAC%A7)bo$VH_Y6c*EP9o17Iq@M#ivR3mF+xK;M zm`f2)?lFds8_=x)R^lnJsR`8NiA@tEnLS{@RFAa7;oL3Vv&v<4CIf_p84xtKa; zQQ(CTnNQPJSo(jWZ|a7Z?7}9)G`y#mUd}Zr>OBQdkeopgEKM-ed zxx{uI?4mBounqo>w`$9z1fhgt3X(Uq&`!Y{i|SypM{cv3dU2_$D*2weAQttg>D3Yu){9srAMH4!Q7=?JGJf+J7#FGno z7H!97a6lNfgq$2JF~g4l>{D@dvIbS4C@PG_!5?&zbfhNHHS$JET8kaVdFp!56K z^TDo8Ob42j^8Sv$6QTBRwKYq+@0FYDSk@t;c8gd5*Y6S zVRa0_RRpy?=iznjfKH$%`(_r2x;_&DVlwJ%N||&!EUr$+WK< zj=Sff?zh5MEd-g3k%Uub1?v;dsqlq^{qv8V;|Mmd)YbJd@cVHmNXzqL;PMxSIrM*_ z;Hn$O5}cW_5{ufc&g(gNy*gM0M-d6w);l0DDnKdm*<+#;RM;t!R3dH4r520TZMmoYXo&l;9R*Gfps4$5*YIF(^h0+HTUf zhP_adD6k_4{>|H)lxQcC6&_y7`3NH)4APIzymr&u)~LLjiJ;4hBYPAGAEN|>(|zpGyjU32LirH3BB6xva@9JjNc z`|%^Z{Ue+lVrOLP?c<0(Pu|%_n#*90@7)c+y6KfSM=jHpL4c!YE!()%34j-@%aZU_ zGO5KOo|I$yFcW)@8_J`-Y50oln%~>KIO3CpAsT%2youTjAbqlr02JjJ;RSw=-t`qXOnDhVuW4Gs0WbD0gRt&MeQE7Nj*au&lhG#y0HhiypAkk8x{7K)6M z-)~l7QR^CVZ02#_KQc^>B#gJ2f$D$?P?CfESkh2t(G7bMvvfhR+pig$HS%{+UZY?v zm7WIxz*9ti%h}7*k?Nq&#k9#NLM9mI15q;yL1=2A^X|0LRcb zFqNjy>r9*F^KljE5ln6L9V^bOqTZdZ$HJ-Wx=T%4 z{Fy}f3+3IIWJ+qh1yqCWl|pVDFj&YQEy$01Ys&5j)q z$8Fp!cb})K=*o(ycoV+Q>O-xPV0U`IW>QxIBaHD@WW_1r&P4ebOg6UTIrI(#q5 zN#i8#T6U5_HeoR~~~{`3ximV{W;t_E4+C3Cu=Dvx4d-j@kWHlN8mN-d*FN zg1^HrZ`df*FFD0rr(KAN^i`i#uC1~O_vO1q6j1XCV5S-j?ID%fhaX%L8}Ex3uD;Jx<=Fnk9paVc%P*tcIqwtYa_U5N^xP}K4`^P za{!wAH1n2d_#;(QerwYWQz~5~YXr#@`IF>o4OhPkQE#k2K`ut@Z{5SD$@#2*8!#d7 zo@c;hBX0R@^GpH#W$Yr6g9&-lBCcWX&eiCdXaU-)k`bQpy8wXN_tLL+j1i(JqAonU zpva0w#MEdu{F2Tq4wP1h>M(Zl3L8pY;8XC`qzTK&@f{QT8Dti&yYz>ihU?=y;yo&8 zSa-gsD8mlr+uceVxGuH({dUY>-g$=uDQ9Uq$&I;j?C=n7;wZQRKU;cB>W}(8n#&tBGvD{K13sv`klo>Om=^Dp71_v99H3&ju=KM8m0cp5 zp1N%ZQM#CcVL;20%ctTcIn~Ouk>JcTNybBv?Pn&o7}Q?f&N?O zbDbRqtmtK+^yBq)X>t7a#BJR^73zloz{G`Ikg7X6(xE-irPSm#NAZg&sn_B7oNlb- zdU}P{pJT;MOu7Smj$l!6%PPUHdcNZ%!*k@D8Wom{UlVNehAq&L(eu$N(2SXW?kEyP zGtJ?`g_$WvPl`&Lv93kvbX`}Ky2L*@CtBiq9Fl^*+9VX$}%fL-IkI?=eCBz=~lz1Rqjtg5uaac`2>eaM~%!O zA2)j~y@cEO2emx%4|HK{mkwG#9ygY}U=i6WJ(OclaJpi5wf{)QwicX$DyhY}%j)q^ zM=S;OsTqCn#XPI2oOXo0U;R+XAE6BmA!!omp~}MIxbfD>HE4I zA$wzSFD)%0G@n)goDaEKpvX7rd!S?pdADwR2VO0;0@DNDzC`*WlWRNPjZ(6uZU{+K zKVG6H-6Q5@B=&tHSb0=4U}iGGyZJ%NT8B0#jjlZHlv$XUr{kJhy=cl6U}jwH4u;B~ zszQqTQnjD3ZUb$i&{N9z>;*6>{<%HpWb+Yp$)81O)VE}o0SRbNPHP;dgwBW_%+_Kh zWzb(c-EJujpPVn&rzQ(ZK^IUDLzq{7LIZ_`RNWHN%E6F8%6;Qgme1`+^Uz$N9^tGn z!g5miSeRmR*2{>B0@W`N94%%VK0?FKf7d6)bI7Prr3Zg%W#jn`DxjZe@!3IkuHNTj zRT8p0gBrIx1{oC$T7qz%?(a8S(!`jN+Njx9Q4wQhdvimA&3`FOG(u=m9GZPy5$@Zt z8!PK}m(NgxGe=b@BB;uwkx}=Y%x*kfWN--xxldyCO=+&0Igl@L`+^Gbw&qzWv(9Eh z?TD$i7m$Ua_K48O)B8ekX~mM{#aD{gt55(zF_4P}G01_0Uz^r$(mVEE?Lh7<6U;hL z00|KKpGf*_N4K{Q*2g3#3pjZ44OOR4SpKXt6MQ99Q6cQ2w84AEv7xqhN21d|=~RLH z^yn0lV=k*{2KYUOR4{^Zh$BvO?j`jaR=^~!2pi?_-kXIN$_Zy z$wiLz`Ig?ULqG!$e1l)^rgUcDL6Px9W;q`vxaV9fdWltr#`8Y@aW@?Dmm2m%vJzB3 zht_|Q;l{hl80wHLNMr<#$&v`p(UGG=!}4Bd4hu++c!)9rQo34H zMEd2L&3x!^o-_)6TDMa!-tc|Mcx7c2F zshNW^c{XwLQ2B-UcU=A)rU?X~cMdfX#py7RV?{Ty7Y;t>%gh0M{-Qc*RWBS%RNMpXPgu->)U?k z4a8L}2U$V0_U0yjt9E2^MWMQOx!@=DLMn6R31$T@-+jQUAqdy0Jz`n6)(`b36+W^{ zHq%TC?*;jg9e|VNPWwwlmpvs5fJc)W!jlD|%U#&P|WqPfcJ7te|VY9)jz6 zy!E)bXmS5!Pdt2h6^*V^TJ-}7V+MTU>K7H<Vjm3ZPt)SDJ0+d zE4AidpG)eupEF4w|4CuX8mAn|R#|o1W_~D6H9YkAMA~9`y1=4cVObWmzb_9{wqZ~~ ziYr=E5~C3<1DsH8WSuvVycQg|9Py%Ozlw?)d!w_M1^EdQS4dJ^<+xJZ)R9Klk?E~a z%%OHYR-Wad#p@+ zLDPND7o%~LD}!N@L9ewL{U6lt+#(?{IQ794{??m@u-JJMIU`0g?o#yiBTz**Mu|}IZ7J$0@b1I*S=OMw7M0|&%Kqp^ zRr8OeCp4jlOMdR9!$!RmqEGW4G1=GSxqXY!HH(W`FSzRS>#JwF12*C+5y6~2Rr;e_ zAF|gbh2&Ts+D7Xn9q)zrU4NsXt~JtWT*!i7_!irUUDQDf#w(U*(CNaHw=uaT3!6kH zv$S81%?UNAwGV?dOvE4{^5!Rsef1D}{k8WirUc#K#Mt@C*OXQ?xc$Al$3bX~x~y(< z{(EANX|^^NOb^&(y*rq^SkLa&MG6(sV74lnXrH^|?Nbp-OQTz~ZQoA!2Uwj;nh-`Z z=P;qj`aH&5L#P4{*9*xtT-@7+y_hB8#VrcZX{4eV;Mb;pm!IBnJ0V{A1PwTvm)SXF zmzJF*f1tCdcV|4Gqc}SkR~i!e_D(1GhT$N5o;%ysc({R+H6QW)i0%XTCc8_%qOMUu z0VfyMx6WOHJMQ4zf@eQCK@i;p(U&RZts5H zF)I5`Hk&m1UQrv8LmGMvhzU8+K_ck6dCp2!0i*3kQHt|n!yPHkdO& zzcK}ahiUZ7M8H-1qfd{pB=wBeEobGKy8CpKBlY8x1Jl;Q6B+cX)JN zh@M%urufu-k~7}7oxtKQy>2Hre<)nT)nn)?k1{bY{>d>A9~G~+;Oz|4N9Y0x2rf?p zNTSb7Zn(b>71xci=1(Qu$&oln2~Z;E1S)eb^>rb18Zchjt`g|Aw$1s7&BV#(x$(G)A?YT;OyR3|y8w+R5Njt~*~d9Zw8V-sT!c zSRr9 zJp28tk*FIO&uy&2yX!H$>C07LTRra{g>Ufga8S5?J#VDNkg=X9>W_ocz{%rGynW1= zDQCDreu-oaGkf&~;FU6xH|&PQAl3R)oA*``_leb=%&gPf=M0yI3Kkl@do?{RP193s zUO)xGX1;UIl;i_-jn4`dGuTfgLr>5cERd4kEwI`up5a)PVh(=&2Oh4TZ;g6%*Q@=fv4kNi=&%aizBXCoeRp z1dGyx!=@(Imsi~`6``6SZ57pGYgLoa^QArg%2)0qc2d7nBJJ}eM6Y(;xtZCer2_u# zO!@5i`Mv%LiI7+^kXwzTx!$l_dpFCB&31(6H_D+wO=!1GpF&Ig;9QsfZN6?m@}`US zFQ(d~q}pzlP4lipL5I1#yb;{F!W?_!>qEWd(!DvGIQ6N*9>1lWS$kCeI$LP$ zW0}-o*;W9KY=Q56RG8#=w249wiCSk=-{38QOv}=M)Tz6fL#M{^06Ul9p!<+vVt9{$ zPqMI%OaLXE48%dKySI;RVzZUkW6z|JZ<>(T$90##}bk>x=fJQdDM7L#x9@TS+R-+cP?_1`DrY|_6M}=>$u;>VPz>1 z)RbWucWn27EI<%>Om?4c{lBeT?|yoRCy(;*ysr!xPow%d_wogS*(DHt8itiS#Hfu)KtAIHJDBP3FXA-joG#dGjNbY1Ch6jaYj6(62d(dmL^?8g?3J~CAr-F6OvpYocg(G%v zAoKC38PmuBtQ}tk>vkXu`5=4A`#Bj9+sYGIQy4^jH3>dK{bdUMa|&fyK)4EMKPcs} zgGIltvc3X%R~|O*))=#t5CpnE)7n4v#Dl}BehKqCuqM&01m7#Fw75+OrTQSysZ648a{}j*3xni;B4#>!wmeii81`Jh5 z_Ine6AJ$0Au-!~CufFj+hhb51n{e&gPU2^?rL+M#SS)CI>}^awCnrVMO;%TTWXcW| zdGMSdU+~BE2G23%bihVN()1!lW62}PO-4Vk%OS)J-w_-5o&^((^$uNmd8yiMPOJ*8 zdnBS+BZ7|r^Mr&uoO0BkBV~DeQLIxRW>wH+_&Q*%%V@@1`dFdi7YUg^_ZheX&h4%N zz(*jRc6#aF7=nKO=x8rD_js+IOpHfwD1Z2(* z1oT}F{_l*g^1nj52kquunYvG3vPk5fV+JHa;VJp7(1BE}WWQ$epEe<8Q1hb&qL7HC z8h_~(L^4UugN5UmxXkd*e6o%tpZ&Q!`|R@46Nur)i>Z5B-EzKYTKcT?x%?Gor9cCt z_OF00jp=WYO#kgZ^YtxW6HJG0;hTj&U|>cqN$C5HokL5rWXM9gUjWcey?PGqgO>g& z6dTTMR5So9xjKHr>@8^#zs@DIxw2{7M508(F%lsyT(y3z!jBMq@|`}Ik0uONi`G6CntMkrC0I+8@^XhMijE`{u>|}qv(kPKG^hOfc+@hBof%M- zzkD@NO9z;-LL4>g&>D348te@c9@E96CObU6?v(CTXxrId9NvHM8$lwA>*R$St8{XD zAXmdzWama(t-VlSo6gZVgEu_IZivk*bxkhC+r_nb2y?4J6p)_y{AbK_<3HE6HsB z?sKWB*{)~084$iDuPLSY;SW(Zjt&`^DDuMCJ$gH6y_r7BJ z8U#s8uhT`!%ZcIT2_YD&hB?ofcrp%ra}INL_Io+#^LrQ3$24QE6mF^1e_UwD{tkYG zHhb+|C?P8y1?eA+f~~l)sEQ81jmJV}Q3;%wF%kOX{3w<2&$)BYd>Sxm0GiKgtYGIt zEP8b62$NCLGSV5df?XVxwa^F5baj9X!bQQ_O&+| z65Gm#ZfO>aS)D|+Tu(zoc}cV!QIu=;Rc8@hjB~F|&>cO0UU@;JdjJ zSA6dk-NU6$A5x(3{?-@kN1zaw@sAn~W`=f=PgLFN{2rKkzYM03{p(pu%!C2WP2-{) zOnjHLMcnk%Gx4WG);_eo&lz!vj%Yyrqg{vvp@EfFZHlP9VeguJ`D%Q>X=4mee!bEaT zIG^N90E9d?2cT)Gwje-DUr3xa;;?p%p=UY6tBEI*P$UYHlCS8NZqt9@EVGmv{ppYh zJwRznbgfEv0F*N1d+-F-T$aH+KQjY>imTmMO>|tv&xBuo4V`^tg&OG%n@$EZ;iON~ z*O1BFWDzr?BS5Fo9%H+i_igMCjE%4p4^DbBcCuS0dJaWheh_3MSrV&u2zXp9TPt!F zk5?yDWxJ4gZNrao)>E7LQCf&8E{_#WTp>Fr7A3qLHXV@5wH&SG_nhVovWGx?(!l$eLEkg5XCSwbf%Brsv6c|cn$qh7mdYt+3w)x2e2I=y1Z zWExLj^Sl?YQFdM)TV)f@yxNZQ)UfSMcV7{Y+$D*c^oc2yht7iX%n4*8)X9|v1@-QBV5wL} zy-CM7^*d9_ArdDzzD$bZzw5(3x(3NqPo?K!SVVUotH&urr0@qp9)wn(S2#n2MuN#@ zr%q$JMX~EaF?rz{Kn`d=M!2WFG7fBzymeQz^1)GBfK-{@R30oUn+mgsAeMW-7qz?R49I z2=$yP5*dm(ap?w&@a^%2BHeS2KR@N8ED-Y=PT&ph`#Yc}H83E>n0QXyfN0Ip92pUGK1 z&$q{5pv3G-<&rN>Zsk4;tG3h1&ht)7$j{up={ggigPk@{s(zNzTa$V+U(9|-mK=4y z9+M`Z^At|wyYa9*MAAP+I0J`EAXg86mdU7LZwG*iG1TrXJ*U+R#dO}^fEeXxnP0}L zQ67z@z)*!wc9-}$Rwl9=7xdwNaDq$jWWt1TytBWt^0U1%gadAAdSJS)%={6)oU?+` zya`j<>Dhk$H66Sz4*^%NE=PkR)r#913GW#aR;K$TcpGv_aHCUa)ag%TA%QNcmB< zcd14p2VOM=qu`LxhrD;j3YU9;!i-X@Ugc(3T7>Chog1f1jvcEc6;MK}8WKXDJ)Ys{bR{ zokZ`lNn?5}*49l{Y=Jje^>7Ucxojj8u68uCYTLrIFeMM0?!F~j_GQ>8 zz}m+oRj(6Xk|yU8)I({d1JHRO&eYwG^R&U472-+pc?x0^`M!vP(O+V?0#KliPb4c% zpy}15wrF|dg2F8!f#M6#xOtss8FTG(Gwvy_W*BLTdf3!yib@9DH}`E1$vadgWtFNn z7-(FtEfG?K4t5b94c;#MS;5U%RJnus^n2%A)-15%53=jEmUq^*@b+kb?Z>C4o|_i0 zbAv=EpYOM{mLGRMy5G63{;CHhmj)jdp}&)nJKP^l_)FM(Y<&O9g)gZlA=6O07tXP9 zRcYog+~BmUwi|gj*mCx#h<#BS7S;%NaaKMz6xyLiHU}DX=?n?lJTizB*=H(06%SNO z(z(EJWd@0o(iI=k;_JM}LBUS*A(?qg%;O^Rq-hNiuqjT}8iMwOKDv|53u~s?msZkx zL3v2w%UQoBYxKMBy`$Jr#AIK%C?#E}o4poumaiiM_`qC(yl6B}w>R?u9x=vPRCJoTTT2 zhw)H;v&bw?6d}RI<7saMV&iuL3Kj22RKaO+Sx)J78B+XC@NbfVJ)RJyS|ZcrOF!T{lK@$B2&ul4L@dKy{l45{Qab}rKdv_CbmZ$ z^BLx@)64J0o?TmV#xXvqa@3)c_1%B_I8)Z3``KeX1yM4Fzx4xj$%}{t%xNeRnabBg z?2o;%j9d`=@XtH~hS_-T)ACK%QC#L0kc1j^n_MXfcw&Z#PqseDPzqAL;9?_d+1ZdO zaqle?KUxEUB6D)|4~9c-X|baDDDhDfc_U>TujVp0H&eMgai3%&_Yg-$XTt=#{E=~Z z`I$Ggq*eYxcu)7&S@k|VcMHb}$o0ygaBI73t&;uyyoc+h(Y(LBS%lMZ8at>UiaYd1 z<#E{aIU^-;(%7ZY%VE*W87xJDuvYi8gJriU|40BB0SUDYq>7s=(ADO^l)011S$4b=l3PNXI#;zy`Lb-uX1PDvw z_g#~EDuOlnsq!2!V;`!~GKVsiWTU3mfj0fRQH983DWHVWHY2%9{5lx+X9hPdAssI_ zGQVr;aC*f+jXCOOp=PvD+;`aBNyAG~r4T{qK%+8g+`#fY<=u6|PRoK}*i8zV`{+k z09|L7cn$x9ztNQbQlWhf*#{pUnJ>_x|MWLuP~WxAs8?V{-c- zE`OW5L5mFMYb1Z6HWOxt?@eRI*W4brG%=b>j7`WcQp?%N(TD^bOxw&ZSpi4d7aiXpaUxc0Q&A(o9b?%%n`)<< zb);(=&A9yA1Q$KffB~&4FL|$91}nIbejK_b^(z5>{w{=Ecm8`9_)PJ#y*xQcCEV_8 z69al=N-G7_IPo$DI7E41GC7_Yg}LtLITlj?5i5DQ!E(Q98ned0^?X=y9JePj+(zGo zq`zMkdmYRnf`E=aNeCfJqh?;U2^N&mnOqZ2Av9bMgDodk>86XANz<4-(hmlz#E1D3y7NxI& z`kxsesjR-y8TFeNZE)Z5~9y<|-caeZXqy1rYG5aBhW#70dajhE87me~#i>Kf1>AM-}T*RJfQNt zXz-o(vF>k!srOUqV0bRYLLYPXvL{#;n_1N=9<^UnOf76BMSt8AhNMjL1KUk8L^t8+b z{ou0a+{*$#v9QczZ{g<;!=8L2u0*Kl;xMXADg>4US%Q8~EenwhsEz}Cd&+pc%sqoI zu;P2kl#;T%p|SUs?ySTTICrY*xm!>~%~;=lM4Y??ikU%sQ~i}%P8TS4D6)2Fcgql6 zPk^Jz6V~`evl~atNwkwP73u2U6#zmRuI{~-!sOpBXC0r_o;0_qAN%S+hB*^n18Xos^a<6>1_Cum*P?5|* zF~R9~cfZ52UIP0XJ5eq3NYSYrW4pqNK@*{29s`uin4f;B>GIPc`=kNM%d8NMdPf06 zY;}isjxa3XIyE>FD@N%2?4((YmXRI)a~sLlH-*TCs~Nm^EZ#TD-Q(0C&I!Rti%T4@ zT?`($cv0z)B~KKR4uOqa1+B{qq5d+xxP5a&2n7g0kzgJ4b~&h$Zq`SHf5fe$QA z1C(t~1vQUQGs;<4>5|hmYrMi_g1gboFN1GB;nMH0WpBFd`s?#%kY4xc2yo?%A)ufT z%HN+`W%i>rY+x0+tFrA3A4;RjD8W2iyf6^QdUqU_p$b#~d+_fq3A^pytwMN>6J^Zo zYU*-9&(RQ`SHmbck!B*TES^ap!$)_ih@%7(7It8kN2qeca#m?bUy{?$1aLDV&!sU; z3SD{jCPQ6IXq%{iwvbH7l=Ao8=kkKotlO(qtjGSO@n|=*PieXFj2Q(yo2EMYe1UUK zEo!Um)-*J{S;?-;YH4MD&px^J8SdxKCXLHzi#^QI&7l(jZzcJPhT0w>jTIWy@w_~? zLCt{Mb|0eArB(gM@QRyIbT3?<}TyF)P z0+=-qeX~b+HQ+gjdh*#2j7}vo8=v~?bt{9AkkZ3c6v|GuE=_cBiO;q`z<(f(-B=i z4Jy~_9L_bBD!Z$CM?WD$Z?6n^^WzOd(csZsE7;-Fm!+jC`tkAFuBs4FJ!0Q_o3W~& zt3-X?ErM`72vVfKvfiYvdjUo$v zBnS?T&pKKSvvXKI$n#ABVB}kzO>ww0KAIO&a5)rKO6*65-1(3)suH~2Z0Xu&3uBAE zfDXI1yjKE^US!LN4s*A*zn4Q{Uf^rVorRLB9dDg+w-ZWzCi+$8XJ%$@8EVv@do_y2 zh!5Hs>1aj|(^BA;v14aB&WR{20%*=Abb7_!HfSV+)-Tc~pcmz|*Gc^zq5^+py29)3 zY&#y(FtiCLO(Aq_2N|bxD%HPJRlzIzWF3^Fb6AZ)_7#!8z<@I6><={p{`|alPv28b z-^=jaO%L^h?q!cIWucN$b}~9F;h^!h6u#pILqa{R+N%^$H{Um=-*AD8d#VHO*ev7Oky@^rr9V?tGyb0qr) zA_SWvXk378nghILSiGfXyzpTtp=>hm2_0j)Sw5j2fb|~Vu{D!uDipmgDU{voZM#5{4VRk0>n|fsQhfq#7--=Q>x-Zz7y~q&P~yl=E#RiW-qv(LgDFNWUW3N@y`??n3*kE z%Deg_p6$J$wz%e4ZWM1 zY7i3)57W=?ZokF9jgk&kyOh)eg_l?9^K_-hh+j=*9h+~WQA-`=v!#1@EJ&oU#$ob% zM0~@Ms8Sml82KgUa7z33hoX5|x4-n(oL$}cMw0MqYG3Gp%F*C)%nl*9?jn7dIu zMZ@AvWOWuy6LA=?kAC04_m6Y?7J;hwYtjAls>J6PbN{KaOM$CS`3&B$+r~;!I7n{Fd&dQg!;vzjug7Q5=PR(^vq<)1Aj9Ww<(uV}& z8RMgbQi<@Ic=P1xSoYRsoY^eVCRr)@z}Ow4O`aeDpG5n@=b&Q@&z7r)@tKbHt%F^v z1fo7D@Rubm1Z?^{@k~UmZRBt%T$O}!1NKtMb;!6%`ef$jnI~n2z1p}rhXwDD_?Hg# zGM(g{KaRSgMZqUGb4&i8zBnDiE|R2%k0pGPx=wpP9R{b4v+C6v+u>+=k zgic~nqiNzV1b$(nu`G&n(F?z^h9#pe4}2l`qAxeLUpGUY2KjJ%ByO$PB}GVIM+2zP z>FkPSVIbjoyo+uQELPRc4R-^e-szOKCq)c~(51aLZ3~J$5m&>rn@k7bJs(@k-MGg( z$KPof(9cEfyt5_k0GN3cf{UG6y(~% zPVD(-`V3gYSRLHC!5Vigg`r7XW89q@G9G`pA~mhF@4r4D&o zx%5*1$ZbqxJvDP}&=f-@?f7t+I2rpJD5LDANb2NbEqY zqKP1zA|QXd)+M!o$M^hzMu+VqS+xrL4hQgfy&w!_3eWx-hGidm$Y!qNe_LcVF3h6? zwG(w$j+&Z6euM^=p67p`jlXpOCfAa4H=rfK8yyLBs2%2e*O(J2)8bG5`8@S-iGTNLGU^#n7w^ZRSr9*-=3LVL-Z6iHj00$3hsx&WlE zu@1Vg2h&(@vL!V$2!M(e;t%~~I4{y;apZ4aN$Ri7bj-gT#iq+fEt zy&4mK^kcN*km>+IWt(4}<=iXoJHB4qt+m#h`iq4B!H*9~XGHhHU|7ZVMi={$T~?`3 z@3ITdCy4fxXVyCtUxGnbmn~M<{OT<>^C0`Dm&EtA

5upaBi&K7%P%^;29U6%4JeX!gL`;vN0X zT@HL5q!K0ODT~fi3J9wu8t`c-tSxgMfXL6?u5K^Vs&+E(?dD+FbM@c%!M4q>eQK2r;nEtn)QFjTtsDJCiYIOHYvn=RkQ-yoQ zWGJp`_#421nU@@p%3dH6PQtNInIwf=*$Bs@3J2gnxRh!XJ(db+z{qY)9t^&iJ9{0H zjqZ`f5e!@2>_+(bm)4~KJJ^rLyLjjFNKyDSa$Pw#jXR3VL!ae^H5sy@d1rzfA{pGR zPb#&ja|$leV}eW4wP+v+%lnMf9_X=!Y;O7bJRR2R z+3#J$j8!)xSFrb|Pqc{MklY14O-<2U$4@&!=Psr=2ikV(Zu*l}vzYgXL?!fR?7t-m zkMkYUgq>j#-|RBGiWC(NR<$(#9uJ4f*$L%nK++ivzd*ah#S(y<_x(WbDLMx!6LAP* zrbe5GdN|zr5=S4*Ts-sK#y(3hfY~uG4j$d{#?|NW1|DL4zRSzB%Q4sT6O6be4eP;^ zd&j+iy?)Wp4a)gR!@QktE?UI-6i%VE6;NSm9{|>m8Px4}T}cm>)1{nE`*2<2o-VxF zn)Ja9^}_JAtIlV$t0W*l4tpWo4{Bw+olJ{tZ^p8=wg#Y#;1qhAjexf|tY@}o>WY~2@AvlbxFu(p9Jql;;iXEn`EO~g0DLDsrBWU1P&1C_6 z$L;Xia4sI~|4OnosHLr)CnTyfE0CuF@=TNJL!vZ+k@L_HuXU zitmaVFk6*6*;W8-T`KEuHI?B!P;%&<5|UCFWQUV+DNt+ym|jHOKdq`^l`)F+?39L%sJFD#cOWVS+(fMdSy7g`8);Ac7D6bqzyP)ALhhy zlcgombJuzYG~*JyhNatoa6fk0!5gODY-I~<1>m=v3hJE8^A5<7$%^r&^dnxmOzdFu zcv_wF(Ubn|S2%4WOF-izd^#$Rz^HO!KUhOqYSRlOF zJZgm;+`EsVWo2!Mxs3cZ8r*|t&`IZ8|9oA2B1QgQ?lW|T#hu|OU_a|!EU22zCCIq* z#Y#=I;Z<6=+h;o6^x;PK1-@jE6RFjXytN2aI=LK`=u-n4vi(9%fbAFVh+Qql?xxB0 zAebM37>z45FO5Pif0<2IO0u#YMr!NK)`bK6X>bi1LpR#J7YFJs`wZ~e4|Jy7Eqqxg zR|*$AdgmhYyxX8q4Q`y7N(t)0oACp)jDe90Sn=$}nwCC1wpN~cIg~?;9QjO5?3&tw zc+L1(&EZ*;sOG4Rp13XD)qTR3opX>E521QwY15+^yg%mlhaiJq->n=e!O)fEXb7@s zvj}%$sE);#RB=cY5}n_k$_CySpAnYdjP4V2t+xglT_?7GkpNDH*EF-+mAakO4dgDP zn1jyp@vSYYR%5?}EL!@R^4bEx|JExQP4qptc)M{nnxUYd*tXCaFi&t>nbOL;-Mx^% z5w<#}yu`-M=*SH|{K2SkG_=RJLRfP9qAG{~Xmj#$&;cLl7WRQTXf3pb0m&4+z3xYZ zr-6Gkb?GvbqtBOFjHSOC3SS+e7<%(A&#quVeHOo5Im>XF5wB;=Hn2S>6+%3I^4UTz zr!#?mi7h~_LWCvfNA_ah*e`8NgYZ{}dV}MW+hiS|r4D%}y#RO@_Os=)>bDsa|0MhS zv|n2&NZQ1!I?YRJDlnDTwIY>c%#Pq}suKyrsVp)0aogCvqurMpZm>@UnmtJ3h1&a; zaD?whpT(`#jY_QTdGBTg5@QzFF?kq&O6yqYI9p`%yI4U3kPPUdT0R%=)y5wzzF0?i zRq!;1B}A;MbH!p}Fxp;=&ii-p2qU%Cb7wVQDINo^VvjSIoE*7r z#(ej8GP>CFg5!2{b|Es!MdFnkK+TVf%b$kflhElgm_I)%3%Nx{*D5}G7}U~pR!r$~ zo}?huBAops6cWn626^h+I<+x2fwAKJ|dI;zWP~njblaZI?q;WKJXXR2z}cI57yV^NZ#j z7U93k`T?~58o)#$L+4vSu&Bf3F#ed3!57J}>bHPnw6#kEq0_?L?ByJ?2@=$Gm`KiY z22%DjB65q(UXG}K`R>DVO+1?rq+bZghCNK#uE`93A(1(32(kInG2MM(f@AkDwx9|h z>}Zmnnq^61ntZ8^8Rg^^DeMulzv<&`=GTGM0nr|BrJ7P+%MN4hmN7OWDTOb~5L%#EJLFcFQ~#wu0|}I6b7GCX9x$kRgoS9 z*4F%u(?3R2eejyGV_Y4wpE4*7q%^Bd5o&GqWqZ84QwA6#==Xrr9RDqxZ!j=6Ul`l7 zEbf-$ncz218Vlzyj>6J|=I4CPLTp!MFqm#hWBaw&d;4xzIUhtAlD&JF_DNl-Yrk`+ z3IUB@$qpt^gd7I$!GS*+R$q|4F8vq+mWTRB0rcW?6u)z+PJ)4vah6)6$f@~#PUL|( z;yB|b`V#4{bwcr0Oc!k7iM6UotZLNu+DC07~lgvC~~GBVYsvK(|E0AXpexs~m+kygNPNohbyDtA?D@6878J zM|cAYd~3X{Zdf3uA0ya})^9iQ1trno+)V(eJa7|XcM!E~UapUGl+xR)tiQbRg`QWx zM^)wYq<}dKZ!IZ^@-$%F^gyCoIGbZBk7^9QiL$N{Pm~62f9O}6X2nK{pR7>u7i=J4PulL*m&Yt#*07=e-?yu)tFIBb z^Tx3VA7V;%`~yb0>Ezu%=ko88mb%*3 z)-4)OG=BC)Zi-c_eAjo?`^MKK^wg?l z7VTLU6_yJs3cO=Y{)Vppu zpBC+2c6=8JS?#awj79kznOA16KMX@O1BXqk&Q99W2UKb_75%G!efGi%xSIM$GaoeSlQ@0VsosKmzU+P~T2ifK>Kkm2lOugj^gp z`56G_IV3$=xqdQt=6n9>Phz;po(R28E*OXN(aIf0ri5BYtikNk(5tZw@PazRcp-Q7 zmN+9H4t8wA@0P-`QRUr4tB9nsc@KjcthipY1`!Ew4cOs?XV;8z??y>5fI8SUxbuRN z*jUvBRix%gi7s3LP#$dTL<{|l8dQd>nW$>vDWfqAP^nSXH10(CX^KZahX5u(gmSzE zxFxWH*i%D_^30~t7k+kD9P1_NN)R5oqxRSc&}Z&T;#hqxJn?@Y03?xgv8DawNr3?kOhv*?iVN9eUJ=?9H6B zHfjre*F~mJx{w-PBI{JP$ukL)G^9Sb2ZBRb*l!urP{erAN0`Z^_jd{quzm{}tX5`^ z3p&Y2U4y+Srit6P`B%0kLG|0JsWhA}Bn!RqmtKh#>X_ccx+oi_kcgp{Hj)OP{4A_pw|xHvqJ(Z7)EE;Z|UF}i&x>2n2M{5AQ|GXFLV?A z(-jAv_fw~e@lpRH^^33iR`I}*sJG-hgtzo5%#z}S`e_>4)+BrfI%Ee{^5Z7^H93_M z9+(iNxYPn$FD!I)RrfB)j5_nR>jHmvVAGqkaT6oY1YuQe{X>h2Ua_(|gq%v&`L-88 z=13q3zITj~_+_E3?<+04D-5u5U}*BjC2qf`IsZ@@Eet$-S7YheHcMugnnEeLgDK9; z^S*`Gs^3w$WX}nxqH#=JLZbYqo!aN;V7|$!*G{D0C8XKvXae}n8Jmon{O%RhWaek` zmibEQf^p3#UKYcw#)c_3g}|C>w<^L-h9D@0laU<(PS=dPe08(dRp|+62f@(@D;V-8nV`@AXbBj9Po-Z~^24@#&!jVe z0s+B(w*=fwZH(>z-&T6G@`PO`6Ouq`$?sWd=VjAW71c&y{d7FF=y@p{1b#74xaIP? zGX7FwRl?Gm8On+qs{$%5u>+%h`@^@ifZdCURQWJ_eS6{CTrnj&NDDTfv;@gi*giEf zpaG$%X$hC?RvF4mHFZlcjXDZNW}Z;58X3eameR;`C_3_NbJ?Cd3DeR0Va+`s8wji& zOqa!LgyxL+9sgMbcld5nsup<*TLA1@?35HvMb%nw2jY%px;57H!)T?|m`xb|{@z|V zxnFNWcVB%YnkF0GtohKhnC+S-nIX=av+!Tr9`Sg*!rSYt!_|Ci;?9jU2#=T#Z=a%qZp*d34LKpP&V2LGmvvs|p2u~`fpujAR7*VDU_r^*wJ+bpt^Ny^` zOx)cF%|WT;VdDcwiMY`XSzNy-#BPh@Qg=gnGTHvVjkuTb6hD&Wro!?>fC%%Pj@~mg zZwOZYoGZy!5Jij@zZ$zwVg?!V~xws`5XV0Nt;hkuoySml88GM3mV!Eeftm&%b&D8u(@CBqG4Ge+?^#Auhe}Bo} z52FA3{NEPne?tFM_Wr+j{|oZ`7Ww{59sMtX??0)38dd*IA%9D!|J$$nPwb!mx_@J< z-#^L!vf2HU`lq|$-xM15e^J)1*8hq9Z*G8pV`n)3X$tsH?th!S|2HRt_wRDezq&R5 zB>!2&`8Rp}^WWtEt>yeD`oEpdf1?=g|KqGGNP~YXEdC|2fChT_KF*IG|9bU*04sOA AW&i*H literal 0 HcmV?d00001 diff --git a/android/Tests/KeyboardHarness/app/src/main/java/com/keyman/android/tests/keyboardHarness/MainActivity.java b/android/Tests/KeyboardHarness/app/src/main/java/com/keyman/android/tests/keyboardHarness/MainActivity.java index 9454e940b7..c09a70c0fe 100644 --- a/android/Tests/KeyboardHarness/app/src/main/java/com/keyman/android/tests/keyboardHarness/MainActivity.java +++ b/android/Tests/KeyboardHarness/app/src/main/java/com/keyman/android/tests/keyboardHarness/MainActivity.java @@ -79,6 +79,22 @@ public class MainActivity extends AppCompatActivity implements OnKeyboardEventLi KMManager.KMDefault_KeyboardFont, KMManager.KMDefault_KeyboardFont); KMManager.addKeyboard(this, platformtestKBbInfo); + + // Final K_ENTER test keyboard + Keyboard finalKBInfo = new Keyboard( + "final", + "final", + "final Keyboard", + "en", + "English", + "1.0", + "", + "", + true, + KMManager.KMDefault_KeyboardFont, + KMManager.KMDefault_KeyboardFont); + KMManager.addKeyboard(this, finalKBInfo); + } @Override From b5708a92515d8415b0f311c08fcd1adfce43f7f6 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Fri, 16 Sep 2022 11:42:16 +0700 Subject: [PATCH 53/59] chore(common): Update to Unicode 15.0 Update to Unicode 15.0 characters from https://www.unicode.org/Public/15.0.0/ucd/ --- .../unicode-character-database/Blocks.txt | 21 +- .../UnicodeData.txt | 300 +++++++++++++++++- 2 files changed, 313 insertions(+), 8 deletions(-) diff --git a/resources/standards-data/unicode-character-database/Blocks.txt b/resources/standards-data/unicode-character-database/Blocks.txt index cc5d61988b..12684594c9 100644 --- a/resources/standards-data/unicode-character-database/Blocks.txt +++ b/resources/standards-data/unicode-character-database/Blocks.txt @@ -1,10 +1,10 @@ -# Blocks-14.0.0.txt -# Date: 2021-01-22, 23:29:00 GMT [KW] -# © 2021 Unicode®, Inc. -# For terms of use, see http://www.unicode.org/terms_of_use.html +# Blocks-15.0.0.txt +# Date: 2022-01-28, 20:58:00 GMT [KW] +# © 2022 Unicode®, Inc. +# For terms of use, see https://www.unicode.org/terms_of_use.html # # Unicode Character Database -# For documentation, see http://www.unicode.org/reports/tr44/ +# For documentation, see https://www.unicode.org/reports/tr44/ # # Format: # Start Code..End Code; Block Name @@ -15,7 +15,7 @@ # and underbars are ignored. # For example, "Latin Extended-A" and "latin extended a" are equivalent. # For more information on the comparison of property values, -# see UAX #44: http://www.unicode.org/reports/tr44/ +# see UAX #44: https://www.unicode.org/reports/tr44/ # # All block ranges start with a value where (cp MOD 16) = 0, # and end with a value where (cp MOD 16) = 15. In other words, @@ -241,6 +241,7 @@ FFF0..FFFF; Specials 10D00..10D3F; Hanifi Rohingya 10E60..10E7F; Rumi Numeral Symbols 10E80..10EBF; Yezidi +10EC0..10EFF; Arabic Extended-C 10F00..10F2F; Old Sogdian 10F30..10F6F; Sogdian 10F70..10FAF; Old Uyghur @@ -272,11 +273,13 @@ FFF0..FFFF; Specials 11A50..11AAF; Soyombo 11AB0..11ABF; Unified Canadian Aboriginal Syllabics Extended-A 11AC0..11AFF; Pau Cin Hau +11B00..11B5F; Devanagari Extended-A 11C00..11C6F; Bhaiksuki 11C70..11CBF; Marchen 11D00..11D5F; Masaram Gondi 11D60..11DAF; Gunjala Gondi 11EE0..11EFF; Makasar +11F00..11F5F; Kawi 11FB0..11FBF; Lisu Supplement 11FC0..11FFF; Tamil Supplement 12000..123FF; Cuneiform @@ -284,7 +287,7 @@ FFF0..FFFF; Specials 12480..1254F; Early Dynastic Cuneiform 12F90..12FFF; Cypro-Minoan 13000..1342F; Egyptian Hieroglyphs -13430..1343F; Egyptian Hieroglyph Format Controls +13430..1345F; Egyptian Hieroglyph Format Controls 14400..1467F; Anatolian Hieroglyphs 16800..16A3F; Bamum Supplement 16A40..16A6F; Mro @@ -309,6 +312,7 @@ FFF0..FFFF; Specials 1D000..1D0FF; Byzantine Musical Symbols 1D100..1D1FF; Musical Symbols 1D200..1D24F; Ancient Greek Musical Notation +1D2C0..1D2DF; Kaktovik Numerals 1D2E0..1D2FF; Mayan Numerals 1D300..1D35F; Tai Xuan Jing Symbols 1D360..1D37F; Counting Rod Numerals @@ -316,9 +320,11 @@ FFF0..FFFF; Specials 1D800..1DAAF; Sutton SignWriting 1DF00..1DFFF; Latin Extended-G 1E000..1E02F; Glagolitic Supplement +1E030..1E08F; Cyrillic Extended-D 1E100..1E14F; Nyiakeng Puachue Hmong 1E290..1E2BF; Toto 1E2C0..1E2FF; Wancho +1E4D0..1E4FF; Nag Mundari 1E7E0..1E7FF; Ethiopic Extended-B 1E800..1E8DF; Mende Kikakui 1E900..1E95F; Adlam @@ -348,6 +354,7 @@ FFF0..FFFF; Specials 2CEB0..2EBEF; CJK Unified Ideographs Extension F 2F800..2FA1F; CJK Compatibility Ideographs Supplement 30000..3134F; CJK Unified Ideographs Extension G +31350..323AF; CJK Unified Ideographs Extension H E0000..E007F; Tags E0100..E01EF; Variation Selectors Supplement F0000..FFFFF; Supplementary Private Use Area-A diff --git a/resources/standards-data/unicode-character-database/UnicodeData.txt b/resources/standards-data/unicode-character-database/UnicodeData.txt index b5abef7ed4..ea963a7162 100644 --- a/resources/standards-data/unicode-character-database/UnicodeData.txt +++ b/resources/standards-data/unicode-character-database/UnicodeData.txt @@ -2975,6 +2975,7 @@ 0CEF;KANNADA DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 0CF1;KANNADA SIGN JIHVAMULIYA;Lo;0;L;;;;;N;;;;; 0CF2;KANNADA SIGN UPADHMANIYA;Lo;0;L;;;;;N;;;;; +0CF3;KANNADA SIGN COMBINING ANUSVARA ABOVE RIGHT;Mc;0;L;;;;;N;;;;; 0D00;MALAYALAM SIGN COMBINING ANUSVARA ABOVE;Mn;0;NSM;;;;;N;;;;; 0D01;MALAYALAM SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;; 0D02;MALAYALAM SIGN ANUSVARA;Mc;0;L;;;;;N;;;;; @@ -3339,6 +3340,7 @@ 0ECB;LAO TONE MAI CATAWA;Mn;122;NSM;;;;;N;;;;; 0ECC;LAO CANCELLATION MARK;Mn;0;NSM;;;;;N;;;;; 0ECD;LAO NIGGAHITA;Mn;0;NSM;;;;;N;;;;; +0ECE;LAO YAMAKKAN;Mn;0;NSM;;;;;N;;;;; 0ED0;LAO DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; 0ED1;LAO DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; 0ED2;LAO DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; @@ -19393,6 +19395,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 10EAD;YEZIDI HYPHENATION MARK;Pd;0;R;;;;;N;;;;; 10EB0;YEZIDI LETTER LAM WITH DOT ABOVE;Lo;0;R;;;;;N;;;;; 10EB1;YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE;Lo;0;R;;;;;N;;;;; +10EFD;ARABIC SMALL LOW WORD SAKTA;Mn;220;NSM;;;;;N;;;;; +10EFE;ARABIC SMALL LOW WORD QASR;Mn;220;NSM;;;;;N;;;;; +10EFF;ARABIC SMALL LOW WORD MADDA;Mn;220;NSM;;;;;N;;;;; 10F00;OLD SOGDIAN LETTER ALEPH;Lo;0;R;;;;;N;;;;; 10F01;OLD SOGDIAN LETTER FINAL ALEPH;Lo;0;R;;;;;N;;;;; 10F02;OLD SOGDIAN LETTER BETH;Lo;0;R;;;;;N;;;;; @@ -20058,6 +20063,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1123C;KHOJKI DOUBLE SECTION MARK;Po;0;L;;;;;N;;;;; 1123D;KHOJKI ABBREVIATION SIGN;Po;0;L;;;;;N;;;;; 1123E;KHOJKI SIGN SUKUN;Mn;0;NSM;;;;;N;;;;; +1123F;KHOJKI LETTER QA;Lo;0;L;;;;;N;;;;; +11240;KHOJKI LETTER SHORT I;Lo;0;L;;;;;N;;;;; +11241;KHOJKI VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;; 11280;MULTANI LETTER A;Lo;0;L;;;;;N;;;;; 11281;MULTANI LETTER I;Lo;0;L;;;;;N;;;;; 11282;MULTANI LETTER U;Lo;0;L;;;;;N;;;;; @@ -21256,6 +21264,16 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11AF6;PAU CIN HAU LOW-FALLING TONE LONG FINAL;Lo;0;L;;;;;N;;;;; 11AF7;PAU CIN HAU LOW-FALLING TONE FINAL;Lo;0;L;;;;;N;;;;; 11AF8;PAU CIN HAU GLOTTAL STOP FINAL;Lo;0;L;;;;;N;;;;; +11B00;DEVANAGARI HEAD MARK;Po;0;L;;;;;N;;;;; +11B01;DEVANAGARI HEAD MARK WITH HEADSTROKE;Po;0;L;;;;;N;;;;; +11B02;DEVANAGARI SIGN BHALE;Po;0;L;;;;;N;;;;; +11B03;DEVANAGARI SIGN BHALE WITH HOOK;Po;0;L;;;;;N;;;;; +11B04;DEVANAGARI SIGN EXTENDED BHALE;Po;0;L;;;;;N;;;;; +11B05;DEVANAGARI SIGN EXTENDED BHALE WITH HOOK;Po;0;L;;;;;N;;;;; +11B06;DEVANAGARI SIGN WESTERN FIVE-LIKE BHALE;Po;0;L;;;;;N;;;;; +11B07;DEVANAGARI SIGN WESTERN NINE-LIKE BHALE;Po;0;L;;;;;N;;;;; +11B08;DEVANAGARI SIGN REVERSED NINE-LIKE BHALE;Po;0;L;;;;;N;;;;; +11B09;DEVANAGARI SIGN MINDU;Po;0;L;;;;;N;;;;; 11C00;BHAIKSUKI LETTER A;Lo;0;L;;;;;N;;;;; 11C01;BHAIKSUKI LETTER AA;Lo;0;L;;;;;N;;;;; 11C02;BHAIKSUKI LETTER I;Lo;0;L;;;;;N;;;;; @@ -21584,6 +21602,92 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11EF6;MAKASAR VOWEL SIGN O;Mc;0;L;;;;;N;;;;; 11EF7;MAKASAR PASSIMBANG;Po;0;L;;;;;N;;;;; 11EF8;MAKASAR END OF SECTION;Po;0;L;;;;;N;;;;; +11F00;KAWI SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;; +11F01;KAWI SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; +11F02;KAWI SIGN REPHA;Lo;0;L;;;;;N;;;;; +11F03;KAWI SIGN VISARGA;Mc;0;L;;;;;N;;;;; +11F04;KAWI LETTER A;Lo;0;L;;;;;N;;;;; +11F05;KAWI LETTER AA;Lo;0;L;;;;;N;;;;; +11F06;KAWI LETTER I;Lo;0;L;;;;;N;;;;; +11F07;KAWI LETTER II;Lo;0;L;;;;;N;;;;; +11F08;KAWI LETTER U;Lo;0;L;;;;;N;;;;; +11F09;KAWI LETTER UU;Lo;0;L;;;;;N;;;;; +11F0A;KAWI LETTER VOCALIC R;Lo;0;L;;;;;N;;;;; +11F0B;KAWI LETTER VOCALIC RR;Lo;0;L;;;;;N;;;;; +11F0C;KAWI LETTER VOCALIC L;Lo;0;L;;;;;N;;;;; +11F0D;KAWI LETTER VOCALIC LL;Lo;0;L;;;;;N;;;;; +11F0E;KAWI LETTER E;Lo;0;L;;;;;N;;;;; +11F0F;KAWI LETTER AI;Lo;0;L;;;;;N;;;;; +11F10;KAWI LETTER O;Lo;0;L;;;;;N;;;;; +11F12;KAWI LETTER KA;Lo;0;L;;;;;N;;;;; +11F13;KAWI LETTER KHA;Lo;0;L;;;;;N;;;;; +11F14;KAWI LETTER GA;Lo;0;L;;;;;N;;;;; +11F15;KAWI LETTER GHA;Lo;0;L;;;;;N;;;;; +11F16;KAWI LETTER NGA;Lo;0;L;;;;;N;;;;; +11F17;KAWI LETTER CA;Lo;0;L;;;;;N;;;;; +11F18;KAWI LETTER CHA;Lo;0;L;;;;;N;;;;; +11F19;KAWI LETTER JA;Lo;0;L;;;;;N;;;;; +11F1A;KAWI LETTER JHA;Lo;0;L;;;;;N;;;;; +11F1B;KAWI LETTER NYA;Lo;0;L;;;;;N;;;;; +11F1C;KAWI LETTER TTA;Lo;0;L;;;;;N;;;;; +11F1D;KAWI LETTER TTHA;Lo;0;L;;;;;N;;;;; +11F1E;KAWI LETTER DDA;Lo;0;L;;;;;N;;;;; +11F1F;KAWI LETTER DDHA;Lo;0;L;;;;;N;;;;; +11F20;KAWI LETTER NNA;Lo;0;L;;;;;N;;;;; +11F21;KAWI LETTER TA;Lo;0;L;;;;;N;;;;; +11F22;KAWI LETTER THA;Lo;0;L;;;;;N;;;;; +11F23;KAWI LETTER DA;Lo;0;L;;;;;N;;;;; +11F24;KAWI LETTER DHA;Lo;0;L;;;;;N;;;;; +11F25;KAWI LETTER NA;Lo;0;L;;;;;N;;;;; +11F26;KAWI LETTER PA;Lo;0;L;;;;;N;;;;; +11F27;KAWI LETTER PHA;Lo;0;L;;;;;N;;;;; +11F28;KAWI LETTER BA;Lo;0;L;;;;;N;;;;; +11F29;KAWI LETTER BHA;Lo;0;L;;;;;N;;;;; +11F2A;KAWI LETTER MA;Lo;0;L;;;;;N;;;;; +11F2B;KAWI LETTER YA;Lo;0;L;;;;;N;;;;; +11F2C;KAWI LETTER RA;Lo;0;L;;;;;N;;;;; +11F2D;KAWI LETTER LA;Lo;0;L;;;;;N;;;;; +11F2E;KAWI LETTER WA;Lo;0;L;;;;;N;;;;; +11F2F;KAWI LETTER SHA;Lo;0;L;;;;;N;;;;; +11F30;KAWI LETTER SSA;Lo;0;L;;;;;N;;;;; +11F31;KAWI LETTER SA;Lo;0;L;;;;;N;;;;; +11F32;KAWI LETTER HA;Lo;0;L;;;;;N;;;;; +11F33;KAWI LETTER JNYA;Lo;0;L;;;;;N;;;;; +11F34;KAWI VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +11F35;KAWI VOWEL SIGN ALTERNATE AA;Mc;0;L;;;;;N;;;;; +11F36;KAWI VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +11F37;KAWI VOWEL SIGN II;Mn;0;NSM;;;;;N;;;;; +11F38;KAWI VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +11F39;KAWI VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;; +11F3A;KAWI VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;; +11F3E;KAWI VOWEL SIGN E;Mc;0;L;;;;;N;;;;; +11F3F;KAWI VOWEL SIGN AI;Mc;0;L;;;;;N;;;;; +11F40;KAWI VOWEL SIGN EU;Mn;0;NSM;;;;;N;;;;; +11F41;KAWI SIGN KILLER;Mc;9;L;;;;;N;;;;; +11F42;KAWI CONJOINER;Mn;9;NSM;;;;;N;;;;; +11F43;KAWI DANDA;Po;0;L;;;;;N;;;;; +11F44;KAWI DOUBLE DANDA;Po;0;L;;;;;N;;;;; +11F45;KAWI PUNCTUATION SECTION MARKER;Po;0;L;;;;;N;;;;; +11F46;KAWI PUNCTUATION ALTERNATE SECTION MARKER;Po;0;L;;;;;N;;;;; +11F47;KAWI PUNCTUATION FLOWER;Po;0;L;;;;;N;;;;; +11F48;KAWI PUNCTUATION SPACE FILLER;Po;0;L;;;;;N;;;;; +11F49;KAWI PUNCTUATION DOT;Po;0;L;;;;;N;;;;; +11F4A;KAWI PUNCTUATION DOUBLE DOT;Po;0;L;;;;;N;;;;; +11F4B;KAWI PUNCTUATION TRIPLE DOT;Po;0;L;;;;;N;;;;; +11F4C;KAWI PUNCTUATION CIRCLE;Po;0;L;;;;;N;;;;; +11F4D;KAWI PUNCTUATION FILLED CIRCLE;Po;0;L;;;;;N;;;;; +11F4E;KAWI PUNCTUATION SPIRAL;Po;0;L;;;;;N;;;;; +11F4F;KAWI PUNCTUATION CLOSING SPIRAL;Po;0;L;;;;;N;;;;; +11F50;KAWI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +11F51;KAWI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +11F52;KAWI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +11F53;KAWI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +11F54;KAWI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +11F55;KAWI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +11F56;KAWI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +11F57;KAWI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +11F58;KAWI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +11F59;KAWI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 11FB0;LISU LETTER YHA;Lo;0;L;;;;;N;;;;; 11FC0;TAMIL FRACTION ONE THREE-HUNDRED-AND-TWENTIETH;No;0;L;;;;1/320;N;;;;; 11FC1;TAMIL FRACTION ONE ONE-HUNDRED-AND-SIXTIETH;No;0;L;;;;1/160;N;;;;; @@ -24040,6 +24144,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1342C;EGYPTIAN HIEROGLYPH AA030;Lo;0;L;;;;;N;;;;; 1342D;EGYPTIAN HIEROGLYPH AA031;Lo;0;L;;;;;N;;;;; 1342E;EGYPTIAN HIEROGLYPH AA032;Lo;0;L;;;;;N;;;;; +1342F;EGYPTIAN HIEROGLYPH V011D;Lo;0;L;;;;;N;;;;; 13430;EGYPTIAN HIEROGLYPH VERTICAL JOINER;Cf;0;L;;;;;N;;;;; 13431;EGYPTIAN HIEROGLYPH HORIZONTAL JOINER;Cf;0;L;;;;;N;;;;; 13432;EGYPTIAN HIEROGLYPH INSERT AT TOP START;Cf;0;L;;;;;N;;;;; @@ -24049,6 +24154,35 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 13436;EGYPTIAN HIEROGLYPH OVERLAY MIDDLE;Cf;0;L;;;;;N;;;;; 13437;EGYPTIAN HIEROGLYPH BEGIN SEGMENT;Cf;0;L;;;;;N;;;;; 13438;EGYPTIAN HIEROGLYPH END SEGMENT;Cf;0;L;;;;;N;;;;; +13439;EGYPTIAN HIEROGLYPH INSERT AT MIDDLE;Cf;0;L;;;;;N;;;;; +1343A;EGYPTIAN HIEROGLYPH INSERT AT TOP;Cf;0;L;;;;;N;;;;; +1343B;EGYPTIAN HIEROGLYPH INSERT AT BOTTOM;Cf;0;L;;;;;N;;;;; +1343C;EGYPTIAN HIEROGLYPH BEGIN ENCLOSURE;Cf;0;L;;;;;N;;;;; +1343D;EGYPTIAN HIEROGLYPH END ENCLOSURE;Cf;0;L;;;;;N;;;;; +1343E;EGYPTIAN HIEROGLYPH BEGIN WALLED ENCLOSURE;Cf;0;L;;;;;N;;;;; +1343F;EGYPTIAN HIEROGLYPH END WALLED ENCLOSURE;Cf;0;L;;;;;N;;;;; +13440;EGYPTIAN HIEROGLYPH MIRROR HORIZONTALLY;Mn;0;NSM;;;;;N;;;;; +13441;EGYPTIAN HIEROGLYPH FULL BLANK;Lo;0;L;;;;;N;;;;; +13442;EGYPTIAN HIEROGLYPH HALF BLANK;Lo;0;L;;;;;N;;;;; +13443;EGYPTIAN HIEROGLYPH LOST SIGN;Lo;0;L;;;;;N;;;;; +13444;EGYPTIAN HIEROGLYPH HALF LOST SIGN;Lo;0;L;;;;;N;;;;; +13445;EGYPTIAN HIEROGLYPH TALL LOST SIGN;Lo;0;L;;;;;N;;;;; +13446;EGYPTIAN HIEROGLYPH WIDE LOST SIGN;Lo;0;L;;;;;N;;;;; +13447;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START;Mn;0;NSM;;;;;N;;;;; +13448;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM START;Mn;0;NSM;;;;;N;;;;; +13449;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT START;Mn;0;NSM;;;;;N;;;;; +1344A;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP END;Mn;0;NSM;;;;;N;;;;; +1344B;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP;Mn;0;NSM;;;;;N;;;;; +1344C;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM START AND TOP END;Mn;0;NSM;;;;;N;;;;; +1344D;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT START AND TOP;Mn;0;NSM;;;;;N;;;;; +1344E;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM END;Mn;0;NSM;;;;;N;;;;; +1344F;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START AND BOTTOM END;Mn;0;NSM;;;;;N;;;;; +13450;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM;Mn;0;NSM;;;;;N;;;;; +13451;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT START AND BOTTOM;Mn;0;NSM;;;;;N;;;;; +13452;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT END;Mn;0;NSM;;;;;N;;;;; +13453;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP AND END;Mn;0;NSM;;;;;N;;;;; +13454;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM AND END;Mn;0;NSM;;;;;N;;;;; +13455;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED;Mn;0;NSM;;;;;N;;;;; 14400;ANATOLIAN HIEROGLYPH A001;Lo;0;L;;;;;N;;;;; 14401;ANATOLIAN HIEROGLYPH A002;Lo;0;L;;;;;N;;;;; 14402;ANATOLIAN HIEROGLYPH A003;Lo;0;L;;;;;N;;;;; @@ -27289,9 +27423,11 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1B120;KATAKANA LETTER ARCHAIC YI;Lo;0;L;;;;;N;;;;; 1B121;KATAKANA LETTER ARCHAIC YE;Lo;0;L;;;;;N;;;;; 1B122;KATAKANA LETTER ARCHAIC WU;Lo;0;L;;;;;N;;;;; +1B132;HIRAGANA LETTER SMALL KO;Lo;0;L;;;;;N;;;;; 1B150;HIRAGANA LETTER SMALL WI;Lo;0;L;;;;;N;;;;; 1B151;HIRAGANA LETTER SMALL WE;Lo;0;L;;;;;N;;;;; 1B152;HIRAGANA LETTER SMALL WO;Lo;0;L;;;;;N;;;;; +1B155;KATAKANA LETTER SMALL KO;Lo;0;L;;;;;N;;;;; 1B164;KATAKANA LETTER SMALL WI;Lo;0;L;;;;;N;;;;; 1B165;KATAKANA LETTER SMALL WE;Lo;0;L;;;;;N;;;;; 1B166;KATAKANA LETTER SMALL WO;Lo;0;L;;;;;N;;;;; @@ -28573,6 +28709,26 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1D243;COMBINING GREEK MUSICAL TETRASEME;Mn;230;NSM;;;;;N;;;;; 1D244;COMBINING GREEK MUSICAL PENTASEME;Mn;230;NSM;;;;;N;;;;; 1D245;GREEK MUSICAL LEIMMA;So;0;ON;;;;;N;;;;; +1D2C0;KAKTOVIK NUMERAL ZERO;No;0;L;;;;0;N;;;;; +1D2C1;KAKTOVIK NUMERAL ONE;No;0;L;;;;1;N;;;;; +1D2C2;KAKTOVIK NUMERAL TWO;No;0;L;;;;2;N;;;;; +1D2C3;KAKTOVIK NUMERAL THREE;No;0;L;;;;3;N;;;;; +1D2C4;KAKTOVIK NUMERAL FOUR;No;0;L;;;;4;N;;;;; +1D2C5;KAKTOVIK NUMERAL FIVE;No;0;L;;;;5;N;;;;; +1D2C6;KAKTOVIK NUMERAL SIX;No;0;L;;;;6;N;;;;; +1D2C7;KAKTOVIK NUMERAL SEVEN;No;0;L;;;;7;N;;;;; +1D2C8;KAKTOVIK NUMERAL EIGHT;No;0;L;;;;8;N;;;;; +1D2C9;KAKTOVIK NUMERAL NINE;No;0;L;;;;9;N;;;;; +1D2CA;KAKTOVIK NUMERAL TEN;No;0;L;;;;10;N;;;;; +1D2CB;KAKTOVIK NUMERAL ELEVEN;No;0;L;;;;11;N;;;;; +1D2CC;KAKTOVIK NUMERAL TWELVE;No;0;L;;;;12;N;;;;; +1D2CD;KAKTOVIK NUMERAL THIRTEEN;No;0;L;;;;13;N;;;;; +1D2CE;KAKTOVIK NUMERAL FOURTEEN;No;0;L;;;;14;N;;;;; +1D2CF;KAKTOVIK NUMERAL FIFTEEN;No;0;L;;;;15;N;;;;; +1D2D0;KAKTOVIK NUMERAL SIXTEEN;No;0;L;;;;16;N;;;;; +1D2D1;KAKTOVIK NUMERAL SEVENTEEN;No;0;L;;;;17;N;;;;; +1D2D2;KAKTOVIK NUMERAL EIGHTEEN;No;0;L;;;;18;N;;;;; +1D2D3;KAKTOVIK NUMERAL NINETEEN;No;0;L;;;;19;N;;;;; 1D2E0;MAYAN NUMERAL ZERO;No;0;L;;;;0;N;;;;; 1D2E1;MAYAN NUMERAL ONE;No;0;L;;;;1;N;;;;; 1D2E2;MAYAN NUMERAL TWO;No;0;L;;;;2;N;;;;; @@ -30404,6 +30560,12 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1DF1C;LATIN SMALL LETTER TESH DIGRAPH WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;; 1DF1D;LATIN SMALL LETTER C WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;; 1DF1E;LATIN SMALL LETTER S WITH CURL;Ll;0;L;;;;;N;;;;; +1DF25;LATIN SMALL LETTER D WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; +1DF26;LATIN SMALL LETTER L WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; +1DF27;LATIN SMALL LETTER N WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; +1DF28;LATIN SMALL LETTER R WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; +1DF29;LATIN SMALL LETTER S WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; +1DF2A;LATIN SMALL LETTER T WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; 1E000;COMBINING GLAGOLITIC LETTER AZU;Mn;230;NSM;;;;;N;;;;; 1E001;COMBINING GLAGOLITIC LETTER BUKY;Mn;230;NSM;;;;;N;;;;; 1E002;COMBINING GLAGOLITIC LETTER VEDE;Mn;230;NSM;;;;;N;;;;; @@ -30442,6 +30604,69 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1E028;COMBINING GLAGOLITIC LETTER BIG YUS;Mn;230;NSM;;;;;N;;;;; 1E029;COMBINING GLAGOLITIC LETTER IOTATED BIG YUS;Mn;230;NSM;;;;;N;;;;; 1E02A;COMBINING GLAGOLITIC LETTER FITA;Mn;230;NSM;;;;;N;;;;; +1E030;MODIFIER LETTER CYRILLIC SMALL A;Lm;0;L; 0430;;;;N;;;;; +1E031;MODIFIER LETTER CYRILLIC SMALL BE;Lm;0;L; 0431;;;;N;;;;; +1E032;MODIFIER LETTER CYRILLIC SMALL VE;Lm;0;L; 0432;;;;N;;;;; +1E033;MODIFIER LETTER CYRILLIC SMALL GHE;Lm;0;L; 0433;;;;N;;;;; +1E034;MODIFIER LETTER CYRILLIC SMALL DE;Lm;0;L; 0434;;;;N;;;;; +1E035;MODIFIER LETTER CYRILLIC SMALL IE;Lm;0;L; 0435;;;;N;;;;; +1E036;MODIFIER LETTER CYRILLIC SMALL ZHE;Lm;0;L; 0436;;;;N;;;;; +1E037;MODIFIER LETTER CYRILLIC SMALL ZE;Lm;0;L; 0437;;;;N;;;;; +1E038;MODIFIER LETTER CYRILLIC SMALL I;Lm;0;L; 0438;;;;N;;;;; +1E039;MODIFIER LETTER CYRILLIC SMALL KA;Lm;0;L; 043A;;;;N;;;;; +1E03A;MODIFIER LETTER CYRILLIC SMALL EL;Lm;0;L; 043B;;;;N;;;;; +1E03B;MODIFIER LETTER CYRILLIC SMALL EM;Lm;0;L; 043C;;;;N;;;;; +1E03C;MODIFIER LETTER CYRILLIC SMALL O;Lm;0;L; 043E;;;;N;;;;; +1E03D;MODIFIER LETTER CYRILLIC SMALL PE;Lm;0;L; 043F;;;;N;;;;; +1E03E;MODIFIER LETTER CYRILLIC SMALL ER;Lm;0;L; 0440;;;;N;;;;; +1E03F;MODIFIER LETTER CYRILLIC SMALL ES;Lm;0;L; 0441;;;;N;;;;; +1E040;MODIFIER LETTER CYRILLIC SMALL TE;Lm;0;L; 0442;;;;N;;;;; +1E041;MODIFIER LETTER CYRILLIC SMALL U;Lm;0;L; 0443;;;;N;;;;; +1E042;MODIFIER LETTER CYRILLIC SMALL EF;Lm;0;L; 0444;;;;N;;;;; +1E043;MODIFIER LETTER CYRILLIC SMALL HA;Lm;0;L; 0445;;;;N;;;;; +1E044;MODIFIER LETTER CYRILLIC SMALL TSE;Lm;0;L; 0446;;;;N;;;;; +1E045;MODIFIER LETTER CYRILLIC SMALL CHE;Lm;0;L; 0447;;;;N;;;;; +1E046;MODIFIER LETTER CYRILLIC SMALL SHA;Lm;0;L; 0448;;;;N;;;;; +1E047;MODIFIER LETTER CYRILLIC SMALL YERU;Lm;0;L; 044B;;;;N;;;;; +1E048;MODIFIER LETTER CYRILLIC SMALL E;Lm;0;L; 044D;;;;N;;;;; +1E049;MODIFIER LETTER CYRILLIC SMALL YU;Lm;0;L; 044E;;;;N;;;;; +1E04A;MODIFIER LETTER CYRILLIC SMALL DZZE;Lm;0;L; A689;;;;N;;;;; +1E04B;MODIFIER LETTER CYRILLIC SMALL SCHWA;Lm;0;L; 04D9;;;;N;;;;; +1E04C;MODIFIER LETTER CYRILLIC SMALL BYELORUSSIAN-UKRAINIAN I;Lm;0;L; 0456;;;;N;;;;; +1E04D;MODIFIER LETTER CYRILLIC SMALL JE;Lm;0;L; 0458;;;;N;;;;; +1E04E;MODIFIER LETTER CYRILLIC SMALL BARRED O;Lm;0;L; 04E9;;;;N;;;;; +1E04F;MODIFIER LETTER CYRILLIC SMALL STRAIGHT U;Lm;0;L; 04AF;;;;N;;;;; +1E050;MODIFIER LETTER CYRILLIC SMALL PALOCHKA;Lm;0;L; 04CF;;;;N;;;;; +1E051;CYRILLIC SUBSCRIPT SMALL LETTER A;Lm;0;L; 0430;;;;N;;;;; +1E052;CYRILLIC SUBSCRIPT SMALL LETTER BE;Lm;0;L; 0431;;;;N;;;;; +1E053;CYRILLIC SUBSCRIPT SMALL LETTER VE;Lm;0;L; 0432;;;;N;;;;; +1E054;CYRILLIC SUBSCRIPT SMALL LETTER GHE;Lm;0;L; 0433;;;;N;;;;; +1E055;CYRILLIC SUBSCRIPT SMALL LETTER DE;Lm;0;L; 0434;;;;N;;;;; +1E056;CYRILLIC SUBSCRIPT SMALL LETTER IE;Lm;0;L; 0435;;;;N;;;;; +1E057;CYRILLIC SUBSCRIPT SMALL LETTER ZHE;Lm;0;L; 0436;;;;N;;;;; +1E058;CYRILLIC SUBSCRIPT SMALL LETTER ZE;Lm;0;L; 0437;;;;N;;;;; +1E059;CYRILLIC SUBSCRIPT SMALL LETTER I;Lm;0;L; 0438;;;;N;;;;; +1E05A;CYRILLIC SUBSCRIPT SMALL LETTER KA;Lm;0;L; 043A;;;;N;;;;; +1E05B;CYRILLIC SUBSCRIPT SMALL LETTER EL;Lm;0;L; 043B;;;;N;;;;; +1E05C;CYRILLIC SUBSCRIPT SMALL LETTER O;Lm;0;L; 043E;;;;N;;;;; +1E05D;CYRILLIC SUBSCRIPT SMALL LETTER PE;Lm;0;L; 043F;;;;N;;;;; +1E05E;CYRILLIC SUBSCRIPT SMALL LETTER ES;Lm;0;L; 0441;;;;N;;;;; +1E05F;CYRILLIC SUBSCRIPT SMALL LETTER U;Lm;0;L; 0443;;;;N;;;;; +1E060;CYRILLIC SUBSCRIPT SMALL LETTER EF;Lm;0;L; 0444;;;;N;;;;; +1E061;CYRILLIC SUBSCRIPT SMALL LETTER HA;Lm;0;L; 0445;;;;N;;;;; +1E062;CYRILLIC SUBSCRIPT SMALL LETTER TSE;Lm;0;L; 0446;;;;N;;;;; +1E063;CYRILLIC SUBSCRIPT SMALL LETTER CHE;Lm;0;L; 0447;;;;N;;;;; +1E064;CYRILLIC SUBSCRIPT SMALL LETTER SHA;Lm;0;L; 0448;;;;N;;;;; +1E065;CYRILLIC SUBSCRIPT SMALL LETTER HARD SIGN;Lm;0;L; 044A;;;;N;;;;; +1E066;CYRILLIC SUBSCRIPT SMALL LETTER YERU;Lm;0;L; 044B;;;;N;;;;; +1E067;CYRILLIC SUBSCRIPT SMALL LETTER GHE WITH UPTURN;Lm;0;L; 0491;;;;N;;;;; +1E068;CYRILLIC SUBSCRIPT SMALL LETTER BYELORUSSIAN-UKRAINIAN I;Lm;0;L; 0456;;;;N;;;;; +1E069;CYRILLIC SUBSCRIPT SMALL LETTER DZE;Lm;0;L; 0455;;;;N;;;;; +1E06A;CYRILLIC SUBSCRIPT SMALL LETTER DZHE;Lm;0;L; 045F;;;;N;;;;; +1E06B;MODIFIER LETTER CYRILLIC SMALL ES WITH DESCENDER;Lm;0;L; 04AB;;;;N;;;;; +1E06C;MODIFIER LETTER CYRILLIC SMALL YERU WITH BACK YER;Lm;0;L; A651;;;;N;;;;; +1E06D;MODIFIER LETTER CYRILLIC SMALL STRAIGHT U WITH STROKE;Lm;0;L; 04B1;;;;N;;;;; +1E08F;COMBINING CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I;Mn;230;NSM;;;;;N;;;;; 1E100;NYIAKENG PUACHUE HMONG LETTER MA;Lo;0;L;;;;;N;;;;; 1E101;NYIAKENG PUACHUE HMONG LETTER TSA;Lo;0;L;;;;;N;;;;; 1E102;NYIAKENG PUACHUE HMONG LETTER NTA;Lo;0;L;;;;;N;;;;; @@ -30603,6 +30828,48 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1E2F8;WANCHO DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; 1E2F9;WANCHO DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 1E2FF;WANCHO NGUN SIGN;Sc;0;ET;;;;;N;;;;; +1E4D0;NAG MUNDARI LETTER O;Lo;0;L;;;;;N;;;;; +1E4D1;NAG MUNDARI LETTER OP;Lo;0;L;;;;;N;;;;; +1E4D2;NAG MUNDARI LETTER OL;Lo;0;L;;;;;N;;;;; +1E4D3;NAG MUNDARI LETTER OY;Lo;0;L;;;;;N;;;;; +1E4D4;NAG MUNDARI LETTER ONG;Lo;0;L;;;;;N;;;;; +1E4D5;NAG MUNDARI LETTER A;Lo;0;L;;;;;N;;;;; +1E4D6;NAG MUNDARI LETTER AJ;Lo;0;L;;;;;N;;;;; +1E4D7;NAG MUNDARI LETTER AB;Lo;0;L;;;;;N;;;;; +1E4D8;NAG MUNDARI LETTER ANY;Lo;0;L;;;;;N;;;;; +1E4D9;NAG MUNDARI LETTER AH;Lo;0;L;;;;;N;;;;; +1E4DA;NAG MUNDARI LETTER I;Lo;0;L;;;;;N;;;;; +1E4DB;NAG MUNDARI LETTER IS;Lo;0;L;;;;;N;;;;; +1E4DC;NAG MUNDARI LETTER IDD;Lo;0;L;;;;;N;;;;; +1E4DD;NAG MUNDARI LETTER IT;Lo;0;L;;;;;N;;;;; +1E4DE;NAG MUNDARI LETTER IH;Lo;0;L;;;;;N;;;;; +1E4DF;NAG MUNDARI LETTER U;Lo;0;L;;;;;N;;;;; +1E4E0;NAG MUNDARI LETTER UC;Lo;0;L;;;;;N;;;;; +1E4E1;NAG MUNDARI LETTER UD;Lo;0;L;;;;;N;;;;; +1E4E2;NAG MUNDARI LETTER UK;Lo;0;L;;;;;N;;;;; +1E4E3;NAG MUNDARI LETTER UR;Lo;0;L;;;;;N;;;;; +1E4E4;NAG MUNDARI LETTER E;Lo;0;L;;;;;N;;;;; +1E4E5;NAG MUNDARI LETTER ENN;Lo;0;L;;;;;N;;;;; +1E4E6;NAG MUNDARI LETTER EG;Lo;0;L;;;;;N;;;;; +1E4E7;NAG MUNDARI LETTER EM;Lo;0;L;;;;;N;;;;; +1E4E8;NAG MUNDARI LETTER EN;Lo;0;L;;;;;N;;;;; +1E4E9;NAG MUNDARI LETTER ETT;Lo;0;L;;;;;N;;;;; +1E4EA;NAG MUNDARI LETTER ELL;Lo;0;L;;;;;N;;;;; +1E4EB;NAG MUNDARI SIGN OJOD;Lm;0;L;;;;;N;;;;; +1E4EC;NAG MUNDARI SIGN MUHOR;Mn;232;NSM;;;;;N;;;;; +1E4ED;NAG MUNDARI SIGN TOYOR;Mn;232;NSM;;;;;N;;;;; +1E4EE;NAG MUNDARI SIGN IKIR;Mn;220;NSM;;;;;N;;;;; +1E4EF;NAG MUNDARI SIGN SUTUH;Mn;230;NSM;;;;;N;;;;; +1E4F0;NAG MUNDARI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +1E4F1;NAG MUNDARI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +1E4F2;NAG MUNDARI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +1E4F3;NAG MUNDARI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +1E4F4;NAG MUNDARI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +1E4F5;NAG MUNDARI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +1E4F6;NAG MUNDARI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +1E4F7;NAG MUNDARI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +1E4F8;NAG MUNDARI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +1E4F9;NAG MUNDARI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 1E7E0;ETHIOPIC SYLLABLE HHYA;Lo;0;L;;;;;N;;;;; 1E7E1;ETHIOPIC SYLLABLE HHYU;Lo;0;L;;;;;N;;;;; 1E7E2;ETHIOPIC SYLLABLE HHYI;Lo;0;L;;;;;N;;;;; @@ -32678,6 +32945,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F6D5;HINDU TEMPLE;So;0;ON;;;;;N;;;;; 1F6D6;HUT;So;0;ON;;;;;N;;;;; 1F6D7;ELEVATOR;So;0;ON;;;;;N;;;;; +1F6DC;WIRELESS;So;0;ON;;;;;N;;;;; 1F6DD;PLAYGROUND SLIDE;So;0;ON;;;;;N;;;;; 1F6DE;WHEEL;So;0;ON;;;;;N;;;;; 1F6DF;RING BUOY;So;0;ON;;;;;N;;;;; @@ -32823,6 +33091,14 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F771;ALCHEMICAL SYMBOL FOR MONTH;So;0;ON;;;;;N;;;;; 1F772;ALCHEMICAL SYMBOL FOR HALF DRAM;So;0;ON;;;;;N;;;;; 1F773;ALCHEMICAL SYMBOL FOR HALF OUNCE;So;0;ON;;;;;N;;;;; +1F774;LOT OF FORTUNE;So;0;ON;;;;;N;;;;; +1F775;OCCULTATION;So;0;ON;;;;;N;;;;; +1F776;LUNAR ECLIPSE;So;0;ON;;;;;N;;;;; +1F77B;HAUMEA;So;0;ON;;;;;N;;;;; +1F77C;MAKEMAKE;So;0;ON;;;;;N;;;;; +1F77D;GONGGONG;So;0;ON;;;;;N;;;;; +1F77E;QUAOAR;So;0;ON;;;;;N;;;;; +1F77F;ORCUS;So;0;ON;;;;;N;;;;; 1F780;BLACK LEFT-POINTING ISOSCELES RIGHT TRIANGLE;So;0;ON;;;;;N;;;;; 1F781;BLACK UP-POINTING ISOSCELES RIGHT TRIANGLE;So;0;ON;;;;;N;;;;; 1F782;BLACK RIGHT-POINTING ISOSCELES RIGHT TRIANGLE;So;0;ON;;;;;N;;;;; @@ -32912,6 +33188,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F7D6;NEGATIVE CIRCLED TRIANGLE;So;0;ON;;;;;N;;;;; 1F7D7;CIRCLED SQUARE;So;0;ON;;;;;N;;;;; 1F7D8;NEGATIVE CIRCLED SQUARE;So;0;ON;;;;;N;;;;; +1F7D9;NINE POINTED WHITE STAR;So;0;ON;;;;;N;;;;; 1F7E0;LARGE ORANGE CIRCLE;So;0;ON;;;;;N;;;;; 1F7E1;LARGE YELLOW CIRCLE;So;0;ON;;;;;N;;;;; 1F7E2;LARGE GREEN CIRCLE;So;0;ON;;;;;N;;;;; @@ -33434,6 +33711,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FA72;BRIEFS;So;0;ON;;;;;N;;;;; 1FA73;SHORTS;So;0;ON;;;;;N;;;;; 1FA74;THONG SANDAL;So;0;ON;;;;;N;;;;; +1FA75;LIGHT BLUE HEART;So;0;ON;;;;;N;;;;; +1FA76;GREY HEART;So;0;ON;;;;;N;;;;; +1FA77;PINK HEART;So;0;ON;;;;;N;;;;; 1FA78;DROP OF BLOOD;So;0;ON;;;;;N;;;;; 1FA79;ADHESIVE BANDAGE;So;0;ON;;;;;N;;;;; 1FA7A;STETHOSCOPE;So;0;ON;;;;;N;;;;; @@ -33446,6 +33726,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FA84;MAGIC WAND;So;0;ON;;;;;N;;;;; 1FA85;PINATA;So;0;ON;;;;;N;;;;; 1FA86;NESTING DOLLS;So;0;ON;;;;;N;;;;; +1FA87;MARACAS;So;0;ON;;;;;N;;;;; +1FA88;FLUTE;So;0;ON;;;;;N;;;;; 1FA90;RINGED PLANET;So;0;ON;;;;;N;;;;; 1FA91;CHAIR;So;0;ON;;;;;N;;;;; 1FA92;RAZOR;So;0;ON;;;;;N;;;;; @@ -33475,6 +33757,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAAA;IDENTIFICATION CARD;So;0;ON;;;;;N;;;;; 1FAAB;LOW BATTERY;So;0;ON;;;;;N;;;;; 1FAAC;HAMSA;So;0;ON;;;;;N;;;;; +1FAAD;FOLDING HAND FAN;So;0;ON;;;;;N;;;;; +1FAAE;HAIR PICK;So;0;ON;;;;;N;;;;; +1FAAF;KHANDA;So;0;ON;;;;;N;;;;; 1FAB0;FLY;So;0;ON;;;;;N;;;;; 1FAB1;WORM;So;0;ON;;;;;N;;;;; 1FAB2;BEETLE;So;0;ON;;;;;N;;;;; @@ -33486,12 +33771,18 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAB8;CORAL;So;0;ON;;;;;N;;;;; 1FAB9;EMPTY NEST;So;0;ON;;;;;N;;;;; 1FABA;NEST WITH EGGS;So;0;ON;;;;;N;;;;; +1FABB;HYACINTH;So;0;ON;;;;;N;;;;; +1FABC;JELLYFISH;So;0;ON;;;;;N;;;;; +1FABD;WING;So;0;ON;;;;;N;;;;; +1FABF;GOOSE;So;0;ON;;;;;N;;;;; 1FAC0;ANATOMICAL HEART;So;0;ON;;;;;N;;;;; 1FAC1;LUNGS;So;0;ON;;;;;N;;;;; 1FAC2;PEOPLE HUGGING;So;0;ON;;;;;N;;;;; 1FAC3;PREGNANT MAN;So;0;ON;;;;;N;;;;; 1FAC4;PREGNANT PERSON;So;0;ON;;;;;N;;;;; 1FAC5;PERSON WITH CROWN;So;0;ON;;;;;N;;;;; +1FACE;MOOSE;So;0;ON;;;;;N;;;;; +1FACF;DONKEY;So;0;ON;;;;;N;;;;; 1FAD0;BLUEBERRIES;So;0;ON;;;;;N;;;;; 1FAD1;BELL PEPPER;So;0;ON;;;;;N;;;;; 1FAD2;OLIVE;So;0;ON;;;;;N;;;;; @@ -33502,6 +33793,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAD7;POURING LIQUID;So;0;ON;;;;;N;;;;; 1FAD8;BEANS;So;0;ON;;;;;N;;;;; 1FAD9;JAR;So;0;ON;;;;;N;;;;; +1FADA;GINGER ROOT;So;0;ON;;;;;N;;;;; +1FADB;PEA POD;So;0;ON;;;;;N;;;;; 1FAE0;MELTING FACE;So;0;ON;;;;;N;;;;; 1FAE1;SALUTING FACE;So;0;ON;;;;;N;;;;; 1FAE2;FACE WITH OPEN EYES AND HAND OVER MOUTH;So;0;ON;;;;;N;;;;; @@ -33510,6 +33803,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAE5;DOTTED LINE FACE;So;0;ON;;;;;N;;;;; 1FAE6;BITING LIP;So;0;ON;;;;;N;;;;; 1FAE7;BUBBLES;So;0;ON;;;;;N;;;;; +1FAE8;SHAKING FACE;So;0;ON;;;;;N;;;;; 1FAF0;HAND WITH INDEX FINGER AND THUMB CROSSED;So;0;ON;;;;;N;;;;; 1FAF1;RIGHTWARDS HAND;So;0;ON;;;;;N;;;;; 1FAF2;LEFTWARDS HAND;So;0;ON;;;;;N;;;;; @@ -33517,6 +33811,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAF4;PALM UP HAND;So;0;ON;;;;;N;;;;; 1FAF5;INDEX POINTING AT THE VIEWER;So;0;ON;;;;;N;;;;; 1FAF6;HEART HANDS;So;0;ON;;;;;N;;;;; +1FAF7;LEFTWARDS PUSHING HAND;So;0;ON;;;;;N;;;;; +1FAF8;RIGHTWARDS PUSHING HAND;So;0;ON;;;;;N;;;;; 1FB00;BLOCK SEXTANT-1;So;0;ON;;;;;N;;;;; 1FB01;BLOCK SEXTANT-2;So;0;ON;;;;;N;;;;; 1FB02;BLOCK SEXTANT-12;So;0;ON;;;;;N;;;;; @@ -33732,7 +34028,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 20000;;Lo;0;L;;;;;N;;;;; 2A6DF;;Lo;0;L;;;;;N;;;;; 2A700;;Lo;0;L;;;;;N;;;;; -2B738;;Lo;0;L;;;;;N;;;;; +2B739;;Lo;0;L;;;;;N;;;;; 2B740;;Lo;0;L;;;;;N;;;;; 2B81D;;Lo;0;L;;;;;N;;;;; 2B820;;Lo;0;L;;;;;N;;;;; @@ -34283,6 +34579,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 2FA1D;CJK COMPATIBILITY IDEOGRAPH-2FA1D;Lo;0;L;2A600;;;;N;;;;; 30000;;Lo;0;L;;;;;N;;;;; 3134A;;Lo;0;L;;;;;N;;;;; +31350;;Lo;0;L;;;;;N;;;;; +323AF;;Lo;0;L;;;;;N;;;;; E0001;LANGUAGE TAG;Cf;0;BN;;;;;N;;;;; E0020;TAG SPACE;Cf;0;BN;;;;;N;;;;; E0021;TAG EXCLAMATION MARK;Cf;0;BN;;;;;N;;;;; From 8c2fb269b674a3cd1b7fa884f56199ddde3cadbc Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 16 Sep 2022 14:26:28 +0700 Subject: [PATCH 54/59] fix(web): fixes unintended auto-acceptance of suggestion after reverting --- web/source/osk/banner.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/web/source/osk/banner.ts b/web/source/osk/banner.ts index e71a2e8f32..c4f28a6e19 100644 --- a/web/source/osk/banner.ts +++ b/web/source/osk/banner.ts @@ -560,13 +560,17 @@ namespace com.keyman.osk { private doAccept(suggestion: BannerSuggestion) { let _this = this; + // Selecting a suggestion or a reversion should both clear selection + // and clear the reversion-displaying state of the banner. + this.selected = null; + this.doRevert = false; + this.revertAcceptancePromise = suggestion.apply(); if(!this.revertAcceptancePromise) { // We get here either if suggestion acceptance fails or if it was a reversion. if(suggestion.suggestion && suggestion.suggestion.tag == 'revert') { // Reversion state management this.recentAccept = false; - this.doRevert = false; this.recentRevert = true; this.doUpdate(); @@ -581,9 +585,7 @@ namespace com.keyman.osk { } }); - this.selected = null; this.recentAccept = true; - this.doRevert = false; this.recentRevert = false; this.swallowPrediction = true; From 42675227e46150cb428f52088408f267b534efb1 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Fri, 16 Sep 2022 14:01:24 -0400 Subject: [PATCH 55/59] auto: increment master version to 16.0.66 --- HISTORY.md | 4 ++++ VERSION.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index ec8afd279d..cf756b6b07 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,9 @@ # Keyman Version History +## 16.0.65 alpha 2022-09-16 + +* chore(linux): Remove unused IBusLookupTable (#7296) + ## 16.0.64 alpha 2022-09-15 * fix(android/engine): Switch keyboard if uninstalling current one (#7291) diff --git a/VERSION.md b/VERSION.md index c3bbc393bb..039dedcefb 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.65 \ No newline at end of file +16.0.66 \ No newline at end of file From df78d25bb333cbbd8bbca6b9dd6fab1362dc271f Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Sat, 17 Sep 2022 14:01:38 -0400 Subject: [PATCH 56/59] auto: increment master version to 16.0.67 --- HISTORY.md | 4 ++++ VERSION.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index cf756b6b07..caaddcb686 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,9 @@ # Keyman Version History +## 16.0.66 alpha 2022-09-17 + +* chore: improve auto labeling (#7288) + ## 16.0.65 alpha 2022-09-16 * chore(linux): Remove unused IBusLookupTable (#7296) diff --git a/VERSION.md b/VERSION.md index 039dedcefb..b90f7346b3 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.66 \ No newline at end of file +16.0.67 \ No newline at end of file From 744354e2a1f75811acf8c7e9add04c6ecc0d3152 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 19 Sep 2022 08:52:50 +0700 Subject: [PATCH 57/59] fix(common/models): fixes reference dropped by git merge --- common/web/lm-worker/src/model-compositor.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/common/web/lm-worker/src/model-compositor.ts b/common/web/lm-worker/src/model-compositor.ts index 5a314f0644..95c2e390fd 100644 --- a/common/web/lm-worker/src/model-compositor.ts +++ b/common/web/lm-worker/src/model-compositor.ts @@ -162,8 +162,9 @@ class ModelCompositor { // The amount of text to 'replace' depends upon whatever sort of context change occurs // from the received input. - let postContextLength = postContextState.tokens.length; - let contextLengthDelta = postContextState.tokens.length - contextState.tokens.length; + const postContextTokens = postContextState.tokens; + let postContextLength = postContextTokens.length; + let contextLengthDelta = postContextTokens.length - contextState.tokens.length; // If the context now has more tokens, the token we'll be 'predicting' didn't originally exist. if(postContextLength == 0 || contextLengthDelta > 0) { // As the word/token being corrected/predicted didn't originally exist, there's no @@ -225,7 +226,7 @@ class ModelCompositor { // // NOTE: we only want this applied word-initially, when any corrections 'correct' // 100% of the word. Things are generally fine once it's not "all or nothing." - let tailToken = contextTokens[contextTokens.length - 1]; + let tailToken = postContextTokens[postContextTokens.length - 1]; const isTokenStart = tailToken.transformDistributions.length <= 1; // TODO: whitespace, backspace filtering. Do it here. From 466b58768f583ef45f1142f213a41e610ea7b206 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 19 Sep 2022 06:39:41 +1000 Subject: [PATCH 58/59] chore(common): update auto labeler configuration --- .github/labeler.yml | 39 +++++++++++++-------------------------- 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index e0abc29f2f..d831af12b6 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -5,15 +5,15 @@ # common ones. The others are commented out. There is still some variance between # folder names and labels; consider this documentation of that ;-) +docs: docs/** + # # Add labels based on changed files using actions/labeler # android/: android/** android/app/: android/KMAPro/** -#android/browser/: android/engine/: android/KMEA/** -#android/resources/: android/samples/: android/Samples/** common/: @@ -33,41 +33,34 @@ core/: developer/: - developer/** - - windows/src/developer/** developer/compilers/: - - windows/src/developer/kmcomp/** - - developer/src/kmlmc/** + - developer/src/kmcomp/** + - developer/src/kmcmpdll/** + - developer/src/kmc/** + - developer/src/kmc-*/** developer/ide/: - - windows/src/developer/TIKE/** + - developer/src/server/** + - developer/src/tike/** -# developer/resources/ -# developer/tools/ ios/: ios/** ios/app/: ios/keyman/** -# ios/browser/ ios/engine/: ios/engine/** -# ios/resources/ ios/samples/: ios/samples/** linux/: linux/** linux/config/: linux/keyman-config/** linux/engine/: - linux/ibus-keyman/** - - linux/ibus-kmfl/** - - linux/kmflcomp/** - - linux/libkmfl/** - - linux/scim_kmfl_imengine/** -# linux/resources/ -# linux/samples/ + - linux/legacy/ibus-kmfl/** + - linux/legacy/kmflcomp/** + - linux/legacy/libkmfl/** mac/: mac/** # mac/config/: -mac/engine/: mac/** -# mac/resources/ -# mac/samples/ +# mac/engine/: mac/** oem/: oem/** oem/fv/: oem/firstvoices/** @@ -78,15 +71,9 @@ oem/fv/windows/: oem/firstvoices/windows/** web/: web/** # web/bookmarklet/ web/engine/: web/source/** -# web/resources/ web/ui/: web/source/kmwui* web/samples/: web/samples/** -# Somewhat messy since we try and exclude Developer :) -windows/: -- any: ['windows/**', '!windows/src/developer/**'] - +windows/: windows/** windows/config/: windows/src/desktop/** windows/engine/: windows/src/engine/** -# windows/resources/ -# windows/samples/ From a4ca7968b9b0800568d9e52d3a3e304e6ddfd681 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 19 Sep 2022 06:39:13 +1000 Subject: [PATCH 59/59] chore(common): make scripts executable and add pre-commit test --- common/windows/cef-checkout.sh | 0 common/windows/mkver.sh | 0 resources/build/build-utils-ci.test.sh | 0 resources/devbox/iis-https/setup-iis-https.sh | 0 resources/devbox/macos/keyman.macos.env.sh | 0 resources/git-hooks/pre-commit | 19 +++++++++++++++++++ windows/src/desktop/locale/crowdin-control.sh | 0 .../i18n-check-unused-strings.sh | 0 8 files changed, 19 insertions(+) mode change 100644 => 100755 common/windows/cef-checkout.sh mode change 100644 => 100755 common/windows/mkver.sh mode change 100644 => 100755 resources/build/build-utils-ci.test.sh mode change 100644 => 100755 resources/devbox/iis-https/setup-iis-https.sh mode change 100644 => 100755 resources/devbox/macos/keyman.macos.env.sh create mode 100644 resources/git-hooks/pre-commit mode change 100644 => 100755 windows/src/desktop/locale/crowdin-control.sh mode change 100644 => 100755 windows/src/test/manual-tests/i18n-check-unused-strings/i18n-check-unused-strings.sh diff --git a/common/windows/cef-checkout.sh b/common/windows/cef-checkout.sh old mode 100644 new mode 100755 diff --git a/common/windows/mkver.sh b/common/windows/mkver.sh old mode 100644 new mode 100755 diff --git a/resources/build/build-utils-ci.test.sh b/resources/build/build-utils-ci.test.sh old mode 100644 new mode 100755 diff --git a/resources/devbox/iis-https/setup-iis-https.sh b/resources/devbox/iis-https/setup-iis-https.sh old mode 100644 new mode 100755 diff --git a/resources/devbox/macos/keyman.macos.env.sh b/resources/devbox/macos/keyman.macos.env.sh old mode 100644 new mode 100755 diff --git a/resources/git-hooks/pre-commit b/resources/git-hooks/pre-commit new file mode 100644 index 0000000000..3fd01c7d02 --- /dev/null +++ b/resources/git-hooks/pre-commit @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# +# We want to make sure that .sh files are executable; however, .inc.sh need not +# be as they are always source-included in scripts. +# +SH_NON_EXECUTABLE=`git ls-files --stage | grep "\\.sh$" | grep -v "\\.inc\\.sh$" | grep -v 755 | cut -f 2 -` + +if [ ! -z "$SH_NON_EXECUTABLE" ]; then + echo "ERROR: The following scripts are not marked as executable:" + echo + echo "$SH_NON_EXECUTABLE" + echo + echo "You can mark them as executable with:" + echo " git add --chmod=+x" + exit 1 +fi + +exit 0 diff --git a/windows/src/desktop/locale/crowdin-control.sh b/windows/src/desktop/locale/crowdin-control.sh old mode 100644 new mode 100755 diff --git a/windows/src/test/manual-tests/i18n-check-unused-strings/i18n-check-unused-strings.sh b/windows/src/test/manual-tests/i18n-check-unused-strings/i18n-check-unused-strings.sh old mode 100644 new mode 100755