diff --git a/HISTORY.md b/HISTORY.md index 48a2da1054..caaddcb686 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,37 @@ # Keyman Version History +## 16.0.66 alpha 2022-09-17 + +* chore: improve auto labeling (#7288) + +## 16.0.65 alpha 2022-09-16 + +* chore(linux): Remove unused IBusLookupTable (#7296) + +## 16.0.64 alpha 2022-09-15 + +* fix(android/engine): Switch keyboard if uninstalling current one (#7291) +* fix(common/models): fixes quote-adjacent pred-text suggestions (#7205) +* fix(common/models): max prediction wait check (#7290) + +## 16.0.63 alpha 2022-09-14 + +* chore(linux): Update debian changelog (#7281) +* fix(linux): Fix ignored error (#7284) + +## 16.0.62 alpha 2022-09-13 + +* fix(developer): hide key-sizes when in desktop layout in touch layout editor (#7225) +* fix(developer): show more useful error if out of space during Setup (#7267) + +## 16.0.61 alpha 2022-09-12 + +* docs(windows): add steps for using testhost debugging (#7263) +* fix(developer): compiler mismatch on currentLine (#7190) +* fix(developer): suppress repeated warnings about unreachable code (#7219) +* chore: try disabling concurrency for browserstack tests (#7258) +* chore(web): disable browserstack on non-web-specific builds (#7260) + ## 16.0.60 alpha 2022-09-10 * fix(web): enhanced timer for prediction algorithm (#7037) diff --git a/VERSION.md b/VERSION.md index d6db1906b4..b90f7346b3 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.61 \ No newline at end of file +16.0.67 \ No newline at end of file diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardPickerActivity.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardPickerActivity.java index 50a478dd30..ec32425899 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardPickerActivity.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardPickerActivity.java @@ -433,7 +433,7 @@ public final class KeyboardPickerActivity extends BaseActivity { if(adapter != null) { adapter.notifyDataSetChanged(); } - if (position == curKbPos && listView != null) { + if (position == curKbPos) { switchKeyboard(0,false); } else if(listView != null) { // A bit of a hack, since LanguageSettingsActivity calls this method too. curKbPos = KeyboardController.getInstance().getKeyboardIndex(KMKeyboard.currentKeyboard()); diff --git a/android/Tests/KeyboardHarness/app/src/main/assets/final.kmp b/android/Tests/KeyboardHarness/app/src/main/assets/final.kmp new file mode 100644 index 0000000000..80d024623b Binary files /dev/null and b/android/Tests/KeyboardHarness/app/src/main/assets/final.kmp differ diff --git a/android/Tests/KeyboardHarness/app/src/main/java/com/keyman/android/tests/keyboardHarness/MainActivity.java b/android/Tests/KeyboardHarness/app/src/main/java/com/keyman/android/tests/keyboardHarness/MainActivity.java index 9454e940b7..c09a70c0fe 100644 --- a/android/Tests/KeyboardHarness/app/src/main/java/com/keyman/android/tests/keyboardHarness/MainActivity.java +++ b/android/Tests/KeyboardHarness/app/src/main/java/com/keyman/android/tests/keyboardHarness/MainActivity.java @@ -79,6 +79,22 @@ public class MainActivity extends AppCompatActivity implements OnKeyboardEventLi KMManager.KMDefault_KeyboardFont, KMManager.KMDefault_KeyboardFont); KMManager.addKeyboard(this, platformtestKBbInfo); + + // Final K_ENTER test keyboard + Keyboard finalKBInfo = new Keyboard( + "final", + "final", + "final Keyboard", + "en", + "English", + "1.0", + "", + "", + true, + KMManager.KMDefault_KeyboardFont, + KMManager.KMDefault_KeyboardFont); + KMManager.addKeyboard(this, finalKBInfo); + } @Override diff --git a/common/models/templates/test/test-tokenization.js b/common/models/templates/test/test-tokenization.js index 4775064b38..a7ae0835e0 100644 --- a/common/models/templates/test/test-tokenization.js +++ b/common/models/templates/test/test-tokenization.js @@ -289,6 +289,82 @@ describe('Tokenization functions', function() { assert.deepEqual(tokenization, expectedResult); }); + + let midLetterNonbreaker = (text) => { + let customization = { + rules: [{ + match: (context) => { + if(context.propertyMatch(null, ["ALetter"], ["MidLetter"], ["eot"])) { + return true; + } else { + return false; + } + }, + breakIfMatch: false + }], + propertyMapping: (char) => { + let hyphens = ['\u002d', '\u2010', '\u058a', '\u30a0']; + if(hyphens.includes(char)) { + return "MidLetter"; + } else { + return null; + } + } + }; + + return wordBreakers.default(text, customization); + } + + it('treats caret as `eot` for pre-caret text', function() { + let context = { + left: "don-", // We use a hyphen here b/c single-quote is hardcoded. + right: " worry", + endOfBuffer: true, + startOfBuffer: true + }; + + let tokenization = models.tokenize(wordBreakers.default, context); + + assert.deepEqual(tokenization, { + left: ["don", "-"], + right: ["worry"], + caretSplitsToken: false + }); + + tokenization = models.tokenize(midLetterNonbreaker, context); + + assert.deepEqual(tokenization, { + left: ["don-"], + right: ["worry"], + caretSplitsToken: false + }); + }); + + it('handles mid-contraction tokenization', function() { + let context = { + left: "don:", + right: "t worry", + endOfBuffer: true, + startOfBuffer: true + }; + + let tokenization = models.tokenize(wordBreakers.default, context); + + assert.deepEqual(tokenization, { + left: ["don", ":"], // This particular case feels like a possible issue. + right: ["t", "worry"], // It'd be a three-way split token, as "don:t" would + // be a single token were it not for the caret in the middle. + caretSplitsToken: false + }) + + tokenization = models.tokenize(midLetterNonbreaker, context); + + assert.deepEqual(tokenization, { + left: ["don:"], + right: ["t", "worry"], + caretSplitsToken: true + }); + }); }); describe('getLastPreCaretToken', function() { diff --git a/common/models/wordbreakers/src/default/data.ts b/common/models/wordbreakers/src/default/data.ts index ffb6b8e79a..16a4d9beb3 100644 --- a/common/models/wordbreakers/src/default/data.ts +++ b/common/models/wordbreakers/src/default/data.ts @@ -4,7 +4,7 @@ export namespace data { /** * Valid values for a word break property. */ -export const enum WordBreakProperty { +export const enum WordBreakProperty { // Scary bit: this does not exist as an object at run-time! Other, LF, Newline, @@ -28,6 +28,32 @@ export const enum WordBreakProperty { eot }; +// Not currently built by the auto-generator tool, but it easily could be. +// If and when we import the data.ts rebuilder, we can add this in. +export const propertyMap = [ + "Other", + "LF", + "Newline", + "CR", + "WSegSpace", + "Double_Quote", + "Single_Quote", + "MidNum", + "MidNumLet", + "Numeric", + "MidLetter", + "ALetter", + "ExtendNumLet", + "Format", + "Extend", + "Hebrew_Letter", + "ZWJ", + "Katakana", + "Regional_Indicator", + "sot", + "eot" +]; + /** * Constants for indexing values in WORD_BREAK_PROPERTY. */ diff --git a/common/models/wordbreakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts index 5f14165c13..715b722932 100644 --- a/common/models/wordbreakers/src/default/index.ts +++ b/common/models/wordbreakers/src/default/index.ts @@ -1,6 +1,34 @@ // Include the word-breaking data here: /// namespace wordBreakers { + /** + * A set of options used to customize and extend the behavior of the default + * Unicode wordbreaker. + */ + export interface DefaultWordBreakerOptions { + /** + * Allows addition of custom wordbreaking rules, which will be applied + * after WB1-WB4 and before all other default wordbreaking rules. + * + * @see `WordbreakerRule` + */ + rules?: WordbreakerRule[]; + + /** + * Allows assignment of characters to different word-breaking properties than + * their standard word-breaking assignment, including to custom properties + * specified within `customProperties`. + * @param char + */ + propertyMapping?(char: string): string; + + /** + * Allows definition of extra word-breaking properties for use with custom + * rules. + */ + customProperties?: string[]; + } + /** * Word breaker based on Unicode Standard Annex #29, Section 4.1: * Default Word Boundary Specification. @@ -8,8 +36,8 @@ namespace wordBreakers { * @see http://unicode.org/reports/tr29/#Word_Boundaries * @see https://github.com/eddieantonio/unicode-default-word-boundary/tree/v12.0.0 */ - export function default_(text: string): Span[] { - let boundaries = findBoundaries(text); + export function default_(text: string, options?: DefaultWordBreakerOptions): Span[] { + let boundaries = findBoundaries(text, options); if (boundaries.length == 0) { return []; } @@ -22,7 +50,7 @@ namespace wordBreakers { let end = boundaries[i + 1]; let span = new LazySpan(text, start, end); - if (isNonSpace(span.text)) { + if (isNonSpace(span.text, options)) { spans.push(span); // Preserve a sequence-final space if it exists. Needed to signal "end of word". } else if (i == boundaries.length - 2) { // if "we just checked the final boundary"... @@ -61,13 +89,198 @@ namespace wordBreakers { } } + /** + * An abstraction supporting custom wordbreaker boundary rules. While this doesn't provide + * support for more complex rules like WB4, WB15, or WB16, this is sufficient for all other + * default word-breaking rules and can be used to define custom rules of similar structure. + * + * @see https://unicode.org/reports/tr29/#WB_Rule_Macros + */ + export interface WordbreakerRule { + /** + * Indicates whether or not the rule applies in the specified context. + * @param context + */ + match(context: BreakerContext): boolean; + + /** + * Indicates whether or not the rule indicates a word boundary at the context's site when it matches. + */ + breakIfMatch: boolean; + } + + /** + * Provides a useful presentation for wordbreaker's context for use in word-breaking rules. + * + * @see https://unicode.org/reports/tr29/#Word_Boundary_Rules + */ + export class BreakerContext { + // Referenced by this object in order to facilitate `lookahead` maintenance. + private readonly text: string; + readonly options?: DefaultWordBreakerOptions; + + /** + * Represents the property of character immediately preceding `left`'s character. + */ + readonly lookbehind: WordBreakProperty = WordBreakProperty.sot; + + /** + * Represents the property of the character immediately preceding the potential word boundary. + */ + readonly left: WordBreakProperty = WordBreakProperty.sot; + + /** + * Represents the property of the character immediately following the potential word boundary. + */ + readonly right: WordBreakProperty = WordBreakProperty.sot; + + /** + * Represents the property of the character immediately following `right`'s character. + */ + readonly lookahead: WordBreakProperty; // Always initialized by constructor. + + /** + * Initializes the word-breaking context at the start of the word-breaker's boundary-detection + * algorithm. + * @param text The text to be word-broken + * @param lookaheadPos The position corresponding to `lookahead`. + */ + constructor(text: string, options: DefaultWordBreakerOptions | undefined, lookaheadPos: number); + /** + * Used internally by the boundary-detection algorithm during context-shifting operations. + * @param text + * @param lookbehind + * @param left + * @param right + * @param lookahead + */ + constructor(text: string, + options: DefaultWordBreakerOptions | undefined, + lookbehind: WordBreakProperty, + left: WordBreakProperty, + right: WordBreakProperty, + lookahead: WordBreakProperty); + constructor(text: string, + options: DefaultWordBreakerOptions | undefined, + prop1: WordBreakProperty | number, + prop2?: WordBreakProperty, + prop3?: WordBreakProperty, + prop4?: WordBreakProperty) { + this.text = text; + this.options = options; + + if(arguments.length == 3) { + this.lookahead = this.wordbreakPropertyAt(prop1);// prop1; + } else /*if(arguments.length == 6)*/ { + this.lookbehind = prop1 as WordBreakProperty; + this.left = prop2 as WordBreakProperty; + this.right = prop3 as WordBreakProperty; + this.lookahead = prop4 as WordBreakProperty; + } + } + + /** + * The general use-case when shifting boundary-check position if WB4 is not active. + * @param lookahead The WordBreakProperty for the character to become `lookahead`. + * @returns + */ + public next(lookaheadPos: number): BreakerContext { + let newLookahead = this.wordbreakPropertyAt(lookaheadPos); + return new BreakerContext(this.text, this.options, this.left, this.right, this.lookahead, newLookahead); + } + + /** + * Used for WB4: when ignoring characters before an intervening linebreak, we + * replace `right` with the current `lookahead`, without affecting `lookbehind` + * or `left`. A new `lookahead` is then needed. + * @param lookahead + * @returns + */ + public ignoringRight(lookaheadPos: number) { + let newLookahead = this.wordbreakPropertyAt(lookaheadPos); + return new BreakerContext(this.text, this.options, this.lookbehind, this.left, this.lookahead, newLookahead); + } + + /** + * Used for WB4: when ignoring characters after an intervening linebreak, it's + * `lookahead` that gets replaced without shifting the other tracked properties. + * @param lookahead + * @returns + */ + public ignoringLookahead(lookaheadPos: number) { + let newLookahead = this.wordbreakPropertyAt(lookaheadPos); + return new BreakerContext(this.text, this.options, this.lookbehind, this.left, this.right, newLookahead); + } + + /** + * Return the value of the Word_Break property at the given string index. + * @param pos position in the text. + */ + private wordbreakPropertyAt(pos: number) { + if (pos < 0) { + return WordBreakProperty.sot; // Always "start of string" before the string starts! + } else if (pos >= this.text.length) { + return WordBreakProperty.eot; // Always "end of string" after the string ends! + } else if (isStartOfSurrogatePair(this.text[pos])) { + // Surrogate pairs the next TWO items from the string! + return property(this.text[pos] + this.text[pos + 1]); + } + return property(this.text[pos], this.options); + } + + /** + * Returns `true` if and only if each member of the context has a property included within + * its corresponding set (when specified). Any set may be replaced with null to disable + * a check against its corresponding property. + * @param lookbehindSet + * @param leftSet + * @param rightSet + * @param lookaheadSet + */ + public match(lookbehindSet: WordBreakProperty[] | null, + leftSet: WordBreakProperty[] | null, + rightSet: WordBreakProperty[] | null, + lookaheadSet: WordBreakProperty[] | null) : boolean { + let result: boolean = lookbehindSet?.includes(this.lookbehind) ?? true; + result = result && (leftSet?.includes(this.left) ?? true); + result = result && (rightSet?.includes(this.right) ?? true); + return result && (lookaheadSet?.includes(this.lookahead) ?? true); + } + + /** + * Returns `true` if and only if each member of the context has a property included within + * its corresponding set (when specified). Any set may be replaced with null to disable + * a check against its corresponding property. + * + * Names should match those found at https://unicode.org/reports/tr29/#Word_Boundary_Rules + * or defined in the word-breaker customization options; matching is case-insensitive. + * Also includes two extra properties: + * - `sot` - start of text + * - `eot` - end of text + * @param lookbehindSet + * @param leftSet + * @param rightSet + * @param lookaheadSet + */ + public propertyMatch(lookbehindSet: string[] | null, + leftSet: string[] | null, + rightSet: string[] | null, + lookaheadSet: string[] | null) : boolean { + const propMapper = (name: string) => propertyVal(name, this.options); + return this.match(lookbehindSet?.map(propMapper) as WordBreakProperty[] | null, + leftSet?.map(propMapper) as WordBreakProperty[] | null, + rightSet?.map(propMapper) as WordBreakProperty[] | null, + lookaheadSet?.map(propMapper) as WordBreakProperty[] | null); + } + } + /** * Returns true when the chunk does not solely consist of whitespace. * * @param chunk a chunk of text. Starts and ends at word boundaries. */ - function isNonSpace(chunk: string): boolean { - return !Array.from(chunk).map(property).every(wb => ( + function isNonSpace(chunk: string, options?: DefaultWordBreakerOptions): boolean { + return !Array.from(chunk).map((char) => property(char, options)).every(wb => ( wb === WordBreakProperty.CR || wb === WordBreakProperty.LF || wb === WordBreakProperty.Newline || @@ -82,13 +295,17 @@ namespace wordBreakers { * * @param text Text to find word boundaries in. */ - function findBoundaries(text: string): number[] { + function findBoundaries(text: string, options?: DefaultWordBreakerOptions): number[] { // WB1 and WB2: no boundaries if given an empty string. if (text.length === 0) { // There are no boundaries in an empty string! return []; } + if(options && !options.rules) { + options.rules = []; + } + // This algorithm works by maintaining a sliding window of four SCALAR VALUES. // // - Scalar values? JavaScript strings are NOT actually a string of @@ -112,10 +329,7 @@ namespace wordBreakers { let rightPos: number; let lookaheadPos = 0; // lookahead, one scalar value to the right of right. // Before the start of the string is also the start of the string. - let lookbehind: WordBreakProperty; - let left = WordBreakProperty.sot; - let right = WordBreakProperty.sot; - let lookahead = wordbreakPropertyAt(0); + let state = new BreakerContext(text, options, lookaheadPos); // Count RIs to make sure we're not splitting emoji flags: let nConsecutiveRegionalIndicators = 0; @@ -124,34 +338,32 @@ namespace wordBreakers { rightPos = lookaheadPos; lookaheadPos = positionAfter(lookaheadPos); // Shift all properties, one scalar value to the right. - [lookbehind, left, right, lookahead] = - [left, right, lookahead, wordbreakPropertyAt(lookaheadPos)]; + state = state.next(lookaheadPos); // Break at the start and end of text, unless the text is empty. // WB1: Break at start of text... - if (left === WordBreakProperty.sot) { + if (state.match(null, [WordBreakProperty.sot], null, null)) { boundaries.push(rightPos); continue; } // WB2: Break at the end of text... - if (right === WordBreakProperty.eot) { + if (state.match(null, null, [WordBreakProperty.eot], null)) { boundaries.push(rightPos); break; // Reached the end of the string. We're done! } // WB3: Do not break within CRLF: - if (left === WordBreakProperty.CR && right === WordBreakProperty.LF) + if (state.match(null, [WordBreakProperty.CR], [WordBreakProperty.LF], null)) { continue; + } + // WB3b: Otherwise, break after... - if (left === WordBreakProperty.Newline || - left === WordBreakProperty.CR || - left === WordBreakProperty.LF) { + const NEWLINE_SET = [WordBreakProperty.Newline, WordBreakProperty.CR, WordBreakProperty.LF]; + if(state.match(null, NEWLINE_SET, null, null)) { boundaries.push(rightPos); continue; } // WB3a: ...and before newlines - if (right === WordBreakProperty.Newline || - right === WordBreakProperty.CR || - right === WordBreakProperty.LF) { + if (state.match(null, null, NEWLINE_SET, null)) { boundaries.push(rightPos); continue; } @@ -163,107 +375,144 @@ namespace wordBreakers { // https://www.unicode.org/Public/emoji/12.0/emoji-zwj-sequences.txt // WB3d: Keep horizontal whitespace together - if (left === WordBreakProperty.WSegSpace && right == WordBreakProperty.WSegSpace) + if (state.match(null, [WordBreakProperty.WSegSpace], [WordBreakProperty.WSegSpace], null)) { continue; + } // WB4: Ignore format and extend characters // This is to keep grapheme clusters together! // See: Section 6.2: https://unicode.org/reports/tr29/#Grapheme_Cluster_and_Format_Rules // N.B.: The rule about "except after sot, CR, LF, and // Newline" already been by WB1, WB2, WB3a, and WB3b above. - while (right === WordBreakProperty.Format || - right === WordBreakProperty.Extend || - right === WordBreakProperty.ZWJ) { + const SET_WB4_IGNORE = [WordBreakProperty.Format, WordBreakProperty.Extend, WordBreakProperty.ZWJ]; + while (state.match(null, null, SET_WB4_IGNORE, null)) { // Continue advancing in the string, as if these // characters do not exist. DO NOT update left and // lookbehind however! [rightPos, lookaheadPos] = [lookaheadPos, positionAfter(lookaheadPos)]; - [right, lookahead] = [lookahead, wordbreakPropertyAt(lookaheadPos)]; + state = state.ignoringRight(lookaheadPos); } // In ignoring the characters in the previous loop, we could // have fallen off the end of the string, so end the loop // prematurely if that happens! - if (right === WordBreakProperty.eot) { + if (state.right === WordBreakProperty.eot) { boundaries.push(rightPos); break; } // WB4 (continued): Lookahead must ALSO ignore these format, // extend, ZWJ characters! - while (lookahead === WordBreakProperty.Format || - lookahead === WordBreakProperty.Extend || - lookahead === WordBreakProperty.ZWJ) { + while (state.match(null, null, null, SET_WB4_IGNORE)) { // Continue advancing in the string, as if these // characters do not exist. DO NOT update left and right, // however! lookaheadPos = positionAfter(lookaheadPos); - lookahead = wordbreakPropertyAt(lookaheadPos); + state = state.ignoringLookahead(lookaheadPos); + } + + // See: https://unicode.org/reports/tr29/#WB_Rule_Macros + const SET_AHLETTER = [WordBreakProperty.ALetter, WordBreakProperty.Hebrew_Letter]; + const SET_MIDNUMLETQ = [WordBreakProperty.MidNumLet, WordBreakProperty.Single_Quote]; + + // Custom rules may override the base ruleset aside from the first few fundamental ones. + if(options?.rules) { + let customMatch: boolean = false; + for(const rule of options.rules) { + customMatch = rule.match(state); + if(customMatch) { + if(rule.breakIfMatch) { + boundaries.push(rightPos); + } + break; // as customMatch == true here, this will trigger the `continue` that follows. + } + } + if(customMatch) { + continue; + } } // WB5: Do not break between most letters. - if (isAHLetter(left) && isAHLetter(right)) + // if (isAHLetter(state.left) && isAHLetter(state.right)) + if(state.match(null, SET_AHLETTER, SET_AHLETTER, null)) { continue; + } // Do not break across certain punctuation // WB6: (Don't break before apostrophes in contractions) - if (isAHLetter(left) && isAHLetter(lookahead) && - (right === WordBreakProperty.MidLetter || isMidNumLetQ(right))) + const SET_ALL_MIDLETTER = [WordBreakProperty.MidLetter, ...SET_MIDNUMLETQ]; + if(state.match(null, SET_AHLETTER, SET_ALL_MIDLETTER, SET_AHLETTER)) { continue; + } // WB7: (Don't break after apostrophes in contractions) - if (isAHLetter(lookbehind) && isAHLetter(right) && - (left === WordBreakProperty.MidLetter || isMidNumLetQ(left))) + if(state.match(SET_AHLETTER, SET_ALL_MIDLETTER, SET_AHLETTER, null)) { continue; + } + // WB7a - if (left === WordBreakProperty.Hebrew_Letter && right === WordBreakProperty.Single_Quote) + if(state.match(null, [WordBreakProperty.Hebrew_Letter], [WordBreakProperty.Single_Quote], null)) { continue; + } // WB7b - if (left === WordBreakProperty.Hebrew_Letter && right === WordBreakProperty.Double_Quote && - lookahead === WordBreakProperty.Hebrew_Letter) + if(state.match(null, + [WordBreakProperty.Hebrew_Letter], + [WordBreakProperty.Double_Quote], + [WordBreakProperty.Hebrew_Letter])) { continue; + } // WB7c - if (lookbehind === WordBreakProperty.Hebrew_Letter && left === WordBreakProperty.Double_Quote && - right === WordBreakProperty.Hebrew_Letter) + if(state.match([WordBreakProperty.Hebrew_Letter], + [WordBreakProperty.Double_Quote], + [WordBreakProperty.Hebrew_Letter], + null)) { continue; + } // Do not break within sequences of digits, or digits adjacent to letters. // e.g., "3a" or "A3" // WB8 - if (left === WordBreakProperty.Numeric && right === WordBreakProperty.Numeric) + if(state.match(null, [WordBreakProperty.Numeric], [WordBreakProperty.Numeric], null)) { continue; + } // WB9 - if (isAHLetter(left) && right === WordBreakProperty.Numeric) + if(state.match(null, SET_AHLETTER, [WordBreakProperty.Numeric], null)) { continue; + } // WB10 - if (left === WordBreakProperty.Numeric && isAHLetter(right)) + if(state.match(null, [WordBreakProperty.Numeric], SET_AHLETTER, null)) { continue; + } // Do not break within sequences, such as 3.2, 3,456.789 // WB11 - if (lookbehind === WordBreakProperty.Numeric && right === WordBreakProperty.Numeric && - (left === WordBreakProperty.MidNum || isMidNumLetQ(left))) + const SET_ALL_MIDNUM = [WordBreakProperty.MidNum, ...SET_MIDNUMLETQ]; + if(state.match([WordBreakProperty.Numeric], SET_ALL_MIDNUM, [WordBreakProperty.Numeric], null)) { continue; + } // WB12 - if (left === WordBreakProperty.Numeric && lookahead === WordBreakProperty.Numeric && - (right === WordBreakProperty.MidNum || isMidNumLetQ(right))) + if(state.match(null, [WordBreakProperty.Numeric], SET_ALL_MIDNUM, [WordBreakProperty.Numeric])) { continue; + } // WB13: Do not break between Katakana - if (left === WordBreakProperty.Katakana && right === WordBreakProperty.Katakana) + if(state.match(null, [WordBreakProperty.Katakana], [WordBreakProperty.Katakana], null)) { continue; + } // Do not break from extenders (e.g., U+202F NARROW NO-BREAK SPACE) // WB13a - if ((isAHLetter(left) || - left === WordBreakProperty.Numeric || - left === WordBreakProperty.Katakana || - left === WordBreakProperty.ExtendNumLet) && - right === WordBreakProperty.ExtendNumLet) + const SET_NUM_KAT_LET = [WordBreakProperty.Katakana, + WordBreakProperty.Numeric, + ...SET_AHLETTER]; + if(state.match(null, SET_NUM_KAT_LET, [WordBreakProperty.ExtendNumLet], null)) { continue; + } + if(state.match(null, [WordBreakProperty.ExtendNumLet], [WordBreakProperty.ExtendNumLet], null)) { + continue; + } // WB13b - if ((isAHLetter(right) || - right === WordBreakProperty.Numeric || - right === WordBreakProperty.Katakana) && left === WordBreakProperty.ExtendNumLet) + if(state.match(null, [WordBreakProperty.ExtendNumLet], SET_NUM_KAT_LET, null)) { continue; + } // WB15 & WB16: // Do not break within emoji flag sequences. That is, do not break between // regional indicator (RI) symbols if there is an odd number of RI // characters before the break point. - if (right === WordBreakProperty.Regional_Indicator) { + if (state.right === WordBreakProperty.Regional_Indicator) { // Emoji flags are actually composed of TWO scalar values, each being a // "regional indicator". These indicators correspond to Latin letters. Put // two of them together, and they spell out an ISO 3166-1-alpha-2 country @@ -300,34 +549,6 @@ namespace wordBreakers { } return pos + 1; } - - /** - * Return the value of the Word_Break property at the given string index. - * @param pos position in the text. - */ - function wordbreakPropertyAt(pos: number) { - if (pos < 0) { - return WordBreakProperty.sot; // Always "start of string" before the string starts! - } else if (pos >= text.length) { - return WordBreakProperty.eot; // Always "end of string" after the string ends! - } else if (isStartOfSurrogatePair(text[pos])) { - // Surrogate pairs the next TWO items from the string! - return property(text[pos] + text[pos + 1]); - } - return property(text[pos]); - } - - // Word_Break rule macros - // See: https://unicode.org/reports/tr29/#WB_Rule_Macros - function isAHLetter(prop: WordBreakProperty): boolean { - return prop === WordBreakProperty.ALetter || - prop === WordBreakProperty.Hebrew_Letter; - } - - function isMidNumLetQ(prop: WordBreakProperty): boolean { - return prop === WordBreakProperty.MidNumLet || - prop === WordBreakProperty.Single_Quote; - } } function isStartOfSurrogatePair(character: string) { @@ -340,13 +561,29 @@ namespace wordBreakers { * Note that * @param character a scalar value */ - function property(character: string): WordBreakProperty { + function property(character: string, options?: DefaultWordBreakerOptions): WordBreakProperty { + // If there is a customized mapping for the character, prioritize that. + if(options?.propertyMapping) { + let propName = options.propertyMapping(character); + if(propName) { + return propertyVal(propName, options); + } + } + // This MUST be a scalar value. // TODO: remove dependence on character.codepointAt()? let codepoint = character.codePointAt(0) as number; + return searchForProperty(codepoint, 0, WORD_BREAK_PROPERTY.length - 1); } + function propertyVal(propName: string, options?: DefaultWordBreakerOptions) { + const matcher = (name: string) => name.toLowerCase() == propName.toLowerCase() + + const customIndex = options?.customProperties?.findIndex(matcher) ?? -1; + return customIndex != -1 ? -customIndex - 1 : data.propertyMap.findIndex(matcher); + } + /** * Binary search for the word break property of a given CODE POINT. * diff --git a/common/models/wordbreakers/test/test-default-word-breaker.js b/common/models/wordbreakers/test/test-default-word-breaker.js index b4db8cb1a3..85129b836c 100644 --- a/common/models/wordbreakers/test/test-default-word-breaker.js +++ b/common/models/wordbreakers/test/test-default-word-breaker.js @@ -8,184 +8,334 @@ const breakWords = require('../build').wordBreakers['default']; const SHY = '\u00AD'; // Other, Format. The "Soft HYphen" - usually invisible unless needed for word-wrapping. describe('The default word breaker', function () { - it('should break multilingual text', function () { - let breaks = breakWords( - `Добрый день! ᑕᐻ᙮ — after working on ka${SHY}wen${SHY}non:${SHY}nis, - let's eat phở! 🥣` - ); - let words = breaks.map(span => span.text); - assert.deepEqual(words, [ - 'Добрый', 'день', '!', 'ᑕᐻ', '᙮', '—', 'after', - 'working', 'on', `ka${SHY}wen${SHY}non:${SHY}nis`, ',', - "let's", 'eat', 'phở', '!', '🥣' - ]); + describe('default configuration', function() { + it('should break multilingual text', function () { + let breaks = breakWords( + `Добрый день! ᑕᐻ᙮ — after working on ka${SHY}wen${SHY}non:${SHY}nis, + let's eat phở! 🥣` + ); + let words = breaks.map(span => span.text); + assert.deepEqual(words, [ + 'Добрый', 'день', '!', 'ᑕᐻ', '᙮', '—', 'after', + 'working', 'on', `ka${SHY}wen${SHY}non:${SHY}nis`, ',', + "let's", 'eat', 'phở', '!', '🥣' + ]); + }); + + it('handles heavily-punctuated English text', function() { + // This test case brought to you by http://unicode.org/reports/tr29/#Word_Boundaries, Figure 1. + let breaks = breakWords( + `The quick ("brown") fox can't jump 32.3 feet, right?` + ); + let words = breaks.map(span => span.text); + assert.deepEqual(words, [ + 'The', 'quick', '(', '"', 'brown', '"', ')', 'fox', "can't", + 'jump', '32.3', 'feet', ',', 'right', '?' + ]); + }); + + // The way these two tests are written is a bit much on the "white-box" style, + // but they do decently cover the boundary rules mentioned. + it('Does not split empty contexts (WB1 + WB2)', function() { + let breaks = breakWords(''); + let words = breaks.map(span => span.text); + assert.deepEqual(words, []); + }); + + it('Does split at context boundaries (WB1 + WB2)', function() { + let breaks = breakWords('a'); + let words = breaks.map(span => span.text); + assert.deepEqual(words, ['a']); + }); + + // WB3, WB3a, WB3b are all handled internally, within the top-level function. + + // iff, as in "if and only if" + it('ignores the zero-width joiner iff appropriate (WB4)', function() { + const zwj = '\u200d'; + + let breaks = breakWords(`a${zwj}b\n${zwj}c${zwj}\nd`); + let words = breaks.map(span => span.text); + + // Does NOT ignore the zwj immediately after a newline - the notable exception + // (the reason for "iff", not "if"). + assert.deepEqual(words, [`a${zwj}b`, `${zwj}`, `c${zwj}`, `d`]); + }) + + it('ignores extend characters iff appropriate (WB4)', function() { + const comboGrave = '\u0300'; // The 'combining grave accent', as used in NFD. + + let breaks = breakWords(`a${comboGrave}e\n${comboGrave}i${comboGrave}\no`); + let words = breaks.map(span => span.text); + + // Does NOT ignore the zwj immediately after a newline - the notable exception + // (the reason for "iff", not "if"). + assert.deepEqual(words, [`a${comboGrave}e`, `${comboGrave}`, `i${comboGrave}`, `o`]); + }); + + it('ignores format characters iff appropriate (WB4)', function() { + // Re-uses `const SHY` from above. + let breaks = breakWords(`a${SHY}e\n${SHY}i${SHY}\no`); + let words = breaks.map(span => span.text); + + // Does NOT ignore the zwj immediately after a newline - the notable exception + // (the reason for "iff", not "if"). + assert.deepEqual(words, [`a${SHY}e`, `${SHY}`, `i${SHY}`, `o`]); + }); + + it('does not break between most alphabetic characters (WB5)', function() { + let breaks = breakWords(`aəσאБ лאʈγX`); // a mix of latin, hebrew, greek, cyrillic, and IPA chars + // for both "words". + let words = breaks.map(span => span.text); + + assert.deepEqual(words, [`aəσאБ`, `лאʈγX`]); + }); + + it('does not break letters across specific punctuation patterns (WB6, WB7)', function() { + // `'`: MidNumLetQ (from Single_Quote) + // '.': MidNumLet + // ':': MidLetter + let breaks = breakWords(`don't b.r.e.a.k t:h:e:s:e`); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, [`don't`, `b.r.e.a.k`, `t:h:e:s:e`]); + + let breaks2 = breakWords(`.drop: :the' 'extras.`); + let words2 = breaks2.map(span => span.text); + + assert.deepEqual(words2, [`.`, `drop`, `:`, `:`, `the`, `'`, `'`, `extras`, `.`]); + + // ',': MidNum (is NOT included by rule!) + let breaks3 = breakWords('do br,eak that'); + let words3 = breaks3.map(span => span.text); + + assert.deepEqual(words3, ['do', 'br', ',', 'eak', 'that']); + }); + + it('treats Hebrew properly (WB7a-c)', function() { + const aleph = 'א'; + const bet = 'ב'; + + // As Hebrew is RTL... this is probably the clearest way for us LTR people to + // clearly see what's going on without ordering mechanics messing up the render. + let breaks = breakWords(`${aleph}' ${aleph}" ${aleph}"${bet}`); + let words = breaks.map(span => span.text); + + // A lingering double-quote isn't cool, but one in the middle's fine. + // Lingering single-quote is fine regardless. + assert.deepEqual(words, [`${aleph}'`, `${aleph}`, `"`, `${aleph}"${bet}`]); + }); + + it(`doesn't break within digit + digit/letter sequences (WB8-10)`, function() { + let breaks = breakWords('a1b2c3 hunter2 ab12cd34 1234567890'); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, ['a1b2c3', 'hunter2', 'ab12cd34', '1234567890']); + }); + + it('does not break within formatted number sequences (WB11-12)', function() { + // Note: `'` fits "MidNumLetQ", part of the two rules! + let breaks = breakWords(`1.2.3 3,458.01 3.45'8,01`); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, [`1.2.3`, `3,458.01`, `3.45'8,01`]); + + let breaks2 = breakWords(`.1' ,3.`); + let words2 = breaks2.map(span => span.text); + + assert.deepEqual(words2, [`.`, `1`, `'`, `,`, `3`, `.`]); + }); + + it('does not break between Katakana (WB13)', function() { + const kataSmA = '\u30a2'; //ァ + const kataA = '\u30a2'; //ア + const kataSound = '\u309b'; // ゛ + + let breaks = breakWords(`${kataSound}${kataA} ${kataSmA}${kataSound}b ${kataA}${kataSound}${kataSmA}`); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, [ + `${kataSound}${kataA}`, + `${kataSmA}${kataSound}`, + 'b', + `${kataA}${kataSound}${kataSmA}` + ]); + }); + + it('does not break form extenders (WB13a-b)', function() { + // The `_` (underscore) fits the ExtendNumLet class this rule focuses on. + const kataA = '\u30a2'; //ア + + let breaks = breakWords(`${kataA}_a__0_b_${kataA} _${kataA} 1_ _c_ ____`); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, [ + `${kataA}_a__0_b_${kataA}`, + `_${kataA}`, + `1_`, + `_c_`, + `____` + ]); + }); + + it('handles emoji flag sequences properly (WB15-16)', function() { + // For clarity on what's being tested... + let CA_FLAG = '\u{1f1e8}\u{1f1e6}' // '🇨🇦' (canadian flag emoji); should not be broken. + let KH_FLAG = '\u{1f1f0}\u{1f1ed}' // '🇰🇭' (khmer flag emoji); same + let X_FLAG_PIECE = '\u{1f1fd}' // '🇽' (half of a flag emoji; '🇽🇽' doesn't match a flag) + let breaks = breakWords(`${CA_FLAG}${KH_FLAG}${X_FLAG_PIECE}${X_FLAG_PIECE}`); + let words = breaks.map(span => span.text); + + // Note that the emoji may not render well within VSCode, but they show up nicely on GitHub. + assert.deepEqual(words, ['🇨🇦', '🇰🇭', '🇽🇽']); + }); + + it('breaks hyphenated words by default', function() { + let breaks = breakWords('Smith-Jones'); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, ['Smith', '-', 'Jones']); + }); }); - it('handles heavily-punctuated English text', function() { - // This test case brought to you by http://unicode.org/reports/tr29/#Word_Boundaries, Figure 1. - let breaks = breakWords( - `The quick ("brown") fox can't jump 32.3 feet, right?` - ); - let words = breaks.map(span => span.text); - assert.deepEqual(words, [ - 'The', 'quick', '(', '"', 'brown', '"', ')', 'fox', "can't", - 'jump', '32.3', 'feet', ',', 'right', '?' - ]); - }); + describe('customization', function() { + // Refer to https://unicode.org/reports/tr29/#Word_Boundary_Rules, third bullet point. + it('custom prop, rule: do not break on letter-adjacent hyphens', function() { + let customization = { + rules: [{ + match: (context) => { + if(context.propertyMatch(null, ["ALetter"], ["Hyphen"], ["ALetter"])) { + return true; + } else if(context.propertyMatch(["ALetter"], ["Hyphen"], ["ALetter"], null)) { + return true; + } else { + return false; + } + }, + breakIfMatch: false + }], + propertyMapping: (char) => { + const validHyphenCodes = [ + '\u002d', '\u2010', '\u058a', '\u30a0' + ]; + if(validHyphenCodes.includes(char)) { + return "Hyphen"; + } - // The way these two tests are written is a bit much on the "white-box" style, - // but they do decently cover the boundary rules mentioned. - it('Does not split empty contexts (WB1 + WB2)', function() { - let breaks = breakWords(''); - let words = breaks.map(span => span.text); - assert.deepEqual(words, []); - }); + return null; + }, + customProperties: ["Hyphen"] + } - it('Does split at context boundaries (WB1 + WB2)', function() { - let breaks = breakWords('a'); - let words = breaks.map(span => span.text); - assert.deepEqual(words, ['a']); - }); + let breaks = breakWords('Smith-Jones', customization); + let words = breaks.map(span => span.text); - // WB3, WB3a, WB3b are all handled internally, within the top-level function. + assert.deepEqual(words, ['Smith-Jones']); + }); - // iff, as in "if and only if" - it('ignores the zero-width joiner iff appropriate (WB4)', function() { - const zwj = '\u200d'; + it('mid-word hyphen via reassignment to MidLetter', function() { + let customization = { + propertyMapping: (char) => { + const validHyphenCodes = [ + '\u002d', '\u2010', '\u058a', '\u30a0' + ]; + if(validHyphenCodes.includes(char)) { + return "MidLetter"; + } - let breaks = breakWords(`a${zwj}b\n${zwj}c${zwj}\nd`); - let words = breaks.map(span => span.text); + return null; + } + } - // Does NOT ignore the zwj immediately after a newline - the notable exception - // (the reason for "iff", not "if"). - assert.deepEqual(words, [`a${zwj}b`, `${zwj}`, `c${zwj}`, `d`]); - }) + let breaks = breakWords('Smith-Jones', customization); + let words = breaks.map(span => span.text); - it('ignores extend characters iff appropriate (WB4)', function() { - const comboGrave = '\u0300'; // The 'combining grave accent', as used in NFD. + assert.deepEqual(words, ['Smith-Jones']); + }); - let breaks = breakWords(`a${comboGrave}e\n${comboGrave}i${comboGrave}\no`); - let words = breaks.map(span => span.text); + // Useful for some regional minority languages that prefer word-breaking spaces. + it('character reassignment: Khmer letters as ALetter', function() { + let customization = { + propertyMapping: (char) => { + if(char >= '\u1780' && char <= '\u17b3') { + return "ALetter"; + } else { + // The other Khmer characters already have useful word-breaking + // property assignments. + return null; + } + } + } - // Does NOT ignore the zwj immediately after a newline - the notable exception - // (the reason for "iff", not "if"). - assert.deepEqual(words, [`a${comboGrave}e`, `${comboGrave}`, `i${comboGrave}`, `o`]); - }); + let breaks = breakWords('ស្រុក ខ្មែរ', customization); + let words = breaks.map(span => span.text); - it('ignores format characters iff appropriate (WB4)', function() { - // Re-uses `const SHY` from above. - let breaks = breakWords(`a${SHY}e\n${SHY}i${SHY}\no`); - let words = breaks.map(span => span.text); + assert.deepEqual(words, ['ស្រុក', 'ខ្មែរ']); + }); - // Does NOT ignore the zwj immediately after a newline - the notable exception - // (the reason for "iff", not "if"). - assert.deepEqual(words, [`a${SHY}e`, `${SHY}`, `i${SHY}`, `o`]); - }); + // See: suggested language-specific WB5a from the spec's notes. + it("french/italian apostrophe / vowel boundaries", function() { + let customization = { + rules: [ + // WB5, but with differentiated consonants (ALetter) and vowels (AVowel) + { + match: (context) => { + if(context.propertyMatch(null, ["ALetter", "AVowel"], ["ALetter", "AVowel"], null)) { + return true; + } else { + return false; + } + }, + breakIfMatch: false + }, + // Proposed WB5a + { + match: (context) => { + if(context.propertyMatch(null, ["Single_Quote"], ["AVowel"], null)) { + return true; + } else { + return false; + } + }, + breakIfMatch: true + }, + // WB6, 7 + { + match: (context) => { + if(context.propertyMatch(null, + ["ALetter", "AVowel"], + ["MidLetter", "MidNumLet", "Single_Quote"], + ["ALetter", "AVowel"])) { + return true; + } else if(context.propertyMatch(["ALetter", "AVowel"], + ["MidLetter", "MidNumLet", "Single_Quote"], + ["ALetter", "AVowel"], + null)) { + return true; + } else { + return false; + } + }, + breakIfMatch: false + } + // Similar extensions to WB9, 10, 13a, and 13b would also be needed for robustness. + // And I kind of left the Hebrew_Letter out of the WB5, 6, and 7 rewrites. + ], + propertyMapping: (char) => { + const vowels = ['a', 'e', 'i', 'o', 'u']; + if(vowels.includes(char)) { + return "AVowel"; + } - it('does not break between most alphabetic characters (WB5)', function() { - let breaks = breakWords(`aəσאБ лאʈγX`); // a mix of latin, hebrew, greek, cyrillic, and IPA chars - // for both "words". - let words = breaks.map(span => span.text); + return null; + }, + customProperties: ["AVowel"] + } - assert.deepEqual(words, [`aəσאБ`, `лאʈγX`]); - }); + let breaks = breakWords("l'objectif aujourd'hui", customization); + let words = breaks.map(span => span.text); - it('does not break letters across specific punctuation patterns (WB6, WB7)', function() { - // `'`: MidNumLetQ (from Single_Quote) - // '.': MidNumLet - // ':': MidLetter - let breaks = breakWords(`don't b.r.e.a.k t:h:e:s:e`); - let words = breaks.map(span => span.text); - - assert.deepEqual(words, [`don't`, `b.r.e.a.k`, `t:h:e:s:e`]); - - let breaks2 = breakWords(`.drop: :the' 'extras.`); - let words2 = breaks2.map(span => span.text); - - assert.deepEqual(words2, [`.`, `drop`, `:`, `:`, `the`, `'`, `'`, `extras`, `.`]); - - // ',': MidNum (is NOT included by rule!) - let breaks3 = breakWords('do br,eak that'); - let words3 = breaks3.map(span => span.text); - - assert.deepEqual(words3, ['do', 'br', ',', 'eak', 'that']); - }); - - it('treats Hebrew properly (WB7a-c)', function() { - const aleph = 'א'; - const bet = 'ב'; - - // As Hebrew is RTL... this is probably the clearest way for us LTR people to - // clearly see what's going on without ordering mechanics messing up the render. - let breaks = breakWords(`${aleph}' ${aleph}" ${aleph}"${bet}`); - let words = breaks.map(span => span.text); - - // A lingering double-quote isn't cool, but one in the middle's fine. - // Lingering single-quote is fine regardless. - assert.deepEqual(words, [`${aleph}'`, `${aleph}`, `"`, `${aleph}"${bet}`]); - }); - - it(`doesn't break within digit + digit/letter sequences (WB8-10)`, function() { - let breaks = breakWords('a1b2c3 hunter2 ab12cd34 1234567890'); - let words = breaks.map(span => span.text); - - assert.deepEqual(words, ['a1b2c3', 'hunter2', 'ab12cd34', '1234567890']); - }); - - it('does not break within formatted number sequences (WB11-12)', function() { - // Note: `'` fits "MidNumLetQ", part of the two rules! - let breaks = breakWords(`1.2.3 3,458.01 3.45'8,01`); - let words = breaks.map(span => span.text); - - assert.deepEqual(words, [`1.2.3`, `3,458.01`, `3.45'8,01`]); - - let breaks2 = breakWords(`.1' ,3.`); - let words2 = breaks2.map(span => span.text); - - assert.deepEqual(words2, [`.`, `1`, `'`, `,`, `3`, `.`]); - }); - - it('does not break between Katakana (WB13)', function() { - const kataSmA = '\u30a2'; //ァ - const kataA = '\u30a2'; //ア - const kataSound = '\u309b'; // ゛ - - let breaks = breakWords(`${kataSound}${kataA} ${kataSmA}${kataSound}b ${kataA}${kataSound}${kataSmA}`); - let words = breaks.map(span => span.text); - - assert.deepEqual(words, [ - `${kataSound}${kataA}`, - `${kataSmA}${kataSound}`, - 'b', - `${kataA}${kataSound}${kataSmA}` - ]); - }); - - it('does not break form extenders (WB13a-b)', function() { - // The `_` (underscore) fits the ExtendNumLet class this rule focuses on. - const kataA = '\u30a2'; //ア - - let breaks = breakWords(`${kataA}_a__0_b_${kataA} _${kataA} 1_ _c_ ____`); - let words = breaks.map(span => span.text); - - assert.deepEqual(words, [ - `${kataA}_a__0_b_${kataA}`, - `_${kataA}`, - `1_`, - `_c_`, - `____` - ]); - }); - - it('handles emoji flag sequences properly (WB15-16)', function() { - // For clarity on what's being tested... - let CA_FLAG = '\u{1f1e8}\u{1f1e6}' // '🇨🇦' (canadian flag emoji); should not be broken. - let KH_FLAG = '\u{1f1f0}\u{1f1ed}' // '🇰🇭' (khmer flag emoji); same - let X_FLAG_PIECE = '\u{1f1fd}' // '🇽' (half of a flag emoji; '🇽🇽' doesn't match a flag) - let breaks = breakWords(`${CA_FLAG}${KH_FLAG}${X_FLAG_PIECE}${X_FLAG_PIECE}`); - let words = breaks.map(span => span.text); - - // Note that the emoji may not render well within VSCode, but they show up nicely on GitHub. - assert.deepEqual(words, ['🇨🇦', '🇰🇭', '🇽🇽']); + assert.deepEqual(words, ["l'", "objectif", "aujourd'hui"]); + }); }); }); diff --git a/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js b/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js index 6a4148b669..fa09ca20de 100644 --- a/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js +++ b/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js @@ -45,7 +45,7 @@ describe('ContextTracker', function() { assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); }); - it("properly matches and aligns when a 'wordbreak' is added'", function() { + it("properly matches and aligns when a 'wordbreak' is added", function() { let existingContext = ["an", "apple", "a", "day", "keeps", "the", "doctor"]; let transform = { insert: ' ', @@ -56,7 +56,7 @@ describe('ContextTracker', function() { let rawTokens = ["an", null, "apple", null, "a", null, "day", null, "keeps", null, "the", null, "doctor", null, ""]; let existingState = ContextTracker.modelContextState(existingContext); - let state = ContextTracker.attemptMatchContext(newContext, existingState, null, toWrapperDistribution(transform)); + let state = ContextTracker.attemptMatchContext(newContext, existingState, toWrapperDistribution(transform)); assert.isNotNull(state); assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); @@ -65,6 +65,26 @@ describe('ContextTracker', function() { assert.isEmpty(state.tokens[state.tokens.length - 1].transformDistributions); }); + it("properly matches and aligns when an implied 'wordbreak' occurs (as when following \"'\")", function() { + let existingContext = ["'"]; + let transform = { + insert: 'a', + deleteLeft: 0 + } + let newContext = Array.from(existingContext); + newContext.push('a'); // The incoming transform should produce a new token WITH TEXT. + let rawTokens = ["'", null, "a"]; + + let existingState = ContextTracker.modelContextState(existingContext); + let state = ContextTracker.attemptMatchContext(newContext, existingState, toWrapperDistribution(transform)); + assert.isNotNull(state); + assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); + + // The 'wordbreak' transform + assert.isEmpty(state.tokens[state.tokens.length - 2].transformDistributions); + assert.isNotEmpty(state.tokens[state.tokens.length - 1].transformDistributions); + }); + it("properly matches and aligns when lead token is removed AND a 'wordbreak' is added'", function() { let existingContext = ["an", "apple", "a", "day", "keeps", "the", "doctor"]; let transform = { @@ -77,7 +97,7 @@ describe('ContextTracker', function() { let rawTokens = ["apple", null, "a", null, "day", null, "keeps", null, "the", null, "doctor", null, ""]; let existingState = ContextTracker.modelContextState(existingContext); - let state = ContextTracker.attemptMatchContext(newContext, existingState, null, toWrapperDistribution(transform)); + let state = ContextTracker.attemptMatchContext(newContext, existingState, toWrapperDistribution(transform)); assert.isNotNull(state); assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); diff --git a/common/predictive-text/unit_tests/headless/transform-utils.js b/common/predictive-text/unit_tests/headless/transform-utils.js new file mode 100644 index 0000000000..b00a24a123 --- /dev/null +++ b/common/predictive-text/unit_tests/headless/transform-utils.js @@ -0,0 +1,58 @@ +var assert = require('chai').assert; + +let TransformUtils = require('../../../web/lm-worker/build/intermediate.js').TransformUtils; + +describe('TransformUtils', function () { + describe('isWhitespace', function () { + it("should not match a string containing standard alphabetic characters", function () { + let testTransforms = [{ + insert: "a ", + deleteLeft: 0 + }, { + insert: " a", + deleteLeft: 0 + }, { + insert: "ab", + deleteLeft: 0 + }]; + + testTransforms.forEach((transform) => assert.isFalse(TransformUtils.isWhitespace(transform), `failed with: '${transform.insert}'`)); + }); + + it("should match a simple ' ' transform", function() { + transform = { + insert: " ", + deleteLeft: 0 + }; + + assert.isTrue(TransformUtils.isWhitespace(transform)); + }); + + it("should match a simple ' ' transform with delete-left", function() { + transform = { + insert: " ", + deleteLeft: 1 + }; + + assert.isTrue(TransformUtils.isWhitespace(transform)); + }); + + it("should match a transform consisting of multiple characters of only whitespace", function() { + transform = { + insert: " \n\r\u00a0\t\u2000 ", + deleteLeft: 0 + }; + + assert.isTrue(TransformUtils.isWhitespace(transform)); + }); + + it("stress tests", function() { + transform = { + insert: " \n\r\u00a0\ta\u2000 ", // the 'a' should cause failure. + deleteLeft: 0 + }; + + assert.isFalse(TransformUtils.isWhitespace(transform)); + }); + }); +}); diff --git a/common/predictive-text/unit_tests/headless/worker-model-compositor.js b/common/predictive-text/unit_tests/headless/worker-model-compositor.js index f7e9173cee..8080b48f47 100644 --- a/common/predictive-text/unit_tests/headless/worker-model-compositor.js +++ b/common/predictive-text/unit_tests/headless/worker-model-compositor.js @@ -50,6 +50,45 @@ describe('ModelCompositor', function() { }); }); + it('strongly avoids corrections for single-character roots', function() { + let compositor = new ModelCompositor(plainModel); + let context = { + left: '', startOfBuffer: true, endOfBuffer: true, + }; + + // The 'weights' involved imply that we have an edge-case fat finger on the bottom of + // the 'q' key, slightly in its favor. + let inputDistribution = [ + {sample: {insert: 'q', deleteLeft: 0}, p: 0.5}, // 'quite' (679) and 'question' (644) are included! + {sample: {insert: 'a', deleteLeft: 0}, p: 0.4} // but at lower weight than 'and' (998). + ]; + + compositor.predict({insert: '', deleteLeft: 0}, context); // Initialize context tracking first! + let suggestions = compositor.predict(inputDistribution, context); + + // remove the keep suggestion; we're not testing that here. + suggestions = suggestions.filter((suggestion) => suggestion.tag != 'keep'); + + suggestions.sort((a, b) => b.p - a.p); + + // There are only 4 suggestions in this limited test model that begin with 'q'. + // We expect more than that, since 'a' is indicated to be very close by. + assert.isAbove(suggestions.length, 4, "fat-finger style corrections needed for test comparisons are missing"); + + // Note: 'and' is (currently) modeled by the text-fixture model to have 9.3x the base probability + // that the worst 'q'-rooted suggestion ('quality') does. Without single-character correction + // avoidance logic, this test _will_ fail. + // + // In case a tweak to test parameters is desired, note that 'and' beats rank #3 - 'questions' - + // at 3.36x base. At the time of writing this test, upping 'a's probability to 0.45 will block + // 'quality' while the top three 'q's (ending with 'questions') remain in place. + let qRange = suggestions.slice(0, 4); + assert.isUndefined(qRange.find((suggestion) => suggestion.transform.insert.charAt(0) != 'q')); + + let aRange = suggestions.slice(4); + assert.isUndefined(aRange.find((suggestion) => suggestion.transform.insert.charAt(0) == 'q')); + }); + it('properly handles suggestions after a backspace', function() { let compositor = new ModelCompositor(plainModel); let context = { @@ -66,12 +105,53 @@ describe('ModelCompositor', function() { // Suggestions always delete the full root of the suggestion. // // After a backspace, that means the text 'the' - 3 chars. - // Char 4 is for the original backspace, as suggstions are built + // Char 4 is for the original backspace, as suggestions are built // based on the context state BEFORE the triggering input - // here, a backspace. assert.equal(suggestion.transform.deleteLeft, 4); }); }); + + it('properly handles suggestions for the first letter after a ` `', function() { + let compositor = new ModelCompositor(plainModel); + let context = { + left: 'the', startOfBuffer: true, endOfBuffer: true, + }; + + let inputTransform = { + insert: ' ', + deleteLeft: 0 + }; + + let suggestions = compositor.predict(inputTransform, context); + suggestions.forEach(function(suggestion) { + // After a space, predictions are based on a new, zero-length root. + // With nothing to replace, .deleteLeft should be zero. + assert.equal(suggestion.transform.deleteLeft, 0); + }); + }); + + it('properly handles suggestions for the first letter after a `\'`', function() { + let compositor = new ModelCompositor(plainModel); + let context = { + left: "the '", startOfBuffer: true, endOfBuffer: true, + }; + + // This results in a new word boundary (between the `'` and the `a`). + // Basically, an implied (but nonexistent) ` `. + let inputTransform = { + insert: "a", + deleteLeft: 0 + }; + + let suggestions = compositor.predict(inputTransform, context); + suggestions.forEach(function(suggestion) { + // Suggestions always delete the full root of the suggestion. + // Which, here, didn't exist before the input. Nothing to + // replace => nothing for the suggestion to delete. + assert.equal(suggestion.transform.deleteLeft, 0); + }); + }); }); describe('applySuggestionCasing', function() { diff --git a/common/web/input-processor/src/text/inputProcessor.ts b/common/web/input-processor/src/text/inputProcessor.ts index 9471c244fe..b45098e538 100644 --- a/common/web/input-processor/src/text/inputProcessor.ts +++ b/common/web/input-processor/src/text/inputProcessor.ts @@ -12,7 +12,7 @@ namespace com.keyman.text { /** * Indicates the device (platform) to be used for non-keystroke events, - * such as those sent to `begin postkeystroke` and `begin newcontext` + * such as those sent to `begin postkeystroke` and `begin newcontext` * entry points. */ private contextDevice: utils.DeviceSpec; @@ -273,6 +273,7 @@ namespace com.keyman.text { let totalMass = 0; // Tracks sum of non-error probabilities. for(let pair of keyDistribution) { if(pair.p < KEYSTROKE_EPSILON) { + totalMass += pair.p; break; } else if(timer && timer() >= TIMEOUT_THRESHOLD) { // Note: it's always possible that the thread _executing_ our JS diff --git a/common/web/keyboard-processor/src/keyboards/activeLayout.ts b/common/web/keyboard-processor/src/keyboards/activeLayout.ts index b1be473bc6..7f0f2f935f 100644 --- a/common/web/keyboard-processor/src/keyboards/activeLayout.ts +++ b/common/web/keyboard-processor/src/keyboards/activeLayout.ts @@ -37,6 +37,7 @@ namespace com.keyman.keyboards { layer: string; displayLayer: string; nextlayer: string; + sp?: ButtonClass; private baseKeyEvent: text.KeyEvent; isMnemonic: boolean = false; @@ -73,6 +74,13 @@ namespace com.keyman.keyboards { return this.id; } + @Enumerable + public get isPadding(): boolean { + // Does not include 9 (class: blank) as that may be an intentional 'catch' for misplaced + // keystrokes. + return this['sp'] == 10; // Button class: hidden. + } + /** * A unique identifier based on both the key ID & the 'desktop layer' to be used for the key. * @@ -337,14 +345,12 @@ namespace com.keyman.keyboards { // Allow for right OSK margin (15 layout units) let rightMargin = ActiveKey.DEFAULT_RIGHT_MARGIN/totalWidth; - totalPercent += rightMargin; // If a single key, and padding is negative, add padding to right align the key if(keys.length == 1 && parseInt(keys[0]['pad'],10) < 0) { keyPercent=parseInt(keys[0]['width'],10)/totalWidth; keys[0]['widthpc']=keyPercent; - totalPercent += keyPercent; - keys[0]['padpc']=1-totalPercent; + keys[0]['padpc']=1-(totalPercent + keyPercent + rightMargin); // compute center's default x-coord (used in headless modes) setProportions(keys[0] as ActiveKey, padPercent, keyPercent, totalPercent); @@ -352,8 +358,7 @@ namespace com.keyman.keyboards { let j=keys.length-1; padPercent=parseInt(keys[j]['pad'],10)/totalWidth; keys[j]['padpc']=padPercent; - totalPercent += padPercent; - keys[j]['widthpc'] = keyPercent = 1-totalPercent; + keys[j]['widthpc'] = keyPercent = 1-(totalPercent + padPercent + rightMargin); // compute center's default x-coord (used in headless modes) setProportions(keys[j] as ActiveKey, padPercent, keyPercent, totalPercent); @@ -515,7 +520,7 @@ namespace com.keyman.keyboards { // Should we wish to allow multiple different transforms for distance -> probability, use a function parameter in place // of the formula in the loop below. for(let key in keyDists) { - totalMass += keyProbs[key] = 1 / (keyDists[key] + 1e-6); // Prevent div-by-0 errors. + totalMass += keyProbs[key] = 1 / (Math.pow(keyDists[key], 2) + 1e-6); // Prevent div-by-0 errors. } for(let key in keyProbs) { @@ -550,6 +555,8 @@ namespace com.keyman.keyboards { // Results in a more optimized distribution. if(text.Codes.isKnownOSKModifierKey(key.baseKeyID)) { return; + } else if(key.isPadding) { // to the user, blank / padding keys do not exist. + return; } } // These represent the within-key distance of the touch from the key's center. diff --git a/common/web/keyboard-processor/src/text/codes.ts b/common/web/keyboard-processor/src/text/codes.ts index 915ae7feed..48fde0d728 100644 --- a/common/web/keyboard-processor/src/text/codes.ts +++ b/common/web/keyboard-processor/src/text/codes.ts @@ -81,10 +81,13 @@ namespace com.keyman.text { case 'K_SHIFT': case 'K_LOPT': case 'K_ROPT': - case 'K_NUMLOCK': // Often used for numeric layers. + case 'K_NUMLOCK': // Often used for numeric layers. case 'K_CAPS': return true; default: + if(Codes.keyCodes[keyID] >= 50000) { // A few are used by `sil_euro_latin`. + return true; // is a 'K_' key defined for layer shifting or 'control' use. + } // Refer to text/codes.ts - these are Keyman-custom "keycodes" used for // layer shifting keys. To be safe, we currently let K_TABBACK and // K_TABFWD through, though we might be able to drop them too. diff --git a/common/web/lm-worker/src/correction/context-tracker.ts b/common/web/lm-worker/src/correction/context-tracker.ts index 52e89bedc8..e17e201d78 100644 --- a/common/web/lm-worker/src/correction/context-tracker.ts +++ b/common/web/lm-worker/src/correction/context-tracker.ts @@ -32,10 +32,6 @@ namespace correction { replacements: TrackedContextSuggestion[]; activeReplacementId: number = -1; - get isNew(): boolean { - return this.transformDistributions.length == 0; - } - get currentText(): string { if(this.replacementText === undefined || this.replacementText === null) { return this.raw; @@ -89,7 +85,7 @@ namespace correction { if(token.replacementText) { copy.replacementText = token.replacementText; } - + return copy; }); this.searchSpace = obj.searchSpace; @@ -139,8 +135,8 @@ namespace correction { // Track the Transform that resulted in the whitespace 'token'. // Will be needed for phrase-level correction/prediction. - whitespaceToken.transformDistributions = [transformDistribution]; - + whitespaceToken.transformDistributions = transformDistribution ? [transformDistribution] : []; + whitespaceToken.raw = null; this.tokens.push(whitespaceToken); } @@ -149,19 +145,19 @@ namespace correction { * Used for 14.0's backspace workaround, which flattens all previous Distribution * entries because of limitations with direct use of backspace transforms. * @param tokenText - * @param transformId + * @param transformId */ replaceTailForBackspace(tokenText: USVString, transformId: number) { this.tokens.pop(); // It's a backspace transform; time for special handling! // - // For now, with 14.0, we simply compress all remaining Transforms for the token into - // multiple single-char transforms. Probabalistically modeling BKSP is quite complex, + // For now, with 14.0, we simply compress all remaining Transforms for the token into + // multiple single-char transforms. Probabalistically modeling BKSP is quite complex, // so we simplify by assuming everything remaining after a BKSP is 'true' and 'intended' text. // // Note that we cannot just use a single, monolithic transform at this point b/c - // of our current edit-distance optimization strategy; diagonalization is currently... + // of our current edit-distance optimization strategy; diagonalization is currently... // not very compatible with that. let backspacedTokenContext: Distribution[] = textToCharTransforms(tokenText, transformId).map(function(transform) { return [{sample: transform, p: 1.0}]; @@ -175,7 +171,7 @@ namespace correction { updateTail(transformDistribution: Distribution, tokenText?: USVString) { let editedToken = this.tail; - + // Preserve existing text if new text isn't specified. tokenText = tokenText || (tokenText === '' ? '' : editedToken.raw); @@ -191,7 +187,7 @@ namespace correction { toRawTokenization() { let sequence: USVString[] = []; - + for(let token of this.tokens) { // Hide any tokens representing wordbreaks. (Thinking ahead to phrase-level possibilities) if(token.currentText !== null) { @@ -281,7 +277,7 @@ namespace correction { /** * Returns items contained within the circular array, ordered from 'oldest' to 'newest' - * the same order in which the items will be dequeued. - * @param index + * @param index */ item(index: number) { if(index >= this.count) { @@ -294,7 +290,7 @@ namespace correction { } export class ContextTracker extends CircularArray { - static attemptMatchContext(tokenizedContext: USVString[], + static attemptMatchContext(tokenizedContext: USVString[], matchState: TrackedContextState, transformDistribution?: Distribution,): TrackedContextState { // Map the previous tokenized state to an edit-distance friendly version. @@ -335,7 +331,7 @@ namespace correction { } // Can happen for the first text input after backspace deletes a wordbreaking character, - // thus the new input continues a previous word while dropping the empty word after + // thus the new input continues a previous word while dropping the empty word after // that prior wordbreaking character. // // We can't handle it reliably from this match state, but a previous entry (without the empty token) @@ -353,7 +349,7 @@ namespace correction { // If we've made it here... success! We have a context match! let state: TrackedContextState; - + if(pushedTail) { // On suggestion acceptance, we should update the previous final token. // We do it first so that the acceptance is replicated in the new TrackedContextState @@ -376,7 +372,9 @@ namespace correction { if(primaryInput && primaryInput.insert == "" && primaryInput.deleteLeft == 0 && !primaryInput.deleteRight) { primaryInput = null; } - const isBackspace = primaryInput && primaryInput.insert == "" && primaryInput.deleteLeft > 0 && !primaryInput.deleteRight; + + const isWhitespace = primaryInput && TransformUtils.isWhitespace(primaryInput); + const isBackspace = primaryInput && TransformUtils.isBackspace(primaryInput); const finalToken = tokenizedContext[tokenizedContext.length-1]; /* Assumption: This is an adequate check for its two sub-branches. @@ -388,7 +386,7 @@ namespace correction { * - Assumption: one keystroke may only cause a single token to be appended to the context * - That is, no "reasonable" keystroke would emit a Transform adding two separate word tokens * - For languages using whitespace to word-break, said keystroke would have to include said whitespace to break the assumption. - */ + */ // If there is/was more than one context token available... if(editPath.length > 1) { @@ -399,17 +397,29 @@ namespace correction { // We're adding an additional context token. if(pushedTail) { - // ASSUMPTION: any transform that triggers this case is a pure-whitespace Transform, as we - // need a word-break before beginning a new word's context. - // Worth note: when invalid, the lm-layer already has problems in other aspects too. - state.pushWhitespaceToTail(transformDistribution); + const tokenizedTail = tokenizedContext[tokenizedContext.length - 1]; + /* + * Common-case: most transforms that trigger this case are from pure-whitespace Transforms. MOST. + * + * Less-common, but noteworthy: some wordbreaks may occur without whitespace. Example: + * `"o` => ['"', 'o']. Make sure to double-check against `tokenizedContext`! + */ + let pushedToken = new TrackedContextToken(); + pushedToken.raw = tokenizedTail; - let emptyToken = new TrackedContextToken(); - emptyToken.raw = ''; - // Continuing the earlier assumption, that 'pure-whitespace Transform' does not emit any initial characters - // for the new word (token), so the input keystrokes do not correspond to the new text token. - emptyToken.transformDistributions = []; - state.pushTail(emptyToken); + if(isWhitespace || !primaryInput) { + state.pushWhitespaceToTail(transformDistribution ?? []); + // Continuing the earlier assumption, that 'pure-whitespace Transform' does not emit any initial characters + // for the new word (token), so the input keystrokes do not correspond to the new text token. + pushedToken.transformDistributions = []; + } else { + state.pushWhitespaceToTail(); + // Assumption: Since we only allow one-transform-at-a-time changes between states, we shouldn't be missing + // any metadata used to construct the new context state token. + pushedToken.transformDistributions = transformDistribution ? [transformDistribution] : []; + } + + state.pushTail(pushedToken); } else { // We're editing the final context token. // TODO: Assumption: we didn't 'miss' any inputs somehow. // As is, may be prone to fragility should the lm-layer's tracked context 'desync' from its host's. @@ -442,7 +452,9 @@ namespace correction { return state; } - static modelContextState(tokenizedContext: USVString[], lexicalModel: LexicalModel): TrackedContextState { + static modelContextState(tokenizedContext: USVString[], + transformDistribution: Distribution, + lexicalModel: LexicalModel): TrackedContextState { let baseTokens = tokenizedContext.map(function(entry) { let token = new TrackedContextToken(); token.raw = entry; @@ -483,13 +495,12 @@ namespace correction { * Compares the current, post-input context against the most recently-seen contexts from previous prediction calls, returning * the most information-rich `TrackedContextState` possible. If a match is found, the state will be annotated with the * input information provided to previous prediction calls and persisted correction-search calculations for re-use. - * - * @param model - * @param context - * @param mainTransform - * @param transformDistribution + * + * @param model + * @param context + * @param transformDistribution */ - analyzeState(model: LexicalModel, + analyzeState(model: LexicalModel, context: Context, transformDistribution?: Distribution): TrackedContextState { if(!model.traverseFromRoot) { @@ -519,7 +530,7 @@ namespace correction { // // Assumption: as a caret needs to move to context before any actual transform distributions occur, // this state is only reached on caret moves; thus, transformDistribution is actually just a single null transform. - let state = ContextTracker.modelContextState(tokenizedContext.left, model); + let state = ContextTracker.modelContextState(tokenizedContext.left, transformDistribution, model); state.taggedContext = context; this.enqueue(state); return state; diff --git a/common/web/lm-worker/src/correction/distance-modeler.ts b/common/web/lm-worker/src/correction/distance-modeler.ts index b0cd2179a9..723cbc93ca 100644 --- a/common/web/lm-worker/src/correction/distance-modeler.ts +++ b/common/web/lm-worker/src/correction/distance-modeler.ts @@ -204,6 +204,15 @@ namespace correction { // TODO: might should also track diagonalWidth. return inputString + models.SENTINEL_CODE_UNIT + matchString; } + + get isFullReplacement(): boolean { + // If the known edit-distance cost is equal to the input length, this means + // that literally every input has been full-on replaced. Thus, this is + // likely not a good 'root' to use for predictions. + // + // Logic exception: 0 cost, 0 length != a "replacement". + return this.knownCost && this.knownCost == this.priorInput.length; + } } class SearchSpaceTier { @@ -653,7 +662,7 @@ namespace correction { shouldTimeout(): boolean { const now = Date.now(); - if(this.start - now > this.maxTrueTime) { + if(now - this.start > this.maxTrueTime) { return true; } @@ -710,7 +719,7 @@ namespace correction { let batcher = new BatchingAssistant(); - const timer = new ExecutionTimer(maxTime*3, maxTime); + const timer = new ExecutionTimer(maxTime*1.5, maxTime); // Stage 1 - if we already have extracted results, build a queue just for them and iterate over it first. let returnedValues = Object.values(this.returnedValues); @@ -721,6 +730,14 @@ namespace correction { timer.startLoop(); while(preprocessedQueue.count > 0) { let entry = preprocessedQueue.dequeue(); + + // Is the entry a reasonable result? + if(entry.isFullReplacement) { + // If the entry's 'match' fully replaces the input string, we consider it + // unreasonable and ignore it. + continue; + } + let batch = batcher.checkAndAdd(entry); timer.markIteration(); @@ -761,6 +778,13 @@ namespace correction { if(newResult.type == 'none') { break; } else if(newResult.type == 'complete') { + // Is the entry a reasonable result? + if(newResult.finalNode.isFullReplacement) { + // If the entry's 'match' fully replaces the input string, we consider it + // unreasonable and ignore it. Also, if we've reached this point... + // we can(?) assume that everything thereafter is as well. + break; + } batch = batcher.checkAndAdd(newResult.finalNode); } diff --git a/common/web/lm-worker/src/index.ts b/common/web/lm-worker/src/index.ts index 7cc3bcbde1..e3c16fa2c8 100644 --- a/common/web/lm-worker/src/index.ts +++ b/common/web/lm-worker/src/index.ts @@ -32,6 +32,7 @@ /// /// /// +/// /** * Encapsulates all the state required for the LMLayer's worker thread. @@ -407,6 +408,7 @@ if (typeof module !== 'undefined' && typeof module.exports !== 'undefined') { module.exports['wordBreakers'] = wordBreakers; /// XXX: export the ModelCompositor for testing. module.exports['ModelCompositor'] = ModelCompositor; + module.exports['TransformUtils'] = TransformUtils; } else if (typeof self !== 'undefined' && 'postMessage' in self && 'importScripts' in self) { // Automatically install if we're in a Web Worker. LMLayerWorker.install(self as any); // really, 'as typeof globalThis', but we're currently getting TS errors from use of that. diff --git a/common/web/lm-worker/src/model-compositor.ts b/common/web/lm-worker/src/model-compositor.ts index 329460776d..95c2e390fd 100644 --- a/common/web/lm-worker/src/model-compositor.ts +++ b/common/web/lm-worker/src/model-compositor.ts @@ -6,6 +6,23 @@ class ModelCompositor { private static readonly MAX_SUGGESTIONS = 12; readonly punctuation: LexicalModelPunctuation; + /** + * Controls the strength of anti-corrective measures for single-character scenarios. + * The base key probability will be raised to this power for this specific case. + * + * Current selection's motivation: (0.5 / 0.4) ^ 16 ~= 35.5. + * - if the most likely has p=0.5 and second-most has p=0.4 - a highly-inaccurate key + * stroke - the net effect will apply a factor of 35.5 to the lexical probability of + * the best key's prediction roots, favoring it in this manner. + * - less extreme edge cases will have a significantly stronger factor, acting as a + * "soft threshold". + * - truly ambiguous, "coin flip" cases will have a lower factor and thus favor the + * more likely words from the pair. + * - Our OSK key-element borders aren't visible to the user, so the 'spot' where + * behavior changes might feel arbitrary to users if we used a hard threshold instead. + */ + private static readonly SINGLE_CHAR_KEY_PROB_EXPONENT = 16; + private SUGGESTION_ID_SEED = 0; constructor(lexicalModel: LexicalModel) { @@ -16,30 +33,6 @@ class ModelCompositor { this.punctuation = ModelCompositor.determinePunctuationFromModel(lexicalModel); } - protected isWhitespace(transform: Transform): boolean { - // Matches prefixed text + any instance of a character with Unicode general property Z* or the following: CR, LF, and Tab. - let whitespaceRemover = /.*[\u0009\u000A\u000D\u0020\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b\u2028\u2029\u202f\u205f\u3000]/i; - - // Filter out null-inserts; their high probability can cause issues. - if(transform.insert == '') { // Can actually register as 'whitespace'. - return false; - } - - let insert = transform.insert; - - insert = insert.replace(whitespaceRemover, ''); - - return insert == ''; - } - - protected isBackspace(transform: Transform): boolean { - return transform.insert == "" && transform.deleteLeft > 0; - } - - protected isEmpty(transform: Transform): boolean { - return transform.insert == '' && transform.deleteLeft == 0; - } - private predictFromCorrections(corrections: ProbabilityMass[], context: Context): Distribution { let returnedPredictions: Distribution = []; @@ -98,8 +91,8 @@ class ModelCompositor { })[0].sample; // Only allow new-word suggestions if space was the most likely keypress. - let allowSpace = this.isWhitespace(inputTransform); - let allowBksp = this.isBackspace(inputTransform); + let allowSpace = TransformUtils.isWhitespace(inputTransform); + let allowBksp = TransformUtils.isBackspace(inputTransform); let postContext = models.applyTransform(inputTransform, context); let keepOptionText = this.wordbreak(postContext); @@ -109,7 +102,7 @@ class ModelCompositor { // Used to restore whitespaces if operations would remove them. let prefixTransform: Transform; - let contextState: correction.TrackedContextState = null; + let postContextState: correction.TrackedContextState = null; // Section 1: determining 'prediction roots'. if(!this.contextTracker) { @@ -124,18 +117,18 @@ class ModelCompositor { predictionRoots = [{sample: inputTransform, p: 1.0}]; prefixTransform = inputTransform; } else { - predictionRoots = transformDistribution.map(function(alt) { + predictionRoots = transformDistribution.map((alt) => { let transform = alt.sample; // Filter out special keys unless they're expected. - if(this.isWhitespace(transform) && !allowSpace) { + if(TransformUtils.isWhitespace(transform) && !allowSpace) { return null; - } else if(this.isBackspace(transform) && !allowBksp) { + } else if(TransformUtils.isBackspace(transform) && !allowBksp) { return null; } return alt; - }, this); + }); } // Remove `null` entries. @@ -144,12 +137,15 @@ class ModelCompositor { // Running in bulk over all suggestions, duplicate entries may be possible. rawPredictions = this.predictFromCorrections(predictionRoots, context); } else { - contextState = this.contextTracker.analyzeState(this.lexicalModel, - postContext, - !this.isEmpty(inputTransform) ? - transformDistribution: - null - ); + // Token replacement benefits greatly from knowledge of the prior context state. + let contextState = this.contextTracker.analyzeState(this.lexicalModel, context, null); + // Corrections and predictions are based upon the post-context state, though. + postContextState = this.contextTracker.analyzeState(this.lexicalModel, + postContext, + !TransformUtils.isEmpty(inputTransform) ? + transformDistribution: + null + ); // TODO: Should we filter backspaces & whitespaces out of the transform distribution? // Ideally, the answer (in the future) will be no, but leaving it in right now may pose an issue. @@ -158,19 +154,81 @@ class ModelCompositor { // let's just note that right now, there will only ever be one. // // The 'eventual' logic will be significantly more complex, though still manageable. - let searchSpace = contextState.searchSpace[0]; + let searchSpace = postContextState.searchSpace[0]; - let newEmptyToken = false; - // Detect if we're starting a new context state. - let contextTokens = contextState.tokens; - if(contextTokens.length == 0 || contextTokens[contextTokens.length - 1].isNew) { - if(this.isEmpty(inputTransform) || this.isWhitespace(inputTransform)) { - newEmptyToken = true; + // No matter the prediction, once we know the root of the prediction, we'll always 'replace' the + // same amount of text. We can handle this before the big 'prediction root' loop. + let deleteLeft = 0; + + // The amount of text to 'replace' depends upon whatever sort of context change occurs + // from the received input. + const postContextTokens = postContextState.tokens; + let postContextLength = postContextTokens.length; + let contextLengthDelta = postContextTokens.length - contextState.tokens.length; + // If the context now has more tokens, the token we'll be 'predicting' didn't originally exist. + if(postContextLength == 0 || contextLengthDelta > 0) { + // As the word/token being corrected/predicted didn't originally exist, there's no + // part of it to 'replace'. + deleteLeft = 0; + + // If the new token is due to whitespace or due to a different input type that would + // likely imply a tokenization boundary... + if(TransformUtils.isWhitespace(inputTransform)) { + /* TODO: consider/implement: the second half of the comment above. + * For example: on input of a `'`, predict new words instead of replacing the `'`. + * (since after a letter, the `'` will be ignored, anyway) + * + * Idea: if the model's most likely prediction (with no root) would make a new + * token if appended to the current token, that's probably a good case. + * Keeps the check simple & quick. + * + * Might need a mixed mode, though: ';' is close enough that `l` is a reasonable + * fat-finger guess. So yeah, we're not addressing this idea right now. + * - so... consider multiple context behavior angles when building prediction roots? + * + * May need something similar to help handle contractions during their construction, + * but that'd be within `ContextTracker`. + * can' => [`can`, `'`] + * can't => [`can't`] (WB6, 7 of https://unicode.org/reports/tr29/#Word_Boundary_Rules) + * + * (Would also helps WB7b+c for Hebrew text) + */ + + // Infer 'new word' mode, even if we received new text when reaching + // this position. That new text didn't exist before, so still - nothing + // to 'replace'. prefixTransform = inputTransform; - context = postContext; // Ensure the whitespace token is preapplied! + context = postContext; // As far as predictions are concerned, the post-context state + // should not be replaced. Predictions are to be rooted on + // text "up for correction" - so we want a null root for this + // branch. + contextState = postContextState; } + // If the tokenized context length is shorter... sounds like a backspace (or similar). + } else if (contextLengthDelta < 0) { + /* Ooh, we've dropped context here. Almost certainly from a backspace. + * Even if we drop multiple tokens... well, we know exactly how many chars + * were actually deleted - `inputTransform.deleteLeft`. + * Since we replace a word being corrected/predicted, we take length of the remaining + * context's tail token in addition to however far was deleted to reach that state. + */ + deleteLeft = this.wordbreak(postContext).kmwLength() + inputTransform.deleteLeft; + } else { + // Suggestions are applied to the pre-input context, so get the token's original length. + // We're on the same token, so just delete its text for the replacement op. + deleteLeft = this.wordbreak(context).kmwLength(); } + // Is the token under construction newly-constructed / is there no pre-existing root? + // If so, we want to strongly avoid overcorrection, even for 'nearby' keys. + // (Strong lexical frequency differences can easily cause overcorrection when only + // one key's available.) + // + // NOTE: we only want this applied word-initially, when any corrections 'correct' + // 100% of the word. Things are generally fine once it's not "all or nothing." + let tailToken = postContextTokens[postContextTokens.length - 1]; + const isTokenStart = tailToken.transformDistributions.length <= 1; + // TODO: whitespace, backspace filtering. Do it here. // Whitespace is probably fine, actually. Less sure about backspace. @@ -192,19 +250,6 @@ class ModelCompositor { finalInput = inputTransform; // A fallback measure. Greatly matters for empty contexts. } - let deleteLeft = 0; - // remove actual token string. If new token, there should be nothing to delete. - if(!newEmptyToken) { - // If this is triggered from a backspace, make sure to use its results - // and also include its left-deletions! It's the one post-input context case. - if(allowBksp) { - deleteLeft = this.wordbreak(postContext).kmwLength() + inputTransform.deleteLeft; - } else { - // Normal case - use the pre-input context. - deleteLeft = this.wordbreak(context).kmwLength(); - } - } - // Replace the existing context with the correction. let correctionTransform: Transform = { insert: correction, // insert correction string @@ -212,9 +257,39 @@ class ModelCompositor { id: inputTransform.id // The correction should always be based on the most recent external transform/transcription ID. } + let rootCost = match.totalCost; + + /* If we're dealing with the FIRST keystroke of a new sequence, we'll **dramatically** boost + * the exponent to ensure only VERY nearby corrections have a chance of winning, and only if + * there are significantly more likely words. We only need this to allow very minor fat-finger + * adjustments for 100% keystroke-sequence corrections in order to prevent finickiness on + * key borders. + * + * Technically, the probabilities this produces won't be normalized as-is... but there's no + * true NEED to do so for it, even if it'd be 'nice to have'. Consistently tracking when + * to apply it could become tricky, so it's simpler to leave out. + * + * Worst-case, it's possible to temporarily add normalization if a code deep-dive + * is needed in the future. + */ + if(isTokenStart) { + /* Suppose a key distribution: most likely with p=0.5, second-most with 0.4 - a pretty + * ambiguous case that would only arise very near the center of the boundary between two keys. + * Raising (0.5/0.4)^16 ~= 35.53. (At time of writing, SINGLE_CHAR_KEY_PROB_EXPONENT = 16.) + * That seems 'within reason' for correction very near boundaries. + * + * So, with the second-most-likely key being that close in probability, its best suggestion + * must be ~ 35.5x more likely than that of the truly-most-likely key to "win". So, it's not + * a HARD cutoff, but more of a 'soft' one. Keeping the principles in mind documented above, + * it's possible to tweak this to a more harsh or lenient setting if desired, rather than + * being totally "all or nothing" on which key is taken for highly-ambiguous keypresses. + */ + rootCost *= ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT; // note the `Math.exp` below. + } + return { sample: correctionTransform, - p: Math.exp(-match.totalCost) + p: Math.exp(-rootCost) }; }, this); @@ -411,8 +486,8 @@ class ModelCompositor { // Store the suggestions on the final token of the current context state (if it exists). // Or, once phrase-level suggestions are possible, on whichever token serves as each prediction's root. - if(contextState) { - contextState.tail.replacements = suggestions.map(function(suggestion) { + if(postContextState) { + postContextState.tail.replacements = suggestions.map(function(suggestion) { return { suggestion: suggestion, tokenWidth: 1 @@ -659,7 +734,7 @@ class ModelCompositor { // than before. if(this.contextTracker) { let tokenizedContext = models.tokenize(this.lexicalModel.wordbreaker || wordBreakers.default, context); - let contextState = correction.ContextTracker.modelContextState(tokenizedContext.left, this.lexicalModel); + let contextState = correction.ContextTracker.modelContextState(tokenizedContext.left, null, this.lexicalModel); this.contextTracker.enqueue(contextState); } } diff --git a/common/web/lm-worker/src/transformUtils.ts b/common/web/lm-worker/src/transformUtils.ts new file mode 100644 index 0000000000..cbb2c151fe --- /dev/null +++ b/common/web/lm-worker/src/transformUtils.ts @@ -0,0 +1,15 @@ +class TransformUtils { + static isWhitespace(transform: Transform): boolean { + // Matches a string that is entirely one or more characters with Unicode general property Z* or the following: CR, LF, and Tab. + const whitespaceRemover = /^[\u0009\u000A\u000D\u0020\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b\u2028\u2029\u202f\u205f\u3000]+$/i; + return transform.insert.match(whitespaceRemover) != null; + } + + static isBackspace(transform: Transform): boolean { + return transform.insert == "" && transform.deleteLeft > 0 && !transform.deleteRight; + } + + static isEmpty(transform: Transform): boolean { + return transform.insert == '' && transform.deleteLeft == 0 && !transform.deleteRight; + } +} \ No newline at end of file diff --git a/common/windows/delphi/setup/SFX.pas b/common/windows/delphi/setup/SFX.pas index 13491802cb..5af3972261 100644 --- a/common/windows/delphi/setup/SFX.pas +++ b/common/windows/delphi/setup/SFX.pas @@ -82,27 +82,38 @@ var fs: TFileStream; ms: TMemoryStream; begin - fs := TFileStream.Create(ParamStr(0), fmOpenRead or fmShareDenyWrite); - ms := TMemoryStream.Create; try - if not FindFirstHeader(fs) then - Exit(False); - - fs.Seek(StartOfFile, TSeekOrigin.soBeginning); - - ms.CopyFrom(fs, fs.Size - StartOfFile); - ms.Position := 0; - - with TZipFile.Create do + fs := TFileStream.Create(ParamStr(0), fmOpenRead or fmShareDenyWrite); + ms := TMemoryStream.Create; try - Open(ms, zmRead); - ExtractAll(ExtPath); + if not FindFirstHeader(fs) then + Exit(False); + + fs.Seek(StartOfFile, TSeekOrigin.soBeginning); + + ms.CopyFrom(fs, fs.Size - StartOfFile); + ms.Position := 0; + + with TZipFile.Create do + try + Open(ms, zmRead); + ExtractAll(ExtPath); + finally + Free; + end; finally - Free; + fs.Free; + ms.Free; + end; + except + on E:Exception do + begin + raise Exception.Create( + 'Failed to extract setup archive. '+ + 'You may have run out of disk space or there may be a '+ + 'problem with the source files.'#13#10#13#10+ + 'The error received was: '+E.Message); end; - finally - fs.Free; - ms.Free; end; Result := True; end; diff --git a/developer/src/tike/xml/layoutbuilder/builder.css b/developer/src/tike/xml/layoutbuilder/builder.css index 60daff8f3c..6e40f5b5e1 100644 --- a/developer/src/tike/xml/layoutbuilder/builder.css +++ b/developer/src/tike/xml/layoutbuilder/builder.css @@ -613,6 +613,11 @@ body:not(.text-controls-in-toolbar) input#inpSubKeyCap { color: white; } +#kbd.desktop .key-size { + /* the layout is fixed on desktop so key size is not useful */ + display: none; +} + /* Position flick keys relative to the flick grid, by hand */ #flick .key { diff --git a/developer/src/tike/xml/layoutbuilder/builder.js b/developer/src/tike/xml/layoutbuilder/builder.js index 561c4d6d0a..a432f8cd97 100644 --- a/developer/src/tike/xml/layoutbuilder/builder.js +++ b/developer/src/tike/xml/layoutbuilder/builder.js @@ -12,10 +12,7 @@ $(function() { this.getPresentation = function () { - var platform = $('#selPlatformPresentation').val(); - //if(platform == 'tablet') return 'tablet-ipad'; - //if(platform == 'phone') return 'phone-iphone5'; - return platform; + return $('#selPlatformPresentation').val(); } this.saveSelection = function() { diff --git a/developer/src/tike/xml/layoutbuilder/constants.js b/developer/src/tike/xml/layoutbuilder/constants.js index 593c7e4ca0..807a8bd30a 100644 --- a/developer/src/tike/xml/layoutbuilder/constants.js +++ b/developer/src/tike/xml/layoutbuilder/constants.js @@ -604,7 +604,8 @@ $(function() { "tablet-ipad-landscape": { "x": 829, "y": 299, "name": "iPad (landscape)" }, // 829x622 = iPad tablet box size; (97,101)-(926,723) "tablet-ipad-portrait": { "x": 605, "y": 300, "name": "iPad (portrait)" }, // 605x806 = iPad tablet box size; (98,94)-(703,900) "phone-iphone5-landscape": { "x": 731, "y": 196, "name": "iPhone 5 (landscape)" }, // 731x412 = iPhone box size; (144,39)-(875,451) - "phone-iphone5-portrait": { "x": 526, "y": 266, "name": "iPhone 5 (portrait)"} // 528x936 = iPhone box size; (90,204)-(618,1040) + "phone-iphone5-portrait": { "x": 526, "y": 266, "name": "iPhone 5 (portrait)"}, // 528x936 = iPhone box size; (90,204)-(618,1040) + "desktop": { "x": 640, "y": 300, "name": "Desktop" }, }; this.keyMargin = 15; diff --git a/linux/debian/changelog b/linux/debian/changelog index a6a93469c5..ab682e3d73 100644 --- a/linux/debian/changelog +++ b/linux/debian/changelog @@ -1,3 +1,10 @@ +keyman (15.0.270-1) unstable; urgency=medium + + * New upstream release. + * Re-release to Debian + + -- Eberhard Beilharz Tue, 13 Sep 2022 11:20:25 +0200 + keyman (15.0.269-1) unstable; urgency=medium * New upstream release. diff --git a/linux/ibus-keyman/src/engine.c b/linux/ibus-keyman/src/engine.c index 3eaeb56f7e..7f514c4203 100644 --- a/linux/ibus-keyman/src/engine.c +++ b/linux/ibus-keyman/src/engine.c @@ -70,7 +70,6 @@ struct _IBusKeymanEngine { gboolean lalt_pressed; gboolean ralt_pressed; gboolean emitting_keystroke; - IBusLookupTable *table; IBusProperty *status_prop; IBusPropList *prop_list; #ifdef GDK_WINDOWING_X11 @@ -262,8 +261,6 @@ ibus_keyman_engine_init(IBusKeymanEngine *keyman) { g_object_ref_sink(keyman->prop_list); ibus_prop_list_append(keyman->prop_list, keyman->status_prop); - keyman->table = ibus_lookup_table_new(9, 0, TRUE, TRUE); - g_object_ref_sink(keyman->table); keyman->state = NULL; #ifdef GDK_WINDOWING_X11 keyman->xdisplay = NULL; @@ -469,11 +466,6 @@ ibus_keyman_engine_destroy (IBusKeymanEngine *keyman) keyman->status_prop = NULL; } - if (keyman->table) { - g_debug("DAR: unref keyman->table"); - g_object_unref (keyman->table); - keyman->table = NULL; - } if (keyman->state) { km_kbp_state_dispose(keyman->state); keyman->state = NULL; diff --git a/linux/keyman-config/Makefile b/linux/keyman-config/Makefile index ee01e2bfc9..cb0fe8ab51 100644 --- a/linux/keyman-config/Makefile +++ b/linux/keyman-config/Makefile @@ -55,8 +55,10 @@ deb: dist man: ./build-help.sh --man --no-reconf -version: +version_reconf: cd .. && ./scripts/reconf.sh keyman-config + +version: version_reconf $(eval VERSION := $(shell python3 -c "from keyman_config import __releaseversion__; print(__releaseversion__)")) # i18n diff --git a/resources/standards-data/unicode-character-database/Blocks.txt b/resources/standards-data/unicode-character-database/Blocks.txt index cc5d61988b..12684594c9 100644 --- a/resources/standards-data/unicode-character-database/Blocks.txt +++ b/resources/standards-data/unicode-character-database/Blocks.txt @@ -1,10 +1,10 @@ -# Blocks-14.0.0.txt -# Date: 2021-01-22, 23:29:00 GMT [KW] -# © 2021 Unicode®, Inc. -# For terms of use, see http://www.unicode.org/terms_of_use.html +# Blocks-15.0.0.txt +# Date: 2022-01-28, 20:58:00 GMT [KW] +# © 2022 Unicode®, Inc. +# For terms of use, see https://www.unicode.org/terms_of_use.html # # Unicode Character Database -# For documentation, see http://www.unicode.org/reports/tr44/ +# For documentation, see https://www.unicode.org/reports/tr44/ # # Format: # Start Code..End Code; Block Name @@ -15,7 +15,7 @@ # and underbars are ignored. # For example, "Latin Extended-A" and "latin extended a" are equivalent. # For more information on the comparison of property values, -# see UAX #44: http://www.unicode.org/reports/tr44/ +# see UAX #44: https://www.unicode.org/reports/tr44/ # # All block ranges start with a value where (cp MOD 16) = 0, # and end with a value where (cp MOD 16) = 15. In other words, @@ -241,6 +241,7 @@ FFF0..FFFF; Specials 10D00..10D3F; Hanifi Rohingya 10E60..10E7F; Rumi Numeral Symbols 10E80..10EBF; Yezidi +10EC0..10EFF; Arabic Extended-C 10F00..10F2F; Old Sogdian 10F30..10F6F; Sogdian 10F70..10FAF; Old Uyghur @@ -272,11 +273,13 @@ FFF0..FFFF; Specials 11A50..11AAF; Soyombo 11AB0..11ABF; Unified Canadian Aboriginal Syllabics Extended-A 11AC0..11AFF; Pau Cin Hau +11B00..11B5F; Devanagari Extended-A 11C00..11C6F; Bhaiksuki 11C70..11CBF; Marchen 11D00..11D5F; Masaram Gondi 11D60..11DAF; Gunjala Gondi 11EE0..11EFF; Makasar +11F00..11F5F; Kawi 11FB0..11FBF; Lisu Supplement 11FC0..11FFF; Tamil Supplement 12000..123FF; Cuneiform @@ -284,7 +287,7 @@ FFF0..FFFF; Specials 12480..1254F; Early Dynastic Cuneiform 12F90..12FFF; Cypro-Minoan 13000..1342F; Egyptian Hieroglyphs -13430..1343F; Egyptian Hieroglyph Format Controls +13430..1345F; Egyptian Hieroglyph Format Controls 14400..1467F; Anatolian Hieroglyphs 16800..16A3F; Bamum Supplement 16A40..16A6F; Mro @@ -309,6 +312,7 @@ FFF0..FFFF; Specials 1D000..1D0FF; Byzantine Musical Symbols 1D100..1D1FF; Musical Symbols 1D200..1D24F; Ancient Greek Musical Notation +1D2C0..1D2DF; Kaktovik Numerals 1D2E0..1D2FF; Mayan Numerals 1D300..1D35F; Tai Xuan Jing Symbols 1D360..1D37F; Counting Rod Numerals @@ -316,9 +320,11 @@ FFF0..FFFF; Specials 1D800..1DAAF; Sutton SignWriting 1DF00..1DFFF; Latin Extended-G 1E000..1E02F; Glagolitic Supplement +1E030..1E08F; Cyrillic Extended-D 1E100..1E14F; Nyiakeng Puachue Hmong 1E290..1E2BF; Toto 1E2C0..1E2FF; Wancho +1E4D0..1E4FF; Nag Mundari 1E7E0..1E7FF; Ethiopic Extended-B 1E800..1E8DF; Mende Kikakui 1E900..1E95F; Adlam @@ -348,6 +354,7 @@ FFF0..FFFF; Specials 2CEB0..2EBEF; CJK Unified Ideographs Extension F 2F800..2FA1F; CJK Compatibility Ideographs Supplement 30000..3134F; CJK Unified Ideographs Extension G +31350..323AF; CJK Unified Ideographs Extension H E0000..E007F; Tags E0100..E01EF; Variation Selectors Supplement F0000..FFFFF; Supplementary Private Use Area-A diff --git a/resources/standards-data/unicode-character-database/UnicodeData.txt b/resources/standards-data/unicode-character-database/UnicodeData.txt index b5abef7ed4..ea963a7162 100644 --- a/resources/standards-data/unicode-character-database/UnicodeData.txt +++ b/resources/standards-data/unicode-character-database/UnicodeData.txt @@ -2975,6 +2975,7 @@ 0CEF;KANNADA DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 0CF1;KANNADA SIGN JIHVAMULIYA;Lo;0;L;;;;;N;;;;; 0CF2;KANNADA SIGN UPADHMANIYA;Lo;0;L;;;;;N;;;;; +0CF3;KANNADA SIGN COMBINING ANUSVARA ABOVE RIGHT;Mc;0;L;;;;;N;;;;; 0D00;MALAYALAM SIGN COMBINING ANUSVARA ABOVE;Mn;0;NSM;;;;;N;;;;; 0D01;MALAYALAM SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;; 0D02;MALAYALAM SIGN ANUSVARA;Mc;0;L;;;;;N;;;;; @@ -3339,6 +3340,7 @@ 0ECB;LAO TONE MAI CATAWA;Mn;122;NSM;;;;;N;;;;; 0ECC;LAO CANCELLATION MARK;Mn;0;NSM;;;;;N;;;;; 0ECD;LAO NIGGAHITA;Mn;0;NSM;;;;;N;;;;; +0ECE;LAO YAMAKKAN;Mn;0;NSM;;;;;N;;;;; 0ED0;LAO DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; 0ED1;LAO DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; 0ED2;LAO DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; @@ -19393,6 +19395,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 10EAD;YEZIDI HYPHENATION MARK;Pd;0;R;;;;;N;;;;; 10EB0;YEZIDI LETTER LAM WITH DOT ABOVE;Lo;0;R;;;;;N;;;;; 10EB1;YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE;Lo;0;R;;;;;N;;;;; +10EFD;ARABIC SMALL LOW WORD SAKTA;Mn;220;NSM;;;;;N;;;;; +10EFE;ARABIC SMALL LOW WORD QASR;Mn;220;NSM;;;;;N;;;;; +10EFF;ARABIC SMALL LOW WORD MADDA;Mn;220;NSM;;;;;N;;;;; 10F00;OLD SOGDIAN LETTER ALEPH;Lo;0;R;;;;;N;;;;; 10F01;OLD SOGDIAN LETTER FINAL ALEPH;Lo;0;R;;;;;N;;;;; 10F02;OLD SOGDIAN LETTER BETH;Lo;0;R;;;;;N;;;;; @@ -20058,6 +20063,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1123C;KHOJKI DOUBLE SECTION MARK;Po;0;L;;;;;N;;;;; 1123D;KHOJKI ABBREVIATION SIGN;Po;0;L;;;;;N;;;;; 1123E;KHOJKI SIGN SUKUN;Mn;0;NSM;;;;;N;;;;; +1123F;KHOJKI LETTER QA;Lo;0;L;;;;;N;;;;; +11240;KHOJKI LETTER SHORT I;Lo;0;L;;;;;N;;;;; +11241;KHOJKI VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;; 11280;MULTANI LETTER A;Lo;0;L;;;;;N;;;;; 11281;MULTANI LETTER I;Lo;0;L;;;;;N;;;;; 11282;MULTANI LETTER U;Lo;0;L;;;;;N;;;;; @@ -21256,6 +21264,16 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11AF6;PAU CIN HAU LOW-FALLING TONE LONG FINAL;Lo;0;L;;;;;N;;;;; 11AF7;PAU CIN HAU LOW-FALLING TONE FINAL;Lo;0;L;;;;;N;;;;; 11AF8;PAU CIN HAU GLOTTAL STOP FINAL;Lo;0;L;;;;;N;;;;; +11B00;DEVANAGARI HEAD MARK;Po;0;L;;;;;N;;;;; +11B01;DEVANAGARI HEAD MARK WITH HEADSTROKE;Po;0;L;;;;;N;;;;; +11B02;DEVANAGARI SIGN BHALE;Po;0;L;;;;;N;;;;; +11B03;DEVANAGARI SIGN BHALE WITH HOOK;Po;0;L;;;;;N;;;;; +11B04;DEVANAGARI SIGN EXTENDED BHALE;Po;0;L;;;;;N;;;;; +11B05;DEVANAGARI SIGN EXTENDED BHALE WITH HOOK;Po;0;L;;;;;N;;;;; +11B06;DEVANAGARI SIGN WESTERN FIVE-LIKE BHALE;Po;0;L;;;;;N;;;;; +11B07;DEVANAGARI SIGN WESTERN NINE-LIKE BHALE;Po;0;L;;;;;N;;;;; +11B08;DEVANAGARI SIGN REVERSED NINE-LIKE BHALE;Po;0;L;;;;;N;;;;; +11B09;DEVANAGARI SIGN MINDU;Po;0;L;;;;;N;;;;; 11C00;BHAIKSUKI LETTER A;Lo;0;L;;;;;N;;;;; 11C01;BHAIKSUKI LETTER AA;Lo;0;L;;;;;N;;;;; 11C02;BHAIKSUKI LETTER I;Lo;0;L;;;;;N;;;;; @@ -21584,6 +21602,92 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11EF6;MAKASAR VOWEL SIGN O;Mc;0;L;;;;;N;;;;; 11EF7;MAKASAR PASSIMBANG;Po;0;L;;;;;N;;;;; 11EF8;MAKASAR END OF SECTION;Po;0;L;;;;;N;;;;; +11F00;KAWI SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;; +11F01;KAWI SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; +11F02;KAWI SIGN REPHA;Lo;0;L;;;;;N;;;;; +11F03;KAWI SIGN VISARGA;Mc;0;L;;;;;N;;;;; +11F04;KAWI LETTER A;Lo;0;L;;;;;N;;;;; +11F05;KAWI LETTER AA;Lo;0;L;;;;;N;;;;; +11F06;KAWI LETTER I;Lo;0;L;;;;;N;;;;; +11F07;KAWI LETTER II;Lo;0;L;;;;;N;;;;; +11F08;KAWI LETTER U;Lo;0;L;;;;;N;;;;; +11F09;KAWI LETTER UU;Lo;0;L;;;;;N;;;;; +11F0A;KAWI LETTER VOCALIC R;Lo;0;L;;;;;N;;;;; +11F0B;KAWI LETTER VOCALIC RR;Lo;0;L;;;;;N;;;;; +11F0C;KAWI LETTER VOCALIC L;Lo;0;L;;;;;N;;;;; +11F0D;KAWI LETTER VOCALIC LL;Lo;0;L;;;;;N;;;;; +11F0E;KAWI LETTER E;Lo;0;L;;;;;N;;;;; +11F0F;KAWI LETTER AI;Lo;0;L;;;;;N;;;;; +11F10;KAWI LETTER O;Lo;0;L;;;;;N;;;;; +11F12;KAWI LETTER KA;Lo;0;L;;;;;N;;;;; +11F13;KAWI LETTER KHA;Lo;0;L;;;;;N;;;;; +11F14;KAWI LETTER GA;Lo;0;L;;;;;N;;;;; +11F15;KAWI LETTER GHA;Lo;0;L;;;;;N;;;;; +11F16;KAWI LETTER NGA;Lo;0;L;;;;;N;;;;; +11F17;KAWI LETTER CA;Lo;0;L;;;;;N;;;;; +11F18;KAWI LETTER CHA;Lo;0;L;;;;;N;;;;; +11F19;KAWI LETTER JA;Lo;0;L;;;;;N;;;;; +11F1A;KAWI LETTER JHA;Lo;0;L;;;;;N;;;;; +11F1B;KAWI LETTER NYA;Lo;0;L;;;;;N;;;;; +11F1C;KAWI LETTER TTA;Lo;0;L;;;;;N;;;;; +11F1D;KAWI LETTER TTHA;Lo;0;L;;;;;N;;;;; +11F1E;KAWI LETTER DDA;Lo;0;L;;;;;N;;;;; +11F1F;KAWI LETTER DDHA;Lo;0;L;;;;;N;;;;; +11F20;KAWI LETTER NNA;Lo;0;L;;;;;N;;;;; +11F21;KAWI LETTER TA;Lo;0;L;;;;;N;;;;; +11F22;KAWI LETTER THA;Lo;0;L;;;;;N;;;;; +11F23;KAWI LETTER DA;Lo;0;L;;;;;N;;;;; +11F24;KAWI LETTER DHA;Lo;0;L;;;;;N;;;;; +11F25;KAWI LETTER NA;Lo;0;L;;;;;N;;;;; +11F26;KAWI LETTER PA;Lo;0;L;;;;;N;;;;; +11F27;KAWI LETTER PHA;Lo;0;L;;;;;N;;;;; +11F28;KAWI LETTER BA;Lo;0;L;;;;;N;;;;; +11F29;KAWI LETTER BHA;Lo;0;L;;;;;N;;;;; +11F2A;KAWI LETTER MA;Lo;0;L;;;;;N;;;;; +11F2B;KAWI LETTER YA;Lo;0;L;;;;;N;;;;; +11F2C;KAWI LETTER RA;Lo;0;L;;;;;N;;;;; +11F2D;KAWI LETTER LA;Lo;0;L;;;;;N;;;;; +11F2E;KAWI LETTER WA;Lo;0;L;;;;;N;;;;; +11F2F;KAWI LETTER SHA;Lo;0;L;;;;;N;;;;; +11F30;KAWI LETTER SSA;Lo;0;L;;;;;N;;;;; +11F31;KAWI LETTER SA;Lo;0;L;;;;;N;;;;; +11F32;KAWI LETTER HA;Lo;0;L;;;;;N;;;;; +11F33;KAWI LETTER JNYA;Lo;0;L;;;;;N;;;;; +11F34;KAWI VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +11F35;KAWI VOWEL SIGN ALTERNATE AA;Mc;0;L;;;;;N;;;;; +11F36;KAWI VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +11F37;KAWI VOWEL SIGN II;Mn;0;NSM;;;;;N;;;;; +11F38;KAWI VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +11F39;KAWI VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;; +11F3A;KAWI VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;; +11F3E;KAWI VOWEL SIGN E;Mc;0;L;;;;;N;;;;; +11F3F;KAWI VOWEL SIGN AI;Mc;0;L;;;;;N;;;;; +11F40;KAWI VOWEL SIGN EU;Mn;0;NSM;;;;;N;;;;; +11F41;KAWI SIGN KILLER;Mc;9;L;;;;;N;;;;; +11F42;KAWI CONJOINER;Mn;9;NSM;;;;;N;;;;; +11F43;KAWI DANDA;Po;0;L;;;;;N;;;;; +11F44;KAWI DOUBLE DANDA;Po;0;L;;;;;N;;;;; +11F45;KAWI PUNCTUATION SECTION MARKER;Po;0;L;;;;;N;;;;; +11F46;KAWI PUNCTUATION ALTERNATE SECTION MARKER;Po;0;L;;;;;N;;;;; +11F47;KAWI PUNCTUATION FLOWER;Po;0;L;;;;;N;;;;; +11F48;KAWI PUNCTUATION SPACE FILLER;Po;0;L;;;;;N;;;;; +11F49;KAWI PUNCTUATION DOT;Po;0;L;;;;;N;;;;; +11F4A;KAWI PUNCTUATION DOUBLE DOT;Po;0;L;;;;;N;;;;; +11F4B;KAWI PUNCTUATION TRIPLE DOT;Po;0;L;;;;;N;;;;; +11F4C;KAWI PUNCTUATION CIRCLE;Po;0;L;;;;;N;;;;; +11F4D;KAWI PUNCTUATION FILLED CIRCLE;Po;0;L;;;;;N;;;;; +11F4E;KAWI PUNCTUATION SPIRAL;Po;0;L;;;;;N;;;;; +11F4F;KAWI PUNCTUATION CLOSING SPIRAL;Po;0;L;;;;;N;;;;; +11F50;KAWI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +11F51;KAWI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +11F52;KAWI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +11F53;KAWI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +11F54;KAWI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +11F55;KAWI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +11F56;KAWI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +11F57;KAWI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +11F58;KAWI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +11F59;KAWI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 11FB0;LISU LETTER YHA;Lo;0;L;;;;;N;;;;; 11FC0;TAMIL FRACTION ONE THREE-HUNDRED-AND-TWENTIETH;No;0;L;;;;1/320;N;;;;; 11FC1;TAMIL FRACTION ONE ONE-HUNDRED-AND-SIXTIETH;No;0;L;;;;1/160;N;;;;; @@ -24040,6 +24144,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1342C;EGYPTIAN HIEROGLYPH AA030;Lo;0;L;;;;;N;;;;; 1342D;EGYPTIAN HIEROGLYPH AA031;Lo;0;L;;;;;N;;;;; 1342E;EGYPTIAN HIEROGLYPH AA032;Lo;0;L;;;;;N;;;;; +1342F;EGYPTIAN HIEROGLYPH V011D;Lo;0;L;;;;;N;;;;; 13430;EGYPTIAN HIEROGLYPH VERTICAL JOINER;Cf;0;L;;;;;N;;;;; 13431;EGYPTIAN HIEROGLYPH HORIZONTAL JOINER;Cf;0;L;;;;;N;;;;; 13432;EGYPTIAN HIEROGLYPH INSERT AT TOP START;Cf;0;L;;;;;N;;;;; @@ -24049,6 +24154,35 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 13436;EGYPTIAN HIEROGLYPH OVERLAY MIDDLE;Cf;0;L;;;;;N;;;;; 13437;EGYPTIAN HIEROGLYPH BEGIN SEGMENT;Cf;0;L;;;;;N;;;;; 13438;EGYPTIAN HIEROGLYPH END SEGMENT;Cf;0;L;;;;;N;;;;; +13439;EGYPTIAN HIEROGLYPH INSERT AT MIDDLE;Cf;0;L;;;;;N;;;;; +1343A;EGYPTIAN HIEROGLYPH INSERT AT TOP;Cf;0;L;;;;;N;;;;; +1343B;EGYPTIAN HIEROGLYPH INSERT AT BOTTOM;Cf;0;L;;;;;N;;;;; +1343C;EGYPTIAN HIEROGLYPH BEGIN ENCLOSURE;Cf;0;L;;;;;N;;;;; +1343D;EGYPTIAN HIEROGLYPH END ENCLOSURE;Cf;0;L;;;;;N;;;;; +1343E;EGYPTIAN HIEROGLYPH BEGIN WALLED ENCLOSURE;Cf;0;L;;;;;N;;;;; +1343F;EGYPTIAN HIEROGLYPH END WALLED ENCLOSURE;Cf;0;L;;;;;N;;;;; +13440;EGYPTIAN HIEROGLYPH MIRROR HORIZONTALLY;Mn;0;NSM;;;;;N;;;;; +13441;EGYPTIAN HIEROGLYPH FULL BLANK;Lo;0;L;;;;;N;;;;; +13442;EGYPTIAN HIEROGLYPH HALF BLANK;Lo;0;L;;;;;N;;;;; +13443;EGYPTIAN HIEROGLYPH LOST SIGN;Lo;0;L;;;;;N;;;;; +13444;EGYPTIAN HIEROGLYPH HALF LOST SIGN;Lo;0;L;;;;;N;;;;; +13445;EGYPTIAN HIEROGLYPH TALL LOST SIGN;Lo;0;L;;;;;N;;;;; +13446;EGYPTIAN HIEROGLYPH WIDE LOST SIGN;Lo;0;L;;;;;N;;;;; +13447;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START;Mn;0;NSM;;;;;N;;;;; +13448;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM START;Mn;0;NSM;;;;;N;;;;; +13449;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT START;Mn;0;NSM;;;;;N;;;;; +1344A;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP END;Mn;0;NSM;;;;;N;;;;; +1344B;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP;Mn;0;NSM;;;;;N;;;;; +1344C;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM START AND TOP END;Mn;0;NSM;;;;;N;;;;; +1344D;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT START AND TOP;Mn;0;NSM;;;;;N;;;;; +1344E;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM END;Mn;0;NSM;;;;;N;;;;; +1344F;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START AND BOTTOM END;Mn;0;NSM;;;;;N;;;;; +13450;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM;Mn;0;NSM;;;;;N;;;;; +13451;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT START AND BOTTOM;Mn;0;NSM;;;;;N;;;;; +13452;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT END;Mn;0;NSM;;;;;N;;;;; +13453;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP AND END;Mn;0;NSM;;;;;N;;;;; +13454;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM AND END;Mn;0;NSM;;;;;N;;;;; +13455;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED;Mn;0;NSM;;;;;N;;;;; 14400;ANATOLIAN HIEROGLYPH A001;Lo;0;L;;;;;N;;;;; 14401;ANATOLIAN HIEROGLYPH A002;Lo;0;L;;;;;N;;;;; 14402;ANATOLIAN HIEROGLYPH A003;Lo;0;L;;;;;N;;;;; @@ -27289,9 +27423,11 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1B120;KATAKANA LETTER ARCHAIC YI;Lo;0;L;;;;;N;;;;; 1B121;KATAKANA LETTER ARCHAIC YE;Lo;0;L;;;;;N;;;;; 1B122;KATAKANA LETTER ARCHAIC WU;Lo;0;L;;;;;N;;;;; +1B132;HIRAGANA LETTER SMALL KO;Lo;0;L;;;;;N;;;;; 1B150;HIRAGANA LETTER SMALL WI;Lo;0;L;;;;;N;;;;; 1B151;HIRAGANA LETTER SMALL WE;Lo;0;L;;;;;N;;;;; 1B152;HIRAGANA LETTER SMALL WO;Lo;0;L;;;;;N;;;;; +1B155;KATAKANA LETTER SMALL KO;Lo;0;L;;;;;N;;;;; 1B164;KATAKANA LETTER SMALL WI;Lo;0;L;;;;;N;;;;; 1B165;KATAKANA LETTER SMALL WE;Lo;0;L;;;;;N;;;;; 1B166;KATAKANA LETTER SMALL WO;Lo;0;L;;;;;N;;;;; @@ -28573,6 +28709,26 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1D243;COMBINING GREEK MUSICAL TETRASEME;Mn;230;NSM;;;;;N;;;;; 1D244;COMBINING GREEK MUSICAL PENTASEME;Mn;230;NSM;;;;;N;;;;; 1D245;GREEK MUSICAL LEIMMA;So;0;ON;;;;;N;;;;; +1D2C0;KAKTOVIK NUMERAL ZERO;No;0;L;;;;0;N;;;;; +1D2C1;KAKTOVIK NUMERAL ONE;No;0;L;;;;1;N;;;;; +1D2C2;KAKTOVIK NUMERAL TWO;No;0;L;;;;2;N;;;;; +1D2C3;KAKTOVIK NUMERAL THREE;No;0;L;;;;3;N;;;;; +1D2C4;KAKTOVIK NUMERAL FOUR;No;0;L;;;;4;N;;;;; +1D2C5;KAKTOVIK NUMERAL FIVE;No;0;L;;;;5;N;;;;; +1D2C6;KAKTOVIK NUMERAL SIX;No;0;L;;;;6;N;;;;; +1D2C7;KAKTOVIK NUMERAL SEVEN;No;0;L;;;;7;N;;;;; +1D2C8;KAKTOVIK NUMERAL EIGHT;No;0;L;;;;8;N;;;;; +1D2C9;KAKTOVIK NUMERAL NINE;No;0;L;;;;9;N;;;;; +1D2CA;KAKTOVIK NUMERAL TEN;No;0;L;;;;10;N;;;;; +1D2CB;KAKTOVIK NUMERAL ELEVEN;No;0;L;;;;11;N;;;;; +1D2CC;KAKTOVIK NUMERAL TWELVE;No;0;L;;;;12;N;;;;; +1D2CD;KAKTOVIK NUMERAL THIRTEEN;No;0;L;;;;13;N;;;;; +1D2CE;KAKTOVIK NUMERAL FOURTEEN;No;0;L;;;;14;N;;;;; +1D2CF;KAKTOVIK NUMERAL FIFTEEN;No;0;L;;;;15;N;;;;; +1D2D0;KAKTOVIK NUMERAL SIXTEEN;No;0;L;;;;16;N;;;;; +1D2D1;KAKTOVIK NUMERAL SEVENTEEN;No;0;L;;;;17;N;;;;; +1D2D2;KAKTOVIK NUMERAL EIGHTEEN;No;0;L;;;;18;N;;;;; +1D2D3;KAKTOVIK NUMERAL NINETEEN;No;0;L;;;;19;N;;;;; 1D2E0;MAYAN NUMERAL ZERO;No;0;L;;;;0;N;;;;; 1D2E1;MAYAN NUMERAL ONE;No;0;L;;;;1;N;;;;; 1D2E2;MAYAN NUMERAL TWO;No;0;L;;;;2;N;;;;; @@ -30404,6 +30560,12 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1DF1C;LATIN SMALL LETTER TESH DIGRAPH WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;; 1DF1D;LATIN SMALL LETTER C WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;; 1DF1E;LATIN SMALL LETTER S WITH CURL;Ll;0;L;;;;;N;;;;; +1DF25;LATIN SMALL LETTER D WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; +1DF26;LATIN SMALL LETTER L WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; +1DF27;LATIN SMALL LETTER N WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; +1DF28;LATIN SMALL LETTER R WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; +1DF29;LATIN SMALL LETTER S WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; +1DF2A;LATIN SMALL LETTER T WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; 1E000;COMBINING GLAGOLITIC LETTER AZU;Mn;230;NSM;;;;;N;;;;; 1E001;COMBINING GLAGOLITIC LETTER BUKY;Mn;230;NSM;;;;;N;;;;; 1E002;COMBINING GLAGOLITIC LETTER VEDE;Mn;230;NSM;;;;;N;;;;; @@ -30442,6 +30604,69 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1E028;COMBINING GLAGOLITIC LETTER BIG YUS;Mn;230;NSM;;;;;N;;;;; 1E029;COMBINING GLAGOLITIC LETTER IOTATED BIG YUS;Mn;230;NSM;;;;;N;;;;; 1E02A;COMBINING GLAGOLITIC LETTER FITA;Mn;230;NSM;;;;;N;;;;; +1E030;MODIFIER LETTER CYRILLIC SMALL A;Lm;0;L; 0430;;;;N;;;;; +1E031;MODIFIER LETTER CYRILLIC SMALL BE;Lm;0;L; 0431;;;;N;;;;; +1E032;MODIFIER LETTER CYRILLIC SMALL VE;Lm;0;L; 0432;;;;N;;;;; +1E033;MODIFIER LETTER CYRILLIC SMALL GHE;Lm;0;L; 0433;;;;N;;;;; +1E034;MODIFIER LETTER CYRILLIC SMALL DE;Lm;0;L; 0434;;;;N;;;;; +1E035;MODIFIER LETTER CYRILLIC SMALL IE;Lm;0;L; 0435;;;;N;;;;; +1E036;MODIFIER LETTER CYRILLIC SMALL ZHE;Lm;0;L; 0436;;;;N;;;;; +1E037;MODIFIER LETTER CYRILLIC SMALL ZE;Lm;0;L; 0437;;;;N;;;;; +1E038;MODIFIER LETTER CYRILLIC SMALL I;Lm;0;L; 0438;;;;N;;;;; +1E039;MODIFIER LETTER CYRILLIC SMALL KA;Lm;0;L; 043A;;;;N;;;;; +1E03A;MODIFIER LETTER CYRILLIC SMALL EL;Lm;0;L; 043B;;;;N;;;;; +1E03B;MODIFIER LETTER CYRILLIC SMALL EM;Lm;0;L; 043C;;;;N;;;;; +1E03C;MODIFIER LETTER CYRILLIC SMALL O;Lm;0;L; 043E;;;;N;;;;; +1E03D;MODIFIER LETTER CYRILLIC SMALL PE;Lm;0;L; 043F;;;;N;;;;; +1E03E;MODIFIER LETTER CYRILLIC SMALL ER;Lm;0;L; 0440;;;;N;;;;; +1E03F;MODIFIER LETTER CYRILLIC SMALL ES;Lm;0;L; 0441;;;;N;;;;; +1E040;MODIFIER LETTER CYRILLIC SMALL TE;Lm;0;L; 0442;;;;N;;;;; +1E041;MODIFIER LETTER CYRILLIC SMALL U;Lm;0;L; 0443;;;;N;;;;; +1E042;MODIFIER LETTER CYRILLIC SMALL EF;Lm;0;L; 0444;;;;N;;;;; +1E043;MODIFIER LETTER CYRILLIC SMALL HA;Lm;0;L; 0445;;;;N;;;;; +1E044;MODIFIER LETTER CYRILLIC SMALL TSE;Lm;0;L; 0446;;;;N;;;;; +1E045;MODIFIER LETTER CYRILLIC SMALL CHE;Lm;0;L; 0447;;;;N;;;;; +1E046;MODIFIER LETTER CYRILLIC SMALL SHA;Lm;0;L; 0448;;;;N;;;;; +1E047;MODIFIER LETTER CYRILLIC SMALL YERU;Lm;0;L; 044B;;;;N;;;;; +1E048;MODIFIER LETTER CYRILLIC SMALL E;Lm;0;L; 044D;;;;N;;;;; +1E049;MODIFIER LETTER CYRILLIC SMALL YU;Lm;0;L; 044E;;;;N;;;;; +1E04A;MODIFIER LETTER CYRILLIC SMALL DZZE;Lm;0;L; A689;;;;N;;;;; +1E04B;MODIFIER LETTER CYRILLIC SMALL SCHWA;Lm;0;L; 04D9;;;;N;;;;; +1E04C;MODIFIER LETTER CYRILLIC SMALL BYELORUSSIAN-UKRAINIAN I;Lm;0;L; 0456;;;;N;;;;; +1E04D;MODIFIER LETTER CYRILLIC SMALL JE;Lm;0;L; 0458;;;;N;;;;; +1E04E;MODIFIER LETTER CYRILLIC SMALL BARRED O;Lm;0;L; 04E9;;;;N;;;;; +1E04F;MODIFIER LETTER CYRILLIC SMALL STRAIGHT U;Lm;0;L; 04AF;;;;N;;;;; +1E050;MODIFIER LETTER CYRILLIC SMALL PALOCHKA;Lm;0;L; 04CF;;;;N;;;;; +1E051;CYRILLIC SUBSCRIPT SMALL LETTER A;Lm;0;L; 0430;;;;N;;;;; +1E052;CYRILLIC SUBSCRIPT SMALL LETTER BE;Lm;0;L; 0431;;;;N;;;;; +1E053;CYRILLIC SUBSCRIPT SMALL LETTER VE;Lm;0;L; 0432;;;;N;;;;; +1E054;CYRILLIC SUBSCRIPT SMALL LETTER GHE;Lm;0;L; 0433;;;;N;;;;; +1E055;CYRILLIC SUBSCRIPT SMALL LETTER DE;Lm;0;L; 0434;;;;N;;;;; +1E056;CYRILLIC SUBSCRIPT SMALL LETTER IE;Lm;0;L; 0435;;;;N;;;;; +1E057;CYRILLIC SUBSCRIPT SMALL LETTER ZHE;Lm;0;L; 0436;;;;N;;;;; +1E058;CYRILLIC SUBSCRIPT SMALL LETTER ZE;Lm;0;L; 0437;;;;N;;;;; +1E059;CYRILLIC SUBSCRIPT SMALL LETTER I;Lm;0;L; 0438;;;;N;;;;; +1E05A;CYRILLIC SUBSCRIPT SMALL LETTER KA;Lm;0;L; 043A;;;;N;;;;; +1E05B;CYRILLIC SUBSCRIPT SMALL LETTER EL;Lm;0;L; 043B;;;;N;;;;; +1E05C;CYRILLIC SUBSCRIPT SMALL LETTER O;Lm;0;L; 043E;;;;N;;;;; +1E05D;CYRILLIC SUBSCRIPT SMALL LETTER PE;Lm;0;L; 043F;;;;N;;;;; +1E05E;CYRILLIC SUBSCRIPT SMALL LETTER ES;Lm;0;L; 0441;;;;N;;;;; +1E05F;CYRILLIC SUBSCRIPT SMALL LETTER U;Lm;0;L; 0443;;;;N;;;;; +1E060;CYRILLIC SUBSCRIPT SMALL LETTER EF;Lm;0;L; 0444;;;;N;;;;; +1E061;CYRILLIC SUBSCRIPT SMALL LETTER HA;Lm;0;L; 0445;;;;N;;;;; +1E062;CYRILLIC SUBSCRIPT SMALL LETTER TSE;Lm;0;L; 0446;;;;N;;;;; +1E063;CYRILLIC SUBSCRIPT SMALL LETTER CHE;Lm;0;L; 0447;;;;N;;;;; +1E064;CYRILLIC SUBSCRIPT SMALL LETTER SHA;Lm;0;L; 0448;;;;N;;;;; +1E065;CYRILLIC SUBSCRIPT SMALL LETTER HARD SIGN;Lm;0;L; 044A;;;;N;;;;; +1E066;CYRILLIC SUBSCRIPT SMALL LETTER YERU;Lm;0;L; 044B;;;;N;;;;; +1E067;CYRILLIC SUBSCRIPT SMALL LETTER GHE WITH UPTURN;Lm;0;L; 0491;;;;N;;;;; +1E068;CYRILLIC SUBSCRIPT SMALL LETTER BYELORUSSIAN-UKRAINIAN I;Lm;0;L; 0456;;;;N;;;;; +1E069;CYRILLIC SUBSCRIPT SMALL LETTER DZE;Lm;0;L; 0455;;;;N;;;;; +1E06A;CYRILLIC SUBSCRIPT SMALL LETTER DZHE;Lm;0;L; 045F;;;;N;;;;; +1E06B;MODIFIER LETTER CYRILLIC SMALL ES WITH DESCENDER;Lm;0;L; 04AB;;;;N;;;;; +1E06C;MODIFIER LETTER CYRILLIC SMALL YERU WITH BACK YER;Lm;0;L; A651;;;;N;;;;; +1E06D;MODIFIER LETTER CYRILLIC SMALL STRAIGHT U WITH STROKE;Lm;0;L; 04B1;;;;N;;;;; +1E08F;COMBINING CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I;Mn;230;NSM;;;;;N;;;;; 1E100;NYIAKENG PUACHUE HMONG LETTER MA;Lo;0;L;;;;;N;;;;; 1E101;NYIAKENG PUACHUE HMONG LETTER TSA;Lo;0;L;;;;;N;;;;; 1E102;NYIAKENG PUACHUE HMONG LETTER NTA;Lo;0;L;;;;;N;;;;; @@ -30603,6 +30828,48 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1E2F8;WANCHO DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; 1E2F9;WANCHO DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 1E2FF;WANCHO NGUN SIGN;Sc;0;ET;;;;;N;;;;; +1E4D0;NAG MUNDARI LETTER O;Lo;0;L;;;;;N;;;;; +1E4D1;NAG MUNDARI LETTER OP;Lo;0;L;;;;;N;;;;; +1E4D2;NAG MUNDARI LETTER OL;Lo;0;L;;;;;N;;;;; +1E4D3;NAG MUNDARI LETTER OY;Lo;0;L;;;;;N;;;;; +1E4D4;NAG MUNDARI LETTER ONG;Lo;0;L;;;;;N;;;;; +1E4D5;NAG MUNDARI LETTER A;Lo;0;L;;;;;N;;;;; +1E4D6;NAG MUNDARI LETTER AJ;Lo;0;L;;;;;N;;;;; +1E4D7;NAG MUNDARI LETTER AB;Lo;0;L;;;;;N;;;;; +1E4D8;NAG MUNDARI LETTER ANY;Lo;0;L;;;;;N;;;;; +1E4D9;NAG MUNDARI LETTER AH;Lo;0;L;;;;;N;;;;; +1E4DA;NAG MUNDARI LETTER I;Lo;0;L;;;;;N;;;;; +1E4DB;NAG MUNDARI LETTER IS;Lo;0;L;;;;;N;;;;; +1E4DC;NAG MUNDARI LETTER IDD;Lo;0;L;;;;;N;;;;; +1E4DD;NAG MUNDARI LETTER IT;Lo;0;L;;;;;N;;;;; +1E4DE;NAG MUNDARI LETTER IH;Lo;0;L;;;;;N;;;;; +1E4DF;NAG MUNDARI LETTER U;Lo;0;L;;;;;N;;;;; +1E4E0;NAG MUNDARI LETTER UC;Lo;0;L;;;;;N;;;;; +1E4E1;NAG MUNDARI LETTER UD;Lo;0;L;;;;;N;;;;; +1E4E2;NAG MUNDARI LETTER UK;Lo;0;L;;;;;N;;;;; +1E4E3;NAG MUNDARI LETTER UR;Lo;0;L;;;;;N;;;;; +1E4E4;NAG MUNDARI LETTER E;Lo;0;L;;;;;N;;;;; +1E4E5;NAG MUNDARI LETTER ENN;Lo;0;L;;;;;N;;;;; +1E4E6;NAG MUNDARI LETTER EG;Lo;0;L;;;;;N;;;;; +1E4E7;NAG MUNDARI LETTER EM;Lo;0;L;;;;;N;;;;; +1E4E8;NAG MUNDARI LETTER EN;Lo;0;L;;;;;N;;;;; +1E4E9;NAG MUNDARI LETTER ETT;Lo;0;L;;;;;N;;;;; +1E4EA;NAG MUNDARI LETTER ELL;Lo;0;L;;;;;N;;;;; +1E4EB;NAG MUNDARI SIGN OJOD;Lm;0;L;;;;;N;;;;; +1E4EC;NAG MUNDARI SIGN MUHOR;Mn;232;NSM;;;;;N;;;;; +1E4ED;NAG MUNDARI SIGN TOYOR;Mn;232;NSM;;;;;N;;;;; +1E4EE;NAG MUNDARI SIGN IKIR;Mn;220;NSM;;;;;N;;;;; +1E4EF;NAG MUNDARI SIGN SUTUH;Mn;230;NSM;;;;;N;;;;; +1E4F0;NAG MUNDARI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +1E4F1;NAG MUNDARI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +1E4F2;NAG MUNDARI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +1E4F3;NAG MUNDARI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +1E4F4;NAG MUNDARI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +1E4F5;NAG MUNDARI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +1E4F6;NAG MUNDARI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +1E4F7;NAG MUNDARI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +1E4F8;NAG MUNDARI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +1E4F9;NAG MUNDARI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 1E7E0;ETHIOPIC SYLLABLE HHYA;Lo;0;L;;;;;N;;;;; 1E7E1;ETHIOPIC SYLLABLE HHYU;Lo;0;L;;;;;N;;;;; 1E7E2;ETHIOPIC SYLLABLE HHYI;Lo;0;L;;;;;N;;;;; @@ -32678,6 +32945,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F6D5;HINDU TEMPLE;So;0;ON;;;;;N;;;;; 1F6D6;HUT;So;0;ON;;;;;N;;;;; 1F6D7;ELEVATOR;So;0;ON;;;;;N;;;;; +1F6DC;WIRELESS;So;0;ON;;;;;N;;;;; 1F6DD;PLAYGROUND SLIDE;So;0;ON;;;;;N;;;;; 1F6DE;WHEEL;So;0;ON;;;;;N;;;;; 1F6DF;RING BUOY;So;0;ON;;;;;N;;;;; @@ -32823,6 +33091,14 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F771;ALCHEMICAL SYMBOL FOR MONTH;So;0;ON;;;;;N;;;;; 1F772;ALCHEMICAL SYMBOL FOR HALF DRAM;So;0;ON;;;;;N;;;;; 1F773;ALCHEMICAL SYMBOL FOR HALF OUNCE;So;0;ON;;;;;N;;;;; +1F774;LOT OF FORTUNE;So;0;ON;;;;;N;;;;; +1F775;OCCULTATION;So;0;ON;;;;;N;;;;; +1F776;LUNAR ECLIPSE;So;0;ON;;;;;N;;;;; +1F77B;HAUMEA;So;0;ON;;;;;N;;;;; +1F77C;MAKEMAKE;So;0;ON;;;;;N;;;;; +1F77D;GONGGONG;So;0;ON;;;;;N;;;;; +1F77E;QUAOAR;So;0;ON;;;;;N;;;;; +1F77F;ORCUS;So;0;ON;;;;;N;;;;; 1F780;BLACK LEFT-POINTING ISOSCELES RIGHT TRIANGLE;So;0;ON;;;;;N;;;;; 1F781;BLACK UP-POINTING ISOSCELES RIGHT TRIANGLE;So;0;ON;;;;;N;;;;; 1F782;BLACK RIGHT-POINTING ISOSCELES RIGHT TRIANGLE;So;0;ON;;;;;N;;;;; @@ -32912,6 +33188,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F7D6;NEGATIVE CIRCLED TRIANGLE;So;0;ON;;;;;N;;;;; 1F7D7;CIRCLED SQUARE;So;0;ON;;;;;N;;;;; 1F7D8;NEGATIVE CIRCLED SQUARE;So;0;ON;;;;;N;;;;; +1F7D9;NINE POINTED WHITE STAR;So;0;ON;;;;;N;;;;; 1F7E0;LARGE ORANGE CIRCLE;So;0;ON;;;;;N;;;;; 1F7E1;LARGE YELLOW CIRCLE;So;0;ON;;;;;N;;;;; 1F7E2;LARGE GREEN CIRCLE;So;0;ON;;;;;N;;;;; @@ -33434,6 +33711,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FA72;BRIEFS;So;0;ON;;;;;N;;;;; 1FA73;SHORTS;So;0;ON;;;;;N;;;;; 1FA74;THONG SANDAL;So;0;ON;;;;;N;;;;; +1FA75;LIGHT BLUE HEART;So;0;ON;;;;;N;;;;; +1FA76;GREY HEART;So;0;ON;;;;;N;;;;; +1FA77;PINK HEART;So;0;ON;;;;;N;;;;; 1FA78;DROP OF BLOOD;So;0;ON;;;;;N;;;;; 1FA79;ADHESIVE BANDAGE;So;0;ON;;;;;N;;;;; 1FA7A;STETHOSCOPE;So;0;ON;;;;;N;;;;; @@ -33446,6 +33726,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FA84;MAGIC WAND;So;0;ON;;;;;N;;;;; 1FA85;PINATA;So;0;ON;;;;;N;;;;; 1FA86;NESTING DOLLS;So;0;ON;;;;;N;;;;; +1FA87;MARACAS;So;0;ON;;;;;N;;;;; +1FA88;FLUTE;So;0;ON;;;;;N;;;;; 1FA90;RINGED PLANET;So;0;ON;;;;;N;;;;; 1FA91;CHAIR;So;0;ON;;;;;N;;;;; 1FA92;RAZOR;So;0;ON;;;;;N;;;;; @@ -33475,6 +33757,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAAA;IDENTIFICATION CARD;So;0;ON;;;;;N;;;;; 1FAAB;LOW BATTERY;So;0;ON;;;;;N;;;;; 1FAAC;HAMSA;So;0;ON;;;;;N;;;;; +1FAAD;FOLDING HAND FAN;So;0;ON;;;;;N;;;;; +1FAAE;HAIR PICK;So;0;ON;;;;;N;;;;; +1FAAF;KHANDA;So;0;ON;;;;;N;;;;; 1FAB0;FLY;So;0;ON;;;;;N;;;;; 1FAB1;WORM;So;0;ON;;;;;N;;;;; 1FAB2;BEETLE;So;0;ON;;;;;N;;;;; @@ -33486,12 +33771,18 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAB8;CORAL;So;0;ON;;;;;N;;;;; 1FAB9;EMPTY NEST;So;0;ON;;;;;N;;;;; 1FABA;NEST WITH EGGS;So;0;ON;;;;;N;;;;; +1FABB;HYACINTH;So;0;ON;;;;;N;;;;; +1FABC;JELLYFISH;So;0;ON;;;;;N;;;;; +1FABD;WING;So;0;ON;;;;;N;;;;; +1FABF;GOOSE;So;0;ON;;;;;N;;;;; 1FAC0;ANATOMICAL HEART;So;0;ON;;;;;N;;;;; 1FAC1;LUNGS;So;0;ON;;;;;N;;;;; 1FAC2;PEOPLE HUGGING;So;0;ON;;;;;N;;;;; 1FAC3;PREGNANT MAN;So;0;ON;;;;;N;;;;; 1FAC4;PREGNANT PERSON;So;0;ON;;;;;N;;;;; 1FAC5;PERSON WITH CROWN;So;0;ON;;;;;N;;;;; +1FACE;MOOSE;So;0;ON;;;;;N;;;;; +1FACF;DONKEY;So;0;ON;;;;;N;;;;; 1FAD0;BLUEBERRIES;So;0;ON;;;;;N;;;;; 1FAD1;BELL PEPPER;So;0;ON;;;;;N;;;;; 1FAD2;OLIVE;So;0;ON;;;;;N;;;;; @@ -33502,6 +33793,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAD7;POURING LIQUID;So;0;ON;;;;;N;;;;; 1FAD8;BEANS;So;0;ON;;;;;N;;;;; 1FAD9;JAR;So;0;ON;;;;;N;;;;; +1FADA;GINGER ROOT;So;0;ON;;;;;N;;;;; +1FADB;PEA POD;So;0;ON;;;;;N;;;;; 1FAE0;MELTING FACE;So;0;ON;;;;;N;;;;; 1FAE1;SALUTING FACE;So;0;ON;;;;;N;;;;; 1FAE2;FACE WITH OPEN EYES AND HAND OVER MOUTH;So;0;ON;;;;;N;;;;; @@ -33510,6 +33803,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAE5;DOTTED LINE FACE;So;0;ON;;;;;N;;;;; 1FAE6;BITING LIP;So;0;ON;;;;;N;;;;; 1FAE7;BUBBLES;So;0;ON;;;;;N;;;;; +1FAE8;SHAKING FACE;So;0;ON;;;;;N;;;;; 1FAF0;HAND WITH INDEX FINGER AND THUMB CROSSED;So;0;ON;;;;;N;;;;; 1FAF1;RIGHTWARDS HAND;So;0;ON;;;;;N;;;;; 1FAF2;LEFTWARDS HAND;So;0;ON;;;;;N;;;;; @@ -33517,6 +33811,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAF4;PALM UP HAND;So;0;ON;;;;;N;;;;; 1FAF5;INDEX POINTING AT THE VIEWER;So;0;ON;;;;;N;;;;; 1FAF6;HEART HANDS;So;0;ON;;;;;N;;;;; +1FAF7;LEFTWARDS PUSHING HAND;So;0;ON;;;;;N;;;;; +1FAF8;RIGHTWARDS PUSHING HAND;So;0;ON;;;;;N;;;;; 1FB00;BLOCK SEXTANT-1;So;0;ON;;;;;N;;;;; 1FB01;BLOCK SEXTANT-2;So;0;ON;;;;;N;;;;; 1FB02;BLOCK SEXTANT-12;So;0;ON;;;;;N;;;;; @@ -33732,7 +34028,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 20000;;Lo;0;L;;;;;N;;;;; 2A6DF;;Lo;0;L;;;;;N;;;;; 2A700;;Lo;0;L;;;;;N;;;;; -2B738;;Lo;0;L;;;;;N;;;;; +2B739;;Lo;0;L;;;;;N;;;;; 2B740;;Lo;0;L;;;;;N;;;;; 2B81D;;Lo;0;L;;;;;N;;;;; 2B820;;Lo;0;L;;;;;N;;;;; @@ -34283,6 +34579,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 2FA1D;CJK COMPATIBILITY IDEOGRAPH-2FA1D;Lo;0;L;2A600;;;;N;;;;; 30000;;Lo;0;L;;;;;N;;;;; 3134A;;Lo;0;L;;;;;N;;;;; +31350;;Lo;0;L;;;;;N;;;;; +323AF;;Lo;0;L;;;;;N;;;;; E0001;LANGUAGE TAG;Cf;0;BN;;;;;N;;;;; E0020;TAG SPACE;Cf;0;BN;;;;;N;;;;; E0021;TAG EXCLAMATION MARK;Cf;0;BN;;;;;N;;;;; diff --git a/web/source/osk/banner.ts b/web/source/osk/banner.ts index e71a2e8f32..c4f28a6e19 100644 --- a/web/source/osk/banner.ts +++ b/web/source/osk/banner.ts @@ -560,13 +560,17 @@ namespace com.keyman.osk { private doAccept(suggestion: BannerSuggestion) { let _this = this; + // Selecting a suggestion or a reversion should both clear selection + // and clear the reversion-displaying state of the banner. + this.selected = null; + this.doRevert = false; + this.revertAcceptancePromise = suggestion.apply(); if(!this.revertAcceptancePromise) { // We get here either if suggestion acceptance fails or if it was a reversion. if(suggestion.suggestion && suggestion.suggestion.tag == 'revert') { // Reversion state management this.recentAccept = false; - this.doRevert = false; this.recentRevert = true; this.doUpdate(); @@ -581,9 +585,7 @@ namespace com.keyman.osk { } }); - this.selected = null; this.recentAccept = true; - this.doRevert = false; this.recentRevert = false; this.swallowPrediction = true;