erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-08-09 10:25:32 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge pull request #7322 from keymanapp/chore/merge-master-to-feature-ldml

Browse source 
chore: merge master into feature-ldml 🙀
This commit is contained in:
Marc Durdin 2022-09-20 13:42:40 +10:00 • committed by GitHub
commit 55f302d929
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
30 changed files with 1563 additions and 408 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
HISTORY.md
View file

@ -1,5 +1,37 @@
# Keyman Version History
## 16.0.66 alpha 2022-09-17
* chore: improve auto labeling (#7288)
## 16.0.65 alpha 2022-09-16
* chore(linux): Remove unused IBusLookupTable (#7296)
## 16.0.64 alpha 2022-09-15
* fix(android/engine): Switch keyboard if uninstalling current one (#7291)
* fix(common/models): fixes quote-adjacent pred-text suggestions (#7205)
* fix(common/models): max prediction wait check (#7290)
## 16.0.63 alpha 2022-09-14
* chore(linux): Update debian changelog (#7281)
* fix(linux): Fix ignored error (#7284)
## 16.0.62 alpha 2022-09-13
* fix(developer): hide key-sizes when in desktop layout in touch layout editor (#7225)
* fix(developer): show more useful error if out of space during Setup (#7267)
## 16.0.61 alpha 2022-09-12
* docs(windows): add steps for using testhost debugging (#7263)
* fix(developer): compiler mismatch on currentLine (#7190)
* fix(developer): suppress repeated warnings about unreachable code (#7219)
* chore: try disabling concurrency for browserstack tests (#7258)
* chore(web): disable browserstack on non-web-specific builds (#7260)
## 16.0.60 alpha 2022-09-10
* fix(web): enhanced timer for prediction algorithm (#7037)

2
VERSION.md
View file

@ -1 +1 @@
16.0.61
16.0.67

2
android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardPickerActivity.java
View file

@ -433,7 +433,7 @@ public final class KeyboardPickerActivity extends BaseActivity {
if(adapter != null) {
adapter.notifyDataSetChanged();
}
if (position == curKbPos && listView != null) {
if (position == curKbPos) {
switchKeyboard(0,false);
} else if(listView != null) { // A bit of a hack, since LanguageSettingsActivity calls this method too.
curKbPos = KeyboardController.getInstance().getKeyboardIndex(KMKeyboard.currentKeyboard());

BIN
android/Tests/KeyboardHarness/app/src/main/assets/final.kmp Normal file
View file

Binary file not shown.

16
android/Tests/KeyboardHarness/app/src/main/java/com/keyman/android/tests/keyboardHarness/MainActivity.java
View file

@ -79,6 +79,22 @@ public class MainActivity extends AppCompatActivity implements OnKeyboardEventLi
KMManager.KMDefault_KeyboardFont,
KMManager.KMDefault_KeyboardFont);
KMManager.addKeyboard(this, platformtestKBbInfo);
// Final K_ENTER test keyboard
Keyboard finalKBInfo = new Keyboard(
"final",
"final",
"final Keyboard",
"en",
"English",
"1.0",
"",
"",
true,
KMManager.KMDefault_KeyboardFont,
KMManager.KMDefault_KeyboardFont);
KMManager.addKeyboard(this, finalKBInfo);
}
@Override

76
common/models/templates/test/test-tokenization.js
View file

@ -289,6 +289,82 @@ describe('Tokenization functions', function() {
assert.deepEqual(tokenization, expectedResult);
});
let midLetterNonbreaker = (text) => {
let customization = {
rules: [{
match: (context) => {
if(context.propertyMatch(null, ["ALetter"], ["MidLetter"], ["eot"])) {
return true;
} else {
return false;
}
},
breakIfMatch: false
}],
propertyMapping: (char) => {
let hyphens = ['\u002d', '\u2010', '\u058a', '\u30a0'];
if(hyphens.includes(char)) {
return "MidLetter";
} else {
return null;
}
}
};
return wordBreakers.default(text, customization);
}
it('treats caret as `eot` for pre-caret text', function() {
let context = {
left: "don-", // We use a hyphen here b/c single-quote is hardcoded.
right: " worry",
endOfBuffer: true,
startOfBuffer: true
};
let tokenization = models.tokenize(wordBreakers.default, context);
assert.deepEqual(tokenization, {
left: ["don", "-"],
right: ["worry"],
caretSplitsToken: false
});
tokenization = models.tokenize(midLetterNonbreaker, context);
assert.deepEqual(tokenization, {
left: ["don-"],
right: ["worry"],
caretSplitsToken: false
});
});
it('handles mid-contraction tokenization', function() {
let context = {
left: "don:",
right: "t worry",
endOfBuffer: true,
startOfBuffer: true
};
let tokenization = models.tokenize(wordBreakers.default, context);
assert.deepEqual(tokenization, {
left: ["don", ":"], // This particular case feels like a possible issue.
right: ["t", "worry"], // It'd be a three-way split token, as "don:t" would
// be a single token were it not for the caret in the middle.
caretSplitsToken: false
})
tokenization = models.tokenize(midLetterNonbreaker, context);
assert.deepEqual(tokenization, {
left: ["don:"],
right: ["t", "worry"],
caretSplitsToken: true
});
});
});
describe('getLastPreCaretToken', function() {

28
common/models/wordbreakers/src/default/data.ts
View file

@ -4,7 +4,7 @@ export namespace data {
/**
* Valid values for a word break property.
*/
export const enum WordBreakProperty {
export const enum WordBreakProperty { // Scary bit: this does not exist as an object at run-time!
Other,
LF,
Newline,
@ -28,6 +28,32 @@ export const enum WordBreakProperty {
eot
};
// Not currently built by the auto-generator tool, but it easily could be.
// If and when we import the data.ts rebuilder, we can add this in.
export const propertyMap = [
"Other",
"LF",
"Newline",
"CR",
"WSegSpace",
"Double_Quote",
"Single_Quote",
"MidNum",
"MidNumLet",
"Numeric",
"MidLetter",
"ALetter",
"ExtendNumLet",
"Format",
"Extend",
"Hebrew_Letter",
"ZWJ",
"Katakana",
"Regional_Indicator",
"sot",
"eot"
];
/**
* Constants for indexing values in WORD_BREAK_PROPERTY.
*/

411
common/models/wordbreakers/src/default/index.ts
View file

@ -1,6 +1,34 @@
// Include the word-breaking data here:
/// <reference path="./data.ts" />
namespace wordBreakers {
/**
* A set of options used to customize and extend the behavior of the default
* Unicode wordbreaker.
*/
export interface DefaultWordBreakerOptions {
/**
* Allows addition of custom wordbreaking rules, which will be applied
* after WB1-WB4 and before all other default wordbreaking rules.
*
* @see `WordbreakerRule`
*/
rules?: WordbreakerRule[];
/**
* Allows assignment of characters to different word-breaking properties than
* their standard word-breaking assignment, including to custom properties
* specified within `customProperties`.
* @param char
*/
propertyMapping?(char: string): string;
/**
* Allows definition of extra word-breaking properties for use with custom
* rules.
*/
customProperties?: string[];
}
/**
* Word breaker based on Unicode Standard Annex #29, Section 4.1:
* Default Word Boundary Specification.
@ -8,8 +36,8 @@ namespace wordBreakers {
* @see http://unicode.org/reports/tr29/#Word_Boundaries
* @see https://github.com/eddieantonio/unicode-default-word-boundary/tree/v12.0.0
*/
export function default_(text: string): Span[] {
let boundaries = findBoundaries(text);
export function default_(text: string, options?: DefaultWordBreakerOptions): Span[] {
let boundaries = findBoundaries(text, options);
if (boundaries.length == 0) {
return [];
}
@ -22,7 +50,7 @@ namespace wordBreakers {
let end = boundaries[i + 1];
let span = new LazySpan(text, start, end);
if (isNonSpace(span.text)) {
if (isNonSpace(span.text, options)) {
spans.push(span);
// Preserve a sequence-final space if it exists. Needed to signal "end of word".
} else if (i == boundaries.length - 2) { // if "we just checked the final boundary"...
@ -61,13 +89,198 @@ namespace wordBreakers {
}
}
/**
* An abstraction supporting custom wordbreaker boundary rules. While this doesn't provide
* support for more complex rules like WB4, WB15, or WB16, this is sufficient for all other
* default word-breaking rules and can be used to define custom rules of similar structure.
*
* @see https://unicode.org/reports/tr29/#WB_Rule_Macros
*/
export interface WordbreakerRule {
/**
* Indicates whether or not the rule applies in the specified context.
* @param context
*/
match(context: BreakerContext): boolean;
/**
* Indicates whether or not the rule indicates a word boundary at the context's site when it matches.
*/
breakIfMatch: boolean;
}
/**
* Provides a useful presentation for wordbreaker's context for use in word-breaking rules.
*
* @see https://unicode.org/reports/tr29/#Word_Boundary_Rules
*/
export class BreakerContext {
// Referenced by this object in order to facilitate `lookahead` maintenance.
private readonly text: string;
readonly options?: DefaultWordBreakerOptions;
/**
* Represents the property of character immediately preceding `left`'s character.
*/
readonly lookbehind: WordBreakProperty = WordBreakProperty.sot;
/**
* Represents the property of the character immediately preceding the potential word boundary.
*/
readonly left: WordBreakProperty = WordBreakProperty.sot;
/**
* Represents the property of the character immediately following the potential word boundary.
*/
readonly right: WordBreakProperty = WordBreakProperty.sot;
/**
* Represents the property of the character immediately following `right`'s character.
*/
readonly lookahead: WordBreakProperty; // Always initialized by constructor.
/**
* Initializes the word-breaking context at the start of the word-breaker's boundary-detection
* algorithm.
* @param text The text to be word-broken
* @param lookaheadPos The position corresponding to `lookahead`.
*/
constructor(text: string, options: DefaultWordBreakerOptions | undefined, lookaheadPos: number);
/**
* Used internally by the boundary-detection algorithm during context-shifting operations.
* @param text
* @param lookbehind
* @param left
* @param right
* @param lookahead
*/
constructor(text: string,
options: DefaultWordBreakerOptions | undefined,
lookbehind: WordBreakProperty,
left: WordBreakProperty,
right: WordBreakProperty,
lookahead: WordBreakProperty);
constructor(text: string,
options: DefaultWordBreakerOptions | undefined,
prop1: WordBreakProperty | number,
prop2?: WordBreakProperty,
prop3?: WordBreakProperty,
prop4?: WordBreakProperty) {
this.text = text;
this.options = options;
if(arguments.length == 3) {
this.lookahead = this.wordbreakPropertyAt(prop1);// prop1;
} else /*if(arguments.length == 6)*/ {
this.lookbehind = prop1 as WordBreakProperty;
this.left = prop2 as WordBreakProperty;
this.right = prop3 as WordBreakProperty;
this.lookahead = prop4 as WordBreakProperty;
}
}
/**
* The general use-case when shifting boundary-check position if WB4 is not active.
* @param lookahead The WordBreakProperty for the character to become `lookahead`.
* @returns
*/
public next(lookaheadPos: number): BreakerContext {
let newLookahead = this.wordbreakPropertyAt(lookaheadPos);
return new BreakerContext(this.text, this.options, this.left, this.right, this.lookahead, newLookahead);
}
/**
* Used for WB4: when ignoring characters before an intervening linebreak, we
* replace `right` with the current `lookahead`, without affecting `lookbehind`
* or `left`. A new `lookahead` is then needed.
* @param lookahead
* @returns
*/
public ignoringRight(lookaheadPos: number) {
let newLookahead = this.wordbreakPropertyAt(lookaheadPos);
return new BreakerContext(this.text, this.options, this.lookbehind, this.left, this.lookahead, newLookahead);
}
/**
* Used for WB4: when ignoring characters after an intervening linebreak, it's
* `lookahead` that gets replaced without shifting the other tracked properties.
* @param lookahead
* @returns
*/
public ignoringLookahead(lookaheadPos: number) {
let newLookahead = this.wordbreakPropertyAt(lookaheadPos);
return new BreakerContext(this.text, this.options, this.lookbehind, this.left, this.right, newLookahead);
}
/**
* Return the value of the Word_Break property at the given string index.
* @param pos position in the text.
*/
private wordbreakPropertyAt(pos: number) {
if (pos < 0) {
return WordBreakProperty.sot; // Always "start of string" before the string starts!
} else if (pos >= this.text.length) {
return WordBreakProperty.eot; // Always "end of string" after the string ends!
} else if (isStartOfSurrogatePair(this.text[pos])) {
// Surrogate pairs the next TWO items from the string!
return property(this.text[pos] + this.text[pos + 1]);
}
return property(this.text[pos], this.options);
}
/**
* Returns `true` if and only if each member of the context has a property included within
* its corresponding set (when specified). Any set may be replaced with null to disable
* a check against its corresponding property.
* @param lookbehindSet
* @param leftSet
* @param rightSet
* @param lookaheadSet
*/
public match(lookbehindSet: WordBreakProperty[] | null,
leftSet: WordBreakProperty[] | null,
rightSet: WordBreakProperty[] | null,
lookaheadSet: WordBreakProperty[] | null) : boolean {
let result: boolean = lookbehindSet?.includes(this.lookbehind) ?? true;
result = result && (leftSet?.includes(this.left) ?? true);
result = result && (rightSet?.includes(this.right) ?? true);
return result && (lookaheadSet?.includes(this.lookahead) ?? true);
}
/**
* Returns `true` if and only if each member of the context has a property included within
* its corresponding set (when specified). Any set may be replaced with null to disable
* a check against its corresponding property.
*
* Names should match those found at https://unicode.org/reports/tr29/#Word_Boundary_Rules
* or defined in the word-breaker customization options; matching is case-insensitive.
* Also includes two extra properties:
* - `sot` - start of text
* - `eot` - end of text
* @param lookbehindSet
* @param leftSet
* @param rightSet
* @param lookaheadSet
*/
public propertyMatch(lookbehindSet: string[] | null,
leftSet: string[] | null,
rightSet: string[] | null,
lookaheadSet: string[] | null) : boolean {
const propMapper = (name: string) => propertyVal(name, this.options);
return this.match(lookbehindSet?.map(propMapper) as WordBreakProperty[] | null,
leftSet?.map(propMapper) as WordBreakProperty[] | null,
rightSet?.map(propMapper) as WordBreakProperty[] | null,
lookaheadSet?.map(propMapper) as WordBreakProperty[] | null);
}
}
/**
* Returns true when the chunk does not solely consist of whitespace.
*
* @param chunk a chunk of text. Starts and ends at word boundaries.
*/
function isNonSpace(chunk: string): boolean {
return !Array.from(chunk).map(property).every(wb => (
function isNonSpace(chunk: string, options?: DefaultWordBreakerOptions): boolean {
return !Array.from(chunk).map((char) => property(char, options)).every(wb => (
wb === WordBreakProperty.CR ||
wb === WordBreakProperty.LF ||
wb === WordBreakProperty.Newline ||
@ -82,13 +295,17 @@ namespace wordBreakers {
*
* @param text Text to find word boundaries in.
*/
function findBoundaries(text: string): number[] {
function findBoundaries(text: string, options?: DefaultWordBreakerOptions): number[] {
// WB1 and WB2: no boundaries if given an empty string.
if (text.length === 0) {
// There are no boundaries in an empty string!
return [];
}
if(options && !options.rules) {
options.rules = [];
}
// This algorithm works by maintaining a sliding window of four SCALAR VALUES.
//
// - Scalar values? JavaScript strings are NOT actually a string of
@ -112,10 +329,7 @@ namespace wordBreakers {
let rightPos: number;
let lookaheadPos = 0; // lookahead, one scalar value to the right of right.
// Before the start of the string is also the start of the string.
let lookbehind: WordBreakProperty;
let left = WordBreakProperty.sot;
let right = WordBreakProperty.sot;
let lookahead = wordbreakPropertyAt(0);
let state = new BreakerContext(text, options, lookaheadPos);
// Count RIs to make sure we're not splitting emoji flags:
let nConsecutiveRegionalIndicators = 0;
@ -124,34 +338,32 @@ namespace wordBreakers {
rightPos = lookaheadPos;
lookaheadPos = positionAfter(lookaheadPos);
// Shift all properties, one scalar value to the right.
[lookbehind, left, right, lookahead] =
[left, right, lookahead, wordbreakPropertyAt(lookaheadPos)];
state = state.next(lookaheadPos);
// Break at the start and end of text, unless the text is empty.
// WB1: Break at start of text...
if (left === WordBreakProperty.sot) {
if (state.match(null, [WordBreakProperty.sot], null, null)) {
boundaries.push(rightPos);
continue;
}
// WB2: Break at the end of text...
if (right === WordBreakProperty.eot) {
if (state.match(null, null, [WordBreakProperty.eot], null)) {
boundaries.push(rightPos);
break; // Reached the end of the string. We're done!
}
// WB3: Do not break within CRLF:
if (left === WordBreakProperty.CR && right === WordBreakProperty.LF)
if (state.match(null, [WordBreakProperty.CR], [WordBreakProperty.LF], null)) {
continue;
}
// WB3b: Otherwise, break after...
if (left === WordBreakProperty.Newline ||
left === WordBreakProperty.CR ||
left === WordBreakProperty.LF) {
const NEWLINE_SET = [WordBreakProperty.Newline, WordBreakProperty.CR, WordBreakProperty.LF];
if(state.match(null, NEWLINE_SET, null, null)) {
boundaries.push(rightPos);
continue;
}
// WB3a: ...and before newlines
if (right === WordBreakProperty.Newline ||
right === WordBreakProperty.CR ||
right === WordBreakProperty.LF) {
if (state.match(null, null, NEWLINE_SET, null)) {
boundaries.push(rightPos);
continue;
}
@ -163,107 +375,144 @@ namespace wordBreakers {
// https://www.unicode.org/Public/emoji/12.0/emoji-zwj-sequences.txt
// WB3d: Keep horizontal whitespace together
if (left === WordBreakProperty.WSegSpace && right == WordBreakProperty.WSegSpace)
if (state.match(null, [WordBreakProperty.WSegSpace], [WordBreakProperty.WSegSpace], null)) {
continue;
}
// WB4: Ignore format and extend characters
// This is to keep grapheme clusters together!
// See: Section 6.2: https://unicode.org/reports/tr29/#Grapheme_Cluster_and_Format_Rules
// N.B.: The rule about "except after sot, CR, LF, and
// Newline" already been by WB1, WB2, WB3a, and WB3b above.
while (right === WordBreakProperty.Format ||
right === WordBreakProperty.Extend ||
right === WordBreakProperty.ZWJ) {
const SET_WB4_IGNORE = [WordBreakProperty.Format, WordBreakProperty.Extend, WordBreakProperty.ZWJ];
while (state.match(null, null, SET_WB4_IGNORE, null)) {
// Continue advancing in the string, as if these
// characters do not exist. DO NOT update left and
// lookbehind however!
[rightPos, lookaheadPos] = [lookaheadPos, positionAfter(lookaheadPos)];
[right, lookahead] = [lookahead, wordbreakPropertyAt(lookaheadPos)];
state = state.ignoringRight(lookaheadPos);
}
// In ignoring the characters in the previous loop, we could
// have fallen off the end of the string, so end the loop
// prematurely if that happens!
if (right === WordBreakProperty.eot) {
if (state.right === WordBreakProperty.eot) {
boundaries.push(rightPos);
break;
}
// WB4 (continued): Lookahead must ALSO ignore these format,
// extend, ZWJ characters!
while (lookahead === WordBreakProperty.Format ||
lookahead === WordBreakProperty.Extend ||
lookahead === WordBreakProperty.ZWJ) {
while (state.match(null, null, null, SET_WB4_IGNORE)) {
// Continue advancing in the string, as if these
// characters do not exist. DO NOT update left and right,
// however!
lookaheadPos = positionAfter(lookaheadPos);
lookahead = wordbreakPropertyAt(lookaheadPos);
state = state.ignoringLookahead(lookaheadPos);
}
// See: https://unicode.org/reports/tr29/#WB_Rule_Macros
const SET_AHLETTER = [WordBreakProperty.ALetter, WordBreakProperty.Hebrew_Letter];
const SET_MIDNUMLETQ = [WordBreakProperty.MidNumLet, WordBreakProperty.Single_Quote];
// Custom rules may override the base ruleset aside from the first few fundamental ones.
if(options?.rules) {
let customMatch: boolean = false;
for(const rule of options.rules) {
customMatch = rule.match(state);
if(customMatch) {
if(rule.breakIfMatch) {
boundaries.push(rightPos);
}
break; // as customMatch == true here, this will trigger the `continue` that follows.
}
}
if(customMatch) {
continue;
}
}
// WB5: Do not break between most letters.
if (isAHLetter(left) && isAHLetter(right))
// if (isAHLetter(state.left) && isAHLetter(state.right))
if(state.match(null, SET_AHLETTER, SET_AHLETTER, null)) {
continue;
}
// Do not break across certain punctuation
// WB6: (Don't break before apostrophes in contractions)
if (isAHLetter(left) && isAHLetter(lookahead) &&
(right === WordBreakProperty.MidLetter || isMidNumLetQ(right)))
const SET_ALL_MIDLETTER = [WordBreakProperty.MidLetter, ...SET_MIDNUMLETQ];
if(state.match(null, SET_AHLETTER, SET_ALL_MIDLETTER, SET_AHLETTER)) {
continue;
}
// WB7: (Don't break after apostrophes in contractions)
if (isAHLetter(lookbehind) && isAHLetter(right) &&
(left === WordBreakProperty.MidLetter || isMidNumLetQ(left)))
if(state.match(SET_AHLETTER, SET_ALL_MIDLETTER, SET_AHLETTER, null)) {
continue;
}
// WB7a
if (left === WordBreakProperty.Hebrew_Letter && right === WordBreakProperty.Single_Quote)
if(state.match(null, [WordBreakProperty.Hebrew_Letter], [WordBreakProperty.Single_Quote], null)) {
continue;
}
// WB7b
if (left === WordBreakProperty.Hebrew_Letter && right === WordBreakProperty.Double_Quote &&
lookahead === WordBreakProperty.Hebrew_Letter)
if(state.match(null,
[WordBreakProperty.Hebrew_Letter],
[WordBreakProperty.Double_Quote],
[WordBreakProperty.Hebrew_Letter])) {
continue;
}
// WB7c
if (lookbehind === WordBreakProperty.Hebrew_Letter && left === WordBreakProperty.Double_Quote &&
right === WordBreakProperty.Hebrew_Letter)
if(state.match([WordBreakProperty.Hebrew_Letter],
[WordBreakProperty.Double_Quote],
[WordBreakProperty.Hebrew_Letter],
null)) {
continue;
}
// Do not break within sequences of digits, or digits adjacent to letters.
// e.g., "3a" or "A3"
// WB8
if (left === WordBreakProperty.Numeric && right === WordBreakProperty.Numeric)
if(state.match(null, [WordBreakProperty.Numeric], [WordBreakProperty.Numeric], null)) {
continue;
}
// WB9
if (isAHLetter(left) && right === WordBreakProperty.Numeric)
if(state.match(null, SET_AHLETTER, [WordBreakProperty.Numeric], null)) {
continue;
}
// WB10
if (left === WordBreakProperty.Numeric && isAHLetter(right))
if(state.match(null, [WordBreakProperty.Numeric], SET_AHLETTER, null)) {
continue;
}
// Do not break within sequences, such as 3.2, 3,456.789
// WB11
if (lookbehind === WordBreakProperty.Numeric && right === WordBreakProperty.Numeric &&
(left === WordBreakProperty.MidNum || isMidNumLetQ(left)))
const SET_ALL_MIDNUM = [WordBreakProperty.MidNum, ...SET_MIDNUMLETQ];
if(state.match([WordBreakProperty.Numeric], SET_ALL_MIDNUM, [WordBreakProperty.Numeric], null)) {
continue;
}
// WB12
if (left === WordBreakProperty.Numeric && lookahead === WordBreakProperty.Numeric &&
(right === WordBreakProperty.MidNum || isMidNumLetQ(right)))
if(state.match(null, [WordBreakProperty.Numeric], SET_ALL_MIDNUM, [WordBreakProperty.Numeric])) {
continue;
}
// WB13: Do not break between Katakana
if (left === WordBreakProperty.Katakana && right === WordBreakProperty.Katakana)
if(state.match(null, [WordBreakProperty.Katakana], [WordBreakProperty.Katakana], null)) {
continue;
}
// Do not break from extenders (e.g., U+202F NARROW NO-BREAK SPACE)
// WB13a
if ((isAHLetter(left) ||
left === WordBreakProperty.Numeric ||
left === WordBreakProperty.Katakana ||
left === WordBreakProperty.ExtendNumLet) &&
right === WordBreakProperty.ExtendNumLet)
const SET_NUM_KAT_LET = [WordBreakProperty.Katakana,
WordBreakProperty.Numeric,
...SET_AHLETTER];
if(state.match(null, SET_NUM_KAT_LET, [WordBreakProperty.ExtendNumLet], null)) {
continue;
}
if(state.match(null, [WordBreakProperty.ExtendNumLet], [WordBreakProperty.ExtendNumLet], null)) {
continue;
}
// WB13b
if ((isAHLetter(right) ||
right === WordBreakProperty.Numeric ||
right === WordBreakProperty.Katakana) && left === WordBreakProperty.ExtendNumLet)
if(state.match(null, [WordBreakProperty.ExtendNumLet], SET_NUM_KAT_LET, null)) {
continue;
}
// WB15 & WB16:
// Do not break within emoji flag sequences. That is, do not break between
// regional indicator (RI) symbols if there is an odd number of RI
// characters before the break point.
if (right === WordBreakProperty.Regional_Indicator) {
if (state.right === WordBreakProperty.Regional_Indicator) {
// Emoji flags are actually composed of TWO scalar values, each being a
// "regional indicator". These indicators correspond to Latin letters. Put
// two of them together, and they spell out an ISO 3166-1-alpha-2 country
@ -300,34 +549,6 @@ namespace wordBreakers {
}
return pos + 1;
}
/**
* Return the value of the Word_Break property at the given string index.
* @param pos position in the text.
*/
function wordbreakPropertyAt(pos: number) {
if (pos < 0) {
return WordBreakProperty.sot; // Always "start of string" before the string starts!
} else if (pos >= text.length) {
return WordBreakProperty.eot; // Always "end of string" after the string ends!
} else if (isStartOfSurrogatePair(text[pos])) {
// Surrogate pairs the next TWO items from the string!
return property(text[pos] + text[pos + 1]);
}
return property(text[pos]);
}
// Word_Break rule macros
// See: https://unicode.org/reports/tr29/#WB_Rule_Macros
function isAHLetter(prop: WordBreakProperty): boolean {
return prop === WordBreakProperty.ALetter ||
prop === WordBreakProperty.Hebrew_Letter;
}
function isMidNumLetQ(prop: WordBreakProperty): boolean {
return prop === WordBreakProperty.MidNumLet ||
prop === WordBreakProperty.Single_Quote;
}
}
function isStartOfSurrogatePair(character: string) {
@ -340,13 +561,29 @@ namespace wordBreakers {
* Note that
* @param character a scalar value
*/
function property(character: string): WordBreakProperty {
function property(character: string, options?: DefaultWordBreakerOptions): WordBreakProperty {
// If there is a customized mapping for the character, prioritize that.
if(options?.propertyMapping) {
let propName = options.propertyMapping(character);
if(propName) {
return propertyVal(propName, options);
}
}
// This MUST be a scalar value.
// TODO: remove dependence on character.codepointAt()?
let codepoint = character.codePointAt(0) as number;
return searchForProperty(codepoint, 0, WORD_BREAK_PROPERTY.length - 1);
}
function propertyVal(propName: string, options?: DefaultWordBreakerOptions) {
const matcher = (name: string) => name.toLowerCase() == propName.toLowerCase()
const customIndex = options?.customProperties?.findIndex(matcher) ?? -1;
return customIndex != -1 ? -customIndex - 1 : data.propertyMap.findIndex(matcher);
}
/**
* Binary search for the word break property of a given CODE POINT.
*

476
common/models/wordbreakers/test/test-default-word-breaker.js
View file

@ -8,184 +8,334 @@ const breakWords = require('../build').wordBreakers['default'];
const SHY = '\u00AD'; // Other, Format. The "Soft HYphen" - usually invisible unless needed for word-wrapping.
describe('The default word breaker', function () {
it('should break multilingual text', function () {
let breaks = breakWords(
`Добрый день! ᑕᐻ᙮ — after working on ka${SHY}wen${SHY}non:${SHY}nis,
let's eat phở! 🥣`
);
let words = breaks.map(span => span.text);
assert.deepEqual(words, [
'Добрый', 'день', '!', 'ᑕᐻ', '᙮', '—', 'after',
'working', 'on', `ka${SHY}wen${SHY}non:${SHY}nis`, ',',
"let's", 'eat', 'phở', '!', '🥣'
]);
describe('default configuration', function() {
it('should break multilingual text', function () {
let breaks = breakWords(
`Добрый день! ᑕᐻ᙮ — after working on ka${SHY}wen${SHY}non:${SHY}nis,
let's eat phở! 🥣`
);
let words = breaks.map(span => span.text);
assert.deepEqual(words, [
'Добрый', 'день', '!', 'ᑕᐻ', '᙮', '—', 'after',
'working', 'on', `ka${SHY}wen${SHY}non:${SHY}nis`, ',',
"let's", 'eat', 'phở', '!', '🥣'
]);
});
it('handles heavily-punctuated English text', function() {
// This test case brought to you by http://unicode.org/reports/tr29/#Word_Boundaries, Figure 1.
let breaks = breakWords(
`The quick ("brown") fox can't jump 32.3 feet, right?`
);
let words = breaks.map(span => span.text);
assert.deepEqual(words, [
'The', 'quick', '(', '"', 'brown', '"', ')', 'fox', "can't",
'jump', '32.3', 'feet', ',', 'right', '?'
]);
});
// The way these two tests are written is a bit much on the "white-box" style,
// but they do decently cover the boundary rules mentioned.
it('Does not split empty contexts (WB1 + WB2)', function() {
let breaks = breakWords('');
let words = breaks.map(span => span.text);
assert.deepEqual(words, []);
});
it('Does split at context boundaries (WB1 + WB2)', function() {
let breaks = breakWords('a');
let words = breaks.map(span => span.text);
assert.deepEqual(words, ['a']);
});
// WB3, WB3a, WB3b are all handled internally, within the top-level function.
// iff, as in "if and only if"
it('ignores the zero-width joiner iff appropriate (WB4)', function() {
const zwj = '\u200d';
let breaks = breakWords(`a${zwj}b\n${zwj}c${zwj}\nd`);
let words = breaks.map(span => span.text);
// Does NOT ignore the zwj immediately after a newline - the notable exception
// (the reason for "iff", not "if").
assert.deepEqual(words, [`a${zwj}b`, `${zwj}`, `c${zwj}`, `d`]);
})
it('ignores extend characters iff appropriate (WB4)', function() {
const comboGrave = '\u0300'; // The 'combining grave accent', as used in NFD.
let breaks = breakWords(`a${comboGrave}e\n${comboGrave}i${comboGrave}\no`);
let words = breaks.map(span => span.text);
// Does NOT ignore the zwj immediately after a newline - the notable exception
// (the reason for "iff", not "if").
assert.deepEqual(words, [`a${comboGrave}e`, `${comboGrave}`, `i${comboGrave}`, `o`]);
});
it('ignores format characters iff appropriate (WB4)', function() {
// Re-uses `const SHY` from above.
let breaks = breakWords(`a${SHY}e\n${SHY}i${SHY}\no`);
let words = breaks.map(span => span.text);
// Does NOT ignore the zwj immediately after a newline - the notable exception
// (the reason for "iff", not "if").
assert.deepEqual(words, [`a${SHY}e`, `${SHY}`, `i${SHY}`, `o`]);
});
it('does not break between most alphabetic characters (WB5)', function() {
let breaks = breakWords(`aəσאБ лאʈγX`); // a mix of latin, hebrew, greek, cyrillic, and IPA chars
// for both "words".
let words = breaks.map(span => span.text);
assert.deepEqual(words, [`aəσאБ`, `лאʈγX`]);
});
it('does not break letters across specific punctuation patterns (WB6, WB7)', function() {
// `'`: MidNumLetQ (from Single_Quote)
// '.': MidNumLet
// ':': MidLetter
let breaks = breakWords(`don't b.r.e.a.k t:h:e:s:e`);
let words = breaks.map(span => span.text);
assert.deepEqual(words, [`don't`, `b.r.e.a.k`, `t:h:e:s:e`]);
let breaks2 = breakWords(`.drop: :the' 'extras.`);
let words2 = breaks2.map(span => span.text);
assert.deepEqual(words2, [`.`, `drop`, `:`, `:`, `the`, `'`, `'`, `extras`, `.`]);
// ',': MidNum (is NOT included by rule!)
let breaks3 = breakWords('do br,eak that');
let words3 = breaks3.map(span => span.text);
assert.deepEqual(words3, ['do', 'br', ',', 'eak', 'that']);
});
it('treats Hebrew properly (WB7a-c)', function() {
const aleph = 'א';
const bet = 'ב';
// As Hebrew is RTL... this is probably the clearest way for us LTR people to
// clearly see what's going on without ordering mechanics messing up the render.
let breaks = breakWords(`${aleph}' ${aleph}" ${aleph}"${bet}`);
let words = breaks.map(span => span.text);
// A lingering double-quote isn't cool, but one in the middle's fine.
// Lingering single-quote is fine regardless.
assert.deepEqual(words, [`${aleph}'`, `${aleph}`, `"`, `${aleph}"${bet}`]);
});
it(`doesn't break within digit + digit/letter sequences (WB8-10)`, function() {
let breaks = breakWords('a1b2c3 hunter2 ab12cd34 1234567890');
let words = breaks.map(span => span.text);
assert.deepEqual(words, ['a1b2c3', 'hunter2', 'ab12cd34', '1234567890']);
});
it('does not break within formatted number sequences (WB11-12)', function() {
// Note: `'` fits "MidNumLetQ", part of the two rules!
let breaks = breakWords(`1.2.3 3,458.01 3.45'8,01`);
let words = breaks.map(span => span.text);
assert.deepEqual(words, [`1.2.3`, `3,458.01`, `3.45'8,01`]);
let breaks2 = breakWords(`.1' ,3.`);
let words2 = breaks2.map(span => span.text);
assert.deepEqual(words2, [`.`, `1`, `'`, `,`, `3`, `.`]);
});
it('does not break between Katakana (WB13)', function() {
const kataSmA = '\u30a2'; //ァ
const kataA = '\u30a2'; //ア
const kataSound = '\u309b'; // ゛
let breaks = breakWords(`${kataSound}${kataA} ${kataSmA}${kataSound}b ${kataA}${kataSound}${kataSmA}`);
let words = breaks.map(span => span.text);
assert.deepEqual(words, [
`${kataSound}${kataA}`,
`${kataSmA}${kataSound}`,
'b',
`${kataA}${kataSound}${kataSmA}`
]);
});
it('does not break form extenders (WB13a-b)', function() {
// The `_` (underscore) fits the ExtendNumLet class this rule focuses on.
const kataA = '\u30a2'; //ア
let breaks = breakWords(`${kataA}_a__0_b_${kataA} _${kataA} 1_ _c_ ____`);
let words = breaks.map(span => span.text);
assert.deepEqual(words, [
`${kataA}_a__0_b_${kataA}`,
`_${kataA}`,
`1_`,
`_c_`,
`____`
]);
});
it('handles emoji flag sequences properly (WB15-16)', function() {
// For clarity on what's being tested...
let CA_FLAG = '\u{1f1e8}\u{1f1e6}' // '🇨🇦' (canadian flag emoji); should not be broken.
let KH_FLAG = '\u{1f1f0}\u{1f1ed}' // '🇰🇭' (khmer flag emoji); same
let X_FLAG_PIECE = '\u{1f1fd}' // '🇽' (half of a flag emoji; '🇽🇽' doesn't match a flag)
let breaks = breakWords(`${CA_FLAG}${KH_FLAG}${X_FLAG_PIECE}${X_FLAG_PIECE}`);
let words = breaks.map(span => span.text);
// Note that the emoji may not render well within VSCode, but they show up nicely on GitHub.
assert.deepEqual(words, ['🇨🇦', '🇰🇭', '🇽🇽']);
});
it('breaks hyphenated words by default', function() {
let breaks = breakWords('Smith-Jones');
let words = breaks.map(span => span.text);
assert.deepEqual(words, ['Smith', '-', 'Jones']);
});
});
it('handles heavily-punctuated English text', function() {
// This test case brought to you by http://unicode.org/reports/tr29/#Word_Boundaries, Figure 1.
let breaks = breakWords(
`The quick ("brown") fox can't jump 32.3 feet, right?`
);
let words = breaks.map(span => span.text);
assert.deepEqual(words, [
'The', 'quick', '(', '"', 'brown', '"', ')', 'fox', "can't",
'jump', '32.3', 'feet', ',', 'right', '?'
]);
});
describe('customization', function() {
// Refer to https://unicode.org/reports/tr29/#Word_Boundary_Rules, third bullet point.
it('custom prop, rule: do not break on letter-adjacent hyphens', function() {
let customization = {
rules: [{
match: (context) => {
if(context.propertyMatch(null, ["ALetter"], ["Hyphen"], ["ALetter"])) {
return true;
} else if(context.propertyMatch(["ALetter"], ["Hyphen"], ["ALetter"], null)) {
return true;
} else {
return false;
}
},
breakIfMatch: false
}],
propertyMapping: (char) => {
const validHyphenCodes = [
'\u002d', '\u2010', '\u058a', '\u30a0'
];
if(validHyphenCodes.includes(char)) {
return "Hyphen";
}
// The way these two tests are written is a bit much on the "white-box" style,
// but they do decently cover the boundary rules mentioned.
it('Does not split empty contexts (WB1 + WB2)', function() {
let breaks = breakWords('');
let words = breaks.map(span => span.text);
assert.deepEqual(words, []);
});
return null;
},
customProperties: ["Hyphen"]
}
it('Does split at context boundaries (WB1 + WB2)', function() {
let breaks = breakWords('a');
let words = breaks.map(span => span.text);
assert.deepEqual(words, ['a']);
});
let breaks = breakWords('Smith-Jones', customization);
let words = breaks.map(span => span.text);
// WB3, WB3a, WB3b are all handled internally, within the top-level function.
assert.deepEqual(words, ['Smith-Jones']);
});
// iff, as in "if and only if"
it('ignores the zero-width joiner iff appropriate (WB4)', function() {
const zwj = '\u200d';
it('mid-word hyphen via reassignment to MidLetter', function() {
let customization = {
propertyMapping: (char) => {
const validHyphenCodes = [
'\u002d', '\u2010', '\u058a', '\u30a0'
];
if(validHyphenCodes.includes(char)) {
return "MidLetter";
}
let breaks = breakWords(`a${zwj}b\n${zwj}c${zwj}\nd`);
let words = breaks.map(span => span.text);
return null;
}
}
// Does NOT ignore the zwj immediately after a newline - the notable exception
// (the reason for "iff", not "if").
assert.deepEqual(words, [`a${zwj}b`, `${zwj}`, `c${zwj}`, `d`]);
})
let breaks = breakWords('Smith-Jones', customization);
let words = breaks.map(span => span.text);
it('ignores extend characters iff appropriate (WB4)', function() {
const comboGrave = '\u0300'; // The 'combining grave accent', as used in NFD.
assert.deepEqual(words, ['Smith-Jones']);
});
let breaks = breakWords(`a${comboGrave}e\n${comboGrave}i${comboGrave}\no`);
let words = breaks.map(span => span.text);
// Useful for some regional minority languages that prefer word-breaking spaces.
it('character reassignment: Khmer letters as ALetter', function() {
let customization = {
propertyMapping: (char) => {
if(char >= '\u1780' && char <= '\u17b3') {
return "ALetter";
} else {
// The other Khmer characters already have useful word-breaking
// property assignments.
return null;
}
}
}
// Does NOT ignore the zwj immediately after a newline - the notable exception
// (the reason for "iff", not "if").
assert.deepEqual(words, [`a${comboGrave}e`, `${comboGrave}`, `i${comboGrave}`, `o`]);
});
let breaks = breakWords('ស្រុក ខ្មែរ', customization);
let words = breaks.map(span => span.text);
it('ignores format characters iff appropriate (WB4)', function() {
// Re-uses `const SHY` from above.
let breaks = breakWords(`a${SHY}e\n${SHY}i${SHY}\no`);
let words = breaks.map(span => span.text);
assert.deepEqual(words, ['ស្រុក', 'ខ្មែរ']);
});
// Does NOT ignore the zwj immediately after a newline - the notable exception
// (the reason for "iff", not "if").
assert.deepEqual(words, [`a${SHY}e`, `${SHY}`, `i${SHY}`, `o`]);
});
// See: suggested language-specific WB5a from the spec's notes.
it("french/italian apostrophe / vowel boundaries", function() {
let customization = {
rules: [
// WB5, but with differentiated consonants (ALetter) and vowels (AVowel)
{
match: (context) => {
if(context.propertyMatch(null, ["ALetter", "AVowel"], ["ALetter", "AVowel"], null)) {
return true;
} else {
return false;
}
},
breakIfMatch: false
},
// Proposed WB5a
{
match: (context) => {
if(context.propertyMatch(null, ["Single_Quote"], ["AVowel"], null)) {
return true;
} else {
return false;
}
},
breakIfMatch: true
},
// WB6, 7
{
match: (context) => {
if(context.propertyMatch(null,
["ALetter", "AVowel"],
["MidLetter", "MidNumLet", "Single_Quote"],
["ALetter", "AVowel"])) {
return true;
} else if(context.propertyMatch(["ALetter", "AVowel"],
["MidLetter", "MidNumLet", "Single_Quote"],
["ALetter", "AVowel"],
null)) {
return true;
} else {
return false;
}
},
breakIfMatch: false
}
// Similar extensions to WB9, 10, 13a, and 13b would also be needed for robustness.
// And I kind of left the Hebrew_Letter out of the WB5, 6, and 7 rewrites.
],
propertyMapping: (char) => {
const vowels = ['a', 'e', 'i', 'o', 'u'];
if(vowels.includes(char)) {
return "AVowel";
}
it('does not break between most alphabetic characters (WB5)', function() {
let breaks = breakWords(`aəσאБ лאʈγX`); // a mix of latin, hebrew, greek, cyrillic, and IPA chars
// for both "words".
let words = breaks.map(span => span.text);
return null;
},
customProperties: ["AVowel"]
}
assert.deepEqual(words, [`aəσאБ`, `лאʈγX`]);
});
let breaks = breakWords("l'objectif aujourd'hui", customization);
let words = breaks.map(span => span.text);
it('does not break letters across specific punctuation patterns (WB6, WB7)', function() {
// `'`: MidNumLetQ (from Single_Quote)
// '.': MidNumLet
// ':': MidLetter
let breaks = breakWords(`don't b.r.e.a.k t:h:e:s:e`);
let words = breaks.map(span => span.text);
assert.deepEqual(words, [`don't`, `b.r.e.a.k`, `t:h:e:s:e`]);
let breaks2 = breakWords(`.drop: :the' 'extras.`);
let words2 = breaks2.map(span => span.text);
assert.deepEqual(words2, [`.`, `drop`, `:`, `:`, `the`, `'`, `'`, `extras`, `.`]);
// ',': MidNum (is NOT included by rule!)
let breaks3 = breakWords('do br,eak that');
let words3 = breaks3.map(span => span.text);
assert.deepEqual(words3, ['do', 'br', ',', 'eak', 'that']);
});
it('treats Hebrew properly (WB7a-c)', function() {
const aleph = 'א';
const bet = 'ב';
// As Hebrew is RTL... this is probably the clearest way for us LTR people to
// clearly see what's going on without ordering mechanics messing up the render.
let breaks = breakWords(`${aleph}' ${aleph}" ${aleph}"${bet}`);
let words = breaks.map(span => span.text);
// A lingering double-quote isn't cool, but one in the middle's fine.
// Lingering single-quote is fine regardless.
assert.deepEqual(words, [`${aleph}'`, `${aleph}`, `"`, `${aleph}"${bet}`]);
});
it(`doesn't break within digit + digit/letter sequences (WB8-10)`, function() {
let breaks = breakWords('a1b2c3 hunter2 ab12cd34 1234567890');
let words = breaks.map(span => span.text);
assert.deepEqual(words, ['a1b2c3', 'hunter2', 'ab12cd34', '1234567890']);
});
it('does not break within formatted number sequences (WB11-12)', function() {
// Note: `'` fits "MidNumLetQ", part of the two rules!
let breaks = breakWords(`1.2.3 3,458.01 3.45'8,01`);
let words = breaks.map(span => span.text);
assert.deepEqual(words, [`1.2.3`, `3,458.01`, `3.45'8,01`]);
let breaks2 = breakWords(`.1' ,3.`);
let words2 = breaks2.map(span => span.text);
assert.deepEqual(words2, [`.`, `1`, `'`, `,`, `3`, `.`]);
});
it('does not break between Katakana (WB13)', function() {
const kataSmA = '\u30a2'; //ァ
const kataA = '\u30a2'; //ア
const kataSound = '\u309b'; // ゛
let breaks = breakWords(`${kataSound}${kataA} ${kataSmA}${kataSound}b ${kataA}${kataSound}${kataSmA}`);
let words = breaks.map(span => span.text);
assert.deepEqual(words, [
`${kataSound}${kataA}`,
`${kataSmA}${kataSound}`,
'b',
`${kataA}${kataSound}${kataSmA}`
]);
});
it('does not break form extenders (WB13a-b)', function() {
// The `_` (underscore) fits the ExtendNumLet class this rule focuses on.
const kataA = '\u30a2'; //ア
let breaks = breakWords(`${kataA}_a__0_b_${kataA} _${kataA} 1_ _c_ ____`);
let words = breaks.map(span => span.text);
assert.deepEqual(words, [
`${kataA}_a__0_b_${kataA}`,
`_${kataA}`,
`1_`,
`_c_`,
`____`
]);
});
it('handles emoji flag sequences properly (WB15-16)', function() {
// For clarity on what's being tested...
let CA_FLAG = '\u{1f1e8}\u{1f1e6}' // '🇨🇦' (canadian flag emoji); should not be broken.
let KH_FLAG = '\u{1f1f0}\u{1f1ed}' // '🇰🇭' (khmer flag emoji); same
let X_FLAG_PIECE = '\u{1f1fd}' // '🇽' (half of a flag emoji; '🇽🇽' doesn't match a flag)
let breaks = breakWords(`${CA_FLAG}${KH_FLAG}${X_FLAG_PIECE}${X_FLAG_PIECE}`);
let words = breaks.map(span => span.text);
// Note that the emoji may not render well within VSCode, but they show up nicely on GitHub.
assert.deepEqual(words, ['🇨🇦', '🇰🇭', '🇽🇽']);
assert.deepEqual(words, ["l'", "objectif", "aujourd'hui"]);
});
});
});

26
common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js
View file

@ -45,7 +45,7 @@ describe('ContextTracker', function() {
assert.deepEqual(state.tokens.map(token => token.raw), rawTokens);
});
it("properly matches and aligns when a 'wordbreak' is added'", function() {
it("properly matches and aligns when a 'wordbreak' is added", function() {
let existingContext = ["an", "apple", "a", "day", "keeps", "the", "doctor"];
let transform = {
insert: ' ',
@ -56,7 +56,7 @@ describe('ContextTracker', function() {
let rawTokens = ["an", null, "apple", null, "a", null, "day", null, "keeps", null, "the", null, "doctor", null, ""];
let existingState = ContextTracker.modelContextState(existingContext);
let state = ContextTracker.attemptMatchContext(newContext, existingState, null, toWrapperDistribution(transform));
let state = ContextTracker.attemptMatchContext(newContext, existingState, toWrapperDistribution(transform));
assert.isNotNull(state);
assert.deepEqual(state.tokens.map(token => token.raw), rawTokens);
@ -65,6 +65,26 @@ describe('ContextTracker', function() {
assert.isEmpty(state.tokens[state.tokens.length - 1].transformDistributions);
});
it("properly matches and aligns when an implied 'wordbreak' occurs (as when following \"'\")", function() {
let existingContext = ["'"];
let transform = {
insert: 'a',
deleteLeft: 0
}
let newContext = Array.from(existingContext);
newContext.push('a'); // The incoming transform should produce a new token WITH TEXT.
let rawTokens = ["'", null, "a"];
let existingState = ContextTracker.modelContextState(existingContext);
let state = ContextTracker.attemptMatchContext(newContext, existingState, toWrapperDistribution(transform));
assert.isNotNull(state);
assert.deepEqual(state.tokens.map(token => token.raw), rawTokens);
// The 'wordbreak' transform
assert.isEmpty(state.tokens[state.tokens.length - 2].transformDistributions);
assert.isNotEmpty(state.tokens[state.tokens.length - 1].transformDistributions);
});
it("properly matches and aligns when lead token is removed AND a 'wordbreak' is added'", function() {
let existingContext = ["an", "apple", "a", "day", "keeps", "the", "doctor"];
let transform = {
@ -77,7 +97,7 @@ describe('ContextTracker', function() {
let rawTokens = ["apple", null, "a", null, "day", null, "keeps", null, "the", null, "doctor", null, ""];
let existingState = ContextTracker.modelContextState(existingContext);
let state = ContextTracker.attemptMatchContext(newContext, existingState, null, toWrapperDistribution(transform));
let state = ContextTracker.attemptMatchContext(newContext, existingState, toWrapperDistribution(transform));
assert.isNotNull(state);
assert.deepEqual(state.tokens.map(token => token.raw), rawTokens);

58
common/predictive-text/unit_tests/headless/transform-utils.js Normal file
View file

@ -0,0 +1,58 @@
var assert = require('chai').assert;
let TransformUtils = require('../../../web/lm-worker/build/intermediate.js').TransformUtils;
describe('TransformUtils', function () {
describe('isWhitespace', function () {
it("should not match a string containing standard alphabetic characters", function () {
let testTransforms = [{
insert: "a ",
deleteLeft: 0
}, {
insert: " a",
deleteLeft: 0
}, {
insert: "ab",
deleteLeft: 0
}];
testTransforms.forEach((transform) => assert.isFalse(TransformUtils.isWhitespace(transform), `failed with: '${transform.insert}'`));
});
it("should match a simple ' ' transform", function() {
transform = {
insert: " ",
deleteLeft: 0
};
assert.isTrue(TransformUtils.isWhitespace(transform));
});
it("should match a simple ' ' transform with delete-left", function() {
transform = {
insert: " ",
deleteLeft: 1
};
assert.isTrue(TransformUtils.isWhitespace(transform));
});
it("should match a transform consisting of multiple characters of only whitespace", function() {
transform = {
insert: " \n\r\u00a0\t\u2000 ",
deleteLeft: 0
};
assert.isTrue(TransformUtils.isWhitespace(transform));
});
it("stress tests", function() {
transform = {
insert: " \n\r\u00a0\ta\u2000 ", // the 'a' should cause failure.
deleteLeft: 0
};
assert.isFalse(TransformUtils.isWhitespace(transform));
});
});
});

82
common/predictive-text/unit_tests/headless/worker-model-compositor.js
View file

@ -50,6 +50,45 @@ describe('ModelCompositor', function() {
});
});
it('strongly avoids corrections for single-character roots', function() {
let compositor = new ModelCompositor(plainModel);
let context = {
left: '', startOfBuffer: true, endOfBuffer: true,
};
// The 'weights' involved imply that we have an edge-case fat finger on the bottom of
// the 'q' key, slightly in its favor.
let inputDistribution = [
{sample: {insert: 'q', deleteLeft: 0}, p: 0.5}, // 'quite' (679) and 'question' (644) are included!
{sample: {insert: 'a', deleteLeft: 0}, p: 0.4} // but at lower weight than 'and' (998).
];
compositor.predict({insert: '', deleteLeft: 0}, context); // Initialize context tracking first!
let suggestions = compositor.predict(inputDistribution, context);
// remove the keep suggestion; we're not testing that here.
suggestions = suggestions.filter((suggestion) => suggestion.tag != 'keep');
suggestions.sort((a, b) => b.p - a.p);
// There are only 4 suggestions in this limited test model that begin with 'q'.
// We expect more than that, since 'a' is indicated to be very close by.
assert.isAbove(suggestions.length, 4, "fat-finger style corrections needed for test comparisons are missing");
// Note: 'and' is (currently) modeled by the text-fixture model to have 9.3x the base probability
// that the worst 'q'-rooted suggestion ('quality') does. Without single-character correction
// avoidance logic, this test _will_ fail.
//
// In case a tweak to test parameters is desired, note that 'and' beats rank #3 - 'questions' -
// at 3.36x base. At the time of writing this test, upping 'a's probability to 0.45 will block
// 'quality' while the top three 'q's (ending with 'questions') remain in place.
let qRange = suggestions.slice(0, 4);
assert.isUndefined(qRange.find((suggestion) => suggestion.transform.insert.charAt(0) != 'q'));
let aRange = suggestions.slice(4);
assert.isUndefined(aRange.find((suggestion) => suggestion.transform.insert.charAt(0) == 'q'));
});
it('properly handles suggestions after a backspace', function() {
let compositor = new ModelCompositor(plainModel);
let context = {
@ -66,12 +105,53 @@ describe('ModelCompositor', function() {
// Suggestions always delete the full root of the suggestion.
//
// After a backspace, that means the text 'the' - 3 chars.
// Char 4 is for the original backspace, as suggstions are built
// Char 4 is for the original backspace, as suggestions are built
// based on the context state BEFORE the triggering input -
// here, a backspace.
assert.equal(suggestion.transform.deleteLeft, 4);
});
});
it('properly handles suggestions for the first letter after a ` `', function() {
let compositor = new ModelCompositor(plainModel);
let context = {
left: 'the', startOfBuffer: true, endOfBuffer: true,
};
let inputTransform = {
insert: ' ',
deleteLeft: 0
};
let suggestions = compositor.predict(inputTransform, context);
suggestions.forEach(function(suggestion) {
// After a space, predictions are based on a new, zero-length root.
// With nothing to replace, .deleteLeft should be zero.
assert.equal(suggestion.transform.deleteLeft, 0);
});
});
it('properly handles suggestions for the first letter after a `\'`', function() {
let compositor = new ModelCompositor(plainModel);
let context = {
left: "the '", startOfBuffer: true, endOfBuffer: true,
};
// This results in a new word boundary (between the `'` and the `a`).
// Basically, an implied (but nonexistent) ` `.
let inputTransform = {
insert: "a",
deleteLeft: 0
};
let suggestions = compositor.predict(inputTransform, context);
suggestions.forEach(function(suggestion) {
// Suggestions always delete the full root of the suggestion.
// Which, here, didn't exist before the input. Nothing to
// replace => nothing for the suggestion to delete.
assert.equal(suggestion.transform.deleteLeft, 0);
});
});
});
describe('applySuggestionCasing', function() {

3
common/web/input-processor/src/text/inputProcessor.ts
View file

@ -12,7 +12,7 @@ namespace com.keyman.text {
/**
* Indicates the device (platform) to be used for non-keystroke events,
* such as those sent to `begin postkeystroke` and `begin newcontext`
* such as those sent to `begin postkeystroke` and `begin newcontext`
* entry points.
*/
private contextDevice: utils.DeviceSpec;
@ -273,6 +273,7 @@ namespace com.keyman.text {
let totalMass = 0; // Tracks sum of non-error probabilities.
for(let pair of keyDistribution) {
if(pair.p < KEYSTROKE_EPSILON) {
totalMass += pair.p;
break;
} else if(timer && timer() >= TIMEOUT_THRESHOLD) {
// Note: it's always possible that the thread _executing_ our JS

19
common/web/keyboard-processor/src/keyboards/activeLayout.ts
View file

@ -37,6 +37,7 @@ namespace com.keyman.keyboards {
layer: string;
displayLayer: string;
nextlayer: string;
sp?: ButtonClass;
private baseKeyEvent: text.KeyEvent;
isMnemonic: boolean = false;
@ -73,6 +74,13 @@ namespace com.keyman.keyboards {
return this.id;
}
@Enumerable
public get isPadding(): boolean {
// Does not include 9 (class: blank) as that may be an intentional 'catch' for misplaced
// keystrokes.
return this['sp'] == 10; // Button class: hidden.
}
/**
* A unique identifier based on both the key ID & the 'desktop layer' to be used for the key.
*
@ -337,14 +345,12 @@ namespace com.keyman.keyboards {
// Allow for right OSK margin (15 layout units)
let rightMargin = ActiveKey.DEFAULT_RIGHT_MARGIN/totalWidth;
totalPercent += rightMargin;
// If a single key, and padding is negative, add padding to right align the key
if(keys.length == 1 && parseInt(keys[0]['pad'],10) < 0) {
keyPercent=parseInt(keys[0]['width'],10)/totalWidth;
keys[0]['widthpc']=keyPercent;
totalPercent += keyPercent;
keys[0]['padpc']=1-totalPercent;
keys[0]['padpc']=1-(totalPercent + keyPercent + rightMargin);
// compute center's default x-coord (used in headless modes)
setProportions(keys[0] as ActiveKey, padPercent, keyPercent, totalPercent);
@ -352,8 +358,7 @@ namespace com.keyman.keyboards {
let j=keys.length-1;
padPercent=parseInt(keys[j]['pad'],10)/totalWidth;
keys[j]['padpc']=padPercent;
totalPercent += padPercent;
keys[j]['widthpc'] = keyPercent = 1-totalPercent;
keys[j]['widthpc'] = keyPercent = 1-(totalPercent + padPercent + rightMargin);
// compute center's default x-coord (used in headless modes)
setProportions(keys[j] as ActiveKey, padPercent, keyPercent, totalPercent);
@ -515,7 +520,7 @@ namespace com.keyman.keyboards {
// Should we wish to allow multiple different transforms for distance -> probability, use a function parameter in place
// of the formula in the loop below.
for(let key in keyDists) {
totalMass += keyProbs[key] = 1 / (keyDists[key] + 1e-6); // Prevent div-by-0 errors.
totalMass += keyProbs[key] = 1 / (Math.pow(keyDists[key], 2) + 1e-6); // Prevent div-by-0 errors.
}
for(let key in keyProbs) {
@ -550,6 +555,8 @@ namespace com.keyman.keyboards {
// Results in a more optimized distribution.
if(text.Codes.isKnownOSKModifierKey(key.baseKeyID)) {
return;
} else if(key.isPadding) { // to the user, blank / padding keys do not exist.
return;
}
}
// These represent the within-key distance of the touch from the key's center.

5
common/web/keyboard-processor/src/text/codes.ts
View file

@ -81,10 +81,13 @@ namespace com.keyman.text {
case 'K_SHIFT':
case 'K_LOPT':
case 'K_ROPT':
case 'K_NUMLOCK': // Often used for numeric layers.
case 'K_NUMLOCK': // Often used for numeric layers.
case 'K_CAPS':
return true;
default:
if(Codes.keyCodes[keyID] >= 50000) { // A few are used by `sil_euro_latin`.
return true; // is a 'K_' key defined for layer shifting or 'control' use.
}
// Refer to text/codes.ts - these are Keyman-custom "keycodes" used for
// layer shifting keys. To be safe, we currently let K_TABBACK and
// K_TABFWD through, though we might be able to drop them too.

85
common/web/lm-worker/src/correction/context-tracker.ts
View file

@ -32,10 +32,6 @@ namespace correction {
replacements: TrackedContextSuggestion[];
activeReplacementId: number = -1;
get isNew(): boolean {
return this.transformDistributions.length == 0;
}
get currentText(): string {
if(this.replacementText === undefined || this.replacementText === null) {
return this.raw;
@ -89,7 +85,7 @@ namespace correction {
if(token.replacementText) {
copy.replacementText = token.replacementText;
}
return copy;
});
this.searchSpace = obj.searchSpace;
@ -139,8 +135,8 @@ namespace correction {
// Track the Transform that resulted in the whitespace 'token'.
// Will be needed for phrase-level correction/prediction.
whitespaceToken.transformDistributions = [transformDistribution];
whitespaceToken.transformDistributions = transformDistribution ? [transformDistribution] : [];
whitespaceToken.raw = null;
this.tokens.push(whitespaceToken);
}
@ -149,19 +145,19 @@ namespace correction {
* Used for 14.0's backspace workaround, which flattens all previous Distribution<Transform>
* entries because of limitations with direct use of backspace transforms.
* @param tokenText
* @param transformId
* @param transformId
*/
replaceTailForBackspace(tokenText: USVString, transformId: number) {
this.tokens.pop();
// It's a backspace transform; time for special handling!
//
// For now, with 14.0, we simply compress all remaining Transforms for the token into
// multiple single-char transforms. Probabalistically modeling BKSP is quite complex,
// For now, with 14.0, we simply compress all remaining Transforms for the token into
// multiple single-char transforms. Probabalistically modeling BKSP is quite complex,
// so we simplify by assuming everything remaining after a BKSP is 'true' and 'intended' text.
//
// Note that we cannot just use a single, monolithic transform at this point b/c
// of our current edit-distance optimization strategy; diagonalization is currently...
// of our current edit-distance optimization strategy; diagonalization is currently...
// not very compatible with that.
let backspacedTokenContext: Distribution<Transform>[] = textToCharTransforms(tokenText, transformId).map(function(transform) {
return [{sample: transform, p: 1.0}];
@ -175,7 +171,7 @@ namespace correction {
updateTail(transformDistribution: Distribution<Transform>, tokenText?: USVString) {
let editedToken = this.tail;
// Preserve existing text if new text isn't specified.
tokenText = tokenText || (tokenText === '' ? '' : editedToken.raw);
@ -191,7 +187,7 @@ namespace correction {
toRawTokenization() {
let sequence: USVString[] = [];
for(let token of this.tokens) {
// Hide any tokens representing wordbreaks. (Thinking ahead to phrase-level possibilities)
if(token.currentText !== null) {
@ -281,7 +277,7 @@ namespace correction {
/**
* Returns items contained within the circular array, ordered from 'oldest' to 'newest' -
* the same order in which the items will be dequeued.
* @param index
* @param index
*/
item(index: number) {
if(index >= this.count) {
@ -294,7 +290,7 @@ namespace correction {
}
export class ContextTracker extends CircularArray<TrackedContextState> {
static attemptMatchContext(tokenizedContext: USVString[],
static attemptMatchContext(tokenizedContext: USVString[],
matchState: TrackedContextState,
transformDistribution?: Distribution<Transform>,): TrackedContextState {
// Map the previous tokenized state to an edit-distance friendly version.
@ -335,7 +331,7 @@ namespace correction {
}
// Can happen for the first text input after backspace deletes a wordbreaking character,
// thus the new input continues a previous word while dropping the empty word after
// thus the new input continues a previous word while dropping the empty word after
// that prior wordbreaking character.
//
// We can't handle it reliably from this match state, but a previous entry (without the empty token)
@ -353,7 +349,7 @@ namespace correction {
// If we've made it here... success! We have a context match!
let state: TrackedContextState;
if(pushedTail) {
// On suggestion acceptance, we should update the previous final token.
// We do it first so that the acceptance is replicated in the new TrackedContextState
@ -376,7 +372,9 @@ namespace correction {
if(primaryInput && primaryInput.insert == "" && primaryInput.deleteLeft == 0 && !primaryInput.deleteRight) {
primaryInput = null;
}
const isBackspace = primaryInput && primaryInput.insert == "" && primaryInput.deleteLeft > 0 && !primaryInput.deleteRight;
const isWhitespace = primaryInput && TransformUtils.isWhitespace(primaryInput);
const isBackspace = primaryInput && TransformUtils.isBackspace(primaryInput);
const finalToken = tokenizedContext[tokenizedContext.length-1];
/* Assumption: This is an adequate check for its two sub-branches.
@ -388,7 +386,7 @@ namespace correction {
* - Assumption: one keystroke may only cause a single token to be appended to the context
* - That is, no "reasonable" keystroke would emit a Transform adding two separate word tokens
* - For languages using whitespace to word-break, said keystroke would have to include said whitespace to break the assumption.
*/
*/
// If there is/was more than one context token available...
if(editPath.length > 1) {
@ -399,17 +397,29 @@ namespace correction {
// We're adding an additional context token.
if(pushedTail) {
// ASSUMPTION: any transform that triggers this case is a pure-whitespace Transform, as we
// need a word-break before beginning a new word's context.
// Worth note: when invalid, the lm-layer already has problems in other aspects too.
state.pushWhitespaceToTail(transformDistribution);
const tokenizedTail = tokenizedContext[tokenizedContext.length - 1];
/*
* Common-case: most transforms that trigger this case are from pure-whitespace Transforms. MOST.
*
* Less-common, but noteworthy: some wordbreaks may occur without whitespace. Example:
* `"o` => ['"', 'o']. Make sure to double-check against `tokenizedContext`!
*/
let pushedToken = new TrackedContextToken();
pushedToken.raw = tokenizedTail;
let emptyToken = new TrackedContextToken();
emptyToken.raw = '';
// Continuing the earlier assumption, that 'pure-whitespace Transform' does not emit any initial characters
// for the new word (token), so the input keystrokes do not correspond to the new text token.
emptyToken.transformDistributions = [];
state.pushTail(emptyToken);
if(isWhitespace || !primaryInput) {
state.pushWhitespaceToTail(transformDistribution ?? []);
// Continuing the earlier assumption, that 'pure-whitespace Transform' does not emit any initial characters
// for the new word (token), so the input keystrokes do not correspond to the new text token.
pushedToken.transformDistributions = [];
} else {
state.pushWhitespaceToTail();
// Assumption: Since we only allow one-transform-at-a-time changes between states, we shouldn't be missing
// any metadata used to construct the new context state token.
pushedToken.transformDistributions = transformDistribution ? [transformDistribution] : [];
}
state.pushTail(pushedToken);
} else { // We're editing the final context token.
// TODO: Assumption: we didn't 'miss' any inputs somehow.
// As is, may be prone to fragility should the lm-layer's tracked context 'desync' from its host's.
@ -442,7 +452,9 @@ namespace correction {
return state;
}
static modelContextState(tokenizedContext: USVString[], lexicalModel: LexicalModel): TrackedContextState {
static modelContextState(tokenizedContext: USVString[],
transformDistribution: Distribution<Transform>,
lexicalModel: LexicalModel): TrackedContextState {
let baseTokens = tokenizedContext.map(function(entry) {
let token = new TrackedContextToken();
token.raw = entry;
@ -483,13 +495,12 @@ namespace correction {
* Compares the current, post-input context against the most recently-seen contexts from previous prediction calls, returning
* the most information-rich `TrackedContextState` possible. If a match is found, the state will be annotated with the
* input information provided to previous prediction calls and persisted correction-search calculations for re-use.
*
* @param model
* @param context
* @param mainTransform
* @param transformDistribution
*
* @param model
* @param context
* @param transformDistribution
*/
analyzeState(model: LexicalModel,
analyzeState(model: LexicalModel,
context: Context,
transformDistribution?: Distribution<Transform>): TrackedContextState {
if(!model.traverseFromRoot) {
@ -519,7 +530,7 @@ namespace correction {
//
// Assumption: as a caret needs to move to context before any actual transform distributions occur,
// this state is only reached on caret moves; thus, transformDistribution is actually just a single null transform.
let state = ContextTracker.modelContextState(tokenizedContext.left, model);
let state = ContextTracker.modelContextState(tokenizedContext.left, transformDistribution, model);
state.taggedContext = context;
this.enqueue(state);
return state;

28
common/web/lm-worker/src/correction/distance-modeler.ts
View file

@ -204,6 +204,15 @@ namespace correction {
// TODO: might should also track diagonalWidth.
return inputString + models.SENTINEL_CODE_UNIT + matchString;
}
get isFullReplacement(): boolean {
// If the known edit-distance cost is equal to the input length, this means
// that literally every input has been full-on replaced. Thus, this is
// likely not a good 'root' to use for predictions.
//
// Logic exception: 0 cost, 0 length != a "replacement".
return this.knownCost && this.knownCost == this.priorInput.length;
}
}
class SearchSpaceTier {
@ -653,7 +662,7 @@ namespace correction {
shouldTimeout(): boolean {
const now = Date.now();
if(this.start - now > this.maxTrueTime) {
if(now - this.start > this.maxTrueTime) {
return true;
}
@ -710,7 +719,7 @@ namespace correction {
let batcher = new BatchingAssistant();
const timer = new ExecutionTimer(maxTime*3, maxTime);
const timer = new ExecutionTimer(maxTime*1.5, maxTime);
// Stage 1 - if we already have extracted results, build a queue just for them and iterate over it first.
let returnedValues = Object.values(this.returnedValues);
@ -721,6 +730,14 @@ namespace correction {
timer.startLoop();
while(preprocessedQueue.count > 0) {
let entry = preprocessedQueue.dequeue();
// Is the entry a reasonable result?
if(entry.isFullReplacement) {
// If the entry's 'match' fully replaces the input string, we consider it
// unreasonable and ignore it.
continue;
}
let batch = batcher.checkAndAdd(entry);
timer.markIteration();
@ -761,6 +778,13 @@ namespace correction {
if(newResult.type == 'none') {
break;
} else if(newResult.type == 'complete') {
// Is the entry a reasonable result?
if(newResult.finalNode.isFullReplacement) {
// If the entry's 'match' fully replaces the input string, we consider it
// unreasonable and ignore it. Also, if we've reached this point...
// we can(?) assume that everything thereafter is as well.
break;
}
batch = batcher.checkAndAdd(newResult.finalNode);
}

2
common/web/lm-worker/src/index.ts
View file

@ -32,6 +32,7 @@
/// <reference types="@keymanapp/lm-message-types" />
/// <reference path="./models/dummy-model.ts" />
/// <reference path="./model-compositor.ts" />
/// <reference path="./transformUtils.ts" />
/**
* Encapsulates all the state required for the LMLayer's worker thread.
@ -407,6 +408,7 @@ if (typeof module !== 'undefined' && typeof module.exports !== 'undefined') {
module.exports['wordBreakers'] = wordBreakers;
/// XXX: export the ModelCompositor for testing.
module.exports['ModelCompositor'] = ModelCompositor;
module.exports['TransformUtils'] = TransformUtils;
} else if (typeof self !== 'undefined' && 'postMessage' in self && 'importScripts' in self) {
// Automatically install if we're in a Web Worker.
LMLayerWorker.install(self as any); // really, 'as typeof globalThis', but we're currently getting TS errors from use of that.

199
common/web/lm-worker/src/model-compositor.ts
View file

@ -6,6 +6,23 @@ class ModelCompositor {
private static readonly MAX_SUGGESTIONS = 12;
readonly punctuation: LexicalModelPunctuation;
/**
* Controls the strength of anti-corrective measures for single-character scenarios.
* The base key probability will be raised to this power for this specific case.
*
* Current selection's motivation: (0.5 / 0.4) ^ 16 ~= 35.5.
* - if the most likely has p=0.5 and second-most has p=0.4 - a highly-inaccurate key
* stroke - the net effect will apply a factor of 35.5 to the lexical probability of
* the best key's prediction roots, favoring it in this manner.
* - less extreme edge cases will have a significantly stronger factor, acting as a
* "soft threshold".
* - truly ambiguous, "coin flip" cases will have a lower factor and thus favor the
* more likely words from the pair.
* - Our OSK key-element borders aren't visible to the user, so the 'spot' where
* behavior changes might feel arbitrary to users if we used a hard threshold instead.
*/
private static readonly SINGLE_CHAR_KEY_PROB_EXPONENT = 16;
private SUGGESTION_ID_SEED = 0;
constructor(lexicalModel: LexicalModel) {
@ -16,30 +33,6 @@ class ModelCompositor {
this.punctuation = ModelCompositor.determinePunctuationFromModel(lexicalModel);
}
protected isWhitespace(transform: Transform): boolean {
// Matches prefixed text + any instance of a character with Unicode general property Z* or the following: CR, LF, and Tab.
let whitespaceRemover = /.*[\u0009\u000A\u000D\u0020\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b\u2028\u2029\u202f\u205f\u3000]/i;
// Filter out null-inserts; their high probability can cause issues.
if(transform.insert == '') { // Can actually register as 'whitespace'.
return false;
}
let insert = transform.insert;
insert = insert.replace(whitespaceRemover, '');
return insert == '';
}
protected isBackspace(transform: Transform): boolean {
return transform.insert == "" && transform.deleteLeft > 0;
}
protected isEmpty(transform: Transform): boolean {
return transform.insert == '' && transform.deleteLeft == 0;
}
private predictFromCorrections(corrections: ProbabilityMass<Transform>[], context: Context): Distribution<Suggestion> {
let returnedPredictions: Distribution<Suggestion> = [];
@ -98,8 +91,8 @@ class ModelCompositor {
})[0].sample;
// Only allow new-word suggestions if space was the most likely keypress.
let allowSpace = this.isWhitespace(inputTransform);
let allowBksp = this.isBackspace(inputTransform);
let allowSpace = TransformUtils.isWhitespace(inputTransform);
let allowBksp = TransformUtils.isBackspace(inputTransform);
let postContext = models.applyTransform(inputTransform, context);
let keepOptionText = this.wordbreak(postContext);
@ -109,7 +102,7 @@ class ModelCompositor {
// Used to restore whitespaces if operations would remove them.
let prefixTransform: Transform;
let contextState: correction.TrackedContextState = null;
let postContextState: correction.TrackedContextState = null;
// Section 1: determining 'prediction roots'.
if(!this.contextTracker) {
@ -124,18 +117,18 @@ class ModelCompositor {
predictionRoots = [{sample: inputTransform, p: 1.0}];
prefixTransform = inputTransform;
} else {
predictionRoots = transformDistribution.map(function(alt) {
predictionRoots = transformDistribution.map((alt) => {
let transform = alt.sample;
// Filter out special keys unless they're expected.
if(this.isWhitespace(transform) && !allowSpace) {
if(TransformUtils.isWhitespace(transform) && !allowSpace) {
return null;
} else if(this.isBackspace(transform) && !allowBksp) {
} else if(TransformUtils.isBackspace(transform) && !allowBksp) {
return null;
}
return alt;
}, this);
});
}
// Remove `null` entries.
@ -144,12 +137,15 @@ class ModelCompositor {
// Running in bulk over all suggestions, duplicate entries may be possible.
rawPredictions = this.predictFromCorrections(predictionRoots, context);
} else {
contextState = this.contextTracker.analyzeState(this.lexicalModel,
postContext,
!this.isEmpty(inputTransform) ?
transformDistribution:
null
);
// Token replacement benefits greatly from knowledge of the prior context state.
let contextState = this.contextTracker.analyzeState(this.lexicalModel, context, null);
// Corrections and predictions are based upon the post-context state, though.
postContextState = this.contextTracker.analyzeState(this.lexicalModel,
postContext,
!TransformUtils.isEmpty(inputTransform) ?
transformDistribution:
null
);
// TODO: Should we filter backspaces & whitespaces out of the transform distribution?
// Ideally, the answer (in the future) will be no, but leaving it in right now may pose an issue.
@ -158,19 +154,81 @@ class ModelCompositor {
// let's just note that right now, there will only ever be one.
//
// The 'eventual' logic will be significantly more complex, though still manageable.
let searchSpace = contextState.searchSpace[0];
let searchSpace = postContextState.searchSpace[0];
let newEmptyToken = false;
// Detect if we're starting a new context state.
let contextTokens = contextState.tokens;
if(contextTokens.length == 0 || contextTokens[contextTokens.length - 1].isNew) {
if(this.isEmpty(inputTransform) || this.isWhitespace(inputTransform)) {
newEmptyToken = true;
// No matter the prediction, once we know the root of the prediction, we'll always 'replace' the
// same amount of text. We can handle this before the big 'prediction root' loop.
let deleteLeft = 0;
// The amount of text to 'replace' depends upon whatever sort of context change occurs
// from the received input.
const postContextTokens = postContextState.tokens;
let postContextLength = postContextTokens.length;
let contextLengthDelta = postContextTokens.length - contextState.tokens.length;
// If the context now has more tokens, the token we'll be 'predicting' didn't originally exist.
if(postContextLength == 0 || contextLengthDelta > 0) {
// As the word/token being corrected/predicted didn't originally exist, there's no
// part of it to 'replace'.
deleteLeft = 0;
// If the new token is due to whitespace or due to a different input type that would
// likely imply a tokenization boundary...
if(TransformUtils.isWhitespace(inputTransform)) {
/* TODO: consider/implement: the second half of the comment above.
* For example: on input of a `'`, predict new words instead of replacing the `'`.
* (since after a letter, the `'` will be ignored, anyway)
*
* Idea: if the model's most likely prediction (with no root) would make a new
* token if appended to the current token, that's probably a good case.
* Keeps the check simple & quick.
*
* Might need a mixed mode, though: ';' is close enough that `l` is a reasonable
* fat-finger guess. So yeah, we're not addressing this idea right now.
* - so... consider multiple context behavior angles when building prediction roots?
*
* May need something similar to help handle contractions during their construction,
* but that'd be within `ContextTracker`.
* can' => [`can`, `'`]
* can't => [`can't`] (WB6, 7 of https://unicode.org/reports/tr29/#Word_Boundary_Rules)
*
* (Would also helps WB7b+c for Hebrew text)
*/
// Infer 'new word' mode, even if we received new text when reaching
// this position. That new text didn't exist before, so still - nothing
// to 'replace'.
prefixTransform = inputTransform;
context = postContext; // Ensure the whitespace token is preapplied!
context = postContext; // As far as predictions are concerned, the post-context state
// should not be replaced. Predictions are to be rooted on
// text "up for correction" - so we want a null root for this
// branch.
contextState = postContextState;
}
// If the tokenized context length is shorter... sounds like a backspace (or similar).
} else if (contextLengthDelta < 0) {
/* Ooh, we've dropped context here. Almost certainly from a backspace.
* Even if we drop multiple tokens... well, we know exactly how many chars
* were actually deleted - `inputTransform.deleteLeft`.
* Since we replace a word being corrected/predicted, we take length of the remaining
* context's tail token in addition to however far was deleted to reach that state.
*/
deleteLeft = this.wordbreak(postContext).kmwLength() + inputTransform.deleteLeft;
} else {
// Suggestions are applied to the pre-input context, so get the token's original length.
// We're on the same token, so just delete its text for the replacement op.
deleteLeft = this.wordbreak(context).kmwLength();
}
// Is the token under construction newly-constructed / is there no pre-existing root?
// If so, we want to strongly avoid overcorrection, even for 'nearby' keys.
// (Strong lexical frequency differences can easily cause overcorrection when only
// one key's available.)
//
// NOTE: we only want this applied word-initially, when any corrections 'correct'
// 100% of the word. Things are generally fine once it's not "all or nothing."
let tailToken = postContextTokens[postContextTokens.length - 1];
const isTokenStart = tailToken.transformDistributions.length <= 1;
// TODO: whitespace, backspace filtering. Do it here.
// Whitespace is probably fine, actually. Less sure about backspace.
@ -192,19 +250,6 @@ class ModelCompositor {
finalInput = inputTransform; // A fallback measure. Greatly matters for empty contexts.
}
let deleteLeft = 0;
// remove actual token string. If new token, there should be nothing to delete.
if(!newEmptyToken) {
// If this is triggered from a backspace, make sure to use its results
// and also include its left-deletions! It's the one post-input context case.
if(allowBksp) {
deleteLeft = this.wordbreak(postContext).kmwLength() + inputTransform.deleteLeft;
} else {
// Normal case - use the pre-input context.
deleteLeft = this.wordbreak(context).kmwLength();
}
}
// Replace the existing context with the correction.
let correctionTransform: Transform = {
insert: correction, // insert correction string
@ -212,9 +257,39 @@ class ModelCompositor {
id: inputTransform.id // The correction should always be based on the most recent external transform/transcription ID.
}
let rootCost = match.totalCost;
/* If we're dealing with the FIRST keystroke of a new sequence, we'll **dramatically** boost
* the exponent to ensure only VERY nearby corrections have a chance of winning, and only if
* there are significantly more likely words. We only need this to allow very minor fat-finger
* adjustments for 100% keystroke-sequence corrections in order to prevent finickiness on
* key borders.
*
* Technically, the probabilities this produces won't be normalized as-is... but there's no
* true NEED to do so for it, even if it'd be 'nice to have'. Consistently tracking when
* to apply it could become tricky, so it's simpler to leave out.
*
* Worst-case, it's possible to temporarily add normalization if a code deep-dive
* is needed in the future.
*/
if(isTokenStart) {
/* Suppose a key distribution: most likely with p=0.5, second-most with 0.4 - a pretty
* ambiguous case that would only arise very near the center of the boundary between two keys.
* Raising (0.5/0.4)^16 ~= 35.53. (At time of writing, SINGLE_CHAR_KEY_PROB_EXPONENT = 16.)
* That seems 'within reason' for correction very near boundaries.
*
* So, with the second-most-likely key being that close in probability, its best suggestion
* must be ~ 35.5x more likely than that of the truly-most-likely key to "win". So, it's not
* a HARD cutoff, but more of a 'soft' one. Keeping the principles in mind documented above,
* it's possible to tweak this to a more harsh or lenient setting if desired, rather than
* being totally "all or nothing" on which key is taken for highly-ambiguous keypresses.
*/
rootCost *= ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT; // note the `Math.exp` below.
}
return {
sample: correctionTransform,
p: Math.exp(-match.totalCost)
p: Math.exp(-rootCost)
};
}, this);
@ -411,8 +486,8 @@ class ModelCompositor {
// Store the suggestions on the final token of the current context state (if it exists).
// Or, once phrase-level suggestions are possible, on whichever token serves as each prediction's root.
if(contextState) {
contextState.tail.replacements = suggestions.map(function(suggestion) {
if(postContextState) {
postContextState.tail.replacements = suggestions.map(function(suggestion) {
return {
suggestion: suggestion,
tokenWidth: 1
@ -659,7 +734,7 @@ class ModelCompositor {
// than before.
if(this.contextTracker) {
let tokenizedContext = models.tokenize(this.lexicalModel.wordbreaker || wordBreakers.default, context);
let contextState = correction.ContextTracker.modelContextState(tokenizedContext.left, this.lexicalModel);
let contextState = correction.ContextTracker.modelContextState(tokenizedContext.left, null, this.lexicalModel);
this.contextTracker.enqueue(contextState);
}
}

15
common/web/lm-worker/src/transformUtils.ts Normal file
View file

@ -0,0 +1,15 @@
class TransformUtils {
static isWhitespace(transform: Transform): boolean {
// Matches a string that is entirely one or more characters with Unicode general property Z* or the following: CR, LF, and Tab.
const whitespaceRemover = /^[\u0009\u000A\u000D\u0020\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b\u2028\u2029\u202f\u205f\u3000]+$/i;
return transform.insert.match(whitespaceRemover) != null;
}
static isBackspace(transform: Transform): boolean {
return transform.insert == "" && transform.deleteLeft > 0 && !transform.deleteRight;
}
static isEmpty(transform: Transform): boolean {
return transform.insert == '' && transform.deleteLeft == 0 && !transform.deleteRight;
}
}

45
common/windows/delphi/setup/SFX.pas
View file

@ -82,27 +82,38 @@ var
fs: TFileStream;
ms: TMemoryStream;
begin
fs := TFileStream.Create(ParamStr(0), fmOpenRead or fmShareDenyWrite);
ms := TMemoryStream.Create;
try
if not FindFirstHeader(fs) then
Exit(False);
fs.Seek(StartOfFile, TSeekOrigin.soBeginning);
ms.CopyFrom(fs, fs.Size - StartOfFile);
ms.Position := 0;
with TZipFile.Create do
fs := TFileStream.Create(ParamStr(0), fmOpenRead or fmShareDenyWrite);
ms := TMemoryStream.Create;
try
Open(ms, zmRead);
ExtractAll(ExtPath);
if not FindFirstHeader(fs) then
Exit(False);
fs.Seek(StartOfFile, TSeekOrigin.soBeginning);
ms.CopyFrom(fs, fs.Size - StartOfFile);
ms.Position := 0;
with TZipFile.Create do
try
Open(ms, zmRead);
ExtractAll(ExtPath);
finally
Free;
end;
finally
Free;
fs.Free;
ms.Free;
end;
except
on E:Exception do
begin
raise Exception.Create(
'Failed to extract setup archive. '+
'You may have run out of disk space or there may be a '+
'problem with the source files.'#13#10#13#10+
'The error received was: '+E.Message);
end;
finally
fs.Free;
ms.Free;
end;
Result := True;
end;

5
developer/src/tike/xml/layoutbuilder/builder.css
View file

@ -613,6 +613,11 @@ body:not(.text-controls-in-toolbar) input#inpSubKeyCap {
color: white;
}
#kbd.desktop .key-size {
/* the layout is fixed on desktop so key size is not useful */
display: none;
}
/* Position flick keys relative to the flick grid, by hand */
#flick .key {

5
developer/src/tike/xml/layoutbuilder/builder.js
View file

@ -12,10 +12,7 @@ $(function() {
this.getPresentation = function () {
var platform = $('#selPlatformPresentation').val();
//if(platform == 'tablet') return 'tablet-ipad';
//if(platform == 'phone') return 'phone-iphone5';
return platform;
return $('#selPlatformPresentation').val();
}
this.saveSelection = function() {

3
developer/src/tike/xml/layoutbuilder/constants.js
View file

@ -604,7 +604,8 @@ $(function() {
"tablet-ipad-landscape": { "x": 829, "y": 299, "name": "iPad (landscape)" }, // 829x622 = iPad tablet box size; (97,101)-(926,723)
"tablet-ipad-portrait": { "x": 605, "y": 300, "name": "iPad (portrait)" }, // 605x806 = iPad tablet box size; (98,94)-(703,900)
"phone-iphone5-landscape": { "x": 731, "y": 196, "name": "iPhone 5 (landscape)" }, // 731x412 = iPhone box size; (144,39)-(875,451)
"phone-iphone5-portrait": { "x": 526, "y": 266, "name": "iPhone 5 (portrait)"} // 528x936 = iPhone box size; (90,204)-(618,1040)
"phone-iphone5-portrait": { "x": 526, "y": 266, "name": "iPhone 5 (portrait)"}, // 528x936 = iPhone box size; (90,204)-(618,1040)
"desktop": { "x": 640, "y": 300, "name": "Desktop" },
};
this.keyMargin = 15;

7
linux/debian/changelog
View file

@ -1,3 +1,10 @@
keyman (15.0.270-1) unstable; urgency=medium
* New upstream release.
* Re-release to Debian
-- Eberhard Beilharz <eb1@sil.org> Tue, 13 Sep 2022 11:20:25 +0200
keyman (15.0.269-1) unstable; urgency=medium
* New upstream release.

8
linux/ibus-keyman/src/engine.c
View file

@ -70,7 +70,6 @@ struct _IBusKeymanEngine {
gboolean lalt_pressed;
gboolean ralt_pressed;
gboolean emitting_keystroke;
IBusLookupTable *table;
IBusProperty *status_prop;
IBusPropList *prop_list;
#ifdef GDK_WINDOWING_X11
@ -262,8 +261,6 @@ ibus_keyman_engine_init(IBusKeymanEngine *keyman) {
g_object_ref_sink(keyman->prop_list);
ibus_prop_list_append(keyman->prop_list, keyman->status_prop);
keyman->table = ibus_lookup_table_new(9, 0, TRUE, TRUE);
g_object_ref_sink(keyman->table);
keyman->state = NULL;
#ifdef GDK_WINDOWING_X11
keyman->xdisplay = NULL;
@ -469,11 +466,6 @@ ibus_keyman_engine_destroy (IBusKeymanEngine *keyman)
keyman->status_prop = NULL;
}
if (keyman->table) {
g_debug("DAR: unref keyman->table");
g_object_unref (keyman->table);
keyman->table = NULL;
}
if (keyman->state) {
km_kbp_state_dispose(keyman->state);
keyman->state = NULL;

4
linux/keyman-config/Makefile
View file

@ -55,8 +55,10 @@ deb: dist
man:
./build-help.sh --man --no-reconf
version:
version_reconf:
cd .. && ./scripts/reconf.sh keyman-config
version: version_reconf
$(eval VERSION := $(shell python3 -c "from keyman_config import __releaseversion__; print(__releaseversion__)"))
# i18n

21
resources/standards-data/unicode-character-database/Blocks.txt
View file

@ -1,10 +1,10 @@
# Blocks-14.0.0.txt
# Date: 2021-01-22, 23:29:00 GMT [KW]
# © 2021 Unicode®, Inc.
# For terms of use, see http://www.unicode.org/terms_of_use.html
# Blocks-15.0.0.txt
# Date: 2022-01-28, 20:58:00 GMT [KW]
# © 2022 Unicode®, Inc.
# For terms of use, see https://www.unicode.org/terms_of_use.html
#
# Unicode Character Database
# For documentation, see http://www.unicode.org/reports/tr44/
# For documentation, see https://www.unicode.org/reports/tr44/
#
# Format:
# Start Code..End Code; Block Name
@ -15,7 +15,7 @@
# and underbars are ignored.
# For example, "Latin Extended-A" and "latin extended a" are equivalent.
# For more information on the comparison of property values,
# see UAX #44: http://www.unicode.org/reports/tr44/
# see UAX #44: https://www.unicode.org/reports/tr44/
#
# All block ranges start with a value where (cp MOD 16) = 0,
# and end with a value where (cp MOD 16) = 15. In other words,
@ -241,6 +241,7 @@ FFF0..FFFF; Specials
10D00..10D3F; Hanifi Rohingya
10E60..10E7F; Rumi Numeral Symbols
10E80..10EBF; Yezidi
10EC0..10EFF; Arabic Extended-C
10F00..10F2F; Old Sogdian
10F30..10F6F; Sogdian
10F70..10FAF; Old Uyghur
@ -272,11 +273,13 @@ FFF0..FFFF; Specials
11A50..11AAF; Soyombo
11AB0..11ABF; Unified Canadian Aboriginal Syllabics Extended-A
11AC0..11AFF; Pau Cin Hau
11B00..11B5F; Devanagari Extended-A
11C00..11C6F; Bhaiksuki
11C70..11CBF; Marchen
11D00..11D5F; Masaram Gondi
11D60..11DAF; Gunjala Gondi
11EE0..11EFF; Makasar
11F00..11F5F; Kawi
11FB0..11FBF; Lisu Supplement
11FC0..11FFF; Tamil Supplement
12000..123FF; Cuneiform
@ -284,7 +287,7 @@ FFF0..FFFF; Specials
12480..1254F; Early Dynastic Cuneiform
12F90..12FFF; Cypro-Minoan
13000..1342F; Egyptian Hieroglyphs
13430..1343F; Egyptian Hieroglyph Format Controls
13430..1345F; Egyptian Hieroglyph Format Controls
14400..1467F; Anatolian Hieroglyphs
16800..16A3F; Bamum Supplement
16A40..16A6F; Mro
@ -309,6 +312,7 @@ FFF0..FFFF; Specials
1D000..1D0FF; Byzantine Musical Symbols
1D100..1D1FF; Musical Symbols
1D200..1D24F; Ancient Greek Musical Notation
1D2C0..1D2DF; Kaktovik Numerals
1D2E0..1D2FF; Mayan Numerals
1D300..1D35F; Tai Xuan Jing Symbols
1D360..1D37F; Counting Rod Numerals
@ -316,9 +320,11 @@ FFF0..FFFF; Specials
1D800..1DAAF; Sutton SignWriting
1DF00..1DFFF; Latin Extended-G
1E000..1E02F; Glagolitic Supplement
1E030..1E08F; Cyrillic Extended-D
1E100..1E14F; Nyiakeng Puachue Hmong
1E290..1E2BF; Toto
1E2C0..1E2FF; Wancho
1E4D0..1E4FF; Nag Mundari
1E7E0..1E7FF; Ethiopic Extended-B
1E800..1E8DF; Mende Kikakui
1E900..1E95F; Adlam
@ -348,6 +354,7 @@ FFF0..FFFF; Specials
2CEB0..2EBEF; CJK Unified Ideographs Extension F
2F800..2FA1F; CJK Compatibility Ideographs Supplement
30000..3134F; CJK Unified Ideographs Extension G
31350..323AF; CJK Unified Ideographs Extension H
E0000..E007F; Tags
E0100..E01EF; Variation Selectors Supplement
F0000..FFFFF; Supplementary Private Use Area-A

300
resources/standards-data/unicode-character-database/UnicodeData.txt
View file

@ -2975,6 +2975,7 @@
0CEF;KANNADA DIGIT NINE;Nd;0;L;;9;9;9;N;;;;;
0CF1;KANNADA SIGN JIHVAMULIYA;Lo;0;L;;;;;N;;;;;
0CF2;KANNADA SIGN UPADHMANIYA;Lo;0;L;;;;;N;;;;;
0CF3;KANNADA SIGN COMBINING ANUSVARA ABOVE RIGHT;Mc;0;L;;;;;N;;;;;
0D00;MALAYALAM SIGN COMBINING ANUSVARA ABOVE;Mn;0;NSM;;;;;N;;;;;
0D01;MALAYALAM SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;;
0D02;MALAYALAM SIGN ANUSVARA;Mc;0;L;;;;;N;;;;;
@ -3339,6 +3340,7 @@
0ECB;LAO TONE MAI CATAWA;Mn;122;NSM;;;;;N;;;;;
0ECC;LAO CANCELLATION MARK;Mn;0;NSM;;;;;N;;;;;
0ECD;LAO NIGGAHITA;Mn;0;NSM;;;;;N;;;;;
0ECE;LAO YAMAKKAN;Mn;0;NSM;;;;;N;;;;;
0ED0;LAO DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;;
0ED1;LAO DIGIT ONE;Nd;0;L;;1;1;1;N;;;;;
0ED2;LAO DIGIT TWO;Nd;0;L;;2;2;2;N;;;;;
@ -19393,6 +19395,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
10EAD;YEZIDI HYPHENATION MARK;Pd;0;R;;;;;N;;;;;
10EB0;YEZIDI LETTER LAM WITH DOT ABOVE;Lo;0;R;;;;;N;;;;;
10EB1;YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE;Lo;0;R;;;;;N;;;;;
10EFD;ARABIC SMALL LOW WORD SAKTA;Mn;220;NSM;;;;;N;;;;;
10EFE;ARABIC SMALL LOW WORD QASR;Mn;220;NSM;;;;;N;;;;;
10EFF;ARABIC SMALL LOW WORD MADDA;Mn;220;NSM;;;;;N;;;;;
10F00;OLD SOGDIAN LETTER ALEPH;Lo;0;R;;;;;N;;;;;
10F01;OLD SOGDIAN LETTER FINAL ALEPH;Lo;0;R;;;;;N;;;;;
10F02;OLD SOGDIAN LETTER BETH;Lo;0;R;;;;;N;;;;;
@ -20058,6 +20063,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1123C;KHOJKI DOUBLE SECTION MARK;Po;0;L;;;;;N;;;;;
1123D;KHOJKI ABBREVIATION SIGN;Po;0;L;;;;;N;;;;;
1123E;KHOJKI SIGN SUKUN;Mn;0;NSM;;;;;N;;;;;
1123F;KHOJKI LETTER QA;Lo;0;L;;;;;N;;;;;
11240;KHOJKI LETTER SHORT I;Lo;0;L;;;;;N;;;;;
11241;KHOJKI VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;;
11280;MULTANI LETTER A;Lo;0;L;;;;;N;;;;;
11281;MULTANI LETTER I;Lo;0;L;;;;;N;;;;;
11282;MULTANI LETTER U;Lo;0;L;;;;;N;;;;;
@ -21256,6 +21264,16 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
11AF6;PAU CIN HAU LOW-FALLING TONE LONG FINAL;Lo;0;L;;;;;N;;;;;
11AF7;PAU CIN HAU LOW-FALLING TONE FINAL;Lo;0;L;;;;;N;;;;;
11AF8;PAU CIN HAU GLOTTAL STOP FINAL;Lo;0;L;;;;;N;;;;;
11B00;DEVANAGARI HEAD MARK;Po;0;L;;;;;N;;;;;
11B01;DEVANAGARI HEAD MARK WITH HEADSTROKE;Po;0;L;;;;;N;;;;;
11B02;DEVANAGARI SIGN BHALE;Po;0;L;;;;;N;;;;;
11B03;DEVANAGARI SIGN BHALE WITH HOOK;Po;0;L;;;;;N;;;;;
11B04;DEVANAGARI SIGN EXTENDED BHALE;Po;0;L;;;;;N;;;;;
11B05;DEVANAGARI SIGN EXTENDED BHALE WITH HOOK;Po;0;L;;;;;N;;;;;
11B06;DEVANAGARI SIGN WESTERN FIVE-LIKE BHALE;Po;0;L;;;;;N;;;;;
11B07;DEVANAGARI SIGN WESTERN NINE-LIKE BHALE;Po;0;L;;;;;N;;;;;
11B08;DEVANAGARI SIGN REVERSED NINE-LIKE BHALE;Po;0;L;;;;;N;;;;;
11B09;DEVANAGARI SIGN MINDU;Po;0;L;;;;;N;;;;;
11C00;BHAIKSUKI LETTER A;Lo;0;L;;;;;N;;;;;
11C01;BHAIKSUKI LETTER AA;Lo;0;L;;;;;N;;;;;
11C02;BHAIKSUKI LETTER I;Lo;0;L;;;;;N;;;;;
@ -21584,6 +21602,92 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
11EF6;MAKASAR VOWEL SIGN O;Mc;0;L;;;;;N;;;;;
11EF7;MAKASAR PASSIMBANG;Po;0;L;;;;;N;;;;;
11EF8;MAKASAR END OF SECTION;Po;0;L;;;;;N;;;;;
11F00;KAWI SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;;
11F01;KAWI SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;;
11F02;KAWI SIGN REPHA;Lo;0;L;;;;;N;;;;;
11F03;KAWI SIGN VISARGA;Mc;0;L;;;;;N;;;;;
11F04;KAWI LETTER A;Lo;0;L;;;;;N;;;;;
11F05;KAWI LETTER AA;Lo;0;L;;;;;N;;;;;
11F06;KAWI LETTER I;Lo;0;L;;;;;N;;;;;
11F07;KAWI LETTER II;Lo;0;L;;;;;N;;;;;
11F08;KAWI LETTER U;Lo;0;L;;;;;N;;;;;
11F09;KAWI LETTER UU;Lo;0;L;;;;;N;;;;;
11F0A;KAWI LETTER VOCALIC R;Lo;0;L;;;;;N;;;;;
11F0B;KAWI LETTER VOCALIC RR;Lo;0;L;;;;;N;;;;;
11F0C;KAWI LETTER VOCALIC L;Lo;0;L;;;;;N;;;;;
11F0D;KAWI LETTER VOCALIC LL;Lo;0;L;;;;;N;;;;;
11F0E;KAWI LETTER E;Lo;0;L;;;;;N;;;;;
11F0F;KAWI LETTER AI;Lo;0;L;;;;;N;;;;;
11F10;KAWI LETTER O;Lo;0;L;;;;;N;;;;;
11F12;KAWI LETTER KA;Lo;0;L;;;;;N;;;;;
11F13;KAWI LETTER KHA;Lo;0;L;;;;;N;;;;;
11F14;KAWI LETTER GA;Lo;0;L;;;;;N;;;;;
11F15;KAWI LETTER GHA;Lo;0;L;;;;;N;;;;;
11F16;KAWI LETTER NGA;Lo;0;L;;;;;N;;;;;
11F17;KAWI LETTER CA;Lo;0;L;;;;;N;;;;;
11F18;KAWI LETTER CHA;Lo;0;L;;;;;N;;;;;
11F19;KAWI LETTER JA;Lo;0;L;;;;;N;;;;;
11F1A;KAWI LETTER JHA;Lo;0;L;;;;;N;;;;;
11F1B;KAWI LETTER NYA;Lo;0;L;;;;;N;;;;;
11F1C;KAWI LETTER TTA;Lo;0;L;;;;;N;;;;;
11F1D;KAWI LETTER TTHA;Lo;0;L;;;;;N;;;;;
11F1E;KAWI LETTER DDA;Lo;0;L;;;;;N;;;;;
11F1F;KAWI LETTER DDHA;Lo;0;L;;;;;N;;;;;
11F20;KAWI LETTER NNA;Lo;0;L;;;;;N;;;;;
11F21;KAWI LETTER TA;Lo;0;L;;;;;N;;;;;
11F22;KAWI LETTER THA;Lo;0;L;;;;;N;;;;;
11F23;KAWI LETTER DA;Lo;0;L;;;;;N;;;;;
11F24;KAWI LETTER DHA;Lo;0;L;;;;;N;;;;;
11F25;KAWI LETTER NA;Lo;0;L;;;;;N;;;;;
11F26;KAWI LETTER PA;Lo;0;L;;;;;N;;;;;
11F27;KAWI LETTER PHA;Lo;0;L;;;;;N;;;;;
11F28;KAWI LETTER BA;Lo;0;L;;;;;N;;;;;
11F29;KAWI LETTER BHA;Lo;0;L;;;;;N;;;;;
11F2A;KAWI LETTER MA;Lo;0;L;;;;;N;;;;;
11F2B;KAWI LETTER YA;Lo;0;L;;;;;N;;;;;
11F2C;KAWI LETTER RA;Lo;0;L;;;;;N;;;;;
11F2D;KAWI LETTER LA;Lo;0;L;;;;;N;;;;;
11F2E;KAWI LETTER WA;Lo;0;L;;;;;N;;;;;
11F2F;KAWI LETTER SHA;Lo;0;L;;;;;N;;;;;
11F30;KAWI LETTER SSA;Lo;0;L;;;;;N;;;;;
11F31;KAWI LETTER SA;Lo;0;L;;;;;N;;;;;
11F32;KAWI LETTER HA;Lo;0;L;;;;;N;;;;;
11F33;KAWI LETTER JNYA;Lo;0;L;;;;;N;;;;;
11F34;KAWI VOWEL SIGN AA;Mc;0;L;;;;;N;;;;;
11F35;KAWI VOWEL SIGN ALTERNATE AA;Mc;0;L;;;;;N;;;;;
11F36;KAWI VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;;
11F37;KAWI VOWEL SIGN II;Mn;0;NSM;;;;;N;;;;;
11F38;KAWI VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;;
11F39;KAWI VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;;
11F3A;KAWI VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;;
11F3E;KAWI VOWEL SIGN E;Mc;0;L;;;;;N;;;;;
11F3F;KAWI VOWEL SIGN AI;Mc;0;L;;;;;N;;;;;
11F40;KAWI VOWEL SIGN EU;Mn;0;NSM;;;;;N;;;;;
11F41;KAWI SIGN KILLER;Mc;9;L;;;;;N;;;;;
11F42;KAWI CONJOINER;Mn;9;NSM;;;;;N;;;;;
11F43;KAWI DANDA;Po;0;L;;;;;N;;;;;
11F44;KAWI DOUBLE DANDA;Po;0;L;;;;;N;;;;;
11F45;KAWI PUNCTUATION SECTION MARKER;Po;0;L;;;;;N;;;;;
11F46;KAWI PUNCTUATION ALTERNATE SECTION MARKER;Po;0;L;;;;;N;;;;;
11F47;KAWI PUNCTUATION FLOWER;Po;0;L;;;;;N;;;;;
11F48;KAWI PUNCTUATION SPACE FILLER;Po;0;L;;;;;N;;;;;
11F49;KAWI PUNCTUATION DOT;Po;0;L;;;;;N;;;;;
11F4A;KAWI PUNCTUATION DOUBLE DOT;Po;0;L;;;;;N;;;;;
11F4B;KAWI PUNCTUATION TRIPLE DOT;Po;0;L;;;;;N;;;;;
11F4C;KAWI PUNCTUATION CIRCLE;Po;0;L;;;;;N;;;;;
11F4D;KAWI PUNCTUATION FILLED CIRCLE;Po;0;L;;;;;N;;;;;
11F4E;KAWI PUNCTUATION SPIRAL;Po;0;L;;;;;N;;;;;
11F4F;KAWI PUNCTUATION CLOSING SPIRAL;Po;0;L;;;;;N;;;;;
11F50;KAWI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;;
11F51;KAWI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;;
11F52;KAWI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;;
11F53;KAWI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;;
11F54;KAWI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;;
11F55;KAWI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;;
11F56;KAWI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;;
11F57;KAWI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;;
11F58;KAWI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;;
11F59;KAWI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;;
11FB0;LISU LETTER YHA;Lo;0;L;;;;;N;;;;;
11FC0;TAMIL FRACTION ONE THREE-HUNDRED-AND-TWENTIETH;No;0;L;;;;1/320;N;;;;;
11FC1;TAMIL FRACTION ONE ONE-HUNDRED-AND-SIXTIETH;No;0;L;;;;1/160;N;;;;;
@ -24040,6 +24144,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1342C;EGYPTIAN HIEROGLYPH AA030;Lo;0;L;;;;;N;;;;;
1342D;EGYPTIAN HIEROGLYPH AA031;Lo;0;L;;;;;N;;;;;
1342E;EGYPTIAN HIEROGLYPH AA032;Lo;0;L;;;;;N;;;;;
1342F;EGYPTIAN HIEROGLYPH V011D;Lo;0;L;;;;;N;;;;;
13430;EGYPTIAN HIEROGLYPH VERTICAL JOINER;Cf;0;L;;;;;N;;;;;
13431;EGYPTIAN HIEROGLYPH HORIZONTAL JOINER;Cf;0;L;;;;;N;;;;;
13432;EGYPTIAN HIEROGLYPH INSERT AT TOP START;Cf;0;L;;;;;N;;;;;
@ -24049,6 +24154,35 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
13436;EGYPTIAN HIEROGLYPH OVERLAY MIDDLE;Cf;0;L;;;;;N;;;;;
13437;EGYPTIAN HIEROGLYPH BEGIN SEGMENT;Cf;0;L;;;;;N;;;;;
13438;EGYPTIAN HIEROGLYPH END SEGMENT;Cf;0;L;;;;;N;;;;;
13439;EGYPTIAN HIEROGLYPH INSERT AT MIDDLE;Cf;0;L;;;;;N;;;;;
1343A;EGYPTIAN HIEROGLYPH INSERT AT TOP;Cf;0;L;;;;;N;;;;;
1343B;EGYPTIAN HIEROGLYPH INSERT AT BOTTOM;Cf;0;L;;;;;N;;;;;
1343C;EGYPTIAN HIEROGLYPH BEGIN ENCLOSURE;Cf;0;L;;;;;N;;;;;
1343D;EGYPTIAN HIEROGLYPH END ENCLOSURE;Cf;0;L;;;;;N;;;;;
1343E;EGYPTIAN HIEROGLYPH BEGIN WALLED ENCLOSURE;Cf;0;L;;;;;N;;;;;
1343F;EGYPTIAN HIEROGLYPH END WALLED ENCLOSURE;Cf;0;L;;;;;N;;;;;
13440;EGYPTIAN HIEROGLYPH MIRROR HORIZONTALLY;Mn;0;NSM;;;;;N;;;;;
13441;EGYPTIAN HIEROGLYPH FULL BLANK;Lo;0;L;;;;;N;;;;;
13442;EGYPTIAN HIEROGLYPH HALF BLANK;Lo;0;L;;;;;N;;;;;
13443;EGYPTIAN HIEROGLYPH LOST SIGN;Lo;0;L;;;;;N;;;;;
13444;EGYPTIAN HIEROGLYPH HALF LOST SIGN;Lo;0;L;;;;;N;;;;;
13445;EGYPTIAN HIEROGLYPH TALL LOST SIGN;Lo;0;L;;;;;N;;;;;
13446;EGYPTIAN HIEROGLYPH WIDE LOST SIGN;Lo;0;L;;;;;N;;;;;
13447;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START;Mn;0;NSM;;;;;N;;;;;
13448;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM START;Mn;0;NSM;;;;;N;;;;;
13449;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT START;Mn;0;NSM;;;;;N;;;;;
1344A;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP END;Mn;0;NSM;;;;;N;;;;;
1344B;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP;Mn;0;NSM;;;;;N;;;;;
1344C;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM START AND TOP END;Mn;0;NSM;;;;;N;;;;;
1344D;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT START AND TOP;Mn;0;NSM;;;;;N;;;;;
1344E;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM END;Mn;0;NSM;;;;;N;;;;;
1344F;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START AND BOTTOM END;Mn;0;NSM;;;;;N;;;;;
13450;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM;Mn;0;NSM;;;;;N;;;;;
13451;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT START AND BOTTOM;Mn;0;NSM;;;;;N;;;;;
13452;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT END;Mn;0;NSM;;;;;N;;;;;
13453;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP AND END;Mn;0;NSM;;;;;N;;;;;
13454;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM AND END;Mn;0;NSM;;;;;N;;;;;
13455;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED;Mn;0;NSM;;;;;N;;;;;
14400;ANATOLIAN HIEROGLYPH A001;Lo;0;L;;;;;N;;;;;
14401;ANATOLIAN HIEROGLYPH A002;Lo;0;L;;;;;N;;;;;
14402;ANATOLIAN HIEROGLYPH A003;Lo;0;L;;;;;N;;;;;
@ -27289,9 +27423,11 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1B120;KATAKANA LETTER ARCHAIC YI;Lo;0;L;;;;;N;;;;;
1B121;KATAKANA LETTER ARCHAIC YE;Lo;0;L;;;;;N;;;;;
1B122;KATAKANA LETTER ARCHAIC WU;Lo;0;L;;;;;N;;;;;
1B132;HIRAGANA LETTER SMALL KO;Lo;0;L;;;;;N;;;;;
1B150;HIRAGANA LETTER SMALL WI;Lo;0;L;;;;;N;;;;;
1B151;HIRAGANA LETTER SMALL WE;Lo;0;L;;;;;N;;;;;
1B152;HIRAGANA LETTER SMALL WO;Lo;0;L;;;;;N;;;;;
1B155;KATAKANA LETTER SMALL KO;Lo;0;L;;;;;N;;;;;
1B164;KATAKANA LETTER SMALL WI;Lo;0;L;;;;;N;;;;;
1B165;KATAKANA LETTER SMALL WE;Lo;0;L;;;;;N;;;;;
1B166;KATAKANA LETTER SMALL WO;Lo;0;L;;;;;N;;;;;
@ -28573,6 +28709,26 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1D243;COMBINING GREEK MUSICAL TETRASEME;Mn;230;NSM;;;;;N;;;;;
1D244;COMBINING GREEK MUSICAL PENTASEME;Mn;230;NSM;;;;;N;;;;;
1D245;GREEK MUSICAL LEIMMA;So;0;ON;;;;;N;;;;;
1D2C0;KAKTOVIK NUMERAL ZERO;No;0;L;;;;0;N;;;;;
1D2C1;KAKTOVIK NUMERAL ONE;No;0;L;;;;1;N;;;;;
1D2C2;KAKTOVIK NUMERAL TWO;No;0;L;;;;2;N;;;;;
1D2C3;KAKTOVIK NUMERAL THREE;No;0;L;;;;3;N;;;;;
1D2C4;KAKTOVIK NUMERAL FOUR;No;0;L;;;;4;N;;;;;
1D2C5;KAKTOVIK NUMERAL FIVE;No;0;L;;;;5;N;;;;;
1D2C6;KAKTOVIK NUMERAL SIX;No;0;L;;;;6;N;;;;;
1D2C7;KAKTOVIK NUMERAL SEVEN;No;0;L;;;;7;N;;;;;
1D2C8;KAKTOVIK NUMERAL EIGHT;No;0;L;;;;8;N;;;;;
1D2C9;KAKTOVIK NUMERAL NINE;No;0;L;;;;9;N;;;;;
1D2CA;KAKTOVIK NUMERAL TEN;No;0;L;;;;10;N;;;;;
1D2CB;KAKTOVIK NUMERAL ELEVEN;No;0;L;;;;11;N;;;;;
1D2CC;KAKTOVIK NUMERAL TWELVE;No;0;L;;;;12;N;;;;;
1D2CD;KAKTOVIK NUMERAL THIRTEEN;No;0;L;;;;13;N;;;;;
1D2CE;KAKTOVIK NUMERAL FOURTEEN;No;0;L;;;;14;N;;;;;
1D2CF;KAKTOVIK NUMERAL FIFTEEN;No;0;L;;;;15;N;;;;;
1D2D0;KAKTOVIK NUMERAL SIXTEEN;No;0;L;;;;16;N;;;;;
1D2D1;KAKTOVIK NUMERAL SEVENTEEN;No;0;L;;;;17;N;;;;;
1D2D2;KAKTOVIK NUMERAL EIGHTEEN;No;0;L;;;;18;N;;;;;
1D2D3;KAKTOVIK NUMERAL NINETEEN;No;0;L;;;;19;N;;;;;
1D2E0;MAYAN NUMERAL ZERO;No;0;L;;;;0;N;;;;;
1D2E1;MAYAN NUMERAL ONE;No;0;L;;;;1;N;;;;;
1D2E2;MAYAN NUMERAL TWO;No;0;L;;;;2;N;;;;;
@ -30404,6 +30560,12 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1DF1C;LATIN SMALL LETTER TESH DIGRAPH WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;;
1DF1D;LATIN SMALL LETTER C WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;;
1DF1E;LATIN SMALL LETTER S WITH CURL;Ll;0;L;;;;;N;;;;;
1DF25;LATIN SMALL LETTER D WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;;
1DF26;LATIN SMALL LETTER L WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;;
1DF27;LATIN SMALL LETTER N WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;;
1DF28;LATIN SMALL LETTER R WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;;
1DF29;LATIN SMALL LETTER S WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;;
1DF2A;LATIN SMALL LETTER T WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;;
1E000;COMBINING GLAGOLITIC LETTER AZU;Mn;230;NSM;;;;;N;;;;;
1E001;COMBINING GLAGOLITIC LETTER BUKY;Mn;230;NSM;;;;;N;;;;;
1E002;COMBINING GLAGOLITIC LETTER VEDE;Mn;230;NSM;;;;;N;;;;;
@ -30442,6 +30604,69 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1E028;COMBINING GLAGOLITIC LETTER BIG YUS;Mn;230;NSM;;;;;N;;;;;
1E029;COMBINING GLAGOLITIC LETTER IOTATED BIG YUS;Mn;230;NSM;;;;;N;;;;;
1E02A;COMBINING GLAGOLITIC LETTER FITA;Mn;230;NSM;;;;;N;;;;;
1E030;MODIFIER LETTER CYRILLIC SMALL A;Lm;0;L;<super> 0430;;;;N;;;;;
1E031;MODIFIER LETTER CYRILLIC SMALL BE;Lm;0;L;<super> 0431;;;;N;;;;;
1E032;MODIFIER LETTER CYRILLIC SMALL VE;Lm;0;L;<super> 0432;;;;N;;;;;
1E033;MODIFIER LETTER CYRILLIC SMALL GHE;Lm;0;L;<super> 0433;;;;N;;;;;
1E034;MODIFIER LETTER CYRILLIC SMALL DE;Lm;0;L;<super> 0434;;;;N;;;;;
1E035;MODIFIER LETTER CYRILLIC SMALL IE;Lm;0;L;<super> 0435;;;;N;;;;;
1E036;MODIFIER LETTER CYRILLIC SMALL ZHE;Lm;0;L;<super> 0436;;;;N;;;;;
1E037;MODIFIER LETTER CYRILLIC SMALL ZE;Lm;0;L;<super> 0437;;;;N;;;;;
1E038;MODIFIER LETTER CYRILLIC SMALL I;Lm;0;L;<super> 0438;;;;N;;;;;
1E039;MODIFIER LETTER CYRILLIC SMALL KA;Lm;0;L;<super> 043A;;;;N;;;;;
1E03A;MODIFIER LETTER CYRILLIC SMALL EL;Lm;0;L;<super> 043B;;;;N;;;;;
1E03B;MODIFIER LETTER CYRILLIC SMALL EM;Lm;0;L;<super> 043C;;;;N;;;;;
1E03C;MODIFIER LETTER CYRILLIC SMALL O;Lm;0;L;<super> 043E;;;;N;;;;;
1E03D;MODIFIER LETTER CYRILLIC SMALL PE;Lm;0;L;<super> 043F;;;;N;;;;;
1E03E;MODIFIER LETTER CYRILLIC SMALL ER;Lm;0;L;<super> 0440;;;;N;;;;;
1E03F;MODIFIER LETTER CYRILLIC SMALL ES;Lm;0;L;<super> 0441;;;;N;;;;;
1E040;MODIFIER LETTER CYRILLIC SMALL TE;Lm;0;L;<super> 0442;;;;N;;;;;
1E041;MODIFIER LETTER CYRILLIC SMALL U;Lm;0;L;<super> 0443;;;;N;;;;;
1E042;MODIFIER LETTER CYRILLIC SMALL EF;Lm;0;L;<super> 0444;;;;N;;;;;
1E043;MODIFIER LETTER CYRILLIC SMALL HA;Lm;0;L;<super> 0445;;;;N;;;;;
1E044;MODIFIER LETTER CYRILLIC SMALL TSE;Lm;0;L;<super> 0446;;;;N;;;;;
1E045;MODIFIER LETTER CYRILLIC SMALL CHE;Lm;0;L;<super> 0447;;;;N;;;;;
1E046;MODIFIER LETTER CYRILLIC SMALL SHA;Lm;0;L;<super> 0448;;;;N;;;;;
1E047;MODIFIER LETTER CYRILLIC SMALL YERU;Lm;0;L;<super> 044B;;;;N;;;;;
1E048;MODIFIER LETTER CYRILLIC SMALL E;Lm;0;L;<super> 044D;;;;N;;;;;
1E049;MODIFIER LETTER CYRILLIC SMALL YU;Lm;0;L;<super> 044E;;;;N;;;;;
1E04A;MODIFIER LETTER CYRILLIC SMALL DZZE;Lm;0;L;<super> A689;;;;N;;;;;
1E04B;MODIFIER LETTER CYRILLIC SMALL SCHWA;Lm;0;L;<super> 04D9;;;;N;;;;;
1E04C;MODIFIER LETTER CYRILLIC SMALL BYELORUSSIAN-UKRAINIAN I;Lm;0;L;<super> 0456;;;;N;;;;;
1E04D;MODIFIER LETTER CYRILLIC SMALL JE;Lm;0;L;<super> 0458;;;;N;;;;;
1E04E;MODIFIER LETTER CYRILLIC SMALL BARRED O;Lm;0;L;<super> 04E9;;;;N;;;;;
1E04F;MODIFIER LETTER CYRILLIC SMALL STRAIGHT U;Lm;0;L;<super> 04AF;;;;N;;;;;
1E050;MODIFIER LETTER CYRILLIC SMALL PALOCHKA;Lm;0;L;<super> 04CF;;;;N;;;;;
1E051;CYRILLIC SUBSCRIPT SMALL LETTER A;Lm;0;L;<sub> 0430;;;;N;;;;;
1E052;CYRILLIC SUBSCRIPT SMALL LETTER BE;Lm;0;L;<sub> 0431;;;;N;;;;;
1E053;CYRILLIC SUBSCRIPT SMALL LETTER VE;Lm;0;L;<sub> 0432;;;;N;;;;;
1E054;CYRILLIC SUBSCRIPT SMALL LETTER GHE;Lm;0;L;<sub> 0433;;;;N;;;;;
1E055;CYRILLIC SUBSCRIPT SMALL LETTER DE;Lm;0;L;<sub> 0434;;;;N;;;;;
1E056;CYRILLIC SUBSCRIPT SMALL LETTER IE;Lm;0;L;<sub> 0435;;;;N;;;;;
1E057;CYRILLIC SUBSCRIPT SMALL LETTER ZHE;Lm;0;L;<sub> 0436;;;;N;;;;;
1E058;CYRILLIC SUBSCRIPT SMALL LETTER ZE;Lm;0;L;<sub> 0437;;;;N;;;;;
1E059;CYRILLIC SUBSCRIPT SMALL LETTER I;Lm;0;L;<sub> 0438;;;;N;;;;;
1E05A;CYRILLIC SUBSCRIPT SMALL LETTER KA;Lm;0;L;<sub> 043A;;;;N;;;;;
1E05B;CYRILLIC SUBSCRIPT SMALL LETTER EL;Lm;0;L;<sub> 043B;;;;N;;;;;
1E05C;CYRILLIC SUBSCRIPT SMALL LETTER O;Lm;0;L;<sub> 043E;;;;N;;;;;
1E05D;CYRILLIC SUBSCRIPT SMALL LETTER PE;Lm;0;L;<sub> 043F;;;;N;;;;;
1E05E;CYRILLIC SUBSCRIPT SMALL LETTER ES;Lm;0;L;<sub> 0441;;;;N;;;;;
1E05F;CYRILLIC SUBSCRIPT SMALL LETTER U;Lm;0;L;<sub> 0443;;;;N;;;;;
1E060;CYRILLIC SUBSCRIPT SMALL LETTER EF;Lm;0;L;<sub> 0444;;;;N;;;;;
1E061;CYRILLIC SUBSCRIPT SMALL LETTER HA;Lm;0;L;<sub> 0445;;;;N;;;;;
1E062;CYRILLIC SUBSCRIPT SMALL LETTER TSE;Lm;0;L;<sub> 0446;;;;N;;;;;
1E063;CYRILLIC SUBSCRIPT SMALL LETTER CHE;Lm;0;L;<sub> 0447;;;;N;;;;;
1E064;CYRILLIC SUBSCRIPT SMALL LETTER SHA;Lm;0;L;<sub> 0448;;;;N;;;;;
1E065;CYRILLIC SUBSCRIPT SMALL LETTER HARD SIGN;Lm;0;L;<sub> 044A;;;;N;;;;;
1E066;CYRILLIC SUBSCRIPT SMALL LETTER YERU;Lm;0;L;<sub> 044B;;;;N;;;;;
1E067;CYRILLIC SUBSCRIPT SMALL LETTER GHE WITH UPTURN;Lm;0;L;<sub> 0491;;;;N;;;;;
1E068;CYRILLIC SUBSCRIPT SMALL LETTER BYELORUSSIAN-UKRAINIAN I;Lm;0;L;<sub> 0456;;;;N;;;;;
1E069;CYRILLIC SUBSCRIPT SMALL LETTER DZE;Lm;0;L;<sub> 0455;;;;N;;;;;
1E06A;CYRILLIC SUBSCRIPT SMALL LETTER DZHE;Lm;0;L;<sub> 045F;;;;N;;;;;
1E06B;MODIFIER LETTER CYRILLIC SMALL ES WITH DESCENDER;Lm;0;L;<super> 04AB;;;;N;;;;;
1E06C;MODIFIER LETTER CYRILLIC SMALL YERU WITH BACK YER;Lm;0;L;<super> A651;;;;N;;;;;
1E06D;MODIFIER LETTER CYRILLIC SMALL STRAIGHT U WITH STROKE;Lm;0;L;<super> 04B1;;;;N;;;;;
1E08F;COMBINING CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I;Mn;230;NSM;;;;;N;;;;;
1E100;NYIAKENG PUACHUE HMONG LETTER MA;Lo;0;L;;;;;N;;;;;
1E101;NYIAKENG PUACHUE HMONG LETTER TSA;Lo;0;L;;;;;N;;;;;
1E102;NYIAKENG PUACHUE HMONG LETTER NTA;Lo;0;L;;;;;N;;;;;
@ -30603,6 +30828,48 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1E2F8;WANCHO DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;;
1E2F9;WANCHO DIGIT NINE;Nd;0;L;;9;9;9;N;;;;;
1E2FF;WANCHO NGUN SIGN;Sc;0;ET;;;;;N;;;;;
1E4D0;NAG MUNDARI LETTER O;Lo;0;L;;;;;N;;;;;
1E4D1;NAG MUNDARI LETTER OP;Lo;0;L;;;;;N;;;;;
1E4D2;NAG MUNDARI LETTER OL;Lo;0;L;;;;;N;;;;;
1E4D3;NAG MUNDARI LETTER OY;Lo;0;L;;;;;N;;;;;
1E4D4;NAG MUNDARI LETTER ONG;Lo;0;L;;;;;N;;;;;
1E4D5;NAG MUNDARI LETTER A;Lo;0;L;;;;;N;;;;;
1E4D6;NAG MUNDARI LETTER AJ;Lo;0;L;;;;;N;;;;;
1E4D7;NAG MUNDARI LETTER AB;Lo;0;L;;;;;N;;;;;
1E4D8;NAG MUNDARI LETTER ANY;Lo;0;L;;;;;N;;;;;
1E4D9;NAG MUNDARI LETTER AH;Lo;0;L;;;;;N;;;;;
1E4DA;NAG MUNDARI LETTER I;Lo;0;L;;;;;N;;;;;
1E4DB;NAG MUNDARI LETTER IS;Lo;0;L;;;;;N;;;;;
1E4DC;NAG MUNDARI LETTER IDD;Lo;0;L;;;;;N;;;;;
1E4DD;NAG MUNDARI LETTER IT;Lo;0;L;;;;;N;;;;;
1E4DE;NAG MUNDARI LETTER IH;Lo;0;L;;;;;N;;;;;
1E4DF;NAG MUNDARI LETTER U;Lo;0;L;;;;;N;;;;;
1E4E0;NAG MUNDARI LETTER UC;Lo;0;L;;;;;N;;;;;
1E4E1;NAG MUNDARI LETTER UD;Lo;0;L;;;;;N;;;;;
1E4E2;NAG MUNDARI LETTER UK;Lo;0;L;;;;;N;;;;;
1E4E3;NAG MUNDARI LETTER UR;Lo;0;L;;;;;N;;;;;
1E4E4;NAG MUNDARI LETTER E;Lo;0;L;;;;;N;;;;;
1E4E5;NAG MUNDARI LETTER ENN;Lo;0;L;;;;;N;;;;;
1E4E6;NAG MUNDARI LETTER EG;Lo;0;L;;;;;N;;;;;
1E4E7;NAG MUNDARI LETTER EM;Lo;0;L;;;;;N;;;;;
1E4E8;NAG MUNDARI LETTER EN;Lo;0;L;;;;;N;;;;;
1E4E9;NAG MUNDARI LETTER ETT;Lo;0;L;;;;;N;;;;;
1E4EA;NAG MUNDARI LETTER ELL;Lo;0;L;;;;;N;;;;;
1E4EB;NAG MUNDARI SIGN OJOD;Lm;0;L;;;;;N;;;;;
1E4EC;NAG MUNDARI SIGN MUHOR;Mn;232;NSM;;;;;N;;;;;
1E4ED;NAG MUNDARI SIGN TOYOR;Mn;232;NSM;;;;;N;;;;;
1E4EE;NAG MUNDARI SIGN IKIR;Mn;220;NSM;;;;;N;;;;;
1E4EF;NAG MUNDARI SIGN SUTUH;Mn;230;NSM;;;;;N;;;;;
1E4F0;NAG MUNDARI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;;
1E4F1;NAG MUNDARI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;;
1E4F2;NAG MUNDARI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;;
1E4F3;NAG MUNDARI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;;
1E4F4;NAG MUNDARI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;;
1E4F5;NAG MUNDARI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;;
1E4F6;NAG MUNDARI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;;
1E4F7;NAG MUNDARI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;;
1E4F8;NAG MUNDARI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;;
1E4F9;NAG MUNDARI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;;
1E7E0;ETHIOPIC SYLLABLE HHYA;Lo;0;L;;;;;N;;;;;
1E7E1;ETHIOPIC SYLLABLE HHYU;Lo;0;L;;;;;N;;;;;
1E7E2;ETHIOPIC SYLLABLE HHYI;Lo;0;L;;;;;N;;;;;
@ -32678,6 +32945,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1F6D5;HINDU TEMPLE;So;0;ON;;;;;N;;;;;
1F6D6;HUT;So;0;ON;;;;;N;;;;;
1F6D7;ELEVATOR;So;0;ON;;;;;N;;;;;
1F6DC;WIRELESS;So;0;ON;;;;;N;;;;;
1F6DD;PLAYGROUND SLIDE;So;0;ON;;;;;N;;;;;
1F6DE;WHEEL;So;0;ON;;;;;N;;;;;
1F6DF;RING BUOY;So;0;ON;;;;;N;;;;;
@ -32823,6 +33091,14 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1F771;ALCHEMICAL SYMBOL FOR MONTH;So;0;ON;;;;;N;;;;;
1F772;ALCHEMICAL SYMBOL FOR HALF DRAM;So;0;ON;;;;;N;;;;;
1F773;ALCHEMICAL SYMBOL FOR HALF OUNCE;So;0;ON;;;;;N;;;;;
1F774;LOT OF FORTUNE;So;0;ON;;;;;N;;;;;
1F775;OCCULTATION;So;0;ON;;;;;N;;;;;
1F776;LUNAR ECLIPSE;So;0;ON;;;;;N;;;;;
1F77B;HAUMEA;So;0;ON;;;;;N;;;;;
1F77C;MAKEMAKE;So;0;ON;;;;;N;;;;;
1F77D;GONGGONG;So;0;ON;;;;;N;;;;;
1F77E;QUAOAR;So;0;ON;;;;;N;;;;;
1F77F;ORCUS;So;0;ON;;;;;N;;;;;
1F780;BLACK LEFT-POINTING ISOSCELES RIGHT TRIANGLE;So;0;ON;;;;;N;;;;;
1F781;BLACK UP-POINTING ISOSCELES RIGHT TRIANGLE;So;0;ON;;;;;N;;;;;
1F782;BLACK RIGHT-POINTING ISOSCELES RIGHT TRIANGLE;So;0;ON;;;;;N;;;;;
@ -32912,6 +33188,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1F7D6;NEGATIVE CIRCLED TRIANGLE;So;0;ON;;;;;N;;;;;
1F7D7;CIRCLED SQUARE;So;0;ON;;;;;N;;;;;
1F7D8;NEGATIVE CIRCLED SQUARE;So;0;ON;;;;;N;;;;;
1F7D9;NINE POINTED WHITE STAR;So;0;ON;;;;;N;;;;;
1F7E0;LARGE ORANGE CIRCLE;So;0;ON;;;;;N;;;;;
1F7E1;LARGE YELLOW CIRCLE;So;0;ON;;;;;N;;;;;
1F7E2;LARGE GREEN CIRCLE;So;0;ON;;;;;N;;;;;
@ -33434,6 +33711,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1FA72;BRIEFS;So;0;ON;;;;;N;;;;;
1FA73;SHORTS;So;0;ON;;;;;N;;;;;
1FA74;THONG SANDAL;So;0;ON;;;;;N;;;;;
1FA75;LIGHT BLUE HEART;So;0;ON;;;;;N;;;;;
1FA76;GREY HEART;So;0;ON;;;;;N;;;;;
1FA77;PINK HEART;So;0;ON;;;;;N;;;;;
1FA78;DROP OF BLOOD;So;0;ON;;;;;N;;;;;
1FA79;ADHESIVE BANDAGE;So;0;ON;;;;;N;;;;;
1FA7A;STETHOSCOPE;So;0;ON;;;;;N;;;;;
@ -33446,6 +33726,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1FA84;MAGIC WAND;So;0;ON;;;;;N;;;;;
1FA85;PINATA;So;0;ON;;;;;N;;;;;
1FA86;NESTING DOLLS;So;0;ON;;;;;N;;;;;
1FA87;MARACAS;So;0;ON;;;;;N;;;;;
1FA88;FLUTE;So;0;ON;;;;;N;;;;;
1FA90;RINGED PLANET;So;0;ON;;;;;N;;;;;
1FA91;CHAIR;So;0;ON;;;;;N;;;;;
1FA92;RAZOR;So;0;ON;;;;;N;;;;;
@ -33475,6 +33757,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1FAAA;IDENTIFICATION CARD;So;0;ON;;;;;N;;;;;
1FAAB;LOW BATTERY;So;0;ON;;;;;N;;;;;
1FAAC;HAMSA;So;0;ON;;;;;N;;;;;
1FAAD;FOLDING HAND FAN;So;0;ON;;;;;N;;;;;
1FAAE;HAIR PICK;So;0;ON;;;;;N;;;;;
1FAAF;KHANDA;So;0;ON;;;;;N;;;;;
1FAB0;FLY;So;0;ON;;;;;N;;;;;
1FAB1;WORM;So;0;ON;;;;;N;;;;;
1FAB2;BEETLE;So;0;ON;;;;;N;;;;;
@ -33486,12 +33771,18 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1FAB8;CORAL;So;0;ON;;;;;N;;;;;
1FAB9;EMPTY NEST;So;0;ON;;;;;N;;;;;
1FABA;NEST WITH EGGS;So;0;ON;;;;;N;;;;;
1FABB;HYACINTH;So;0;ON;;;;;N;;;;;
1FABC;JELLYFISH;So;0;ON;;;;;N;;;;;
1FABD;WING;So;0;ON;;;;;N;;;;;
1FABF;GOOSE;So;0;ON;;;;;N;;;;;
1FAC0;ANATOMICAL HEART;So;0;ON;;;;;N;;;;;
1FAC1;LUNGS;So;0;ON;;;;;N;;;;;
1FAC2;PEOPLE HUGGING;So;0;ON;;;;;N;;;;;
1FAC3;PREGNANT MAN;So;0;ON;;;;;N;;;;;
1FAC4;PREGNANT PERSON;So;0;ON;;;;;N;;;;;
1FAC5;PERSON WITH CROWN;So;0;ON;;;;;N;;;;;
1FACE;MOOSE;So;0;ON;;;;;N;;;;;
1FACF;DONKEY;So;0;ON;;;;;N;;;;;
1FAD0;BLUEBERRIES;So;0;ON;;;;;N;;;;;
1FAD1;BELL PEPPER;So;0;ON;;;;;N;;;;;
1FAD2;OLIVE;So;0;ON;;;;;N;;;;;
@ -33502,6 +33793,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1FAD7;POURING LIQUID;So;0;ON;;;;;N;;;;;
1FAD8;BEANS;So;0;ON;;;;;N;;;;;
1FAD9;JAR;So;0;ON;;;;;N;;;;;
1FADA;GINGER ROOT;So;0;ON;;;;;N;;;;;
1FADB;PEA POD;So;0;ON;;;;;N;;;;;
1FAE0;MELTING FACE;So;0;ON;;;;;N;;;;;
1FAE1;SALUTING FACE;So;0;ON;;;;;N;;;;;
1FAE2;FACE WITH OPEN EYES AND HAND OVER MOUTH;So;0;ON;;;;;N;;;;;
@ -33510,6 +33803,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1FAE5;DOTTED LINE FACE;So;0;ON;;;;;N;;;;;
1FAE6;BITING LIP;So;0;ON;;;;;N;;;;;
1FAE7;BUBBLES;So;0;ON;;;;;N;;;;;
1FAE8;SHAKING FACE;So;0;ON;;;;;N;;;;;
1FAF0;HAND WITH INDEX FINGER AND THUMB CROSSED;So;0;ON;;;;;N;;;;;
1FAF1;RIGHTWARDS HAND;So;0;ON;;;;;N;;;;;
1FAF2;LEFTWARDS HAND;So;0;ON;;;;;N;;;;;
@ -33517,6 +33811,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
1FAF4;PALM UP HAND;So;0;ON;;;;;N;;;;;
1FAF5;INDEX POINTING AT THE VIEWER;So;0;ON;;;;;N;;;;;
1FAF6;HEART HANDS;So;0;ON;;;;;N;;;;;
1FAF7;LEFTWARDS PUSHING HAND;So;0;ON;;;;;N;;;;;
1FAF8;RIGHTWARDS PUSHING HAND;So;0;ON;;;;;N;;;;;
1FB00;BLOCK SEXTANT-1;So;0;ON;;;;;N;;;;;
1FB01;BLOCK SEXTANT-2;So;0;ON;;;;;N;;;;;
1FB02;BLOCK SEXTANT-12;So;0;ON;;;;;N;;;;;
@ -33732,7 +34028,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
20000;<CJK Ideograph Extension B, First>;Lo;0;L;;;;;N;;;;;
2A6DF;<CJK Ideograph Extension B, Last>;Lo;0;L;;;;;N;;;;;
2A700;<CJK Ideograph Extension C, First>;Lo;0;L;;;;;N;;;;;
2B738;<CJK Ideograph Extension C, Last>;Lo;0;L;;;;;N;;;;;
2B739;<CJK Ideograph Extension C, Last>;Lo;0;L;;;;;N;;;;;
2B740;<CJK Ideograph Extension D, First>;Lo;0;L;;;;;N;;;;;
2B81D;<CJK Ideograph Extension D, Last>;Lo;0;L;;;;;N;;;;;
2B820;<CJK Ideograph Extension E, First>;Lo;0;L;;;;;N;;;;;
@ -34283,6 +34579,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;;
2FA1D;CJK COMPATIBILITY IDEOGRAPH-2FA1D;Lo;0;L;2A600;;;;N;;;;;
30000;<CJK Ideograph Extension G, First>;Lo;0;L;;;;;N;;;;;
3134A;<CJK Ideograph Extension G, Last>;Lo;0;L;;;;;N;;;;;
31350;<CJK Ideograph Extension H, First>;Lo;0;L;;;;;N;;;;;
323AF;<CJK Ideograph Extension H, Last>;Lo;0;L;;;;;N;;;;;
E0001;LANGUAGE TAG;Cf;0;BN;;;;;N;;;;;
E0020;TAG SPACE;Cf;0;BN;;;;;N;;;;;
E0021;TAG EXCLAMATION MARK;Cf;0;BN;;;;;N;;;;;

8
web/source/osk/banner.ts
View file

@ -560,13 +560,17 @@ namespace com.keyman.osk {
private doAccept(suggestion: BannerSuggestion) {
let _this = this;
// Selecting a suggestion or a reversion should both clear selection
// and clear the reversion-displaying state of the banner.
this.selected = null;
this.doRevert = false;
this.revertAcceptancePromise = suggestion.apply();
if(!this.revertAcceptancePromise) {
// We get here either if suggestion acceptance fails or if it was a reversion.
if(suggestion.suggestion && suggestion.suggestion.tag == 'revert') {
// Reversion state management
this.recentAccept = false;
this.doRevert = false;
this.recentRevert = true;
this.doUpdate();
@ -581,9 +585,7 @@ namespace com.keyman.osk {
}
});
this.selected = null;
this.recentAccept = true;
this.doRevert = false;
this.recentRevert = false;
this.swallowPrediction = true;