From ef7d87ace121465b6a069e7f77e7a864b9e73b15 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Sun, 10 Mar 2024 05:48:28 +0700 Subject: [PATCH 01/40] refactor(common/models): moves TS priority-queue implementation to web-utils --- common/models/templates/src/index.ts | 1 - common/models/templates/src/trie-model.ts | 3 +-- .../lm-worker/src/main/correction/distance-modeler.ts | 3 ++- .../test/mocha/cases/edit-distance/distance-modeler.js | 9 +++++---- common/web/utils/src/index.ts | 2 ++ .../templates => web/utils}/src/priority-queue.ts | 8 ++++---- .../utils/src/test/priorityQueue.js} | 2 +- 7 files changed, 15 insertions(+), 13 deletions(-) rename common/{models/templates => web/utils}/src/priority-queue.ts (96%) rename common/{models/templates/test/test-priority-queue.js => web/utils/src/test/priorityQueue.js} (97%) diff --git a/common/models/templates/src/index.ts b/common/models/templates/src/index.ts index 52650ba9bb..8563004fef 100644 --- a/common/models/templates/src/index.ts +++ b/common/models/templates/src/index.ts @@ -2,7 +2,6 @@ export { SENTINEL_CODE_UNIT, applyTransform, buildMergedTransform, isHighSurrogate, isLowSurrogate, isSentinel, transformToSuggestion, defaultApplyCasing } from "./common.js"; -export { default as PriorityQueue, Comparator } from "./priority-queue.js"; export { default as QuoteBehavior } from "./quote-behavior.js"; export { Tokenization, tokenize, getLastPreCaretToken, wordbreak } from "./tokenization.js"; export { default as TrieModel, TrieModelOptions } from "./trie-model.js"; \ No newline at end of file diff --git a/common/models/templates/src/trie-model.ts b/common/models/templates/src/trie-model.ts index 0a571517c7..b2297a27a3 100644 --- a/common/models/templates/src/trie-model.ts +++ b/common/models/templates/src/trie-model.ts @@ -26,12 +26,11 @@ // Should probably make a 'lm-utils' submodule. // Allows the kmwstring bindings to resolve. -import { extendString } from "@keymanapp/web-utils"; +import { extendString, PriorityQueue } from "@keymanapp/web-utils"; import { default as defaultWordBreaker } from "@keymanapp/models-wordbreakers"; import { applyTransform, isHighSurrogate, isSentinel, SENTINEL_CODE_UNIT, transformToSuggestion } from "./common.js"; import { getLastPreCaretToken } from "./tokenization.js"; -import PriorityQueue from "./priority-queue.js"; extendString(); diff --git a/common/web/lm-worker/src/main/correction/distance-modeler.ts b/common/web/lm-worker/src/main/correction/distance-modeler.ts index b4e25200bc..f414c446c2 100644 --- a/common/web/lm-worker/src/main/correction/distance-modeler.ts +++ b/common/web/lm-worker/src/main/correction/distance-modeler.ts @@ -1,4 +1,5 @@ -import { Comparator, isHighSurrogate, SENTINEL_CODE_UNIT, PriorityQueue } from '@keymanapp/models-templates'; +import { isHighSurrogate, SENTINEL_CODE_UNIT } from '@keymanapp/models-templates'; +import { QueueComparator as Comparator, PriorityQueue } from '@keymanapp/web-utils'; import { ClassicalDistanceCalculation, EditToken } from './classical-calculation.js'; import { ExecutionTimer, STANDARD_TIME_BETWEEN_DEFERS } from './execution-timer.js'; diff --git a/common/web/lm-worker/src/test/mocha/cases/edit-distance/distance-modeler.js b/common/web/lm-worker/src/test/mocha/cases/edit-distance/distance-modeler.js index daa3fa5fcf..b49b511e37 100644 --- a/common/web/lm-worker/src/test/mocha/cases/edit-distance/distance-modeler.js +++ b/common/web/lm-worker/src/test/mocha/cases/edit-distance/distance-modeler.js @@ -2,6 +2,7 @@ import { assert } from 'chai'; import * as models from '#./models/index.js'; import * as correction from '#./correction/index.js'; +import { PriorityQueue } from '@keymanapp/web-utils'; import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; function buildTestTimer() { @@ -142,7 +143,7 @@ describe('Correction Distance Modeler', function() { assert.equal(edges.length, expectedChildCount); // One final bit, which is a bit of integration - we know the top two nodes that should result. - let queue = new models.PriorityQueue(correction.QUEUE_NODE_COMPARATOR, edges); + let queue = new PriorityQueue(correction.QUEUE_NODE_COMPARATOR, edges); let firstEdge = queue.dequeue(); assert.equal(firstEdge.priorInput[0].sample.insert, 't'); @@ -185,13 +186,13 @@ describe('Correction Distance Modeler', function() { ]; let layer1Edges = rootNode.buildSubstitutionEdges(synthDistribution1); - let layer1Queue = new models.PriorityQueue(correction.QUEUE_NODE_COMPARATOR, layer1Edges); + let layer1Queue = new PriorityQueue(correction.QUEUE_NODE_COMPARATOR, layer1Edges); let tEdge = layer1Queue.dequeue(); assertEdgeChars(tEdge, 't', 't'); let layer2Edges = tEdge.buildSubstitutionEdges(synthDistribution2); - let layer2Queue = new models.PriorityQueue(correction.QUEUE_NODE_COMPARATOR, layer2Edges); + let layer2Queue = new PriorityQueue(correction.QUEUE_NODE_COMPARATOR, layer2Edges); let eEdge = layer2Queue.dequeue(); assertEdgeChars(eEdge, 'e', 'e'); @@ -208,7 +209,7 @@ describe('Correction Distance Modeler', function() { let layer3eEdges = eEdge.buildSubstitutionEdges(synthDistribution3); let layer3hEdges = hEdge.buildSubstitutionEdges(synthDistribution3); let layer3ehEdges = ehEdge.buildSubstitutionEdges(synthDistribution3); - let layer3Queue = new models.PriorityQueue(correction.QUEUE_NODE_COMPARATOR, layer3eEdges.concat(layer3hEdges).concat(layer3ehEdges)); + let layer3Queue = new PriorityQueue(correction.QUEUE_NODE_COMPARATOR, layer3eEdges.concat(layer3hEdges).concat(layer3ehEdges)); // Find the first result with an actual word directly represented. let bestEdge; diff --git a/common/web/utils/src/index.ts b/common/web/utils/src/index.ts index 1d1e540a80..ca7bb2ba55 100644 --- a/common/web/utils/src/index.ts +++ b/common/web/utils/src/index.ts @@ -23,6 +23,8 @@ export { default as TimeoutPromise, timedPromise } from "./timeoutPromise.js"; export { Uni_IsSurrogate1, Uni_IsSurrogate2 } from "./surrogates.js"; +export { default as PriorityQueue, QueueComparator } from "./priority-queue.js" + // // Uncomment the following line and run the bundled output to verify successful // // esbuild bundling of this submodule: // console.log(Version.CURRENT.toString()); \ No newline at end of file diff --git a/common/models/templates/src/priority-queue.ts b/common/web/utils/src/priority-queue.ts similarity index 96% rename from common/models/templates/src/priority-queue.ts rename to common/web/utils/src/priority-queue.ts index 6a1344fd89..f0b16f0e25 100644 --- a/common/models/templates/src/priority-queue.ts +++ b/common/web/utils/src/priority-queue.ts @@ -11,11 +11,11 @@ * - value > 0 if `b` should come before `a` * - 0 if they should be treated equally. */ -export type Comparator = (a: Type, b: Type) => number; +export type QueueComparator = (a: Type, b: Type) => number; export default class PriorityQueue { - private comparator: Comparator; + private comparator: QueueComparator; private heap: Type[]; /** @@ -29,8 +29,8 @@ export default class PriorityQueue { * the first parameter should precede the second parameter. * @param initialEntries */ - constructor(comparator: Comparator, initialEntries?: Type[]); - constructor(arg1: Comparator | PriorityQueue, initialEntries?: Type[]) { + constructor(comparator: QueueComparator, initialEntries?: Type[]); + constructor(arg1: QueueComparator | PriorityQueue, initialEntries?: Type[]) { if(typeof arg1 != 'function') { this.comparator = arg1.comparator; // Shallow-copies are fine. diff --git a/common/models/templates/test/test-priority-queue.js b/common/web/utils/src/test/priorityQueue.js similarity index 97% rename from common/models/templates/test/test-priority-queue.js rename to common/web/utils/src/test/priorityQueue.js index 0887a19d22..0aa4c50b96 100644 --- a/common/models/templates/test/test-priority-queue.js +++ b/common/web/utils/src/test/priorityQueue.js @@ -3,7 +3,7 @@ */ import { assert } from 'chai'; -import { PriorityQueue } from '@keymanapp/models-templates'; +import { PriorityQueue } from '@keymanapp/web-utils'; describe('Priority queue', function() { it('can act as a min-heap', function () { From abb326490f5cd8dfc82f30082f70d19034abbb71 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Sun, 10 Mar 2024 01:48:49 +0700 Subject: [PATCH 02/40] feat(common/models): lexicon traversals - probability access --- common/models/templates/src/trie-model.ts | 40 +++++++++++++++++------ common/models/types/index.d.ts | 8 ++++- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/common/models/templates/src/trie-model.ts b/common/models/templates/src/trie-model.ts index b2297a27a3..48078b6756 100644 --- a/common/models/templates/src/trie-model.ts +++ b/common/models/templates/src/trie-model.ts @@ -94,13 +94,21 @@ class Traversal implements LexiconTraversal { */ root: Node; - constructor(root: Node, prefix: string) { + /** + * The max weight for the Trie being 'traversed'. Needed for probability + * calculations. + */ + totalWeight: number; + + constructor(root: Node, prefix: string, maxWeight: number) { this.root = root; this.prefix = prefix; + this.totalWeight = maxWeight; } *children(): Generator<{char: string, traversal: () => LexiconTraversal}> { let root = this.root; + const totalWeight = this.totalWeight; if(root.type == 'internal') { for(let entry of root.values) { @@ -119,7 +127,7 @@ class Traversal implements LexiconTraversal { let prefix = this.prefix + entry + lowSurrogate; yield { char: entry + lowSurrogate, - traversal: function() { return new Traversal(internalNode.children[lowSurrogate], prefix) } + traversal: function() { return new Traversal(internalNode.children[lowSurrogate], prefix, totalWeight) } } } } else { @@ -130,7 +138,7 @@ class Traversal implements LexiconTraversal { yield { char: entry, - traversal: function () {return new Traversal(entryNode, prefix)} + traversal: function () {return new Traversal(entryNode, prefix, totalWeight)} } } } else if(isSentinel(entry)) { @@ -142,7 +150,7 @@ class Traversal implements LexiconTraversal { let prefix = this.prefix + entry; yield { char: entry, - traversal: function() { return new Traversal(entryNode, prefix)} + traversal: function() { return new Traversal(entryNode, prefix, totalWeight)} } } } @@ -164,30 +172,42 @@ class Traversal implements LexiconTraversal { } yield { char: nodeKey, - traversal: function() { return new Traversal(root, prefix + nodeKey)} + traversal: function() { return new Traversal(root, prefix + nodeKey, totalWeight)} } }; return; } } - get entries(): string[] { + get entries() { + const totalWeight = this.totalWeight; + const entryMapper = function(value: Entry) { + return { + text: value.content, + p: value.weight / totalWeight + } + } + if(this.root.type == 'leaf') { let prefix = this.prefix; let matches = this.root.entries.filter(function(entry) { return entry.key == prefix; }); - return matches.map(function(value) { return value.content }); + return matches.map(entryMapper); } else { let matchingLeaf = this.root.children[SENTINEL_CODE_UNIT]; if(matchingLeaf && matchingLeaf.type == 'leaf') { - return matchingLeaf.entries.map(function(value) { return value.content }); + return matchingLeaf.entries.map(entryMapper); } else { return []; } } } + + get maxP(): number { + return this.root.weight / this.totalWeight; + } } /** @@ -285,7 +305,7 @@ export default class TrieModel implements LexicalModel { } public traverseFromRoot(): LexiconTraversal { - return new Traversal(this._trie['root'], ''); + return new Traversal(this._trie['root'], '', this._trie.totalWeight); } }; @@ -368,7 +388,7 @@ interface Entry { class Trie { private root: Node; /** The total weight of the entire trie. */ - private totalWeight: number; + readonly totalWeight: number; /** * Converts arbitrary strings to a search key. The trie is built up of * search keys; not each entry's word form! diff --git a/common/models/types/index.d.ts b/common/models/types/index.d.ts index 30aba3ba3d..5b2b85e683 100644 --- a/common/models/types/index.d.ts +++ b/common/models/types/index.d.ts @@ -70,7 +70,13 @@ declare interface LexiconTraversal { * - prefix of 'crepe': ['crêpe', 'crêpé'] * - other examples: https://www.thoughtco.com/french-accent-homographs-1371072 */ - entries: USVString[]; + entries: { text: USVString, p: number }[]; + + /** + * Gives the probability of the highest-frequency lexical entry that is either a member or + * descendent of the represented trie `Node`. + */ + maxP: number; } /** From 8e058a507822ff70b8304415cd8facbee5c1f4cd Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Sun, 10 Mar 2024 03:02:19 +0700 Subject: [PATCH 03/40] fix(common/models): basic unit test patchup --- common/models/templates/test/test-trie-traversal.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/models/templates/test/test-trie-traversal.js b/common/models/templates/test/test-trie-traversal.js index d3d0496316..194aeec1ca 100644 --- a/common/models/templates/test/test-trie-traversal.js +++ b/common/models/templates/test/test-trie-traversal.js @@ -21,7 +21,7 @@ describe('Trie traversal abstractions', function() { assert.isDefined(rootTraversal); let rootKeys = ['t', 'o', 'a', 'i', 'w', 'h', 'f', 'b', 'n', 'y', 's', 'm', - 'u', 'c', 'd', 'l', 'e', 'j', 'p', 'g', 'v', 'k', 'r', 'q'] + 'u', 'c', 'd', 'l', 'e', 'j', 'p', 'g', 'v', 'k', 'r', 'q']; for(let child of rootTraversal.children()) { let keyIndex = rootKeys.indexOf(child.char); @@ -66,7 +66,7 @@ describe('Trie traversal abstractions', function() { assert.isDefined(traversalInner3); assert.isDefined(traversalInner3.entries); - assert.equal(traversalInner3.entries[0], "the"); + assert.equal(traversalInner3.entries[0].text, "the"); for(let eChild of traversalInner3.children()) { let keyIndex = eKeys.indexOf(eChild.char); @@ -140,7 +140,7 @@ describe('Trie traversal abstractions', function() { } else { let finalTraversal = curChild.traversal(); assert.isDefined(finalTraversal.entries); - assert.equal(finalTraversal.entries[0], 'trouble'); + assert.equal(finalTraversal.entries[0].text, 'trouble'); eSuccess = true; } } while (leafChildSequence.length > 0); @@ -232,7 +232,7 @@ describe('Trie traversal abstractions', function() { } else { let finalTraversal = curChild.traversal(); assert.isDefined(finalTraversal.entries); - assert.equal(finalTraversal.entries[0], smpA + smpP + 'pl' + smpE); + assert.equal(finalTraversal.entries[0].text, smpA + smpP + 'pl' + smpE); eSuccess = true; } } while (leafChildSequence.length > 0); From 17fefaa86f0ea0cfbc11a2e19e914369fdc204b9 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Sun, 10 Mar 2024 03:21:39 +0700 Subject: [PATCH 04/40] feat(common/models/templates): unit test enhancements for new feature --- .../templates/test/test-trie-traversal.js | 65 ++++++++++++++----- 1 file changed, 50 insertions(+), 15 deletions(-) diff --git a/common/models/templates/test/test-trie-traversal.js b/common/models/templates/test/test-trie-traversal.js index 194aeec1ca..ad3f9b2c87 100644 --- a/common/models/templates/test/test-trie-traversal.js +++ b/common/models/templates/test/test-trie-traversal.js @@ -35,6 +35,10 @@ describe('Trie traversal abstractions', function() { it('traversal with simple internal nodes', function() { var model = new TrieModel(jsonFixture('tries/english-1000')); + // Prob: entry weight / total weight + // "the" is the highest-weighted word in the fixture. + const PROB_OF_THE = 1000 / 500500; + let rootTraversal = model.traverseFromRoot(); assert.isDefined(rootTraversal); @@ -50,6 +54,7 @@ describe('Trie traversal abstractions', function() { assert.isDefined(traversalInner1); assert.isArray(child.traversal().entries); assert.isEmpty(child.traversal().entries); + assert.equal(traversalInner1.maxP, PROB_OF_THE); for(let tChild of traversalInner1.children()) { if(tChild.char == 'h') { @@ -58,19 +63,28 @@ describe('Trie traversal abstractions', function() { assert.isDefined(traversalInner2); assert.isEmpty(tChild.traversal().entries); assert.isArray(tChild.traversal().entries); + assert.equal(traversalInner2.maxP, PROB_OF_THE); for(let hChild of traversalInner2.children()) { if(hChild.char == 'e') { eSuccess = true; let traversalInner3 = hChild.traversal(); assert.isDefined(traversalInner3); - assert.isDefined(traversalInner3.entries); - assert.equal(traversalInner3.entries[0].text, "the"); + assert.deepEqual(traversalInner3.entries, [ + { + text: "the", + p: PROB_OF_THE + } + ]); + assert.equal(traversalInner3.maxP, PROB_OF_THE); for(let eChild of traversalInner3.children()) { let keyIndex = eKeys.indexOf(eChild.char); assert.notEqual(keyIndex, -1, "Did not find char '" + eChild.char + "' in array!"); + + // THE is not accessible if any of the sub-tries of our 'e' node (traversalInner3). + assert.isBelow(eChild.traversal().maxP, PROB_OF_THE); eKeys.splice(keyIndex, 1); } } @@ -89,6 +103,7 @@ describe('Trie traversal abstractions', function() { it('traversal over compact leaf node', function() { var model = new TrieModel(jsonFixture('tries/english-1000')); + const PROB_OF_TROUBLE = 267 / 500500; let rootTraversal = model.traverseFromRoot(); assert.isDefined(rootTraversal); @@ -102,6 +117,7 @@ describe('Trie traversal abstractions', function() { assert.isDefined(traversalInner1); assert.isArray(child.traversal().entries); assert.isEmpty(child.traversal().entries); + assert.equal(traversalInner1.maxP, 1000 / 500500 /* prob of 'the' */); for(let tChild of traversalInner1.children()) { if(tChild.char == 'r') { @@ -109,6 +125,7 @@ describe('Trie traversal abstractions', function() { assert.isDefined(traversalInner2); assert.isArray(tChild.traversal().entries); assert.isEmpty(tChild.traversal().entries); + assert.equal(traversalInner2.maxP, 607 / 500500 /* prob of 'true', the best 'tr-' entry */); for(let rChild of traversalInner2.children()) { if(rChild.char == 'o') { @@ -137,10 +154,17 @@ describe('Trie traversal abstractions', function() { if(leafChildSequence.length > 0) { assert.isArray(curChild.traversal().entries); assert.isEmpty(curChild.traversal().entries); + assert.equal(curChild.traversal().maxP, PROB_OF_TROUBLE); } else { let finalTraversal = curChild.traversal(); + assert.equal(finalTraversal.maxP, PROB_OF_TROUBLE); assert.isDefined(finalTraversal.entries); - assert.equal(finalTraversal.entries[0].text, 'trouble'); + assert.deepEqual(finalTraversal.entries, [ + { + text: 'trouble', + p: PROB_OF_TROUBLE + } + ]); eSuccess = true; } } while (leafChildSequence.length > 0); @@ -179,18 +203,20 @@ describe('Trie traversal abstractions', function() { for(let child of rootTraversal.children()) { if(child.char == smpA) { aSuccess = true; - let traversalInner1 = child.traversal(); + const traversalInner1 = child.traversal(); assert.isDefined(traversalInner1); - assert.isArray(child.traversal().entries); - assert.isEmpty(child.traversal().entries); + assert.isArray(traversalInner1.entries); + assert.isEmpty(traversalInner1.entries); + assert.equal(traversalInner1.maxP, 0.5); // The two entries are equally weighted. for(let aChild of traversalInner1.children()) { if(aChild.char == smpP) { pSuccess = true; - let traversalInner2 = aChild.traversal(); + const traversalInner2 = aChild.traversal(); assert.isDefined(traversalInner2); - assert.isArray(aChild.traversal().entries); - assert.isEmpty(aChild.traversal().entries); + assert.isArray(traversalInner2.entries); + assert.isEmpty(traversalInner2.entries); + assert.equal(traversalInner2.maxP, 0.5); for(let pChild of traversalInner2.children()) { let keyIndex = pKeys.indexOf(pChild.char); @@ -198,10 +224,11 @@ describe('Trie traversal abstractions', function() { pKeys.splice(keyIndex, 1); if(pChild.char == 'p') { // We'll test traversal with the 'mixed' entry from here. - let traversalInner3 = pChild.traversal(); + const traversalInner3 = pChild.traversal(); assert.isDefined(traversalInner3); - assert.isArray(pChild.traversal().entries); - assert.isEmpty(pChild.traversal().entries); + assert.isArray(traversalInner3.entries); + assert.isEmpty(traversalInner3.entries); + assert.equal(traversalInner3.maxP, 0.5); // Now to handle the rest, knowing it's backed by a leaf node. let curChild = pChild; @@ -227,12 +254,20 @@ describe('Trie traversal abstractions', function() { // Conditional test - if that was not the final character, entries should be undefined. if(leafChildSequence.length > 0) { - assert.isArray(curChild.traversal().entries); - assert.isEmpty(curChild.traversal().entries); + const nextTraversal = curChild.traversal() + assert.isArray(nextTraversal.entries); + assert.isEmpty(nextTraversal.entries); + assert.equal(nextTraversal.maxP, 0.5); } else { let finalTraversal = curChild.traversal(); assert.isDefined(finalTraversal.entries); - assert.equal(finalTraversal.entries[0].text, smpA + smpP + 'pl' + smpE); + assert.deepEqual(finalTraversal.entries, [ + { + text: smpA + smpP + 'pl' + smpE, + p: 1/2 + } + ]); + assert.equal(finalTraversal.maxP, 0.5); eSuccess = true; } } while (leafChildSequence.length > 0); From 0c47fcdbe61d0b0ab69fc84e5ecc90153cf829c5 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 14 Mar 2024 21:44:27 +0700 Subject: [PATCH 05/40] change(web): maxP -> p, to ensure common field name for node and entry --- common/models/templates/src/trie-model.ts | 2 +- common/models/types/index.d.ts | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/common/models/templates/src/trie-model.ts b/common/models/templates/src/trie-model.ts index 48078b6756..1f37f72931 100644 --- a/common/models/templates/src/trie-model.ts +++ b/common/models/templates/src/trie-model.ts @@ -205,7 +205,7 @@ class Traversal implements LexiconTraversal { } } - get maxP(): number { + get p(): number { return this.root.weight / this.totalWeight; } } diff --git a/common/models/types/index.d.ts b/common/models/types/index.d.ts index 5b2b85e683..668e99552d 100644 --- a/common/models/types/index.d.ts +++ b/common/models/types/index.d.ts @@ -25,7 +25,7 @@ declare type CasingForm = 'lower' | 'initial' | 'upper'; */ declare interface LexiconTraversal { /** - * Provides an iterable pattern used to search for words with a prefix matching + * Provides an iterable pattern used to search for words with a 'keyed' prefix matching * the current traversal state's prefix when a new character is appended. Iterating * across `children` provides 'breadth' to a lexical search. * @@ -70,13 +70,25 @@ declare interface LexiconTraversal { * - prefix of 'crepe': ['crêpe', 'crêpé'] * - other examples: https://www.thoughtco.com/french-accent-homographs-1371072 */ - entries: { text: USVString, p: number }[]; + entries: { + /** + * A lexical entry (word) offered by the model. + * + * Note: not the search-term keyed part. This will match the actual, unkeyed form. + */ + text: USVString, + /** + * The probability of the lexical entry, directly based upon its frequency. + */ + p: number + }[]; + // Note: `p`, not `maxP` - we want to see the same name for `this.entries.p` and `this.p` /** * Gives the probability of the highest-frequency lexical entry that is either a member or * descendent of the represented trie `Node`. */ - maxP: number; + p: number; } /** From 27bc02137c2415e3635a0c140a755a15cfd24084 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 14 Mar 2024 21:51:43 +0700 Subject: [PATCH 06/40] fix(common/models/templates): unit test patchup after maxP -> p --- .../templates/test/test-trie-traversal.js | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/common/models/templates/test/test-trie-traversal.js b/common/models/templates/test/test-trie-traversal.js index ad3f9b2c87..a64e73c66b 100644 --- a/common/models/templates/test/test-trie-traversal.js +++ b/common/models/templates/test/test-trie-traversal.js @@ -54,7 +54,7 @@ describe('Trie traversal abstractions', function() { assert.isDefined(traversalInner1); assert.isArray(child.traversal().entries); assert.isEmpty(child.traversal().entries); - assert.equal(traversalInner1.maxP, PROB_OF_THE); + assert.equal(traversalInner1.p, PROB_OF_THE); for(let tChild of traversalInner1.children()) { if(tChild.char == 'h') { @@ -63,7 +63,7 @@ describe('Trie traversal abstractions', function() { assert.isDefined(traversalInner2); assert.isEmpty(tChild.traversal().entries); assert.isArray(tChild.traversal().entries); - assert.equal(traversalInner2.maxP, PROB_OF_THE); + assert.equal(traversalInner2.p, PROB_OF_THE); for(let hChild of traversalInner2.children()) { if(hChild.char == 'e') { @@ -77,14 +77,14 @@ describe('Trie traversal abstractions', function() { p: PROB_OF_THE } ]); - assert.equal(traversalInner3.maxP, PROB_OF_THE); + assert.equal(traversalInner3.p, PROB_OF_THE); for(let eChild of traversalInner3.children()) { let keyIndex = eKeys.indexOf(eChild.char); assert.notEqual(keyIndex, -1, "Did not find char '" + eChild.char + "' in array!"); // THE is not accessible if any of the sub-tries of our 'e' node (traversalInner3). - assert.isBelow(eChild.traversal().maxP, PROB_OF_THE); + assert.isBelow(eChild.traversal().p, PROB_OF_THE); eKeys.splice(keyIndex, 1); } } @@ -117,7 +117,7 @@ describe('Trie traversal abstractions', function() { assert.isDefined(traversalInner1); assert.isArray(child.traversal().entries); assert.isEmpty(child.traversal().entries); - assert.equal(traversalInner1.maxP, 1000 / 500500 /* prob of 'the' */); + assert.equal(traversalInner1.p, 1000 / 500500 /* prob of 'the' */); for(let tChild of traversalInner1.children()) { if(tChild.char == 'r') { @@ -125,7 +125,7 @@ describe('Trie traversal abstractions', function() { assert.isDefined(traversalInner2); assert.isArray(tChild.traversal().entries); assert.isEmpty(tChild.traversal().entries); - assert.equal(traversalInner2.maxP, 607 / 500500 /* prob of 'true', the best 'tr-' entry */); + assert.equal(traversalInner2.p, 607 / 500500 /* prob of 'true', the best 'tr-' entry */); for(let rChild of traversalInner2.children()) { if(rChild.char == 'o') { @@ -154,10 +154,10 @@ describe('Trie traversal abstractions', function() { if(leafChildSequence.length > 0) { assert.isArray(curChild.traversal().entries); assert.isEmpty(curChild.traversal().entries); - assert.equal(curChild.traversal().maxP, PROB_OF_TROUBLE); + assert.equal(curChild.traversal().p, PROB_OF_TROUBLE); } else { let finalTraversal = curChild.traversal(); - assert.equal(finalTraversal.maxP, PROB_OF_TROUBLE); + assert.equal(finalTraversal.p, PROB_OF_TROUBLE); assert.isDefined(finalTraversal.entries); assert.deepEqual(finalTraversal.entries, [ { @@ -207,7 +207,7 @@ describe('Trie traversal abstractions', function() { assert.isDefined(traversalInner1); assert.isArray(traversalInner1.entries); assert.isEmpty(traversalInner1.entries); - assert.equal(traversalInner1.maxP, 0.5); // The two entries are equally weighted. + assert.equal(traversalInner1.p, 0.5); // The two entries are equally weighted. for(let aChild of traversalInner1.children()) { if(aChild.char == smpP) { @@ -216,7 +216,7 @@ describe('Trie traversal abstractions', function() { assert.isDefined(traversalInner2); assert.isArray(traversalInner2.entries); assert.isEmpty(traversalInner2.entries); - assert.equal(traversalInner2.maxP, 0.5); + assert.equal(traversalInner2.p, 0.5); for(let pChild of traversalInner2.children()) { let keyIndex = pKeys.indexOf(pChild.char); @@ -228,7 +228,7 @@ describe('Trie traversal abstractions', function() { assert.isDefined(traversalInner3); assert.isArray(traversalInner3.entries); assert.isEmpty(traversalInner3.entries); - assert.equal(traversalInner3.maxP, 0.5); + assert.equal(traversalInner3.p, 0.5); // Now to handle the rest, knowing it's backed by a leaf node. let curChild = pChild; @@ -257,7 +257,7 @@ describe('Trie traversal abstractions', function() { const nextTraversal = curChild.traversal() assert.isArray(nextTraversal.entries); assert.isEmpty(nextTraversal.entries); - assert.equal(nextTraversal.maxP, 0.5); + assert.equal(nextTraversal.p, 0.5); } else { let finalTraversal = curChild.traversal(); assert.isDefined(finalTraversal.entries); @@ -267,7 +267,7 @@ describe('Trie traversal abstractions', function() { p: 1/2 } ]); - assert.equal(finalTraversal.maxP, 0.5); + assert.equal(finalTraversal.p, 0.5); eSuccess = true; } } while (leafChildSequence.length > 0); From 15f49f25511b5bd7fb4e20efa752927291f51ca0 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Sun, 10 Mar 2024 02:53:10 +0700 Subject: [PATCH 07/40] feat(common/models): direct, non-iterative traversal to specific children --- common/models/templates/src/trie-model.ts | 55 ++++++++++++++++++- .../templates/test/test-trie-traversal.js | 36 ++++++++++++ common/models/types/index.d.ts | 13 +++++ 3 files changed, 101 insertions(+), 3 deletions(-) diff --git a/common/models/templates/src/trie-model.ts b/common/models/templates/src/trie-model.ts index 1f37f72931..71ce1ce90e 100644 --- a/common/models/templates/src/trie-model.ts +++ b/common/models/templates/src/trie-model.ts @@ -100,13 +100,62 @@ class Traversal implements LexiconTraversal { */ totalWeight: number; - constructor(root: Node, prefix: string, maxWeight: number) { + constructor(root: Node, prefix: string, totalWeight: number) { this.root = root; this.prefix = prefix; - this.totalWeight = maxWeight; + this.totalWeight = totalWeight; } - *children(): Generator<{char: string, traversal: () => LexiconTraversal}> { + child(char: USVString): LexiconTraversal | undefined { + /* + Note: would otherwise return the current instance if `char == ''`. If + such a call is happening, it's probably indicative of an implementation + issue elsewhere - let's signal now in order to catch such stuff early. + */ + if(char == '') { + return undefined; + } + + // Split into individual code units. + let steps = char.split(''); + let traversal: ReturnType = this; + + while(steps.length > 0 && traversal) { + const step: string = steps.shift()!; + traversal = traversal._child(step); + } + + return traversal; + } + + // Handles one code unit at a time. + private _child(char: USVString): Traversal | undefined { + const root = this.root; + const totalWeight = this.totalWeight; + const nextPrefix = this.prefix + char; + + if(root.type == 'internal') { + let childNode = root.children[char]; + if(!childNode) { + return undefined; + } + + return new Traversal(childNode, nextPrefix, totalWeight); + } else { + // root.type == 'leaf'; + const legalChildren = root.entries.filter(function(entry) { + return entry.content.indexOf(nextPrefix) == 0; + }); + + if(!legalChildren.length) { + return undefined; + } + + return new Traversal(root, nextPrefix, totalWeight); + } + } + + *children(): Generator<{char: USVString, traversal: () => LexiconTraversal}> { let root = this.root; const totalWeight = this.totalWeight; diff --git a/common/models/templates/test/test-trie-traversal.js b/common/models/templates/test/test-trie-traversal.js index a64e73c66b..07730aedf0 100644 --- a/common/models/templates/test/test-trie-traversal.js +++ b/common/models/templates/test/test-trie-traversal.js @@ -23,6 +23,10 @@ describe('Trie traversal abstractions', function() { let rootKeys = ['t', 'o', 'a', 'i', 'w', 'h', 'f', 'b', 'n', 'y', 's', 'm', 'u', 'c', 'd', 'l', 'e', 'j', 'p', 'g', 'v', 'k', 'r', 'q']; + rootKeys.forEach((entry) => assert.isOk(rootTraversal.child(entry))); + assert.isNotOk(rootTraversal.child('x')); + assert.isNotOk(rootTraversal.child('z')); + for(let child of rootTraversal.children()) { let keyIndex = rootKeys.indexOf(child.char); assert.notEqual(keyIndex, -1); @@ -101,6 +105,38 @@ describe('Trie traversal abstractions', function() { assert.isEmpty(eKeys); }); + it('direct traversal with simple internal nodes', function() { + var model = new TrieModel(jsonFixture('tries/english-1000')); + + let rootTraversal = model.traverseFromRoot(); + assert.isDefined(rootTraversal); + + let eKeys = ['y', 'r', 'i', 'm', 's', 'n', 'o']; + + const tNode = rootTraversal.child('t'); + assert.isOk(tNode); + assert.isDefined(tNode); + assert.isArray(tNode.entries); + assert.isEmpty(tNode.entries); + + const hNode = tNode.child('h'); + assert.isOk(hNode); + assert.isDefined(hNode); + assert.isArray(hNode.entries); + assert.isEmpty(hNode.entries); + + const eNode = hNode.child('e'); + assert.isOk(eNode); + assert.isDefined(eNode); + assert.isArray(eNode.entries); + assert.isNotEmpty(eNode.entries); + assert.equal(eNode.entries[0].text, "the"); + + for(let key of eKeys) { + assert.isOk(eNode.child(key)); + } + }); + it('traversal over compact leaf node', function() { var model = new TrieModel(jsonFixture('tries/english-1000')); const PROB_OF_TROUBLE = 267 / 500500; diff --git a/common/models/types/index.d.ts b/common/models/types/index.d.ts index 668e99552d..9b1f2de9d9 100644 --- a/common/models/types/index.d.ts +++ b/common/models/types/index.d.ts @@ -50,6 +50,19 @@ declare interface LexiconTraversal { */ children(): Generator<{char: USVString, traversal: () => LexiconTraversal}>; + /** + * Allows direct access to the traversal state that results when appending a + * `char` representing a single UTF-16 codepoint to the current traversal + * state's prefix. This bypasses the need to iterate among all legal child + * Traversals. + * + * If such a traversal state is not supported, returns `undefined`. + * Implementations may choose to return `undefined` if more than one UTF-16 + * codepoint is appended, even if such a descendant exists. + * @param char + */ + child(char: USVString): LexiconTraversal | undefined; + /** * Any entries directly keyed by the currently-represented lookup prefix. Entries and * children may exist simultaneously, but `entries` must always exist when no children are From b58945e2551bdee0a629c965bbe2a468dc7c00e4 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Sun, 10 Mar 2024 03:35:54 +0700 Subject: [PATCH 08/40] feat(common/models/templates): unit test for new feature --- .../templates/test/test-trie-traversal.js | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/common/models/templates/test/test-trie-traversal.js b/common/models/templates/test/test-trie-traversal.js index 07730aedf0..a62d6ff54a 100644 --- a/common/models/templates/test/test-trie-traversal.js +++ b/common/models/templates/test/test-trie-traversal.js @@ -214,7 +214,6 @@ describe('Trie traversal abstractions', function() { assert.isTrue(eSuccess); }); - it('traversal with SMP entries', function() { // Two entries, both of which read "apple" to native English speakers. // One solely uses SMP characters, the other of which uses a mix of SMP and standard. @@ -320,4 +319,52 @@ describe('Trie traversal abstractions', function() { assert.isEmpty(pKeys); }); + + it('direct traversal with SMP entries', function() { + // Two entries, both of which read "apple" to native English speakers. + // One solely uses SMP characters, the other of which uses a mix of SMP and standard. + var model = new TrieModel(jsonFixture('tries/smp-apple')); + + let rootTraversal = model.traverseFromRoot(); + assert.isDefined(rootTraversal); + + let smpA = smpForUnicode(0x1d5ba); + let smpP = smpForUnicode(0x1d5c9); + let smpL = smpForUnicode(0x1d5c5); + let smpE = smpForUnicode(0x1d5be); + + // Just to be sure our utility function is working right. + assert.equal(smpA + smpP + 'pl' + smpE, "𝖺𝗉pl𝖾"); + + let pKeys = ['p', smpP]; + let leafChildSequence = ['l', smpE]; + + const aNode = rootTraversal.child(smpA); + assert.isOk(aNode); + assert.isNotOk(rootTraversal.child('a')); + + const pNode1 = aNode.child(smpP); + assert.isOk(pNode1); + assert.isNotOk(aNode.child('p')); + + const pNode2 = pNode1.child('p'); + assert.isOk(pNode2); + assert.isOk(pNode1.child(smpP)); // Both exist for this step. + + const lNode = pNode2.child('l'); + assert.isOk(lNode); + assert.isNotOk(pNode2.child(smpL)); + + const eNode = lNode.child(smpE); + assert.isOk(eNode); + assert.isNotOk(lNode.child('e')); + + assert.deepEqual(eNode.entries, [ + { + text: smpA + smpP + 'pl' + smpE, + p: 1/2 + } + ]); + assert.equal(eNode.maxP, 0.5); + }); }); From 64efabbc49658062a54969e06a8e867778d23c75 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 14 Mar 2024 21:02:14 +0700 Subject: [PATCH 09/40] change(common/models/templates): traversal on Trie --- common/models/templates/src/trie-model.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/common/models/templates/src/trie-model.ts b/common/models/templates/src/trie-model.ts index 71ce1ce90e..f6f09907b9 100644 --- a/common/models/templates/src/trie-model.ts +++ b/common/models/templates/src/trie-model.ts @@ -354,7 +354,7 @@ export default class TrieModel implements LexicalModel { } public traverseFromRoot(): LexiconTraversal { - return new Traversal(this._trie['root'], '', this._trie.totalWeight); + return this._trie.traverseFromRoot(); } }; @@ -450,6 +450,10 @@ class Trie { this.totalWeight = totalWeight; } + public traverseFromRoot(): LexiconTraversal { + return new Traversal(this.root, '', this.totalWeight); + } + /** * Lookups an arbitrary prefix (a query) in the trie. Returns the top 3 * results in sorted order. From 6de595394bf2ea95779728058df3b2778be69fbd Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 14 Mar 2024 21:47:51 +0700 Subject: [PATCH 10/40] docs(common/models/templates): a bit more for traversal `child` method --- common/models/types/index.d.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/common/models/types/index.d.ts b/common/models/types/index.d.ts index 9b1f2de9d9..e94d758b0b 100644 --- a/common/models/types/index.d.ts +++ b/common/models/types/index.d.ts @@ -59,6 +59,10 @@ declare interface LexiconTraversal { * If such a traversal state is not supported, returns `undefined`. * Implementations may choose to return `undefined` if more than one UTF-16 * codepoint is appended, even if such a descendant exists. + * + * Note: traversals navigate and represent the lexicon in its "keyed" state, + * as produced by use of the search-term keying function defined for the model. + * That is, if a model "keys" `è` to `e`, there will be no `è` child. * @param char */ child(char: USVString): LexiconTraversal | undefined; From e0892deccf4ee059348bf44b94a26ce9bdb3a600 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 14 Mar 2024 21:52:40 +0700 Subject: [PATCH 11/40] fix(common/models/templates): unit test patchup post maxP -> p --- common/models/templates/test/test-trie-traversal.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/models/templates/test/test-trie-traversal.js b/common/models/templates/test/test-trie-traversal.js index a62d6ff54a..aed35d654b 100644 --- a/common/models/templates/test/test-trie-traversal.js +++ b/common/models/templates/test/test-trie-traversal.js @@ -365,6 +365,6 @@ describe('Trie traversal abstractions', function() { p: 1/2 } ]); - assert.equal(eNode.maxP, 0.5); + assert.equal(eNode.p, 0.5); }); }); From aef1dacc59776e20d2e30b32e681597ec4bf87f3 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 15 Mar 2024 20:01:37 +0700 Subject: [PATCH 12/40] fix(common/models): traverse by key, not true wordform --- common/models/templates/src/trie-model.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/models/templates/src/trie-model.ts b/common/models/templates/src/trie-model.ts index f6f09907b9..df94e5cb83 100644 --- a/common/models/templates/src/trie-model.ts +++ b/common/models/templates/src/trie-model.ts @@ -144,7 +144,7 @@ class Traversal implements LexiconTraversal { } else { // root.type == 'leaf'; const legalChildren = root.entries.filter(function(entry) { - return entry.content.indexOf(nextPrefix) == 0; + return entry.key.indexOf(nextPrefix) == 0; }); if(!legalChildren.length) { From fe31917bb5f1de71cdde1fa8dfbd26c8c3a510b5 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 15 Mar 2024 20:26:19 +0700 Subject: [PATCH 13/40] change(common/models/templates): predict methods now utilize traversals --- common/models/templates/src/trie-model.ts | 124 ++++------------------ common/models/types/index.d.ts | 38 ++++--- 2 files changed, 43 insertions(+), 119 deletions(-) diff --git a/common/models/templates/src/trie-model.ts b/common/models/templates/src/trie-model.ts index df94e5cb83..605bbc7c3f 100644 --- a/common/models/templates/src/trie-model.ts +++ b/common/models/templates/src/trie-model.ts @@ -73,15 +73,6 @@ export interface TrieModelOptions { punctuation?: LexicalModelPunctuation; } -/** - * Used to determine the probability of an entry from the trie. - */ -type TextWithProbability = { - text: string; - // TODO: use negative-log scaling instead? - p: number; // real-number weight, from 0 to 1 -} - class Traversal implements LexiconTraversal { /** * The lexical prefix corresponding to the current traversal state. @@ -375,11 +366,10 @@ export default class TrieModel implements LexicalModel { type SearchKey = string & { _: 'SearchKey'}; /** - * The priority queue will always pop the most weighted item. There can only - * be two kinds of items right now: nodes, and entries; both having a weight - * attribute. + * The priority queue will always pop the most probable item - be it a Traversal + * state or a lexical entry reached via Traversal. */ -type Weighted = Node | Entry; +type TraversableWithProb = TextWithProbability | LexiconTraversal; /** * A function that converts a string (word form or query) into a search key @@ -462,12 +452,8 @@ class Trie { */ lookup(prefix: string): TextWithProbability[] { let searchKey = this.toKey(prefix); - let lowestCommonNode = findPrefix(this.root, searchKey); - if (lowestCommonNode === null) { - return []; - } - - return getSortedResults(lowestCommonNode, searchKey, this.totalWeight); + let rootTraversal = this.traverseFromRoot().child(searchKey); + return rootTraversal ? getSortedResults(rootTraversal) : []; } /** @@ -475,36 +461,10 @@ class Trie { * @param n How many suggestions, maximum, to return. */ firstN(n: number): TextWithProbability[] { - return getSortedResults(this.root, '' as SearchKey, this.totalWeight, n); + return getSortedResults(this.traverseFromRoot(), n); } } -/** - * Finds the deepest descendent in the trie with the given prefix key. - * - * This means that a search in the trie for a given prefix has a best-case - * complexity of O(m) where m is the length of the prefix. - * - * @param key The prefix to search for. - * @param index The index in the prefix. Initially 0. - */ -function findPrefix(node: Node, key: SearchKey, index: number = 0): Node | null { - // An important note - the Trie itself is built on a per-JS-character basis, - // not on a UTF-8 character-code basis. - if (node.type === 'leaf' || index === key.length) { - return node; - } - - // So, for SMP models, we need to match each char of the supplementary pair - // in sequence. Each has its own node in the Trie. - let char = key[index]; - if (node.children[char]) { - return findPrefix(node.children[char], key, index + 1); - } - - return null; -} - /** * Returns all entries matching the given prefix, in descending order of * weight. @@ -513,72 +473,32 @@ function findPrefix(node: Node, key: SearchKey, index: number = 0): Node | null * @param results the current results * @param queue */ -function getSortedResults(node: Node, prefix: SearchKey, N: number, limit = MAX_SUGGESTIONS): TextWithProbability[] { - let queue = new PriorityQueue(function(a: Weighted, b: Weighted) { +function getSortedResults(traversal: LexiconTraversal, limit = MAX_SUGGESTIONS): TextWithProbability[] { + let queue = new PriorityQueue(function(a: TraversableWithProb, b: TraversableWithProb) { // In case of Trie compilation issues that emit `null` or `undefined` - return (b ? b.weight : 0) - (a ? a.weight : 0); + return (b ? b.p : 0) - (a ? a.p : 0); }); let results: TextWithProbability[] = []; - if (node.type === 'leaf') { - // Assuming the values are sorted, we can just add all of the values in the - // leaf, until we reach the limit. - for (let item of node.entries) { - // String.startsWith is not supported on certain Android (5.0) devices we wish to support. - // Requires a minimum of Chrome 36, as opposed to 5.0's default of 35. - if (item.key.indexOf(prefix) == 0) { - let { content, weight } = item; - results.push({ - text: content, - p: weight / N - }); + queue.enqueue(traversal); - if (results.length >= limit) { - return results; - } - } - } - } else { - queue.enqueue(node); - let next: Weighted | undefined; + while(queue.count > 0) { + const entry = queue.dequeue(); - while (next = queue.dequeue()) { - if (isNode(next)) { - // When a node is next up in the queue, that means that next least - // likely suggestion is among its decsendants. - // So we search all of its descendants! - if (next.type === 'leaf') { - queue.enqueueAll(next.entries); - } else { - // XXX: alias `next` so that TypeScript can be SURE that internal is - // in fact an internal node. Because of the callback binding to the - // original definition of node (i.e., a Node | Entry), this will not - // type-check otherwise. - let internal = next; - queue.enqueueAll(next.values.map(char => { - return internal.children[char]; - })); - } - } else { - // When an entry is up next in the queue, we just add its contents to - // the results! - results.push({ - text: next.content, - p: next.weight / N - }); - if (results.length >= limit) { - return results; - } + if((entry as TextWithProbability)!.text !== undefined) { + const lexicalEntry = entry as TextWithProbability; + results.push(lexicalEntry); + if(results.length >= limit) { + return results; } + } else { + const traversal = entry as LexiconTraversal; + queue.enqueueAll(traversal.entries); + queue.enqueueAll(Array.from(traversal.children()).map((entry) => entry.traversal())); } } + return results; - -} - -/** TypeScript type guard that returns whether the thing is a Node. */ -function isNode(x: Entry | Node): x is Node { - return 'type' in x; } /** diff --git a/common/models/types/index.d.ts b/common/models/types/index.d.ts index e94d758b0b..2fc1c92cb3 100644 --- a/common/models/types/index.d.ts +++ b/common/models/types/index.d.ts @@ -19,6 +19,23 @@ declare type USVString = string; declare type CasingForm = 'lower' | 'initial' | 'upper'; +/** + * Represents one lexical entry and its probability.. + */ +type TextWithProbability = { + /** + * A lexical entry (word) offered by the model. + * + * Note: not the search-term keyed part. This will match the actual, unkeyed form. + */ + text: string; + + /** + * The probability of the lexical entry, directly based upon its frequency. + */ + p: number; // real-number weight, from 0 to 1 +} + /** * Used to facilitate edit-distance calculations by allowing the LMLayer to * efficiently search the model's lexicon in a Trie-like manner. @@ -52,13 +69,11 @@ declare interface LexiconTraversal { /** * Allows direct access to the traversal state that results when appending a - * `char` representing a single UTF-16 codepoint to the current traversal - * state's prefix. This bypasses the need to iterate among all legal child - * Traversals. + * `char` representing one or more individual UTF-16 codepoints to the + * current traversal state's prefix. This bypasses the need to iterate + * among all legal child Traversals. * * If such a traversal state is not supported, returns `undefined`. - * Implementations may choose to return `undefined` if more than one UTF-16 - * codepoint is appended, even if such a descendant exists. * * Note: traversals navigate and represent the lexicon in its "keyed" state, * as produced by use of the search-term keying function defined for the model. @@ -87,18 +102,7 @@ declare interface LexiconTraversal { * - prefix of 'crepe': ['crêpe', 'crêpé'] * - other examples: https://www.thoughtco.com/french-accent-homographs-1371072 */ - entries: { - /** - * A lexical entry (word) offered by the model. - * - * Note: not the search-term keyed part. This will match the actual, unkeyed form. - */ - text: USVString, - /** - * The probability of the lexical entry, directly based upon its frequency. - */ - p: number - }[]; + entries: TextWithProbability[]; // Note: `p`, not `maxP` - we want to see the same name for `this.entries.p` and `this.p` /** From 02956db5e8692a7995c3e58535976fc3575781d1 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Thu, 20 Jun 2024 08:24:09 -0500 Subject: [PATCH 14/40] chore(android): Add crowdin for Polytonic Greek --- .../src/main/res/values-b+el/strings.xml | 164 +++++++++++++++++ .../com/keyman/engine/DisplayLanguages.java | 1 + .../app/src/main/res/values-b+el/strings.xml | 172 ++++++++++++++++++ crowdin.yml | 4 +- 4 files changed, 340 insertions(+), 1 deletion(-) create mode 100644 android/KMAPro/kMAPro/src/main/res/values-b+el/strings.xml create mode 100644 android/KMEA/app/src/main/res/values-b+el/strings.xml diff --git a/android/KMAPro/kMAPro/src/main/res/values-b+el/strings.xml b/android/KMAPro/kMAPro/src/main/res/values-b+el/strings.xml new file mode 100644 index 0000000000..43e43eb4ca --- /dev/null +++ b/android/KMAPro/kMAPro/src/main/res/values-b+el/strings.xml @@ -0,0 +1,164 @@ + + + + + Μοιρασθῆτε + + Φυλλομετρητής + + Μέγεθος κειμένου + + Περισσότερα + + Διαγραφὴ κειμένου + + Πληροφορίες + + Ρυθμίσεις + + Εγκατάσταση Ενημερώσεων + + Ἔκδοση %1$s + + Τὸ Keyman ἀπαιτεῖ ἔκδοση Chrome 57 ἢ νεώτερη. + + Ἐνημερῶστε τὸ Chrome + + Ἀρχίστε νὰ γράφετε ἐδῶ… + + + + Μέγεθος κειμένου: %1$d + + Μεγαλῶστε τὸ κείμενο + + Μεγαλῶστε τὸ κείμενο + + Ρύθμιση μεγέθους κειμένου + + \nΤὸ κείμενο θὰ ἐκκαθαρισθεῖ πλήρως\n + + Μικρύνετε τὸ κείμενο + + Προσθέστε πληκτρολόγιο γιὰ τὴν γλῶσσα σας + + Ὁρίστε τὸ Κῆμαν ὡς παν-συστημικὸ πληκτρολόγιο + + Ὁρίστε τὸ Κῆμαν ὡς προεπιλεγμένο πληκτρολόγιο + + Περισσότερες πληροφορίες + + Κατὰ τὴν ἔναρξη νὰ προβάλλεται τὸ \"%1$s + + Γιὰ νὰ ἐγκαταστήσετε πακέτα πληκτρολογίων, ἐπιτρέψτε στὸ Κῆμαν νὰ διαβάζει ἐξωτερικὸ χῶρο ἀποθήκευσης. + + Ἀπερρίφθη αἴτημα χώρου ἀποθήκευσης. Πιθανὴ ἀποτυχία ἐγκατάστασης πακέτου πληκτρολογίου + Ἀπερρίφθη αἴτημα χώρου ἀποθηκεύσεως. Δοκιμάστε τὶς ρυθμίσεις Κῆμαν - Ἐγκατάσταση ἀπὸ τοπικὸ ἀρχεῖο + + Ρυθμίσεις + + + Ἐγκατεστημένες γλῶσσες (%1$d) + Ἐγκατεστημένες γλῶσσες (%1$d) + + + Ἐγκαταστῆστε πληκτρολόγιο ἢ λεξικό + + Γλῶσσα προβολῆς + + Μεταβολὴ ὕψους πληκτρολογίου + + Spacebar caption + + Πληκτρολόγιο + + Γλῶσσα + + Γλῶσσα + Πληκτρολόγιο + + Κενό + + \'Ονομα πληκτρολογίου στὸ πλῆκτρο διαστήματος + + \'Ονομα γλώσσας στὸ πλῆκτρο διαστήματος + + \'Ονομα πληκτρολογίου καὶ γλώσσας στὸ πλῆκτρο διαστήματος + + Καμμία λεζάντα στὸ πλῆκτρο διαστήματος + + Δὸνηση κατὰ τὴν πληκτρολόγηση + + Νὰ ἐμφανίζεται πάντα banner + + Πρὸς ὑλοποίησιν + + Ὅταν εἶναι off, ἐμφανίζεται μόνο ὅταν ἔχει ἐνεργοποιηθεῖ τὸ προγνωστικὸ κείμενο + + Ἐπιτρέψτε τὴν ἀποστολὴ ἀναφορῶν κατάρρευσης μέσῳ δικτύου + + Ὅταν εἶναι ΟΝ, θὰ ἀποστέλλονται ἀναφορὲς κατάρρευσης + + Ὅταν εἶναι off, δὲν θὰ ἀποστέλλονται ἀναφορὲς κατάρρευσης + + Ἐγκατάσταση ἀπὸ τὸ keyman.com + + Ἐγκατάσταση ἀπὸ τοπικὸ ἀρχεῖο + + Ἐγκατάσταση ἀπὸ ἄλλη συσκευή + + Προσθέστε γλῶσσες σὲ ἐγκατεστημένο πληκτρολόγιο + + (ἀπὸ πακέτο πληκτρολογίου) + + Ἐπιλέξτε Πακέτο Πληκτρολογίου + + Ἐπιλέξτε γλῶσσες γιὰ τὸ %1$s + + Προσετέθη ἡ γλῶσσα %1$s στὸ %2$s + + Ὅλες οἱ γλῶσσες ἔχουν ἤδη ἐγκατασταθεῖ + + Σύρετε τὸ πληκτρολόγιο γιὰ νὰ ἀλλάξετε τὸ ὕψος + + Περιστρέψτε τὴν συσκευὴ γιὰ λειτουργία πορτραίτου καὶ τοπίου + + Ἐπαναφέρετε τὶς προεπιλεγμένες ρυθμίσεις + + Ἀναζητῆστε ἢ πληκτρολογῆστε URL + + Σελιδοδεῖκτες + + Δὲν ὑπάρχουν σελιδοδεῖκτες + + Προσθέστε σελιδοδείκτη + + Τίτλος + + URL + + Τὸ πακέτο %1$s ἀπέτυχε νὰ ἐγκατασταθεῖ + + Λήψη πακέτου πληκτρολογίου\n%1$s… + + Ἀποτυχία ἐξαγωγῆς + + Ἐγκαταστῆστε πληκτρολόγιο + + Ἐγκαταστῆστε Λεξικό + + Τὸ %1$s δὲν εἶναι ἔγκυρο ἀρχεῖο πακέτου Κῆμαν.\n%2$s\" + + Τὸ πακέτο πληκτρολογίου δὲν ἔχει βελτιστοποιημένα πληκτρολόγια ἀφῆς πρὸς ἐγκατάστασιν + + Δὲν ὑπάρχει νέο προγνωστικό κείμενο πρὸς ἐγκατάστασιν + + Δὲν ὑπάρχουν πληκτρολόγια ἢ προγνωστικό κείμενο πρὸς ἐγκατάστασιν + + Τὸ πακέτο πληκτρολογίου δὲν ἔχει σχετικὲς μὲ αὐτὸ γλῶσσες πρὸς ἐγκατάστασιν + + Ἄκυρα/ἐλλιπῆ μεταδεδομένα στὸ πακέτο + + Τὸ πληκτρολόγιο ἀπαιτεῖ νεώτερη ἔκδοση τοῦ Κῆμαν + + Ἀδυναμία ἐκκινήσεως φυλλομετρητῆ + diff --git a/android/KMEA/app/src/main/java/com/keyman/engine/DisplayLanguages.java b/android/KMEA/app/src/main/java/com/keyman/engine/DisplayLanguages.java index 27c462e95a..6a159523a2 100644 --- a/android/KMEA/app/src/main/java/com/keyman/engine/DisplayLanguages.java +++ b/android/KMEA/app/src/main/java/com/keyman/engine/DisplayLanguages.java @@ -62,6 +62,7 @@ public class DisplayLanguages { new DisplayLanguageType("nl-NL", "Nederlands (Dutch)"), new DisplayLanguageType("ann", "Obolo"), new DisplayLanguageType("pl-PL", "Polski (Polish)"), + new DisplayLanguageType("el", "Polytonic Greek"), new DisplayLanguageType("pt-PT", "Português do Portugal"), new DisplayLanguageType("ff-ZA", "Pulaar-Fulfulde"), // or Fulah new DisplayLanguageType("ru-RU", "Pyccĸий (Russian)"), diff --git a/android/KMEA/app/src/main/res/values-b+el/strings.xml b/android/KMEA/app/src/main/res/values-b+el/strings.xml new file mode 100644 index 0000000000..871171d63f --- /dev/null +++ b/android/KMEA/app/src/main/res/values-b+el/strings.xml @@ -0,0 +1,172 @@ + + + + + + + Πληκτρολόγιο + Πληκτρολόγια + + + + Ἄλλη μέθοδος εἰσαγωγῆς + Ἄλλες Μέθοδοι Εἰσαγωγῆς + + + Προσθέστε νέο Πληκτρολόγιο + + Ἐγκατεστημένες γλῶσσες + + Ρυθμίσεις %1$s + + Προσθέστε + + Πίσω + + Ἀκυρῶστε + + Κλεῖστε + + Κλεῖστε τὸ Κῆμαν + + Προχωρῆστε + + Ἑπόμενη Μέθοδος Εἰσαγωγῆς + + Λήψη + + Ἐγκαταστῆστε + + Ἀργότερα + + Ἑπόμενο + + OK + + Ἐνημερῶστε + + Δὲν ὑπάρχει σύνδεση μὲ τὸ Διαδίκτυο + + Ἀδυναμία συνδέσεως μὲ τὸν διακομιστὴ τοῦ Κῆμαν! + + Θέλετε νὰ διαγράψετε αὐτὸ τὸ πληκτρολόγιο; + + Θὰ θέλατε νὰ κατεβάσετε τὴν τελευταία ἔκδοση αὐτοῦ τοῦ πληκτρολογίου; + + Θά θέλατε νὰ ἐνημερώσετε τώρα πληκτρολόγια καὶ λεξικά; + + Ἐνημερώσεις Πόρων + + Διαθέσιμες Ἐνημερώσεις Πόρων + + %1$s (Διαθέσιμη Ἐνημέρωση) + + Διαθέσιμες ἐνημερώσεις γιὰ τὸ πληκτρολόγιο %1$s: %2$s + + Διαθέσιμες ἐνημερώσεις γιὰ τὸ λεξικὸ %1$s: %2$s + + Ἔκδοση πληκτρολογίου + + Σύνδεσμος βοηθείας + + Ἀπεγκαταστῆστε πληκτρολόγιο + + [νέο] %1$s + + Σαρῶστε αὐτὸν τὸν κωδικό γιὰ νὰ φορτώσετε\nαὐτὸ τὸ πληκτρολόγιο σὲ ἄλλη συσκευή + + Καλωσορίσατε στὸ %1$s + + Ἀπαιτεῖται βιβλιοθήκη FileProvider γιὰ νὰ δεῖτε ἀρχεῖο βοηθείας: %1$s + + Μοιραῖο σφάλμα πληκτρολογίου στὸ %1$s:%2$s γιὰ τὴν %3$s γλῶσσα. Φορτώνεται προεπιλεγμένο πληκτρολόγιο. + + Error in keyboard %1$s:%2$s for %3$s language. + + Ἔλεγχος συσχετισμένου λεξικοῦ πρὸς λῆψιν + Ἀδυναμία συνδέσεως μὲ τὸν διακομιστὴ Κῆμαν γιὰ τὸν ἔλεγχο συσχετισμένου λεξικοῦ πρὸς λῆψιν + + Θὰ θέλατε νὰ κατεβάσετε τὴν τελευταία ἔκδοση αὐτοῦ τοῦ λεξικοῦ; + + Δὲν ὑπάρχει λεξικὸ πρὸς λῆψιν + + Μὴ διαθέσιμος κατάλογος πόρων + + Ἔχει ξεκινήσει ἐνημέρωση καταλόγου στὸ παρασκήνιο + + Ἡ λήψη τοῦ καταλόγου συνεχίζεται· παρακαλοῦμε ξαναδοκιμάστε σὲ λίγο! + + Ἔλεγχος πόρου σὲ ἐξέλιξη + + Ἡ λήψη τοῦ πληκτρολογίου ἔχει ξεκινήσει στὸ παρασκήνιο + + Ἡ λήψη τοῦ ἐπιλεγέντος πληκτρολογίου βρίσκεται σὲ ἐξέλιξη· παρακαλοῦμε ξαναδοκιμάστε σὲ λίγο! + + Ἡ λήψη τοῦ πληκτρολογίου ὁλοκληρώθηκε! + + Ἡ λήψη τοῦ λεξικοῦ ἔχει ξεκινήσει στὸ παρασκήνιο + + Ἡ λήψη τοῦ ἐπιλεγέντος λεξικοῦ βρίσκεται σὲ ἐξέλιξη· παρακαλοῦμε ξαναδοκιμάστε σὲ λίγο! + + Ἡ λήψη τοῦ λεξικοῦ ὁλοκληρώθηκε. + + Ἡ λήψη ἀπέτυχε + + Ἀποτυχία ἀνακτήσεως ληφθέντος ἀρχείου + + Ἀποτυχία προσβάσεως στὸν διακομιστή! + + "Ὅλοι οἱ πόροι ἔχουν ἐνημερωθεῖ!" + + Ἕνας ἢ περισσότεροι πόροι ἀπέτυχαν νὰ ἐνημερωθοῦν! + + Οἱ πόροι ἐνημερώθηκαν ἐπιτυχῶς! + + Ἔκδοση λεξικοῦ + + Ἀπεγκαταστῆτε λεξικό + + Θὰ θέλατε νὰ διαγράψετε αὐτὸ τὸ λεξικό; + + Τὸ λεξικὸ διεγράφη + + Τὸ πληκτρολόγιο %1$s ἐγκατεστάθη + + Τὸ πληκτρολόγιο διεγράφη + + Ἐνεργοποιῆστε τὶς διορθώσεις + + Ἐνεργοποιῆστε προβλέψεις + + Λεξικά + + Λεξικό + Λεξικά + + + Ἔλεγχος διαθεσίμου λεξικοῦ + Ἔλεγχος λεξικῶν ὀνλάϊν + + Λεξικό: %1$s + + %1$s λεξικά + + + Τὸ λεξικὸ ἐγκατεστάθη + + + (%1$d πληκτρολόγιο) + (%1$d πληκτρολόγια) + + + Προεπιλεγμένη Γλῶσσα + + + + Διαγράψτε + + + Κτυπῆστε ἐδῶ γιὰ νὰ ἀλλάξετε πληκτρολόγιο + + Ἀδυναμία ἐκκινήσεως φυλλομετρητῆ + diff --git a/crowdin.yml b/crowdin.yml index f8b9785bc3..d96609f7b8 100644 --- a/crowdin.yml +++ b/crowdin.yml @@ -34,6 +34,7 @@ files: languages_mapping: # Prevent invalid region pap-rPAP. Leaving "in" for Indonesian android_code: + el-polyton: b+el # TODO: figure out polyton variant es-419: b+es+419 pap: pap shu-latn-n: b+shu+latn @@ -44,6 +45,7 @@ files: languages_mapping: # Prevent invalid region pap-rPAP android_code: + el-polyton: b+el # TODO: figure out polyton variant es-419: b+es+419 pap: pap shu-latn-n: b+shu+latn @@ -136,7 +138,7 @@ files: osx_code: pt-PT: pt-PT.lproj - - source: /mac/Keyman4MacIM/Keyman4MacIM/KMKeyboardHelpWindow/en.lproj/KMKeyboardHelpWindowController.strings + - source: /mac/Keyman4MacIM/Keyman4MacIM/KMKeyboardHelpWindow/en.lproj/KMKeyboardHelpWindowController.strings dest: /mac/app/KMKeyboardHelpWindowController.strings translation: /mac/Keyman4MacIM/Keyman4MacIM/KMKeyboardHelpWindow/%osx_code%/%original_file_name% languages_mapping: From 276c736467a579b741f651ac7de81976a81b3384 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Mon, 24 Jun 2024 01:37:32 -0500 Subject: [PATCH 15/40] chore(ios): add crowdin for Polytonic Greek --- .../el-polyton.lproj/Localizable.stringsdict | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 ios/engine/KMEI/KeymanEngine/el-polyton.lproj/Localizable.stringsdict diff --git a/ios/engine/KMEI/KeymanEngine/el-polyton.lproj/Localizable.stringsdict b/ios/engine/KMEI/KeymanEngine/el-polyton.lproj/Localizable.stringsdict new file mode 100644 index 0000000000..afaf5ae03d --- /dev/null +++ b/ios/engine/KMEI/KeymanEngine/el-polyton.lproj/Localizable.stringsdict @@ -0,0 +1,118 @@ + + + + + menu-langsettings-lexical-model-count + + NSStringLocalizedFormatKey + %#@count@ + count + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + u + one + Τὸ λεξικὸ %u ἐγκατεστάθη + other + %u λεξικὰ ἐγκατεστάθησαν + + + notification-update-failed + + NSStringLocalizedFormatKey + %#@count@ + count + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + u + one + %u ἐνημέρωση ἀπέτυχε + other + %u ἐνημερώσεις ἀπέτυχαν + + + notification-update-success + + NSStringLocalizedFormatKey + %#@count@ + count + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + u + one + %u ἐπιτυχὴς ἐνημέρωση + other + %u ἐπιτυχεῖς ἐνημερώσεις + + + settings-keyboards-installed-count + + NSStringLocalizedFormatKey + %#@count@ + count + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + u + one + %u ἐγκατεστημένο πληκτρολόγιο + other + %u ἐγκατεστημένα πληκτρολόγια + + + settings-languages-installed-count + + NSStringLocalizedFormatKey + %#@count@ + count + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + u + one + %u ἐγκατεστημένη γλῶσσα + other + %u ἐγκατεστημένες γλῶσσες + + + package-default-found-keyboards + + NSStringLocalizedFormatKey + %#@count@ + count + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + u + one + Εὑρέθη %u πληκτρολόγιο στὸ πακέτο: + other + Εὑρέθησαν %u πληκτρολόγια στὸ πακέτο: + + + package-default-found-lexical-models + + NSStringLocalizedFormatKey + %#@count@ + count + + NSStringFormatSpecTypeKey + NSStringPluralRuleType + NSStringFormatValueTypeKey + u + one + Εὑρέθη %u λεξικὸ στὸ πακέτο: + other + Εὑρέθησαν %u λεξικὰ στὸ πακέτο: + + + + From d70928ca2742376df9a300b67a32549a65cd41f4 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Mon, 24 Jun 2024 07:48:44 -0500 Subject: [PATCH 16/40] fix(ios): Use el instead of el-polyton locale --- crowdin.yml | 4 + .../Classes/el.lproj/ResourceInfoView.strings | 2 + .../KeymanEngine/el.lproj/Localizable.strings | 257 ++++++++++++++++++ .../Localizable.stringsdict | 0 .../Keyman/el.lproj/Localizable.strings | 81 ++++++ 5 files changed, 344 insertions(+) create mode 100644 ios/engine/KMEI/KeymanEngine/Classes/el.lproj/ResourceInfoView.strings create mode 100644 ios/engine/KMEI/KeymanEngine/el.lproj/Localizable.strings rename ios/engine/KMEI/KeymanEngine/{el-polyton.lproj => el.lproj}/Localizable.stringsdict (100%) create mode 100644 ios/keyman/Keyman/Keyman/el.lproj/Localizable.strings diff --git a/crowdin.yml b/crowdin.yml index d96609f7b8..3c95e40186 100644 --- a/crowdin.yml +++ b/crowdin.yml @@ -81,6 +81,7 @@ files: languages_mapping: osx_code: pt-PT: pt-PT.lproj + el-polyton: el.lproj - source: /ios/engine/KMEI/KeymanEngine/en.lproj/Localizable.strings dest: /ios/engine/Localizable.strings @@ -88,6 +89,7 @@ files: languages_mapping: osx_code: pt-PT: pt-PT.lproj + el-polyton: el.lproj - source: /ios/engine/KMEI/KeymanEngine/en.lproj/Localizable.stringsdict dest: /ios/engine/Localizable.stringsdict @@ -95,6 +97,7 @@ files: languages_mapping: osx_code: pt-PT: pt-PT.lproj + el-polyton: el.lproj - source: /ios/keyman/Keyman/Keyman/en.lproj/Localizable.strings dest: /ios/app/Localizable.strings @@ -102,6 +105,7 @@ files: languages_mapping: osx_code: pt-PT: pt-PT.lproj + el-polyton: el.lproj # Linux files diff --git a/ios/engine/KMEI/KeymanEngine/Classes/el.lproj/ResourceInfoView.strings b/ios/engine/KMEI/KeymanEngine/Classes/el.lproj/ResourceInfoView.strings new file mode 100644 index 0000000000..6288e1237c --- /dev/null +++ b/ios/engine/KMEI/KeymanEngine/Classes/el.lproj/ResourceInfoView.strings @@ -0,0 +1,2 @@ +/* Class = "UILabel"; text = "Scan this code to load this keyboard on another device"; ObjectID = "z2O-MT-IoV"; */ +"z2O-MT-IoV.text" = "Σαρῶστε αὐτὸν τὸν κωδικό γιὰ νὰ φορτώσετε αὐτὸ τὸ πληκτρολόγιο σὲ ἄλλη συσκευή"; diff --git a/ios/engine/KMEI/KeymanEngine/el.lproj/Localizable.strings b/ios/engine/KMEI/KeymanEngine/el.lproj/Localizable.strings new file mode 100644 index 0000000000..47b3aa9de0 --- /dev/null +++ b/ios/engine/KMEI/KeymanEngine/el.lproj/Localizable.strings @@ -0,0 +1,257 @@ +/* A descriptive message used for errors when the app is already busy downloading */ +"alert-download-error-busy" = "Λήψη ἤδη σὲ ἐξέλιξη."; + +/* A descriptive message used when a download fails */ +"alert-download-error-detail" = "Παρουσιάσθηκε σφάλμα κατὰ τὴν λήψη ἢ ἐγκατάσταση."; + +/* Title for a "download failed" alert */ +"alert-download-error-title" = "Σφάλμα κατὰ τὴν λήψη"; + +/* Title for a general alert about errors */ +"alert-error-title" = "Σφάλμα"; + +/* A descriptive message used when no internet connection is detected */ +"alert-no-connection-detail" = "Ἀδυναμία προσβάσεως στὸν διακομιστὴ Κῆμαν. Παρακαλοῦμε ξαναδοκιμάστε ἀργότερα."; + +/* Title for a "no connection" alert */ +"alert-no-connection-title" = "Σφάλμα συνδέσεως"; + +/* Short text for a 'back' navigational command. Used to return to the previous screen */ +"command-back" = "Πίσω"; + +/* Short text for a 'cancel' command. Used to back out of a menu or install process without making changes */ +"command-cancel" = "Ἀκυρῶστε"; + +/* Short text for a 'done' command. Used to back out of settings menus after changes have been made */ +"command-done" = "Ἕτοιμοι"; + +/* Short text for an 'install' command. Used to confirm installation of a package. */ +"command-install" = "Ἐγκαταστῆστε"; + +/* Short text for a 'next' navigational command. Used to advance to the next screen */ +"command-next" = "Ἑπόμενο"; + +/* Short text for an 'OK' command. Used for some error alerts */ +"command-ok" = "OK"; + +/* Short text for confirmining 'uninstall' commands. */ +"command-uninstall" = "Ἀπεγκαταστῆστε"; + +/* Text for the command to uninstall a keyboard */ +"command-uninstall-keyboard" = "Ἀπεγκαταστῆστε πληκτρολόγιο"; + +/* Confirmation text to display before uninstalling a keyboard */ +"command-uninstall-keyboard-confirm" = "Θὰ θέλατε νὰ ἀπεγκαταστήσετε αὐτὸ τὸ πληκτρολόγιο;"; + +/* Text for the command to uninstall a lexical model */ +"command-uninstall-lexical-model" = "Ἀπεγκαταστῆτε λεξικό"; + +/* Confirmation text to display before uninstalling a lexical model */ +"command-uninstall-lexical-model-confirm" = "Θὰ θέλατε νὰ ἀπεγκαταστήσετε αὐτὸ τὸ λεξικό;"; + +/* Text for error when a keyboard cannot load properly */ +"error-loading-keyboard" = "Ἀδυνατοῦμε νὰ φορτώσομε τὸ πληκτρολόγιο ποὺ ζητήσατε"; + +/* Text for error when a lexical model cannot load properly */ +"error-loading-lexical-model" = "Ἀδυνατοῦμε νὰ φορτώσομε τὸ λεξικὸ ποὺ ζητήσατε"; + +/* Text for error when an installed file is unexpectedly missing */ +"error-missing-file" = "Δὲν βρέθηκε κάποιο ἀπαραίτητο ἀρχεῖο."; + +/* Text for error when an installed file is unexpectedly missing */ +"error-missing-file-critical" = "Κάποιο σημαντικὸ ἀρχεῖο δὲν εὑρέθη. Παρακαλοῦμε προσπαθῆστε νὰ ἐγκαταστήσετε ἐκ νέου τὴν ἐφαρμογή."; + +/* Text for errors in data received from search queries */ +"error-query-decoding" = "Αὐτὴν τὴν στιγμὴ ὑπάρχουν τεχνικὰ προβλήματα στὸν διακομιστή"; + +/* A descriptive message used when a download fails */ +"error-query-general" = "Σημειώθηκε σφάλμα κατὰ τὴν ἐπικοινωνία μὲ τὸν διακομιστή"; + +/* Text for error when a search query is unexpectedly empty */ +"error-query-no-data" = "Ὁ διακομιστὴς δὲν ἀποκρίθηκε"; + +/* Text for general errors where not much information is known */ +"error-unknown" = "Σημειώθηκε ἀπροσδόκητο σφάλμα"; + +/* Text for error when updating a resource: cannot download because no source is available */ +"error-update-no-link" = "Δὲν ὑπάρχει πηγαῖος κώδικας πρὸς ἐνημέρωσιν αὐτοῦ τοῦ πακέτου"; + +/* Text for error when updating a resource: Keyman does not know how to update the resource */ +"error-update-not-managed" = "Ἀδυνατοῦμε νὰ ἐνημερώσομε πόρο μὴ διαχειριζόμενο ἀπὸ τὴν μηχανὴ Κῆμαν"; + +/* Text for the command to open the help page of a keyboard, as shown on resource information views */ +"info-command-help-keyboard" = "Βοήθεια γιὰ τὸ πληκτρολόγιο"; + +/* Text for the command to open the help page of a lexical model, as shown on resource information views */ +"info-command-help-lexical-model" = "Βοήθεια γιὰ τὸ λεξικό"; + +/* Text label for displaying a keyboard's version, as shown on resource information views */ +"info-label-version-keyboard" = "Ἔκδοση πληκτρολογίου"; + +/* Text label for displaying a lexical model's version, as shown on resource information views */ +"info-label-version-lexical-model" = "Ἔκδοση λεξικοῦ"; + +/* Label used for the "package info" / readme tab with the package installation prompt on phones. */ +"installer-label-package-info" = "Πληροφορίες πακέτου"; + +/* Label used for the language-selection tab with the package installation prompt on phones. */ +"installer-label-select-languages" = "Ἐπιλέξτε γλῶσσα(-ες)"; + +/* Label used with a package's version, as seen within the package installation prompt. Example: "Version: 14.0.0" */ +"installer-label-version" = "Ἔκδοση: %@"; + +/* Section header for languages supported by a package, as seen within the package installation prompt */ +"installer-section-available-languages" = "Διαθέσιμες γλῶσσες"; + +/* Text for the help popup for changing keyboards with the globe key */ +"keyboard-help-change" = "Κτυπῆστε ἐδῶ γιὰ νὰ ἀλλάξετε πληκτρολόγιο"; + +/* Text for the command to exit the Keyman keyboard in favor of other keyboards installed on the system */ +"keyboard-menu-exit" = "Κλεῖστε τὸ %@"; + +/* Error installing a Keyman package - could not allocate a location to install it */ +"kmp-error-file-system" = "Σφάλμα κατά τὴν ἐγκατάσταση πακέτου - σφάλμα συστήματος ἀρχείων"; + +/* Error installing a Keyman package - could not copy package files */ +"kmp-error-file-copying" = "Σφάλμα κατά τὴν ἐγκατάσταση πακέτου - ἀδυναμία ἀντιγραφῆς ἀπαραιτήτων ἀρχείων"; + +/* Error opening a Keyman package - package is not valid */ +"kmp-error-invalid" = "Κατεστραμμένο ἀρχεῖο πακέτου."; + +/* Error opening a Keyman package - it does not exist / the specified location is wrong */ +"kmp-error-missing" = "Τὸ πακέτο ποὺ ὁρίσατε δὲν ὑπάρχει."; + +/* Error installing a Keyman package - expected resource (keyboard or dictionary) is missing */ +"kmp-error-missing-resource" = "Αὐτὸ τὸ πακέτο δὲν περιέχει τὸ πληκτρολόγιο ἢ λεξικὸ ποὺ ζητήσατε."; + +/* Error installing a Keyman package with a version of Keyman that does not support it */ +"kmp-error-unsupported-keyman-version" = "Αὐτὸ τὸ πακέτο ἀπαιτεῖ νεώτερη ἔκδοση τοῦ Κῆμαν."; + +/* Error opening a Keyman package - cannot parse contents */ +"kmp-error-no-metadata" = "Αὐτὸ τὸ πακέτο δὲν ἔχει κατασκευασθεῖ σωστά - ὑπάρχουν ἄγνωστα περιεχόμενα."; + +/* Error opening a Keyman package - package's contents are for desktop platforms only */ +"kmp-error-unsupported" = "Αὐτὸ τὸ πακέτο δὲν συμπεριλαμβάνει ὑποστήριξη γιὰ τὴν συσκευή σας."; + +/* Error opening a Keyman package - package contains unexpected resource (keyboard or dictionary) type */ +"kmp-error-wrong-type" = "Αὐτὸ τὸ πακέτο δὲν περιέχει τὸν ἀναμενόμενο τύπο πόρου."; + +/* Title for the Installed Languages menu */ +"menu-installed-languages-title" = "Ἐγκατεστημένες γλῶσσες"; + +/* Section header for lexical models within a language-specific settings menu */ +"menu-langsettings-label-lexical-models" = "Λεξικά"; + +/* Section header for keyboards within a language-specific settings menu */ +"menu-langsettings-section-keyboards" = "Πληκτρολόγια"; + +/* Section header for the settings toggles within a language-specific settings menu */ +"menu-langsettings-section-settings" = "Ρυθμίσεις γλώσσας"; + +/* Title for the language-specific settings menus */ +"menu-langsettings-title" = "Ρυθμίσεις %@"; + +/* Label for the toggle that enables corrections that is displayed within a language-specific settings menu */ +"menu-langsettings-toggle-correct" = "Ἐνεργοποιῆστε τὶς διορθώσεις"; + +/* Label for the toggle that enables predictions that is displayed within a language-specific settings menu */ +"menu-langsettings-toggle-predict" = "Ἐνεργοποιῆστε προβλέψεις"; + +/* Help message for a prompt that appears for confirming a lexical model download: language (1): lexical model (dictionary) name (2) */ +"menu-lexical-model-install-message" = "Θὰ θέλατε νὰ ἐγκαταστήσετε αὐτὸ τὸ λεξικό;"; + +/* Title for a prompt that appears for confirming a lexical model download: language (1): lexical model (dictionary) name (2) */ +"menu-lexical-model-install-title" = "%1$@: %2$@"; + +/* Text for an info alert indicating that no lexical models are available */ +"menu-lexical-model-none-message" = "Δὲν ὑπάρχουν διαθέσιμα λεξικά"; + +/* Title for the lexical model menu, a submenu of the language-specific settings menus */ +"menu-lexical-model-title" = "%@ Λεξικά"; + +/* Title for the keyboard picker */ +"menu-picker-title" = "Πληκτρολόγια"; + +/* Primary text for the Settings menu option to report keyboard crashes */ +"menu-settings-error-report" = "Ἐπιτρέψτε ἀναφορὲς σφαλμάτων"; + +/* Secondary text for the Settings menu option to report keyboard crashes */ +"menu-settings-error-report-description" = "Μπορεῖ νὰ ἀπαιτηθεῖ \"πλήρης πρόσβαση\""; + +/* Primary text for the Settings menu local-file package installation option */ +"menu-settings-install-from-file" = "Ἐγκαταστῆστε ἀπὸ ἀρχεῖο"; + +/* Secondary text for the Settings menu local-file package installation option */ +"menu-settings-install-from-file-description" = "Ψάξτε ἀρχεῖα .kmp"; + +/* Primary text for the Settings menu option that displays the iOS system menu options for the app's keyboard */ +"menu-settings-system-keyboard-menu" = "Ρυθμίσεις πληκτρολογίου συστήματος"; + +/* Label for the "Show Banner" toggle on the main settings screen */ +"menu-settings-show-banner" = "Νὰ ἐμφανίζεται Banner"; + +/* Label for the "Get Started" automatic display toggle seen in the Settings menu */ +"menu-settings-startup-get-started" = "Νὰ ἐμφανίζεται ἡ ἐναρκτήρια ὀθόνη κατὰ τὴν ἐκκίνηση"; + +/* Title for the main Settings menu */ +"menu-settings-title" = "Ρυθμίσεις Κῆμαν"; + +/* Secondary text showing current setting for spacebar caption - blank */ +"menu-settings-spacebar-hint-blank" = "Καμμία λεζάντα στὸ πλῆκτρο διαστήματος"; + +/* Secondary text showing current setting for spacebar caption - keyboard */ +"menu-settings-spacebar-hint-keyboard" = "Νὰ ἐμφανίζεται στὸ στὸ πλῆκτρο διαστήματος τὸ ὄνομα τοῦ πληκτρολογίου"; + +/* Secondary text showing current setting for spacebar caption - language */ +"menu-settings-spacebar-hint-language" = "Νὰ ἐμφανίζεται στὸ στὸ πλῆκτρο διαστήματος τὸ ὄνομα τῆς γλώσσας"; + +/* Secondary text showing current setting for spacebar caption - language + keyboard */ +"menu-settings-spacebar-hint-languageKeyboard" = "Νὰ ἐμφανίζεται στὸ στὸ πλῆκτρο διαστήματος τὸ ὄνομα τοῦ πληκτρολογίου καὶ τῆς γλώσσας"; + +/* Label for the "Spacebar Caption" item on the main settings screen */ +"menu-settings-spacebar-text" = "Λεζάντα Πλήκτρου Διαστήματος"; + +/* Title for the "Spacebar Caption" settings screen */ +"menu-settings-spacebar-title" = "Λεζάντα Πλήκτρου Διαστήματος"; + +/* Text showing name of spacebar caption - blank */ +"menu-settings-spacebar-item-blank" = "Κενό"; + +/* Text showing name of spacebar caption - keyboard */ +"menu-settings-spacebar-item-keyboard" = "Πληκτρολόγιο"; + +/* Text showing name of spacebar caption - language */ +"menu-settings-spacebar-item-language" = "Γλῶσσα"; + +/* Text showing name of spacebar caption - language + keyboard */ +"menu-settings-spacebar-item-languageKeyboard" = "Γλῶσσα καὶ πληκτρολόγιο"; + +/* Short text for notification: download failure for keyboard */ +"notification-download-failure-keyboard" = "Ἡ μεταφόρτωση τοῦ πληκτρολογίου ἀπέτυχε"; + +/* Short text for notification: download failure for lexical model */ +"notification-download-failure-lexical-model" = "Ἡ μεταφόρτωση τοῦ λεξικοῦ ἀπέτυχε"; + +/* Short text for notification: download success for keyboard */ +"notification-download-success-keyboard" = "Τὸ πληκτρολόγιο μεταφορτώθηκε ἐπιτυχῶς"; + +/* Short text for notification: download success for lexical model */ +"notification-download-success-lexical-model" = "Τὸ λεξιλογικὸ μοντέλλο μεταφορτώθηκε ἐπιτυχῶς"; + +/* Short text for notification: downloading a keyboard */ +"notification-downloading-keyboard" = "Μεταφορτώνεται πληκτρολόγιο\U2026"; + +/* Short text for notification: downloading a lexical model */ +"notification-downloading-lexical-model" = "Μεταφορτώνεται λεξικό\U2026"; + +/* Short text for notification: an update is available */ +"notification-update-available" = "Διαθέσιμη Ἐνημέρωση"; + +/* Short text for notification: currently updating */ +"notification-update-processing" = "Ἐνημέρωση σὲ ἐξέλιξη\U2026"; + +/* Text indicating success at installing new keyboards or dictionaries */ +"success-install" = "Ἐπιτυχὴς ἐγκατάσταση."; + +/* A title to use in alerts indicating 'success' at whatever task the user requested */ +"success-title" = "Ἐπιτυχία"; diff --git a/ios/engine/KMEI/KeymanEngine/el-polyton.lproj/Localizable.stringsdict b/ios/engine/KMEI/KeymanEngine/el.lproj/Localizable.stringsdict similarity index 100% rename from ios/engine/KMEI/KeymanEngine/el-polyton.lproj/Localizable.stringsdict rename to ios/engine/KMEI/KeymanEngine/el.lproj/Localizable.stringsdict diff --git a/ios/keyman/Keyman/Keyman/el.lproj/Localizable.strings b/ios/keyman/Keyman/Keyman/el.lproj/Localizable.strings new file mode 100644 index 0000000000..2df136fccc --- /dev/null +++ b/ios/keyman/Keyman/Keyman/el.lproj/Localizable.strings @@ -0,0 +1,81 @@ +/* Title for the bookmarks menu within the embedded browser */ +"browser-bookmarks-add-title" = "Προσθέστε σελιδοδείκτη"; + +/* Title for the bookmarks menu within the embedded browser */ +"browser-bookmarks-none" = "Δὲν ὑπάρχουν σελιδοδεῖκτες"; + +/* Title for the bookmarks menu within the embedded browser */ +"browser-bookmarks-title" = "Σελιδοδεῖκτες"; + +/* Used to confirm a user's desire to install a package */ +"confirm-install" = "Ἐγκαταστῆστε"; + +/* The label for the toggle to stop automatically showing the "Get Started" tutorial popup. */ +"disable-get-started" = "Νὰ μὴν ξαναεμφανισθεῖ"; + +/* Indicates an error opening a web page requested by a user */ +"error-opening-page" = "Ἀδυναμία ἀνοίγματος σελίδας"; + +/* Long-form used when installing a font in order to display a language properly. */ +"font-install-description" = "Πατῆστε τὸ πλῆκτρο ἐγκατάστασης γιὰ νὰ ἐμφανίζεται ἡ %@ σωστὰ σὲ ὅλες τὶς ἐφαρμογές σας"; + +/* The name of the iOS Settings menu option for keyboards */ +"ios-settings-keyboards" = "Πληκτρολόγια"; + +/* The name of the iOS Settings menu option for giving the keyboard full access. (The menu entry +underneath the app's name within the app-specific keyboard menu.) */ +"ios-settings-allow-full-access" = "Ἐπιστρέψτε πλήρη πρόσβαση"; + +/* Short-form used when installing a font in order to display a language properly. */ +"language-for-font" = "Γραμματοσειρά %@"; + +/* Used for 'add' options within menus */ +"menu-add" = "Προσθήκη"; + +/* Used to indicate a sequence of menu options that a user needs to copy. May be chained. Example: "Keyboards (1) > Enable Keyman (2)" */ +"menu-breadcrumbing" = "%1$@ > %2$@"; + +/* Used to exit a menu without choosing an option */ +"menu-cancel" = "Ἀκύρωση"; + +/* Menu option that erases all previously-typed text. */ +"menu-clear-text" = "Διαγραφὴ κειμένου"; + +/* Menu option that displays a list designed to help users start using the app */ +"menu-get-started" = "Ξεκινῆστε"; + +/* Menu option that displays help for the app */ +"menu-help" = "Πληροφορίες"; + +/* Menu option that displays the main screen's drop-down menu */ +"menu-more" = "Περισσότερα"; + +/* Menu option used to edit keyboard and dictionary settings */ +"menu-settings" = "Ρυθμίσεις"; + +/* Menu option that displays the iOS Share menu */ +"menu-share" = "Μοιρασθῆτε"; + +/* Menu option that displays an embedded web browser. */ +"menu-show-browser" = "Φυλλομετρητής"; + +/* Menu option used to control in-app font size. */ +"menu-text-size" = "Μέγεθος κειμένου"; + +/* Used to describe the current font size */ +"text-size-label" = "Μέγεθος κειμένου: %i"; + +/* Text to indicate that a user should set a toggle within iOS Settings to 'active'. */ +"toggle-to-enable" = "Ἐνεργοποιῆστε %@"; + +/* First option on the Get Started tutorial - Add a keyboard for your language */ +"tutorial-add-keyboard" = "Προσθέστε πληκτρολόγιο γιὰ τὴν γλῶσσα σας"; + +/* Third option on the Get Started tutorial - Displays app help. */ +"tutorial-show-help" = "Περισσότερες πληροφορίες"; + +/* Second option on the Get Started tutorial - Set up Keyman as system-wide keyboard */ +"tutorial-system-keyboard" = "Ὁρίστε τὸ Κῆμαν ὡς παν-συστημικὸ πληκτρολόγιο"; + +/* Used to display app version (as in \"Version: 1.0.2\" */ +"version-label" = "Ἔκδοση: %@"; From 9f5719011ff42d01aa6e9aa8389e55b3ad3c844f Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 25 Jun 2024 08:20:49 +0700 Subject: [PATCH 17/40] chore(ios): link 'el' localization data to iOS engine + app --- ios/engine/KMEI/KeymanEngine.xcodeproj/project.pbxproj | 7 +++++++ ios/keyman/Keyman/Keyman.xcodeproj/project.pbxproj | 5 ++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/ios/engine/KMEI/KeymanEngine.xcodeproj/project.pbxproj b/ios/engine/KMEI/KeymanEngine.xcodeproj/project.pbxproj index eaacec0439..d1c711368f 100644 --- a/ios/engine/KMEI/KeymanEngine.xcodeproj/project.pbxproj +++ b/ios/engine/KMEI/KeymanEngine.xcodeproj/project.pbxproj @@ -493,6 +493,9 @@ CEA9670A24BEC8030035AACF /* EngineStateBundler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EngineStateBundler.swift; sourceTree = ""; }; CEA9670C24BEEFF80035AACF /* khmer_angkor update-base.bundle */ = {isa = PBXFileReference; lastKnownFileType = "wrapper.plug-in"; path = "khmer_angkor update-base.bundle"; sourceTree = ""; }; CEA9670E24BEF05A0035AACF /* Updates.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Updates.swift; sourceTree = ""; }; + CEAC2A222C2A503600C99ABD /* el */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = el; path = el.lproj/ResourceInfoView.strings; sourceTree = ""; }; + CEAC2A232C2A505700C99ABD /* el */ = {isa = PBXFileReference; lastKnownFileType = text.plist.stringsdict; name = el; path = el.lproj/Localizable.stringsdict; sourceTree = ""; }; + CEAC2A242C2A508000C99ABD /* el */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = el; path = el.lproj/Localizable.strings; sourceTree = ""; }; CEACC90125F07C81006EAB45 /* de */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = de; path = de.lproj/ResourceInfoView.strings; sourceTree = ""; }; CEACC90325F07CAF006EAB45 /* de */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = de; path = de.lproj/Localizable.strings; sourceTree = ""; }; CEACC90425F07CB6006EAB45 /* km */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = km; path = km.lproj/Localizable.strings; sourceTree = ""; }; @@ -1205,6 +1208,7 @@ uk, ru, es, + el, ); mainGroup = F243887314BBD43000A3E055; productRefGroup = F243887F14BBD43000A3E055 /* Products */; @@ -1602,6 +1606,7 @@ 298566D32980D39F004ACA95 /* uk */, 298566DF2980E48C004ACA95 /* ru */, CEFFECD72A417F2E00D58C36 /* es */, + CEAC2A232C2A505700C99ABD /* el */, ); name = Localizable.stringsdict; sourceTree = ""; @@ -1630,6 +1635,7 @@ 298566D22980D39A004ACA95 /* uk */, 298566DE2980E486004ACA95 /* ru */, CEFFECD62A417F2900D58C36 /* es */, + CEAC2A242C2A508000C99ABD /* el */, ); name = Localizable.strings; sourceTree = ""; @@ -1659,6 +1665,7 @@ 298566D12980D390004ACA95 /* uk */, 298566DD2980E477004ACA95 /* ru */, CEFFECD52A417F1300D58C36 /* es */, + CEAC2A222C2A503600C99ABD /* el */, ); name = ResourceInfoView.xib; sourceTree = ""; diff --git a/ios/keyman/Keyman/Keyman.xcodeproj/project.pbxproj b/ios/keyman/Keyman/Keyman.xcodeproj/project.pbxproj index fdeae450f0..6211f26573 100644 --- a/ios/keyman/Keyman/Keyman.xcodeproj/project.pbxproj +++ b/ios/keyman/Keyman/Keyman.xcodeproj/project.pbxproj @@ -3,7 +3,7 @@ archiveVersion = 1; classes = { }; - objectVersion = 52; + objectVersion = 54; objects = { /* Begin PBXBuildFile section */ @@ -332,6 +332,7 @@ CE7FF1EF239A0293007859D9 /* PackageBrowserViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PackageBrowserViewController.swift; sourceTree = ""; }; CE80AD32257F2B4A008D2150 /* KeymanEngine.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; path = KeymanEngine.framework; sourceTree = BUILT_PRODUCTS_DIR; }; CE98E6BD2615593300F3F2C0 /* ff */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = ff; path = ff.lproj/Localizable.strings; sourceTree = ""; }; + CEAC2A272C2A50E100C99ABD /* el */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = el; path = el.lproj/Localizable.strings; sourceTree = ""; }; CEACC90E25F07D58006EAB45 /* fr */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = fr; path = fr.lproj/Localizable.strings; sourceTree = ""; }; CEACC90F25F07D5A006EAB45 /* km */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = km; path = km.lproj/Localizable.strings; sourceTree = ""; }; CEACC91225F07D77006EAB45 /* de */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = de; path = de.lproj/Localizable.strings; sourceTree = ""; }; @@ -793,6 +794,7 @@ sv, uk, ru, + el, ); mainGroup = 98ABADA7176935E400B62590; productRefGroup = 98ABADB1176935E400B62590 /* Products */; @@ -1106,6 +1108,7 @@ 298566C42980C493004ACA95 /* sv */, 298566D02980D354004ACA95 /* uk */, 298566DC2980E416004ACA95 /* ru */, + CEAC2A272C2A50E100C99ABD /* el */, ); name = Localizable.strings; sourceTree = ""; From 3423c63bc0a16ddcee6a6021327ea5360be6aa62 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 26 Jun 2024 11:48:20 +0700 Subject: [PATCH 18/40] change(web): avoid use of Array.from with iterators (in TrieModel.predict) --- common/models/templates/src/trie-model.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/common/models/templates/src/trie-model.ts b/common/models/templates/src/trie-model.ts index 605bbc7c3f..cbbe918162 100644 --- a/common/models/templates/src/trie-model.ts +++ b/common/models/templates/src/trie-model.ts @@ -494,7 +494,11 @@ function getSortedResults(traversal: LexiconTraversal, limit = MAX_SUGGESTIONS): } else { const traversal = entry as LexiconTraversal; queue.enqueueAll(traversal.entries); - queue.enqueueAll(Array.from(traversal.children()).map((entry) => entry.traversal())); + let children: LexiconTraversal[] = [] + for(let child of traversal.children()) { + children.push(child.traversal()); + } + queue.enqueueAll(children); } } From ba5b0f6c911a61c17295211030b3f9ec2ec6efbd Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 24 Jun 2024 14:32:20 +0700 Subject: [PATCH 19/40] change(web): track the base correction when generating predictions --- .../lm-worker/src/main/model-compositor.ts | 117 +++++++++++------- .../src/test/mocha/cases/worker-predict.js | 13 +- 2 files changed, 83 insertions(+), 47 deletions(-) diff --git a/common/web/lm-worker/src/main/model-compositor.ts b/common/web/lm-worker/src/main/model-compositor.ts index b2d85b4ded..befdae1826 100644 --- a/common/web/lm-worker/src/main/model-compositor.ts +++ b/common/web/lm-worker/src/main/model-compositor.ts @@ -4,6 +4,12 @@ import * as correction from './correction/index.js' import TransformUtils from './transformUtils.js'; +type CorrectionPredictionTuple = { + prediction: ProbabilityMass, + correction: ProbabilityMass, + totalProb: number; +}; + export default class ModelCompositor { private lexicalModel: LexicalModel; private contextTracker?: correction.ContextTracker; @@ -39,9 +45,13 @@ export default class ModelCompositor { private SUGGESTION_ID_SEED = 0; - private testMode: boolean = false + private testMode: boolean = false; + private verbose: boolean = true; - constructor(lexicalModel: LexicalModel, testMode?: boolean) { + constructor( + lexicalModel: LexicalModel, + testMode?: boolean + ) { this.lexicalModel = lexicalModel; if(lexicalModel.traverseFromRoot) { this.contextTracker = new correction.ContextTracker(); @@ -50,23 +60,32 @@ export default class ModelCompositor { this.testMode = !!testMode; } - private predictFromCorrections(corrections: ProbabilityMass[], context: Context): Distribution { - let returnedPredictions: Distribution = []; + private predictFromCorrections(corrections: ProbabilityMass[], context: Context): CorrectionPredictionTuple[] { + let returnedPredictions: CorrectionPredictionTuple[] = []; for(let correction of corrections) { let predictions = this.lexicalModel.predict(correction.sample, context); + const { sample: correctionTransform, p: correctionProb } = correction; + const correctionRoot = this.wordbreak(models.applyTransform(correction.sample, context)); + let predictionSet = predictions.map(function(pair: ProbabilityMass) { - let transform = correction.sample; - let inputProb = correction.p; + // Let's not rely on the model to copy transform IDs. // Only bother is there IS an ID to copy. - if(transform.id !== undefined) { - pair.sample.transformId = transform.id; + if(correctionTransform.id !== undefined) { + pair.sample.transformId = correctionTransform.id; } - let prediction = {sample: pair.sample, p: pair.p * inputProb}; - return prediction; + let tuple: CorrectionPredictionTuple = { + prediction: pair, + correction: { + sample: correctionRoot, + p: correctionProb + }, + totalProb: pair.p * correctionProb + }; + return tuple; }, this); returnedPredictions = returnedPredictions.concat(predictionSet); @@ -76,7 +95,7 @@ export default class ModelCompositor { } async predict(transformDistribution: Transform | Distribution, context: Context): Promise { - let suggestionDistribution: Distribution = []; + let suggestionDistribution: CorrectionPredictionTuple[] = []; let lexicalModel = this.lexicalModel; let punctuation = this.punctuation; @@ -119,7 +138,7 @@ export default class ModelCompositor { let keepOptionText = this.wordbreak(postContext); let keepOption: Outcome = null; - let rawPredictions: Distribution = []; + let rawPredictions: CorrectionPredictionTuple[] = []; // Used to restore whitespaces if operations would remove them. let prefixTransform: Transform; @@ -318,10 +337,10 @@ export default class ModelCompositor { // If we're getting the same prediction again, it's lower-cost. Update! let oldPredictionSet = correctionPredictionMap[match.matchString]; if(oldPredictionSet) { - rawPredictions = rawPredictions.filter((entry) => !oldPredictionSet.find((match) => entry == match)) + rawPredictions = rawPredictions.filter((entry) => !oldPredictionSet.find((match) => entry.prediction.sample == match.sample)); } - correctionPredictionMap[match.matchString] = predictions; + correctionPredictionMap[match.matchString] = predictions.map((entry) => entry.prediction); rawPredictions = rawPredictions.concat(predictions); @@ -337,13 +356,13 @@ export default class ModelCompositor { } else { // Sort the prediction list; we need them in descending order for the next check. rawPredictions.sort(function(a, b) { - return b.p - a.p; + return b.totalProb - a.totalProb; }); // If the best suggestion from the search's current tier fails to beat the worst // pending suggestion from previous tiers, assume all further corrections will // similarly fail to win; terminate the search-loop. - if(rawPredictions[ModelCompositor.MAX_SUGGESTIONS-1].p > Math.exp(-correctionCost)) { + if(rawPredictions[ModelCompositor.MAX_SUGGESTIONS-1].totalProb > Math.exp(-correctionCost)) { break; } } @@ -361,7 +380,7 @@ export default class ModelCompositor { // Section 2 - post-analysis for our generated predictions, managing 'keep'. // Assumption: Duplicated 'displayAs' properties indicate duplicated Suggestions. // When true, we can use an 'associative array' to de-duplicate everything. - let suggestionDistribMap: {[key: string]: ProbabilityMass} = {}; + let suggestionDistribMap: {[key: string]: CorrectionPredictionTuple} = {}; let currentCasing: CasingForm = null; if(lexicalModel.languageUsesCasing) { currentCasing = this.detectCurrentCasing(postContext); @@ -370,9 +389,12 @@ export default class ModelCompositor { let baseWord = this.wordbreak(context); // Deduplicator + annotator of 'keep' suggestions. - for(let prediction of rawPredictions) { + for(let tuple of rawPredictions) { + const prediction = tuple.prediction.sample; + const prob = tuple.totalProb; + // Combine duplicate samples. - let displayText = prediction.sample.displayAs; + let displayText = prediction.displayAs; let preserveAsKeep = displayText == keepOptionText; // De-duplication should be case-insensitive, but NOT @@ -384,7 +406,7 @@ export default class ModelCompositor { if(preserveAsKeep) { // Preserve the original, pre-keyed version of the text. if(!keepOption) { - let baseTransform = prediction.sample.transform; + let baseTransform = prediction.transform; let keepTransform = { insert: keepOptionText, @@ -393,15 +415,15 @@ export default class ModelCompositor { id: baseTransform.id } - let intermediateKeep = models.transformToSuggestion(keepTransform, prediction.p); + let intermediateKeep = models.transformToSuggestion(keepTransform, prob); keepOption = this.toAnnotatedSuggestion(intermediateKeep, 'keep', models.QuoteBehavior.noQuotes); keepOption.matchesModel = true; // Since we replaced the original Suggestion with a keep-annotated one, // we must manually preserve the transform ID. - keepOption.transformId = prediction.sample.transformId; - } else if(keepOption.p && prediction.p) { - keepOption.p += prediction.p; + keepOption.transformId = prediction.transformId; + } else if(keepOption.p && prob) { + keepOption.p += prob; } } else { // Apply capitalization rules now; facilitates de-duplication of suggestions @@ -409,16 +431,16 @@ export default class ModelCompositor { // // Example: "apple" and "Apple" are separate when 'lower', but identical for 'initial' and 'upper'. if(currentCasing && currentCasing != 'lower') { - this.applySuggestionCasing(prediction.sample, baseWord, currentCasing); + this.applySuggestionCasing(prediction, baseWord, currentCasing); // update the mapping string, too. - displayText = prediction.sample.displayAs; + displayText = prediction.displayAs; } let existingSuggestion = suggestionDistribMap[displayText]; if(existingSuggestion) { - existingSuggestion.p += prediction.p; + existingSuggestion.totalProb += prob; } else { - suggestionDistribMap[displayText] = prediction; + suggestionDistribMap[displayText] = tuple; } } } @@ -448,28 +470,31 @@ export default class ModelCompositor { } suggestionDistribution = suggestionDistribution.sort(function(a, b) { - return b.p - a.p; // Use descending order - we want the largest probabilty suggestions first! + return b.totalProb - a.totalProb; // Use descending order - we want the largest probabilty suggestions first! }); - let suggestions = suggestionDistribution.splice(0, ModelCompositor.MAX_SUGGESTIONS).map(function(value) { - let sample: Suggestion & { - p?: number, - "lexical-p"?: number, - "correction-p"?: number - } = value.sample; + let suggestions = suggestionDistribution.splice(0, ModelCompositor.MAX_SUGGESTIONS).map((tuple) => { + const prediction = tuple.prediction; - if(sample['p']) { - // For analysis / debugging - sample['lexical-p'] = sample['p']; - sample['correction-p'] = value.p / sample['p']; - // Use of the Trie model always exposed the lexical model's probability for a word to KMW. - // It's useful for debugging right now, so may as well repurpose it as the posterior. - // - // We still condition on 'p' existing so that test cases aren't broken. - sample['p'] = value.p; + if(!this.verbose) { + return { + ...prediction.sample, + p: tuple.totalProb + }; + } else { + const sample: Suggestion & { + p?: number, + "lexical-p"?: number, + "correction-p"?: number + } = { + ...prediction.sample, + p: tuple.totalProb, + "lexical-p": tuple.prediction.p, + "correction-p": tuple.correction.p + } + + return sample; } - // - return sample; }); if(keepOption) { diff --git a/common/web/lm-worker/src/test/mocha/cases/worker-predict.js b/common/web/lm-worker/src/test/mocha/cases/worker-predict.js index cbc4d433d3..89334aa0b8 100644 --- a/common/web/lm-worker/src/test/mocha/cases/worker-predict.js +++ b/common/web/lm-worker/src/test/mocha/cases/worker-predict.js @@ -74,7 +74,18 @@ describe('LMLayerWorker', function () { sinon.assert.calledWithMatch(fakePostMessage.lastCall, { message: 'suggestions', token: token, - suggestions: hazel[0] + suggestions: hazel[0].map((entry) => { + return { + ...entry, + // Dummy-model predictions all claim probability 1; there's no actual probability stuff + // used here. + 'lexical-p': 1, + // We're predicting from a single transform, not a distribution, so probability 1. + 'correction-p': 1, + // Multiply 'em together. + p: 1, + } + }) }); }); From 34986503c9f0d1a0c3bc0c3243d3d538d0076356 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 27 Jun 2024 09:49:09 +0700 Subject: [PATCH 20/40] fix(web): patch-up automated tests that utilize dummy models The dummy model never set a prediction's probability on the suggestion, so it never emitted p, lexical-p, or correction-p values before this branch's changes. --- .../cases/worker-dummy-integration.spec.ts | 12 ++++++++++++ common/test/resources/model-helpers.mjs | 13 ++++++++++++- .../src/test/mocha/cases/worker-predict.js | 9 +-------- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.spec.ts b/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.spec.ts index a0f7d77290..2ac5f62d00 100644 --- a/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.spec.ts +++ b/common/predictive-text/unit_tests/in_browser/cases/worker-dummy-integration.spec.ts @@ -36,6 +36,18 @@ describe('LMLayer using dummy model', function () { // Since Firefox can't do JSON imports quite yet. const hazelFixture = await fetch(new URL(`${domain}/resources/json/models/future_suggestions/i_got_distracted_by_hazel.json`)); hazelModel = await hazelFixture.json(); + hazelModel = hazelModel.map((set) => set.map((entry) => { + return { + ...entry, + // Dummy-model predictions all claim probability 1; there's no actual probability stuff + // used here. + 'lexical-p': 1, + // We're predicting from a single transform, not a distribution, so probability 1. + 'correction-p': 1, + // Multiply 'em together. + p: 1, + } + })); }); describe('Prediction', function () { diff --git a/common/test/resources/model-helpers.mjs b/common/test/resources/model-helpers.mjs index 1de10bdb15..b2ad083f96 100644 --- a/common/test/resources/model-helpers.mjs +++ b/common/test/resources/model-helpers.mjs @@ -113,7 +113,18 @@ export function randomToken() { } export function iGotDistractedByHazel() { - return jsonFixture('models/future_suggestions/i_got_distracted_by_hazel'); + return jsonFixture('models/future_suggestions/i_got_distracted_by_hazel').map((set) => set.map((entry) => { + return { + ...entry, + // Dummy-model predictions all claim probability 1; there's no actual probability stuff + // used here. + 'lexical-p': 1, + // We're predicting from a single transform, not a distribution, so probability 1. + 'correction-p': 1, + // Multiply 'em together. + p: 1, + } + })); } export function jsonFixture(name, root, import_root) { diff --git a/common/web/lm-worker/src/test/mocha/cases/worker-predict.js b/common/web/lm-worker/src/test/mocha/cases/worker-predict.js index 89334aa0b8..2ef2510d30 100644 --- a/common/web/lm-worker/src/test/mocha/cases/worker-predict.js +++ b/common/web/lm-worker/src/test/mocha/cases/worker-predict.js @@ -76,14 +76,7 @@ describe('LMLayerWorker', function () { token: token, suggestions: hazel[0].map((entry) => { return { - ...entry, - // Dummy-model predictions all claim probability 1; there's no actual probability stuff - // used here. - 'lexical-p': 1, - // We're predicting from a single transform, not a distribution, so probability 1. - 'correction-p': 1, - // Multiply 'em together. - p: 1, + ...entry } }) }); From 60772bfdfa50a5d7a6f63bc1791c101bad2b01a8 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 27 Jun 2024 09:50:40 +0700 Subject: [PATCH 21/40] chore(web): minor cleanup after generalizing fix with prior commit --- common/web/lm-worker/src/test/mocha/cases/worker-predict.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/common/web/lm-worker/src/test/mocha/cases/worker-predict.js b/common/web/lm-worker/src/test/mocha/cases/worker-predict.js index 2ef2510d30..cbc4d433d3 100644 --- a/common/web/lm-worker/src/test/mocha/cases/worker-predict.js +++ b/common/web/lm-worker/src/test/mocha/cases/worker-predict.js @@ -74,11 +74,7 @@ describe('LMLayerWorker', function () { sinon.assert.calledWithMatch(fakePostMessage.lastCall, { message: 'suggestions', token: token, - suggestions: hazel[0].map((entry) => { - return { - ...entry - } - }) + suggestions: hazel[0] }); }); From 7bda1e5d93f419ef8438dfd1a7706ce6953c8266 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 26 Jun 2024 09:59:29 +0700 Subject: [PATCH 22/40] feat(web): determine suggestion to use for auto-correct --- common/models/types/index.d.ts | 5 ++ .../lm-worker/src/main/model-compositor.ts | 62 ++++++++++++++++++- 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/common/models/types/index.d.ts b/common/models/types/index.d.ts index 2fc1c92cb3..af1b17c3ae 100644 --- a/common/models/types/index.d.ts +++ b/common/models/types/index.d.ts @@ -333,6 +333,11 @@ declare interface Suggestion { * to the input text. Ex: 'keep', 'emoji', 'correction', etc. */ tag?: SuggestionTag; + + /** + * Set to true if this suggestion is a valid auto-accept target. + */ + autoAccept?: boolean } interface Reversion extends Suggestion { diff --git a/common/web/lm-worker/src/main/model-compositor.ts b/common/web/lm-worker/src/main/model-compositor.ts index befdae1826..f8416d6c35 100644 --- a/common/web/lm-worker/src/main/model-compositor.ts +++ b/common/web/lm-worker/src/main/model-compositor.ts @@ -69,8 +69,7 @@ export default class ModelCompositor { const { sample: correctionTransform, p: correctionProb } = correction; const correctionRoot = this.wordbreak(models.applyTransform(correction.sample, context)); - let predictionSet = predictions.map(function(pair: ProbabilityMass) { - + let predictionSet = predictions.map((pair: ProbabilityMass) => { // Let's not rely on the model to copy transform IDs. // Only bother is there IS an ID to copy. if(correctionTransform.id !== undefined) { @@ -473,6 +472,16 @@ export default class ModelCompositor { return b.totalProb - a.totalProb; // Use descending order - we want the largest probabilty suggestions first! }); + // Section 4: Auto-correction + finalization. + if(keepOption && keepOption.matchesModel) { + // Auto-select it for auto-acceptance; we don't correct away from perfectly-valid + // lexical entries, even if they are comparatively low-frequency. + keepOption.autoAccept = true; + } else { + this.predictionAutoSelect(suggestionDistribution); + } + + // Now that we've marked the suggestion to auto-select, we can finalize the suggestions. let suggestions = suggestionDistribution.splice(0, ModelCompositor.MAX_SUGGESTIONS).map((tuple) => { const prediction = tuple.prediction; @@ -489,7 +498,7 @@ export default class ModelCompositor { } = { ...prediction.sample, p: tuple.totalProb, - "lexical-p": tuple.prediction.p, + "lexical-p": prediction.p, "correction-p": tuple.correction.p } @@ -559,6 +568,53 @@ export default class ModelCompositor { return suggestions; } + private predictionAutoSelect(suggestionDistribution: CorrectionPredictionTuple[]) { + if(suggestionDistribution.length == 0) { + return; + } + + if(suggestionDistribution.length == 1) { + // Mark for auto-acceptance; there are no alternatives. + suggestionDistribution[0].prediction.sample.autoAccept = true; + return; + } + + // Is it reasonable to auto-accept any of our suggestions? + const bestSuggestion = suggestionDistribution[0]; + + const baseCorrection = bestSuggestion.correction.sample; + if(baseCorrection.length == 0) { + // If the correction is rooted on an empty root, there's no basis for + // auto-correcting to this suggestion. + return; + } + + // Find the highest probability for any correction that led to a valid prediction. + // No need to full-on re-sort everything, though. + const bestCorrection = suggestionDistribution.reduce((prev, current) => prev?.correction.p > current.correction.p ? prev : current, null).correction; + if(bestCorrection.p > bestSuggestion.correction.p) { + // Here, the best suggestion didn't come from the best correction. + // Is it actually reasonable to auto-correct? We're probably just very + // biased toward its frequency. (Maybe a threshold should be considered?) + return; + } + + // compare best vs other probabilities. + const probSum = suggestionDistribution.reduce((accum, current) => accum + current.totalProb, 0); + const proportionOfBest = bestSuggestion.totalProb / probSum; + if(proportionOfBest < .66) { + return; + } + + // compare correction-cost aspects? We disable if the base correction is lower than best, + // but should we do other comparisons too? + + // const nextSuggestion = suggestionDistribution[1]; + // baseCorrection + + bestSuggestion.prediction.sample.autoAccept = true; + } + // Responsible for applying casing rules to suggestions. private applySuggestionCasing(suggestion: Suggestion, baseWord: USVString, casingForm: CasingForm) { // Step 1: does the suggestion replace the whole word? If not, we should extend the suggestion to do so. From 017d0e4521c9c2758f18c96840fece98e46201ac Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 25 Jun 2024 10:07:35 +0700 Subject: [PATCH 23/40] feat(web): apply auto-correct suggestions when applicable --- .../src/text/prediction/languageProcessor.ts | 19 ++++++++------- .../src/text/prediction/predictionContext.ts | 24 +++++++++++++------ .../engine/osk/src/banner/suggestionBanner.ts | 11 +++++++++ 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/common/web/input-processor/src/text/prediction/languageProcessor.ts b/common/web/input-processor/src/text/prediction/languageProcessor.ts index a24938fe27..921f4abf3c 100644 --- a/common/web/input-processor/src/text/prediction/languageProcessor.ts +++ b/common/web/input-processor/src/text/prediction/languageProcessor.ts @@ -19,7 +19,7 @@ export type StateChangeHandler = (state: StateChangeEnum) => any; /** * Covers 'tryaccept' events. */ -export type TryUIHandler = (source: string) => boolean; +export type TryUIHandler = (source: string, returnObj: {shouldSwallow: boolean}) => boolean; export type InvalidateSourceEnum = 'new'|'context'; @@ -168,9 +168,9 @@ export default class LanguageProcessor extends EventEmitter { + private doTryAccept = (source: string, returnObj: {shouldSwallow: boolean}): void => { //let keyman = com.keyman.singleton; if(!this.recentAccept && this.selected) { this.accept(this.selected); - // returnObj.shouldSwallow = true; + returnObj.shouldSwallow = true; + // No need to swallow the next keystroke's whitespace; we triggered FROM a space, + // which this matches. + // Standard applications from the banner, those we DO want to swallow the first time. + this.recentAccept = false; } else if(this.recentAccept && source == 'space') { this.recentAccept = false; - // // If the model doesn't insert wordbreaks, don't swallow the space. If it does, - // // we consider that insertion to be the results of the first post-accept space. - // returnObj.shouldSwallow = !!keyman.core.languageProcessor.wordbreaksAfterSuggestions; // can be handed outside + // If the model doesn't insert wordbreaks, don't swallow the space. If it does, + // we consider that insertion to be the results of the first post-accept space. + returnObj.shouldSwallow = !!this.langProcessor.wordbreaksAfterSuggestions; // can be handed outside } else { - // returnObj.shouldSwallow = false; + returnObj.shouldSwallow = false; } } @@ -270,6 +274,7 @@ export default class PredictionContext extends EventEmitter { // By default, we assume that the context is the same until we notice otherwise. this.initNewContext = false; + this.selected = null; if(!this.swallowPrediction || source == 'context') { this.recentAccept = false; @@ -312,6 +317,7 @@ export default class PredictionContext extends EventEmitter