erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-08-27 10:47:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 12

chore(common/models): defines Token type per review suggestion

Browse source
This commit is contained in:
Joshua A. Horton 2024-08-01 08:28:57 +07:00
parent cfbf973f49
commit e6cf85baf5
3 changed files with 10 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
common/models/templates/src/index.ts
View file

@ -3,5 +3,5 @@ export {
transformToSuggestion, defaultApplyCasing
} from "./common.js";
export { default as QuoteBehavior } from "./quote-behavior.js";
export { Tokenization, tokenize, getLastPreCaretToken, wordbreak } from "./tokenization.js";
export { getLastPreCaretToken, Token, Tokenization, tokenize, wordbreak } from "./tokenization.js";
export { default as TrieModel, TrieModelOptions } from "./trie-model.js";

15
common/models/templates/src/tokenization.ts
View file

@ -1,24 +1,23 @@
// While we _could_ define this within @keymanapp/models-wordbreakers instead, it's probably
// better to leave that package as _just_ the wordbreakers.
export interface Token {
text: string,
isWhitespace?: boolean
}
export interface Tokenization {
/**
* An array of tokens to the left of the caret. If the caret is in the middle of a token,
* only the part to the left of the caret is included.
*/
left: {
text: USVString,
isWhitespace?: boolean
}[],
left: Token[],
/**
* An array of tokens to the right of the caret. If the caret is in the middle of a token,
* only the part to the right of the caret is included.
*/
right: {
text: USVString,
isWhitespace?: boolean
}[],
right: Token[],
/**
* A flag indicating whether or not the caret's position in the context caused a token

4
common/web/lm-worker/src/main/correction/context-tracker.ts
View file

@ -1,4 +1,4 @@
import { applyTransform } from '@keymanapp/models-templates';
import { Token, applyTransform } from '@keymanapp/models-templates';
import { ClassicalDistanceCalculation } from './classical-calculation.js';
import { SearchSpace } from './distance-modeler.js';
@ -290,7 +290,7 @@ class CircularArray<Item> {
export class ContextTracker extends CircularArray<TrackedContextState> {
static attemptMatchContext(
tokenizedContext: { text: USVString, isWhitespace?: boolean } [],
tokenizedContext: Token[],
matchState: TrackedContextState,
transformDistribution?: Distribution<Transform>
): TrackedContextState {