diff --git a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts index 66e634d8af..9a115b3e58 100644 --- a/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts +++ b/web/src/engine/predictive-text/worker-thread/src/main/correction/context-tokenization.ts @@ -1,3 +1,12 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2025-07-30 + * + * Represents cached data about one potential tokenization of contents of + * the sliding context window for one specific instance of context state. + */ + import { ContextToken } from './context-token.js'; import { TrackedContextStateAlignment } from './context-tracker.js'; @@ -36,7 +45,7 @@ export class ContextTokenization { */ get exampleInput(): string[] { return this.tokens - // Hide any tokens representing wordbreaks. (Thinking ahead to phrase-level possibilities) + // Hide any tokens representing invisible wordbreaks. (Thinking ahead to phrase-level possibilities) .filter(token => token.exampleInput !== null) .map(token => token.exampleInput); } diff --git a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.tests.js similarity index 90% rename from web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js rename to web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.tests.js index 8ba132ed56..6b0e0fffa2 100644 --- a/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.js +++ b/web/src/engine/predictive-text/worker-thread/src/tests/mocha/cases/edit-distance/context-tokenization.tests.js @@ -1,13 +1,21 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by jahorton on 2025-07-30 + * + * This file contains low-level tests designed to validate the behavior of the + * of the ContextTokenization class and its integration with the lower-level + * classes that it utilizes. + */ + import { assert } from 'chai'; -import { ContextToken } from '#./correction/context-token.js'; -import { ContextTokenization } from '#./correction/context-tokenization.js'; - -import * as models from '#./models/index.js'; import { default as defaultBreaker } from '@keymanapp/models-wordbreakers'; import { jsonFixture } from '@keymanapp/common-test-resources/model-helpers.mjs'; -var TrieModel = models.TrieModel; +import { ContextToken } from '#./correction/context-token.js'; +import { ContextTokenization } from '#./correction/context-tokenization.js'; +import { TrieModel } from '#./models/index.js'; var plainModel = new TrieModel(jsonFixture('models/tries/english-1000'), {wordBreaker: defaultBreaker});