From 60113a314769008f3e0a544c2dae5bb44d4bfe41 Mon Sep 17 00:00:00 2001 From: jahorton Date: Tue, 10 Nov 2020 10:51:06 +0700 Subject: [PATCH 1/4] change(developer/compilers): ModelPseudoclosure -> ModelDefinitions --- .../lexical-model-compiler.ts | 12 ++++++------ ...el-pseudoclosure.ts => model-definitions.ts} | 17 +++++++---------- ...eudoclosure.ts => test-model-definitions.ts} | 10 +++++----- 3 files changed, 18 insertions(+), 21 deletions(-) rename developer/js/source/lexical-model-compiler/{model-pseudoclosure.ts => model-definitions.ts} (95%) rename developer/js/tests/{test-model-pseudoclosure.ts => test-model-definitions.ts} (95%) diff --git a/developer/js/source/lexical-model-compiler/lexical-model-compiler.ts b/developer/js/source/lexical-model-compiler/lexical-model-compiler.ts index 2797701efa..6e9cdf2676 100644 --- a/developer/js/source/lexical-model-compiler/lexical-model-compiler.ts +++ b/developer/js/source/lexical-model-compiler/lexical-model-compiler.ts @@ -9,7 +9,7 @@ import * as ts from "typescript"; import * as fs from "fs"; import * as path from "path"; import { createTrieDataStructure } from "./build-trie"; -import { ModelPseudoclosure } from "./model-pseudoclosure"; +import { ModelDefinitions } from "./model-definitions"; import {decorateWithJoin} from "./join-word-breaker-decorator"; import {decorateWithScriptOverrides} from "./script-overrides-decorator"; @@ -50,28 +50,28 @@ export default class LexicalModelCompiler { // file, rather than the current working directory. let filenames = modelSource.sources.map(filename => path.join(sourcePath, filename)); - let pseudoclosure = new ModelPseudoclosure(modelSource); + let definitions = new ModelDefinitions(modelSource); - func += pseudoclosure.compilePseudoclosure(); + func += definitions.compileDefinitions(); // Needs the actual searchTermToKey closure... // Which needs the actual applyCasing closure as well. func += `LMLayerWorker.loadModel(new models.TrieModel(${ - createTrieDataStructure(filenames, pseudoclosure.searchTermToKey) + createTrieDataStructure(filenames, definitions.searchTermToKey) }, {\n`; let wordBreakerSourceCode = compileWordBreaker(normalizeWordBreakerSpec(modelSource.wordBreaker)); func += ` wordBreaker: ${wordBreakerSourceCode},\n`; // START - the lexical mapping option block - func += ` searchTermToKey: ${pseudoclosure.compileSearchTermToKey()},\n`; + func += ` searchTermToKey: ${definitions.compileSearchTermToKey()},\n`; if(modelSource.languageUsesCasing != null) { func += ` languageUsesCasing: ${modelSource.languageUsesCasing},\n`; } // else leave undefined. if(modelSource.languageUsesCasing) { - func += ` applyCasing: ${pseudoclosure.compileApplyCasing()},\n`; + func += ` applyCasing: ${definitions.compileApplyCasing()},\n`; } // END - the lexical mapping option block. diff --git a/developer/js/source/lexical-model-compiler/model-pseudoclosure.ts b/developer/js/source/lexical-model-compiler/model-definitions.ts similarity index 95% rename from developer/js/source/lexical-model-compiler/model-pseudoclosure.ts rename to developer/js/source/lexical-model-compiler/model-definitions.ts index 83d5ff4f2d..a031a65a31 100644 --- a/developer/js/source/lexical-model-compiler/model-pseudoclosure.ts +++ b/developer/js/source/lexical-model-compiler/model-definitions.ts @@ -12,7 +12,7 @@ import { defaultApplyCasing, * will very closely match the organizational patterns of this class in order to * facilitate the maintenance of this approach. */ -export class ModelPseudoclosure { +export class ModelDefinitions { static readonly COMPILED_NAME = 'definitions'; /** * A closure fully implementing the model's defined `applyCasing` behavior with @@ -124,9 +124,8 @@ export class ModelPseudoclosure { * This should be written to the file within the same IIFE as the model but BEFORE * the model itself, as the model will need to refer to the definitions herein. */ - compilePseudoclosure(): string { + compileDefinitions(): string { let defn: string = ''; - let PSEUDOCLOSURE = ModelPseudoclosure.COMPILED_NAME; defn += `let ${PSEUDOCLOSURE} = {\n` // ---------------------- @@ -201,8 +200,6 @@ export class ModelPseudoclosure { * compiled pseudoclosure. */ compileSearchTermToKey(): string { - let PSEUDOCLOSURE = ModelPseudoclosure.COMPILED_NAME; - // Simply point the model to the constructed closure defined by `compilePseudoclosure`. // See "START - compiled closures" section. return `${PSEUDOCLOSURE}.searchTermToKey`; @@ -212,11 +209,11 @@ export class ModelPseudoclosure { * Compiles the model-options entry for `applyCasing` in reference to the * compiled pseudoclosure. */ - compileApplyCasing(): string { - let PSEUDOCLOSURE = ModelPseudoclosure.COMPILED_NAME; - - // Simply point the model to the constructed closure defined by `compilePseudoclosure`. + compileApplyCasing(): string {// Simply point the model to the constructed closure defined by `compilePseudoclosure`. // See "START - compiled closures" section. return `${PSEUDOCLOSURE}.applyCasing`; } -} \ No newline at end of file +} + +// Because it references the class field, this line must come afterward. +const PSEUDOCLOSURE = ModelDefinitions.COMPILED_NAME; \ No newline at end of file diff --git a/developer/js/tests/test-model-pseudoclosure.ts b/developer/js/tests/test-model-definitions.ts similarity index 95% rename from developer/js/tests/test-model-pseudoclosure.ts rename to developer/js/tests/test-model-definitions.ts index 2a94f647f2..5ed1ea4673 100644 --- a/developer/js/tests/test-model-pseudoclosure.ts +++ b/developer/js/tests/test-model-definitions.ts @@ -1,6 +1,6 @@ import 'mocha'; import { assert } from 'chai'; -import { ModelPseudoclosure } from '../dist/lexical-model-compiler/model-pseudoclosure'; +import { ModelDefinitions } from '../dist/lexical-model-compiler/model-definitions'; describe('Model definition pseudoclosures', function () { describe('14.0 defaults', function() { @@ -13,7 +13,7 @@ describe('Model definition pseudoclosures', function () { format: 'trie-1.0' }; - let pseudoclosure = new ModelPseudoclosure(modelSource); + let pseudoclosure = new ModelDefinitions(modelSource); const testCases: [string, string, string][] = [ // Note: not written the Turkish way. Turns out 'İ'.toLowerCase() decomposes the result, @@ -61,7 +61,7 @@ describe('Model definition pseudoclosures', function () { format: 'trie-1.0' }; - let pseudoclosure = new ModelPseudoclosure(modelSource); + let pseudoclosure = new ModelDefinitions(modelSource); const testCases: [string, string][] = [ // Note: not written the Turkish way. Turns out 'İ'.toLowerCase() decomposes the result, @@ -99,7 +99,7 @@ describe('Model definition pseudoclosures', function () { format: 'trie-1.0' }; - let pseudoclosure = new ModelPseudoclosure(modelSource); + let pseudoclosure = new ModelDefinitions(modelSource); const testCases: [string, string][] = [ // Note: not written the Turkish way. Turns out 'İ'.toLowerCase() decomposes the result, @@ -163,7 +163,7 @@ describe('Model definition pseudoclosures', function () { format: 'trie-1.0' }; - let pseudoclosure = new ModelPseudoclosure(modelSource); + let pseudoclosure = new ModelDefinitions(modelSource); const testCases: [string, string, string][] = [ ['İstanbul', 'istanbul', 'istanbul'], From cbf5d4f7da3ec6672b4caba0bc221e8006b66f2b Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Tue, 10 Nov 2020 10:53:03 +0700 Subject: [PATCH 2/4] Update developer/js/source/lexical-model-compiler/lexical-model.ts Co-authored-by: Eddie Antonio Santos --- developer/js/source/lexical-model-compiler/lexical-model.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/developer/js/source/lexical-model-compiler/lexical-model.ts b/developer/js/source/lexical-model-compiler/lexical-model.ts index bbc66d3052..e1b55266d0 100644 --- a/developer/js/source/lexical-model-compiler/lexical-model.ts +++ b/developer/js/source/lexical-model-compiler/lexical-model.ts @@ -100,8 +100,7 @@ interface LexicalModelSource extends LexicalModelDeclaration { readonly rootClass?: string /** - * Indicates that the language being modeled has syntactic casing rules. When set to - * `true`, suggestions will attempt to match the case of the input text even if + * When set to `true`, suggestions will attempt to match the case of the input text even if * the lexicon entries use a different casing scheme due to search term keying effects. * @since 14.0 */ From cb42240360e5f46ce22ea918764745bf5a376f93 Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Wed, 11 Nov 2020 08:44:49 +0700 Subject: [PATCH 3/4] chore: Update common/models/templates/src/common.ts Co-authored-by: Eddie Antonio Santos --- common/models/templates/src/common.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/models/templates/src/common.ts b/common/models/templates/src/common.ts index 1ea964817d..b36a692039 100644 --- a/common/models/templates/src/common.ts +++ b/common/models/templates/src/common.ts @@ -118,7 +118,7 @@ namespace models { if(lowSurrogateCode >= 0xDC00 && lowSurrogateCode <= 0xDFFF) { // We have a surrogate pair; this pair is the 'first' character. - headUnitLength++; + headUnitLength = 2; } } From 761494996b054c13b95e801cbe8f9ad61fbcae43 Mon Sep 17 00:00:00 2001 From: jahorton Date: Wed, 11 Nov 2020 08:55:26 +0700 Subject: [PATCH 4/4] chore(common/models/types): CasingEnum -> CasingForm --- common/models/templates/src/common.ts | 2 +- common/models/types/index.d.ts | 6 +++--- .../js/source/lexical-model-compiler/model-defaults.ts | 2 +- .../js/source/lexical-model-compiler/model-definitions.ts | 2 +- developer/js/tests/test-compile-model-with-pseudoclosure.ts | 2 +- developer/js/tests/test-default-search-term-to-key.ts | 4 ++-- developer/js/tests/test-model-definitions.ts | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/common/models/templates/src/common.ts b/common/models/templates/src/common.ts index b36a692039..84d544c238 100644 --- a/common/models/templates/src/common.ts +++ b/common/models/templates/src/common.ts @@ -100,7 +100,7 @@ namespace models { return suggestion; } - export function defaultApplyCasing(casing: CasingEnum, text: string): string { + export function defaultApplyCasing(casing: CasingForm, text: string): string { switch(casing) { case 'lower': return text.toLowerCase(); diff --git a/common/models/types/index.d.ts b/common/models/types/index.d.ts index 73429634ed..bab450a8a0 100644 --- a/common/models/types/index.d.ts +++ b/common/models/types/index.d.ts @@ -17,7 +17,7 @@ */ declare type USVString = string; -declare type CasingEnum = 'lower' | 'initial' | 'upper'; +declare type CasingForm = 'lower' | 'initial' | 'upper'; /** * Used to facilitate edit-distance calculations by allowing the LMLayer to @@ -117,7 +117,7 @@ declare interface LexicalModel { * @param form * @param text */ - applyCasing?(form: CasingEnum, text: string): string + applyCasing?(form: CasingForm, text: string): string /** * Indicates a mapping function used by the model to simplify lookup operations @@ -496,7 +496,7 @@ declare interface WordBreakingFunction { } declare interface CasingFunction { - (caseToApply: CasingEnum, text: string, defaultApplyCasing?: CasingFunction): string; + (caseToApply: CasingForm, text: string, defaultApplyCasing?: CasingFunction): string; } /** diff --git a/developer/js/source/lexical-model-compiler/model-defaults.ts b/developer/js/source/lexical-model-compiler/model-defaults.ts index 8e8e0a4573..6880969b43 100644 --- a/developer/js/source/lexical-model-compiler/model-defaults.ts +++ b/developer/js/source/lexical-model-compiler/model-defaults.ts @@ -69,7 +69,7 @@ export function defaultCasedSearchTermToKey(wordform: string, applyCasing: Casin * This may be overwritten as appropriate in model-specific implementations. * @param text The text to be modified. */ -export function defaultApplyCasing(casing: CasingEnum, text: string): string { +export function defaultApplyCasing(casing: CasingForm, text: string): string { switch(casing) { case 'lower': return text.toLowerCase(); diff --git a/developer/js/source/lexical-model-compiler/model-definitions.ts b/developer/js/source/lexical-model-compiler/model-definitions.ts index a031a65a31..4b5b47edf1 100644 --- a/developer/js/source/lexical-model-compiler/model-definitions.ts +++ b/developer/js/source/lexical-model-compiler/model-definitions.ts @@ -74,7 +74,7 @@ export class ModelDefinitions { // Since the defined casing function may expect to take our default implementation // as a parameter, we can define the full implementation via closure capture. - this.applyCasing = function(casing: CasingEnum, text: string) { + this.applyCasing = function(casing: CasingForm, text: string) { return _this.model.applyCasing(casing, text, _this.defaults.applyCasing); }; } else { diff --git a/developer/js/tests/test-compile-model-with-pseudoclosure.ts b/developer/js/tests/test-compile-model-with-pseudoclosure.ts index 1c2c9f41e2..baba6059c9 100644 --- a/developer/js/tests/test-compile-model-with-pseudoclosure.ts +++ b/developer/js/tests/test-compile-model-with-pseudoclosure.ts @@ -10,7 +10,7 @@ describe('LexicalModelCompiler - pseudoclosure compilation + use', function () { const PATH = path.join(__dirname, 'fixtures', MODEL_ID); describe('specifying custom methods: applyCasing and searchTermToKey', function () { - let casingWithPrependedSymbols: CasingFunction = function(casingName: CasingEnum, text: string, defaultApplyCasing: CasingFunction) { + let casingWithPrependedSymbols: CasingFunction = function(casingName: CasingForm, text: string, defaultApplyCasing: CasingFunction) { switch(casingName) { // Use of symbols, and of the `casingName` name, exist to serve as regex targets. case 'lower': diff --git a/developer/js/tests/test-default-search-term-to-key.ts b/developer/js/tests/test-default-search-term-to-key.ts index 2e2f69bfd4..5b90b0e48c 100644 --- a/developer/js/tests/test-default-search-term-to-key.ts +++ b/developer/js/tests/test-default-search-term-to-key.ts @@ -101,7 +101,7 @@ describe('The default searchTermToKey() function', function () { // as U+0130's default handling is... not ideal in Turkish. // // Instead, we can get a simple-enough test with inverted casing. - let customCasing = function(caseToApply: CasingEnum, + let customCasing = function(caseToApply: CasingForm, text: string, defaultApplyCasing: CasingFunction): string { switch(caseToApply) { @@ -116,7 +116,7 @@ describe('The default searchTermToKey() function', function () { } } - let customCasingClosure = function(caseToApply: CasingEnum, text: string): string { + let customCasingClosure = function(caseToApply: CasingForm, text: string): string { return customCasing(caseToApply, text, defaultApplyCasing); } diff --git a/developer/js/tests/test-model-definitions.ts b/developer/js/tests/test-model-definitions.ts index 5ed1ea4673..bacd236762 100644 --- a/developer/js/tests/test-model-definitions.ts +++ b/developer/js/tests/test-model-definitions.ts @@ -132,7 +132,7 @@ describe('Model definition pseudoclosures', function () { describe('Model-defined applyCasing + (dependent) searchTermToKey', function() { // Note: this test only implements enough Turkish-related stuff to facilitate // a functional test. Not guaranteed to be sufficient for actual Turkish use. - let turkishCasing = function(form: CasingEnum, text: string, defaultApplyCasing: (form: CasingEnum, text: string) => string): string { + let turkishCasing = function(form: CasingForm, text: string, defaultApplyCasing: (form: CasingForm, text: string) => string): string { switch(form) { case 'lower': return defaultApplyCasing(form, text