From 5c9c2bd8e6fdbbfb29a9d08f99ce89bf45b52b76 Mon Sep 17 00:00:00 2001 From: jahorton Date: Wed, 4 Nov 2020 10:52:41 +0700 Subject: [PATCH] feat(developer/compilers): initial applyCasing compilation, new code reorg --- common/models/templates/src/common.ts | 28 +++ .../lexical-model-compiler.ts | 60 +---- ...-mapping-defaults.ts => model-defaults.ts} | 25 +- .../model-pseudoclosure.ts | 220 ++++++++++++++++++ .../tests/test-default-search-term-to-key.ts | 2 +- 5 files changed, 279 insertions(+), 56 deletions(-) rename developer/js/source/lexical-model-compiler/{lexical-mapping-defaults.ts => model-defaults.ts} (76%) create mode 100644 developer/js/source/lexical-model-compiler/model-pseudoclosure.ts diff --git a/common/models/templates/src/common.ts b/common/models/templates/src/common.ts index 476e0f85ad..fba3a0ee4f 100644 --- a/common/models/templates/src/common.ts +++ b/common/models/templates/src/common.ts @@ -99,4 +99,32 @@ namespace models { } return suggestion; } + + export function defaultApplyCasing(casing: CasingEnum, text: string): string { + switch(casing) { + case 'lower': + return text.toLowerCase(); + case 'upper': + return text.toUpperCase(); + case 'initial': + let headCode = text.charCodeAt(0); + // The length of the first code unit, as measured in code points. + let headUnitLength = 1; + + // Is the first character a high surrogate, indicating possible use of UTF-16 + // surrogate pairs? Also, is the string long enough for there to BE a pair? + if(text.length > 1 && headCode >= 0xD800 && headCode <= 0xDBFF) { + // It's possible, so now we check for low surrogates. + let lowSurrogateCode = text.charCodeAt(1); + + if(lowSurrogateCode >= 0xDC00 && lowSurrogateCode <= 0xDFFF) { + // We have a surrogate pair; this pair is the 'first' character. + headUnitLength++; + } + } + + // Capitalizes the first code unit of the string, leaving the rest intact. + return text.substring(0, headUnitLength).toUpperCase() .concat(text.substring(headUnitLength)); + } + } } diff --git a/developer/js/source/lexical-model-compiler/lexical-model-compiler.ts b/developer/js/source/lexical-model-compiler/lexical-model-compiler.ts index ab9f4900b4..4ce6a57764 100644 --- a/developer/js/source/lexical-model-compiler/lexical-model-compiler.ts +++ b/developer/js/source/lexical-model-compiler/lexical-model-compiler.ts @@ -9,9 +9,7 @@ import * as ts from "typescript"; import * as fs from "fs"; import * as path from "path"; import { createTrieDataStructure } from "./build-trie"; -import { defaultSearchTermToKey, - defaultCasedSearchTermToKey, - defaultApplyCasing } from "./lexical-mapping-defaults"; +import { ModelPseudoclosure } from "./model-pseudoclosure"; import {decorateWithJoin} from "./join-word-breaker-decorator"; import {decorateWithScriptOverrides} from "./script-overrides-decorator"; @@ -52,55 +50,22 @@ export default class LexicalModelCompiler { // file, rather than the current working directory. let filenames = modelSource.sources.map(filename => path.join(sourcePath, filename)); - // Determine the model's `applyCasing` function / implementation. - // Conceptually correct... but closures make actual compilation far trickier. - // casingClosure.toString() will not yield a proper compilation. - let applyCasing: CasingFunction = defaultApplyCasing; - if(modelSource.languageUsesCasing) { - applyCasing = modelSource.applyCasing || defaultApplyCasing; + let pseudoclosure = new ModelPseudoclosure(modelSource); - // Since the defined casing function may expect to take our default implementation - // as a parameter, we can define the full implementation via closure capture. - let casingClosure: CasingFunction = function(casing: CasingEnum, text: string) { - return applyCasing(casing, text, defaultApplyCasing); - } - // Kept as a separate line for easier comparison / debugging. - applyCasing = casingClosure; - } - - // Use the default search term to key function, if left unspecified. - let searchTermToKey: WordformToKeySpec = modelSource.searchTermToKey - if(!searchTermToKey) { - if(modelSource.languageUsesCasing) { - // applyCasing is defined here. - // Unfortunately, this only works conceptually. .toString on a closure - // does not result in proper compilation. - searchTermToKey = function(text: string) { - return defaultCasedSearchTermToKey(text, applyCasing); - } - } else if(modelSource.languageUsesCasing == false) { - searchTermToKey = defaultSearchTermToKey; - } else { - // If languageUsesCasing is not defined, then we use pre-14.0 behavior, - // which expects a lowercased default. - // Unfortunately, this only works conceptually. .toString on a closure - // does not result in proper compilation. - searchTermToKey = function(text: string) { - return defaultCasedSearchTermToKey(text, defaultApplyCasing); - } - } - } + // TODO: the connection here is still rough, may not be fully syntactically correct. + func += pseudoclosure.compilePseudoclosure(); // Needs the actual searchTermToKey closure... // Which needs the actual applyCasing closure as well. func += `LMLayerWorker.loadModel(new models.TrieModel(${ - createTrieDataStructure(filenames, searchTermToKey) + createTrieDataStructure(filenames, pseudoclosure.searchTermToKey) }, {\n`; let wordBreakerSourceCode = compileWordBreaker(normalizeWordBreakerSpec(modelSource.wordBreaker)); func += ` wordBreaker: ${wordBreakerSourceCode},\n`; - func += ` searchTermToKey: ${searchTermToKey.toString()},\n`; + // START - the lexical mapping option block + func += ` searchTermToKey: ${pseudoclosure.compileSearchTermToKey()},\n`; if(modelSource.languageUsesCasing != null) { func += ` languageUsesCasing: ${modelSource.languageUsesCasing},\n`; @@ -109,8 +74,9 @@ export default class LexicalModelCompiler { } if(modelSource.languageUsesCasing) { - func += ` applyCasing: ${compileApplyCasing(applyCasing)},\n`; + func += ` applyCasing: ${pseudoclosure.compileApplyCasing()},\n`; } + // END - the lexical mapping option block. if (modelSource.punctuation) { func += ` punctuation: ${JSON.stringify(modelSource.punctuation)},\n`; @@ -190,14 +156,6 @@ function compileInnerWordBreaker(spec: SimpleWordBreakerSpec): string { } } -/** - * Compiles the `applyCasing` function, returning the source code of the - * full closure's JavaScript representation. - */ -function compileApplyCasing(closure: CasingFunction) { - return closure.toString().replace(/^applyCasing\b/, 'function'); -} - /** * Given a word breaker specification in any of the messy ways, * normalizes it to a common form that the compiler can deal with. diff --git a/developer/js/source/lexical-model-compiler/lexical-mapping-defaults.ts b/developer/js/source/lexical-model-compiler/model-defaults.ts similarity index 76% rename from developer/js/source/lexical-model-compiler/lexical-mapping-defaults.ts rename to developer/js/source/lexical-model-compiler/model-defaults.ts index 36e25d9615..8e8e0a4573 100644 --- a/developer/js/source/lexical-model-compiler/lexical-mapping-defaults.ts +++ b/developer/js/source/lexical-model-compiler/model-defaults.ts @@ -48,11 +48,27 @@ export function defaultSearchTermToKey(wordform: string): string { * language. There is a chance the default will work properly out of the box. */ export function defaultCasedSearchTermToKey(wordform: string, applyCasing: CasingFunction): string { - return Array.from(defaultSearchTermToKey(wordform)) - .map(c => applyCasing('lower', c)) - .join(''); + // While this is a bit WET, as the basic `defaultSearchTermToKey` exists and performs some of + // the same functions, repetition is the easiest way to allow the function to be safely compiled + // with ease by use of `.toString()`. + return Array.from(wordform + .normalize('NFKD') + // Remove any combining diacritics (if input is in NFKD) + .replace(/[\u0300-\u036F]/g, '') + ) // end of `Array.from` + .map(c => applyCasing('lower', c)) + .join(''); } +/** + * Specifies default casing behavior for lexical models when `languageUsesCasing` is + * set to true. + * @param casing One of 'lower' (lowercased), 'upper' (uppercased), or 'initial'. + * + * 'initial' is designed to cover cases like sentence-initial & proper noun capitalization in English. + * This may be overwritten as appropriate in model-specific implementations. + * @param text The text to be modified. + */ export function defaultApplyCasing(casing: CasingEnum, text: string): string { switch(casing) { case 'lower': @@ -77,6 +93,7 @@ export function defaultApplyCasing(casing: CasingEnum, text: string): string { } // Capitalizes the first code unit of the string, leaving the rest intact. - return text.substring(0, headUnitLength).toUpperCase().concat(text.substring(headUnitLength)); + return text.substring(0, headUnitLength).toUpperCase() // head - uppercased + .concat(text.substring(headUnitLength)); // tail - lowercased } } \ No newline at end of file diff --git a/developer/js/source/lexical-model-compiler/model-pseudoclosure.ts b/developer/js/source/lexical-model-compiler/model-pseudoclosure.ts new file mode 100644 index 0000000000..d4210b55b5 --- /dev/null +++ b/developer/js/source/lexical-model-compiler/model-pseudoclosure.ts @@ -0,0 +1,220 @@ +import { defaultApplyCasing, + defaultCasedSearchTermToKey, + defaultSearchTermToKey + } from "./model-defaults"; + +/** + * Processes certain defined model behaviors in such a way that the needed closures + * may be safely compiled to a JS file and loaded within the LMLayer. + * + * This is accomplished by writing out a 'pseudoclosure' within the model's IIFE, + * then used to build _actual_ closures at LMLayer load time. This 'pseudoclosure' + * will very closely match the organizational patterns of this class in order to + * facilitate the maintenance of this approach. + */ +export class ModelPseudoclosure { + static readonly COMPILED_NAME = 'definitions'; + /** + * A closure fully implementing the model's defined `applyCasing` behavior with + * the function parameter preset to the version-appropriate default. + * `defaults.applyCasing` is captured as part of the closure. + * + * During compilation of some models (such as Trie-based wordlist templated models), + * this closure will be directly used as part of searchTermToKey. + * + * In compiled code, this will instead be defined in-line as an autogenerated closure + * using the other properties of the pseudoclosure. + */ + applyCasing?: CasingFunction; + + /** + * A closure fully implementing the model's defined `searchTermToKey` behavior + * based upon the model's specified casing rules. The `applyCasing` closure is + * itself captured within this closure. + * + * During compilation of some models (such as Trie-based wordlist templated models), + * this closure will be directly utilized when compiling the lexicon. + * + * In compiled code, this will instead be defined in-line as an autogenerated closure + * using the other properties of the pseudoclosure. + */ + searchTermToKey?: WordformToKeySpec; + + /** + * Contains embedded 'default' implementations that may be needed for + * closures in the compiled version, annotated with the current version + * of Developer. + */ + private defaults: { + version: string; + applyCasing?: CasingFunction; + } = { + version: process.env.npm_package_version + }; + + /** + * Contains the model-specific definitions specified in the model's source. + * + * These definitions may expect `defaults.applyCasing` as a parameter in + * their final closures. + */ + private model: { + applyCasing?: CasingFunction; + searchTermToKey?: WordformToKeySpec; + } = {}; + + constructor(modelSource: LexicalModelSource) { + // Determine the model's `applyCasing` function / implementation. + if(modelSource.languageUsesCasing) { + this.defaults.applyCasing = defaultApplyCasing; + + if(modelSource.applyCasing) { + this.model.applyCasing = modelSource.applyCasing; + let _this = this; + + // Since the defined casing function may expect to take our default implementation + // as a parameter, we can define the full implementation via closure capture. + this.applyCasing = function(casing: CasingEnum, text: string) { + return _this.model.applyCasing(casing, text, _this.defaults.applyCasing); + }; + } else { + this.applyCasing = this.defaults.applyCasing; + } + } + + // START: if(model type uses keying)... + + // Use the default search term to key function, if left unspecified. + if(modelSource.searchTermToKey) { + this.model.searchTermToKey = modelSource.searchTermToKey; + } else if(modelSource.languageUsesCasing) { + // applyCasing is defined here. + // Unfortunately, this only works conceptually. .toString on a closure + // does not result in proper compilation. + this.model.searchTermToKey = defaultCasedSearchTermToKey; + } else if(modelSource.languageUsesCasing == false) { + this.model.searchTermToKey = defaultSearchTermToKey; + } else { + // If languageUsesCasing is not defined, then we use pre-14.0 behavior, + // which expects a lowercased default. + this.model.searchTermToKey = defaultCasedSearchTermToKey; + // Needed to provide pre-14.0 default lowercasing as part of the + // search-term keying operation. + this.defaults.applyCasing = defaultApplyCasing; + // For compile-time use. + this.applyCasing = this.defaults.applyCasing; + } + + let _this = this; + this.searchTermToKey = function(text: string) { + return _this.model.searchTermToKey(text, _this.applyCasing); + } + + // END: if(model type uses keying)... + } + + // ------------ end: common compile-time / run-time code --------------- + + // START: handwritten compilation code (to accomplish the 'common' pattern defined above) + + /** + * Writes out a compiled JS version of the pseudoclosure, preserving all function + * implementations. + * + * This should be written to the file within the same IIFE as the model but BEFORE + * the model itself, as the model will need to refer to the definitions herein. + */ + compilePseudoclosure(): string { + let defn: string = ''; + let PSEUDOCLOSURE = ModelPseudoclosure.COMPILED_NAME; + defn += `let ${PSEUDOCLOSURE} = {\n` + + // ---------------------- + // START - the 'defaults', which are common within the same Developer version. + defn += ` defaults: {\n version: "${this.defaults.version}"`; + + // Only write out `applyCasing` if and when it is needed. + if(this.defaults.applyCasing) { + defn += `,\n applyCasing: ${this.defaults.applyCasing.toString()}`; + } + + // Finalizes `defaults` + defn += `\n },`; + // END - the 'defaults' + + // ---------------------- + // START - model-specific definitions (when defined) + defn += ` model: {\n`; + defn += ` searchTermToKey: ${this.model.searchTermToKey.toString()}`; + + if(this.model.applyCasing) { + defn += `,\n applyCasing: ${this.model.applyCasing.toString()}`; + } + defn += `\n }` + // END - model-specific definitions + + // ---------------------- + // START - compiled closures. Given those definitions, write out the + // pseudoclosure-referencing closures for the needed methods. + + // We should be able to define these closures in-line with the object's + // initialization. Worst-case, we simply move the definitions outside + // of the pseudoclosure's init and THEN define/assign these closures to + // the object, as references will be available then for sure. + if(this.model.applyCasing) { + // A major potential issue: if the user wants to call extra custom functions that they've written. + // + // `applyCasing` recursion SHOULD be fine if they write `this.applyCasing() and forward all arguments + // appropriately, as it will be known as `applyCasing` on the runtime `this` (`model`) object. + // + // Similarly, as long as any helper functions are similarly compiled and stored as part of `model`, + // they should be accessible too. The issue would be to actually allow use of extra custom funcs + // and include them as part of this object as part of compilation. + defn += `,\n applyCasing: function(caseToApply, text) { + return ${PSEUDOCLOSURE}.model.applyCasing(caseToApply, text, ${PSEUDOCLOSURE}.defaults.applyCasing); + }`; + } else if(this.defaults.applyCasing) { + // We can't directly assign from `.defaults`, as initialization-time field reads + // are not permitted within JS. Function references, however, are valid. + defn += `,\n applyCasing: function(caseToApply, text) { + return ${PSEUDOCLOSURE}.defaults.applyCasing(caseToApply, text); + }`; + } + + // if(this.searchTermToKey) { + defn += `,\n searchTermToKey: function(text) { + return ${PSEUDOCLOSURE}.model.searchTermToKey(text, ${PSEUDOCLOSURE}.applyCasing); + }`; + // } + + // END - compiled closures. + + // ---------------------- + // Finalize the definition of... `definitions`. + defn += `\n};\n`; + + return defn; + } + + /** + * Compiles the model-options entry for `searchTermToKey` in reference to the + * compiled pseudoclosure. + */ + compileSearchTermToKey(): string { + let PSEUDOCLOSURE = ModelPseudoclosure.COMPILED_NAME; + + // TODO: consider - should it write the full line, or just the JSON value for the entry? + return `${PSEUDOCLOSURE}.searchTermToKey`; + } + + /** + * Compiles the model-options entry for `applyCasing` in reference to the + * compiled pseudoclosure. + */ + compileApplyCasing(): string { + let PSEUDOCLOSURE = ModelPseudoclosure.COMPILED_NAME; + + // TODO: consider - should it write the full line, or just the JSON value for the entry? + return `${PSEUDOCLOSURE}.applyCasing`; + } +} \ No newline at end of file diff --git a/developer/js/tests/test-default-search-term-to-key.ts b/developer/js/tests/test-default-search-term-to-key.ts index 33e4f2527e..c040cb045a 100644 --- a/developer/js/tests/test-default-search-term-to-key.ts +++ b/developer/js/tests/test-default-search-term-to-key.ts @@ -1,7 +1,7 @@ import 'mocha'; import {assert} from 'chai'; -import { defaultSearchTermToKey } from '../dist/lexical-model-compiler/build-trie'; +import { defaultSearchTermToKey } from '../dist/lexical-model-compiler/model-defaults'; describe('The default searchTermToKey() function', function () {