feat(developer/compilers): initial applyCasing compilation, new code reorg

common/models/templates/src/common.ts

@@ -99,4 +99,32 @@ namespace models {
     }
     return suggestion;
   }
+
+  export function defaultApplyCasing(casing: CasingEnum, text: string): string {
+    switch(casing) {
+      case 'lower':
+        return text.toLowerCase();
+      case 'upper':
+        return text.toUpperCase();
+      case 'initial':
+        let headCode = text.charCodeAt(0);
+        // The length of the first code unit, as measured in code points.
+        let headUnitLength = 1;
+  
+        // Is the first character a high surrogate, indicating possible use of UTF-16 
+        // surrogate pairs?  Also, is the string long enough for there to BE a pair?
+        if(text.length > 1 && headCode >= 0xD800 && headCode <= 0xDBFF) {
+          // It's possible, so now we check for low surrogates.
+          let lowSurrogateCode = text.charCodeAt(1);
+  
+          if(lowSurrogateCode >= 0xDC00 && lowSurrogateCode <= 0xDFFF) {
+            // We have a surrogate pair; this pair is the 'first' character.
+            headUnitLength++;
+          }
+        }
+  
+        // Capitalizes the first code unit of the string, leaving the rest intact.
+        return text.substring(0, headUnitLength).toUpperCase() .concat(text.substring(headUnitLength));
+    }
+  }
 }

developer/js/source/lexical-model-compiler/lexical-model-compiler.ts

@@ -9,9 +9,7 @@ import * as ts from "typescript";
 import * as fs from "fs";
 import * as path from "path";
 import { createTrieDataStructure } from "./build-trie";
-import { defaultSearchTermToKey, 
-         defaultCasedSearchTermToKey,
-         defaultApplyCasing } from "./lexical-mapping-defaults";
+import { ModelPseudoclosure } from "./model-pseudoclosure";
 import {decorateWithJoin} from "./join-word-breaker-decorator";
 import {decorateWithScriptOverrides} from "./script-overrides-decorator";
 
@@ -52,55 +50,22 @@ export default class LexicalModelCompiler {
         // file, rather than the current working directory.
         let filenames = modelSource.sources.map(filename => path.join(sourcePath, filename));
 
-        // Determine the model's `applyCasing` function / implementation.
-        // Conceptually correct... but closures make actual compilation far trickier.
-        // casingClosure.toString() will not yield a proper compilation.
-        let applyCasing: CasingFunction = defaultApplyCasing;
-        if(modelSource.languageUsesCasing) {
-          applyCasing = modelSource.applyCasing || defaultApplyCasing;
+        let pseudoclosure = new ModelPseudoclosure(modelSource);
 
-          // Since the defined casing function may expect to take our default implementation
-          // as a parameter, we can define the full implementation via closure capture.
-          let casingClosure: CasingFunction = function(casing: CasingEnum, text: string) {
-            return applyCasing(casing, text, defaultApplyCasing);
-          }
-          // Kept as a separate line for easier comparison / debugging.
-          applyCasing = casingClosure;
-        }
-
-        // Use the default search term to key function, if left unspecified.
-        let searchTermToKey: WordformToKeySpec = modelSource.searchTermToKey 
-        if(!searchTermToKey) {
-          if(modelSource.languageUsesCasing) {
-            // applyCasing is defined here.
-            // Unfortunately, this only works conceptually.  .toString on a closure
-            // does not result in proper compilation.
-            searchTermToKey = function(text: string) {
-              return defaultCasedSearchTermToKey(text, applyCasing);
-            }
-          } else if(modelSource.languageUsesCasing == false) {
-            searchTermToKey = defaultSearchTermToKey;
-          } else {
-            // If languageUsesCasing is not defined, then we use pre-14.0 behavior, 
-            // which expects a lowercased default.
-            // Unfortunately, this only works conceptually.  .toString on a closure
-            // does not result in proper compilation.
-            searchTermToKey = function(text: string) {
-              return defaultCasedSearchTermToKey(text, defaultApplyCasing);
-            }
-          }
-        }
+        // TODO: the connection here is still rough, may not be fully syntactically correct.
+        func += pseudoclosure.compilePseudoclosure();
 
         // Needs the actual searchTermToKey closure...
         // Which needs the actual applyCasing closure as well.
         func += `LMLayerWorker.loadModel(new models.TrieModel(${
-          createTrieDataStructure(filenames, searchTermToKey)
+          createTrieDataStructure(filenames, pseudoclosure.searchTermToKey)
         }, {\n`;
 
         let wordBreakerSourceCode = compileWordBreaker(normalizeWordBreakerSpec(modelSource.wordBreaker));
         func += `  wordBreaker: ${wordBreakerSourceCode},\n`;
 
-        func += `  searchTermToKey: ${searchTermToKey.toString()},\n`;
+        // START - the lexical mapping option block
+        func += `  searchTermToKey: ${pseudoclosure.compileSearchTermToKey()},\n`;
 
         if(modelSource.languageUsesCasing != null) {
           func += `  languageUsesCasing: ${modelSource.languageUsesCasing},\n`;
@@ -109,8 +74,9 @@ export default class LexicalModelCompiler {
         }
 
         if(modelSource.languageUsesCasing) {
-          func += `  applyCasing: ${compileApplyCasing(applyCasing)},\n`;
+          func += `  applyCasing: ${pseudoclosure.compileApplyCasing()},\n`;
         }
+        // END - the lexical mapping option block.
 
         if (modelSource.punctuation) {
           func += `  punctuation: ${JSON.stringify(modelSource.punctuation)},\n`;
@@ -190,14 +156,6 @@ function compileInnerWordBreaker(spec: SimpleWordBreakerSpec): string {
   }
 }
 
-/**
- * Compiles the `applyCasing` function, returning the source code of the
- * full closure's JavaScript representation.
- */
-function compileApplyCasing(closure: CasingFunction) {
-  return closure.toString().replace(/^applyCasing\b/, 'function');
-}
-
 /**
  * Given a word breaker specification in any of the messy ways,
  * normalizes it to a common form that the compiler can deal with.

developer/js/source/lexical-model-compiler/model-defaults.ts

@@ -48,11 +48,27 @@ export function defaultSearchTermToKey(wordform: string): string {
  * language. There is a chance the default will work properly out of the box.
  */
 export function defaultCasedSearchTermToKey(wordform: string, applyCasing: CasingFunction): string {
-  return Array.from(defaultSearchTermToKey(wordform))
-              .map(c => applyCasing('lower', c))
-              .join('');
+  // While this is a bit WET, as the basic `defaultSearchTermToKey` exists and performs some of
+  // the same functions, repetition is the easiest way to allow the function to be safely compiled 
+  // with ease by use of `.toString()`.
+  return Array.from(wordform
+        .normalize('NFKD')
+        // Remove any combining diacritics (if input is in NFKD)
+        .replace(/[\u0300-\u036F]/g, '')
+      ) // end of `Array.from`
+      .map(c => applyCasing('lower', c))
+      .join('');
 }
 
+/**
+ * Specifies default casing behavior for lexical models when `languageUsesCasing` is
+ * set to true.
+ * @param casing One of 'lower' (lowercased), 'upper' (uppercased), or 'initial'.  
+ * 
+ * 'initial' is designed to cover cases like sentence-initial & proper noun capitalization in English.
+ * This may be overwritten as appropriate in model-specific implementations.
+ * @param text The text to be modified.
+ */
 export function defaultApplyCasing(casing: CasingEnum, text: string): string {
   switch(casing) {
     case 'lower':
@@ -77,6 +93,7 @@ export function defaultApplyCasing(casing: CasingEnum, text: string): string {
       }
 
       // Capitalizes the first code unit of the string, leaving the rest intact.
-      return text.substring(0, headUnitLength).toUpperCase().concat(text.substring(headUnitLength));
+      return text.substring(0, headUnitLength).toUpperCase() // head - uppercased
+             .concat(text.substring(headUnitLength));        // tail - lowercased
   }
 }

developer/js/source/lexical-model-compiler/model-pseudoclosure.ts

@@ -0,0 +1,220 @@
+import { defaultApplyCasing, 
+         defaultCasedSearchTermToKey, 
+         defaultSearchTermToKey
+       } from "./model-defaults";
+
+/**
+ * Processes certain defined model behaviors in such a way that the needed closures
+ * may be safely compiled to a JS file and loaded within the LMLayer.
+ * 
+ * This is accomplished by writing out a 'pseudoclosure' within the model's IIFE,
+ * then used to build _actual_ closures at LMLayer load time.  This 'pseudoclosure'
+ * will very closely match the organizational patterns of this class in order to 
+ * facilitate the maintenance of this approach.
+ */
+export class ModelPseudoclosure {
+  static readonly COMPILED_NAME = 'definitions';
+  /**
+   * A closure fully implementing the model's defined `applyCasing` behavior with
+   * the function parameter preset to the version-appropriate default.
+   * `defaults.applyCasing` is captured as part of the closure.
+   * 
+   * During compilation of some models (such as Trie-based wordlist templated models),
+   * this closure will be directly used as part of searchTermToKey.
+   * 
+   * In compiled code, this will instead be defined in-line as an autogenerated closure
+   * using the other properties of the pseudoclosure.
+   */
+  applyCasing?: CasingFunction;
+
+  /**
+   * A closure fully implementing the model's defined `searchTermToKey` behavior
+   * based upon the model's specified casing rules.  The `applyCasing` closure is
+   * itself captured within this closure.
+   *
+   * During compilation of some models (such as Trie-based wordlist templated models),
+   * this closure will be directly utilized when compiling the lexicon.
+   * 
+   * In compiled code, this will instead be defined in-line as an autogenerated closure
+   * using the other properties of the pseudoclosure.
+   */
+  searchTermToKey?: WordformToKeySpec;
+
+  /**
+   * Contains embedded 'default' implementations that may be needed for
+   * closures in the compiled version, annotated with the current version 
+   * of Developer.
+   */
+  private defaults: {
+    version: string;
+    applyCasing?: CasingFunction;
+  } = {
+    version: process.env.npm_package_version
+  };
+
+  /**
+   * Contains the model-specific definitions specified in the model's source.
+   * 
+   * These definitions may expect `defaults.applyCasing` as a parameter in
+   * their final closures.
+   */
+  private model: {
+    applyCasing?: CasingFunction;
+    searchTermToKey?: WordformToKeySpec;
+  } = {};
+
+  constructor(modelSource: LexicalModelSource) {
+    // Determine the model's `applyCasing` function / implementation.
+    if(modelSource.languageUsesCasing) {
+      this.defaults.applyCasing = defaultApplyCasing;
+
+      if(modelSource.applyCasing) {
+        this.model.applyCasing = modelSource.applyCasing;
+        let _this = this;
+
+        // Since the defined casing function may expect to take our default implementation
+        // as a parameter, we can define the full implementation via closure capture.
+        this.applyCasing = function(casing: CasingEnum, text: string) {
+          return _this.model.applyCasing(casing, text, _this.defaults.applyCasing);
+        };
+      } else {
+        this.applyCasing = this.defaults.applyCasing;
+      }
+    }
+
+    // START: if(model type uses keying)...
+
+    // Use the default search term to key function, if left unspecified.
+    if(modelSource.searchTermToKey) {
+      this.model.searchTermToKey = modelSource.searchTermToKey;
+    } else if(modelSource.languageUsesCasing) {
+      // applyCasing is defined here.
+      // Unfortunately, this only works conceptually.  .toString on a closure
+      // does not result in proper compilation.
+      this.model.searchTermToKey = defaultCasedSearchTermToKey;
+    } else if(modelSource.languageUsesCasing == false) {
+      this.model.searchTermToKey = defaultSearchTermToKey;
+    } else {
+      // If languageUsesCasing is not defined, then we use pre-14.0 behavior, 
+      // which expects a lowercased default.
+      this.model.searchTermToKey = defaultCasedSearchTermToKey;
+      // Needed to provide pre-14.0 default lowercasing as part of the 
+      // search-term keying operation.
+      this.defaults.applyCasing = defaultApplyCasing;
+      // For compile-time use.
+      this.applyCasing = this.defaults.applyCasing;
+    }
+
+    let _this = this;
+    this.searchTermToKey = function(text: string) {
+      return _this.model.searchTermToKey(text, _this.applyCasing);
+    }
+
+    // END: if(model type uses keying)...
+  }
+
+  // ------------ end:  common compile-time / run-time code ---------------
+
+  // START:  handwritten compilation code (to accomplish the 'common' pattern defined above)
+
+  /**
+   * Writes out a compiled JS version of the pseudoclosure, preserving all function
+   * implementations.
+   * 
+   * This should be written to the file within the same IIFE as the model but BEFORE
+   * the model itself, as the model will need to refer to the definitions herein.
+   */
+  compilePseudoclosure(): string {
+    let defn: string = '';
+    let PSEUDOCLOSURE = ModelPseudoclosure.COMPILED_NAME;
+    defn += `let ${PSEUDOCLOSURE} = {\n`
+
+    // ----------------------
+    // START - the 'defaults', which are common within the same Developer version.
+    defn += `  defaults: {\n    version: "${this.defaults.version}"`;
+    
+    // Only write out `applyCasing` if and when it is needed.
+    if(this.defaults.applyCasing) {
+      defn += `,\n    applyCasing: ${this.defaults.applyCasing.toString()}`;
+    }
+
+    // Finalizes `defaults`
+    defn += `\n  },`;
+    // END - the 'defaults'
+
+    // ----------------------
+    // START - model-specific definitions (when defined)
+    defn += `  model: {\n`;
+    defn += `    searchTermToKey: ${this.model.searchTermToKey.toString()}`;
+
+    if(this.model.applyCasing) {
+      defn += `,\n    applyCasing: ${this.model.applyCasing.toString()}`;
+    }
+    defn += `\n  }`
+    // END - model-specific definitions
+
+    // ----------------------
+    // START - compiled closures.  Given those definitions, write out the 
+    // pseudoclosure-referencing closures for the needed methods.
+
+    // We should be able to define these closures in-line with the object's
+    // initialization.  Worst-case, we simply move the definitions outside
+    // of the pseudoclosure's init and THEN define/assign these closures to
+    // the object, as references will be available then for sure.
+    if(this.model.applyCasing) {
+      // A major potential issue:  if the user wants to call extra custom functions that they've written.
+      //
+      // `applyCasing` recursion SHOULD be fine if they write `this.applyCasing() and forward all arguments 
+      // appropriately, as it will be known as `applyCasing` on the runtime `this` (`model`) object.
+      //
+      // Similarly, as long as any helper functions are similarly compiled and stored as part of `model`,
+      // they should be accessible too.  The issue would be to actually allow use of extra custom funcs
+      // and include them as part of this object as part of compilation.
+      defn += `,\n  applyCasing: function(caseToApply, text) {
+        return ${PSEUDOCLOSURE}.model.applyCasing(caseToApply, text, ${PSEUDOCLOSURE}.defaults.applyCasing);
+      }`;
+    } else if(this.defaults.applyCasing) {
+      // We can't directly assign from `.defaults`, as initialization-time field reads
+      // are not permitted within JS.  Function references, however, are valid.
+      defn += `,\n  applyCasing: function(caseToApply, text) {
+        return ${PSEUDOCLOSURE}.defaults.applyCasing(caseToApply, text);
+      }`;
+    }
+
+    // if(this.searchTermToKey) {
+    defn += `,\n  searchTermToKey: function(text) {
+      return ${PSEUDOCLOSURE}.model.searchTermToKey(text, ${PSEUDOCLOSURE}.applyCasing);
+    }`;
+    // }
+
+    // END - compiled closures.
+
+    // ----------------------
+    // Finalize the definition of... `definitions`.
+    defn += `\n};\n`;
+
+    return defn;
+  }
+
+  /**
+   * Compiles the model-options entry for `searchTermToKey` in reference to the
+   * compiled pseudoclosure.
+   */
+  compileSearchTermToKey(): string {
+    let PSEUDOCLOSURE = ModelPseudoclosure.COMPILED_NAME;
+
+    // TODO:  consider - should it write the full line, or just the JSON value for the entry?
+    return `${PSEUDOCLOSURE}.searchTermToKey`;
+  }
+
+  /**
+   * Compiles the model-options entry for `applyCasing` in reference to the
+   * compiled pseudoclosure.
+   */
+  compileApplyCasing(): string {
+    let PSEUDOCLOSURE = ModelPseudoclosure.COMPILED_NAME;
+
+    // TODO:  consider - should it write the full line, or just the JSON value for the entry?
+    return `${PSEUDOCLOSURE}.applyCasing`;
+  }
+}

developer/js/tests/test-default-search-term-to-key.ts

@@ -1,7 +1,7 @@
 
 import 'mocha';
 import {assert} from 'chai';
-import { defaultSearchTermToKey } from '../dist/lexical-model-compiler/build-trie';
+import { defaultSearchTermToKey } from '../dist/lexical-model-compiler/model-defaults';
 
 
 describe('The default searchTermToKey() function', function () {