Merge pull request #10690 from keymanapp/feat/web/wordbreaker-property-data-gen

feat(web): import the generator for the pred-text wordbreaker's Unicode-property data-table ⚡

common/models/wordbreakers/.gitignore

@@ -1 +1,2 @@
-build/
+build/
+src/main/default/data.inc.ts

common/models/wordbreakers/build.sh

@@ -12,6 +12,8 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")"
 
 ################################ Main script ################################
 
+# Note:  the raw text files used for data.inc.ts are found within
+# /resources/standards-data/unicode-character-database.
 builder_describe "Builds the predictive-text wordbreaker implementation module" \
   "clean" \
   "configure" \
@@ -20,16 +22,26 @@ builder_describe "Builds the predictive-text wordbreaker implementation module"
   "--ci"
 
 builder_describe_outputs \
-  configure          /node_modules \
-  build              build/obj/index.js
+  configure          src/main/default/data.inc.ts \
+  build              build/main/obj/index.js
 
 builder_parse "$@"
 
+function do_configure() {
+  verify_npm_setup
+
+  # This is a script used to build the data.inc.ts file needed by the
+  # default wordbreaker.  We rarely update the backing data, but it
+  # is needed _before_ the `build` action's compilation step.
+  tsc -b tools/data-compiler/tsconfig.json
+  node ./build/tools/data-compiler/obj/index.js
+}
+
 function do_build() {
-  tsc -b
+  tsc -b ./tsconfig.json
 
   # Declaration bundling.
-  tsc --emitDeclarationOnly --outFile ./build/lib/index.d.ts
+  tsc -p ./tsconfig.json --emitDeclarationOnly --outFile ./build/main/lib/index.d.ts
 }
 
 function do_test() {
@@ -40,7 +52,7 @@ function do_test() {
   fi
 }
 
-builder_run_action configure  verify_npm_setup
+builder_run_action configure  do_configure
 builder_run_action clean      rm -rf build/
 builder_run_action build      do_build
 builder_run_action test       do_test

common/models/wordbreakers/package.json

@@ -14,17 +14,23 @@
   ],
   "homepage": "https://github.com/keymanapp/keyman",
   "license": "MIT",
-  "main": "build/obj/index.js",
-  "types": "build/obj/index.d.ts",
+  "main": "build/main/obj/index.js",
+  "types": "build/main/obj/index.d.ts",
   "exports": {
     ".": {
-      "es6-bundling": "./src/index.ts",
-      "default": "./build/obj/index.js"
+      "es6-bundling": "./src/main/index.ts",
+      "default": "./build/main/obj/index.js"
     },
     "./lib": {
-      "types": "./build/lib/index.d.ts"
+      "types": "./build/main/lib/index.d.ts"
     },
-    "./obj/*.js": "./build/obj/*.js"
+    "./obj/*.js": "./build/main/obj/*.js",
+    "./test-index": {
+      "default": "./build/main/obj/test-index.js"
+    },
+    "./README.md": {
+      "default": "./README.md"
+    }
   },
   "directories": {
     "lib": "lib",

common/models/wordbreakers/src/main/default/index.ts

@@ -1,4 +1,4 @@
-import { WordBreakProperty, WORD_BREAK_PROPERTY, I, propertyMap } from "./data.js";
+import { WordBreakProperty, WORD_BREAK_PROPERTY, I, propertyMap } from "./data.inc.js";
 
 /**
  * A set of options used to customize and extend the behavior of the default

common/models/wordbreakers/src/main/test-index.ts

@@ -0,0 +1,5 @@
+// Include all standard exports.
+export * from './index.js';
+
+// Exposes some internal properties for unit-test accessibility
+export { WordBreakProperty } from './default/data.inc.js';

common/models/wordbreakers/tools/data-compiler/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2019 National Research Council Canada
+Copyright (c) 2024 Eddie Antonio Santos
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

common/models/wordbreakers/tools/data-compiler/index.ts

@@ -0,0 +1,358 @@
+#!/usr/bin/env node
+
+// Original version found at: https://github.com/eddieantonio/unicode-default-word-boundary/blob/master/libexec/compile-word-break.js
+
+// TODO:  Adapt to produce two string-encoded arrays - one for BMP chars, one for non-BMP chars.
+
+import fs from 'fs';
+import path from 'path';
+
+import { createRequire } from 'module';
+const require = createRequire(import.meta.url);
+
+/*
+ * Generates the TypeScript file for data required for the word boundary
+ * function:
+ *
+ *  - a sorted array to facilitate binary search of the Word_Break property.
+ *  - a regular expression that matches characters that have Extended_Pictographic=Yes.
+ *
+ * For internal use only. Please keep away from children.
+ *
+ * The generated file is saved to ../../src/main/default/data.inc.ts
+ */
+
+const MAX_CODE_POINT = 0x10FFFF;
+
+// Where to get the data:
+//  - https://www.unicode.org/reports/tr51/#emoji_data
+//  - https://www.unicode.org/reports/tr41/#Props0
+
+//////////////////////////////////// Main ////////////////////////////////////
+
+const projectDir = path.dirname(require.resolve("@keymanapp/models-wordbreakers/README.md"));
+const generatedFilename = path.join(projectDir, 'src', 'main', 'default', 'data.inc.ts');
+
+// The data files should be in this repository, with names matching the
+// Unicode version.
+const wordBoundaryFilename = path.join(projectDir, `../../../resources/standards-data/unicode-character-database/WordBreakProperty.txt`);
+const emojiDataFilename = path.join(projectDir, `../../../resources/standards-data/unicode-character-database/emoji-data.txt`);
+
+///////////////////////////// Word_Boundary file /////////////////////////////
+
+interface DataRange {
+  start: number;
+  end: number;
+  property: string;
+}
+
+// Extract the ranges IN ASCENDING ORDER from the file.
+// This will be the big binary search table.
+let ranges = readCharacterPropertyFile(wordBoundaryFilename)
+  .sort((a, b) => {
+    return a.start - b.start;
+  });
+
+// The list of ranges are initially sparse — having gaps between assigned
+// ranges. Fill in those gaps:
+ranges = makeDense(ranges);
+ensureDense(ranges);
+
+// The possible Word_Break property assignments.
+let categories = new Set<string>();
+for (let {property} of ranges) {
+  categories.add(property);
+}
+// Also add pseudo-categories of start-of-text and end-of-text:
+categories.add('sot');
+categories.add('eot');
+
+///////////////////////// Extended_Pictographic=Yes //////////////////////////
+
+let extendedPictographicCodePoints = readCharacterPropertyFile(emojiDataFilename)
+  .filter(({property}) => property === 'Extended_Pictographic');
+
+// Try generating the regular expression both in a way that is
+// backwards-compatbile and one that only works in ES6+.
+// let extendedPictographicRegExp;
+let compatibleRegexp = utf16AlternativesStrategy();
+let es6Regexp = unicodeRangeStrategy();
+
+// Choose the shortest regular expression.
+// In my experience, the ES6 regexp is an order of magnitude smaller!
+if (es6Regexp.length < compatibleRegexp.length) {
+  // extendedPictographicRegExp = es6Regexp;
+  console.warn(`Using ES6 regexp [${es6Regexp.length} chars]`);
+} else {
+  // extendedPictographicRegExp = compatibleRegexp;
+  console.warn(`Using compatibility regexp [${compatibleRegexp.length} chars]`);
+}
+
+let catIndexSeed = 0;
+const categoryMap = new Map<string, number>();
+
+for(let cat of categories) {
+  categoryMap.set(cat, catIndexSeed++);
+}
+
+//////////////////////// Creating the generated file /////////////////////////
+
+// Save the output in the gen/ directory.
+let stream = fs.createWriteStream(generatedFilename);
+
+// // Former entry in the original version by Eddie that was never included in our repo:
+// export const extendedPictographic = ${extendedPictographicRegExp};
+
+// Generate the file!
+stream.write(`// Automatically generated file. DO NOT MODIFY.
+// The generator script is defined at /common/models/wordbreakers/src/data-compiler/index.ts.
+
+/**
+ * Valid values for a word break property.
+ *
+ * Is optimized away at compile-time; use \`propertyMap\` to find the mapped
+ * value at runtime for a property name if needed.
+ */
+export const enum WordBreakProperty {
+${ /* Create enum values for each word break property */
+  Array.from(categories)
+    .map(x => `  ${x}`)
+    .join(',\n')
+}
+};
+
+/**
+ * Contains property names per associated index, as this is compiled away
+ * by TypeScript for \`const enum\` cases like \`WordBreakProperty\`.
+ */
+export const propertyMap = [
+${ /* Enumerate the plain-text names for ease of lookup at runtime */
+  Array.from(categories)
+  .map(x => `  "${x}"`)
+  .join(',\n')
+}
+];
+
+/**
+ * Constants for indexing values in WORD_BREAK_PROPERTY.
+ */
+export const enum I {
+  Start = 0,
+  Value = 1
+}
+
+/**
+ * Defines a mapping of all characters to their assigned word-breaking
+ * property type.
+ *
+ * There are implicit buckets starting at the char with specified code \`number\`
+ * of an entry up to, but not including, the value in the next entry.  All
+ * entries in each bucket share the same property value.
+ *
+ * Consider the following two consecutive buckets:
+ * - [0x0041, WordBreakProperty.ALetter]
+ * - [0x005B, WordBreakProperty.Other]
+ *
+ * For this example, all characters from 0x0041 to 0x005B (that is, 'A'-'Z')
+ * have the wordbreaking property \`ALetter\`.
+ */
+export const WORD_BREAK_PROPERTY: [number, WordBreakProperty][] = [
+${
+  // TODO:  Two versions:  one that's BMP-encoded, one that's non-BMP encoded.
+    ranges.map(({start, property}) => (`  [` +
+      `/*start*/ 0x${start.toString(16).toUpperCase()}, ` +
+      `WordBreakProperty.${property}],`
+    )).join('\n')
+}
+];
+`);
+
+/**
+ * Reads a Unicode character property file.
+ *
+ * Character property files are composed of comment lines, empty lines, and
+ * property lines.  Comments lines begin with '#' and should be ignored, as
+ * well as empty lines.
+ *
+ * Property lines have a code point or a code point range, followed by a
+ * semi-colon, followed by the property text. e.g.,
+ *
+ *    1F600         ; Emoji                #  6.1  [1] (😀)        grinning face
+ *    26C4..26C5    ; Emoji_Presentation   #  5.2  [2] (⛄..⛅)    snowman without snow..sun behind butt
+ *
+ * This will read the file at the given filename, and return an ordered array
+ * or property lines, with attributes:
+ *
+ *  {start: number, end: number, property: string}
+ *
+ * If the property specifies a single code point (i.e., not a range of code
+ * points), then end === start.
+ */
+function readCharacterPropertyFile(filename: string) {
+  let textContents = fs.readFileSync(filename, { encoding: 'utf8'});
+
+  return textContents.split('\n')
+    .filter(line => !line.startsWith('#') && line.trim())
+    .map(line => {
+      let [_, startText, endText, property] = line.match(
+        // Parse lines that look like this:
+        // 0000             .. 0000               ;   CategoryName
+        /^([0-9A-F]{4,6})(?:..([0-9A-F]{4,6}))?\s+;\s+([A-Za-z_]+)/
+      );
+
+      let start = parseCodepoint(startText);
+      let end = endText !== undefined ? parseCodepoint(endText) : start;
+
+      return { start, end, property };
+    });
+}
+
+/**
+ * Parses a code point, expressed as a 4 or 6 digit hexadecimal string.
+ * Does some bounds checking in order to determine if the string is in fact a
+ * valid code point.
+ */
+function parseCodepoint(hexString: string) {
+  let number = parseInt(hexString, 16);
+  if (Number.isNaN(number)) {
+    throw new SyntaxError(`Cannot parse codepoint: ${hexString}`);
+  }
+
+  if (number < 0 || number > MAX_CODE_POINT) {
+    throw new RangeError(`Codepoint out of range: ${number}`);
+  }
+
+  return number;
+}
+
+function toUnicodeEscape(codePoint: number) {
+  let isBMP = codePoint <= 0xFFFF;
+  let simpleConversion = codePoint.toString(16).toUpperCase();
+
+  let padding = (isBMP ? 4 : 6) - simpleConversion.length;
+  let digits = '0'.repeat(padding) + simpleConversion;
+
+  if (isBMP) {
+    return '\\u' + digits;
+  } else {
+    return `\\u{${digits}}`;
+  }
+}
+
+function utf16AlternativesStrategy() {
+  let codePoints = [];
+  for (let {start, end} of extendedPictographicCodePoints) {
+    for (let current = start; current <= end; current ++) {
+      codePoints.push(current);
+    }
+  }
+
+  let alternatives = codePoints.map(codePointToUTF16Escape);
+  return `/^(?:${alternatives.join('|')})/`;
+}
+
+function codePointToUTF16Escape(codePoint: number): string {
+  // Scalar values remain the same
+  if (codePoint <= 0xFFFF) {
+    return toUnicodeEscape(codePoint);
+  }
+
+  const LOWEST_TEN_BITS_MASK = 0x03FF;
+  let astralBits = codePoint - 0x10000;
+
+  let highSurrogate = 0xD800 + (astralBits >>> 10);
+  let lowSurrogate = 0xDC00 + (astralBits & LOWEST_TEN_BITS_MASK);
+
+  console.assert(highSurrogate <= 0xDBFF);
+  console.assert(lowSurrogate <= 0xDFFF);
+  console.assert(String.fromCharCode(highSurrogate) + String.fromCharCode(lowSurrogate) ===
+                 String.fromCodePoint(codePoint));
+  return codePointToUTF16Escape(highSurrogate) + codePointToUTF16Escape(lowSurrogate);
+}
+
+function unicodeRangeStrategy() {
+  let regexp = '';
+  for (let {start, end} of extendedPictographicCodePoints) {
+    if (start === end) {
+      regexp += toUnicodeEscape(start);
+    } else {
+      regexp += toUnicodeEscape(start) + '-' + toUnicodeEscape(end);
+    }
+  }
+  return `/^[${regexp}]/u`;
+}
+
+function makeDense(ranges: DataRange[]) {
+  return joinSameAdjacentProperties(fillInGaps(ranges));
+}
+
+function ensureDense(ranges: DataRange[]) {
+  let lastEnd = -1;
+  let lastProperty = 'sot';
+  for (let range of ranges) {
+    let {start, end, property} = range
+
+    if (lastEnd + 1 !== start) {
+      throw new Error(`Non-adjacent range: ${JSON.stringify(range)}`);
+    }
+
+    if (lastProperty === property) {
+      throw new Error(`adjacent ranges have same property: ${JSON.stringify(range)}`);
+    }
+
+    lastEnd = end;
+    lastProperty = property;
+  }
+}
+
+
+function joinSameAdjacentProperties(ranges: DataRange[]) {
+  console.assert(ranges.length > 1);
+
+  let conjoinedRanges = [];
+  conjoinedRanges.push(ranges.shift());
+
+  for (let range of ranges) {
+    let lastRange = conjoinedRanges[conjoinedRanges.length - 1];
+    if (range.property === lastRange.property) {
+      lastRange.end = range.end;
+    } else {
+      conjoinedRanges.push(range);
+    }
+  }
+
+  return conjoinedRanges;
+}
+
+function fillInGaps(ranges: DataRange[]) {
+  console.assert(ranges.length > 1);
+
+  let denseRanges = [];
+  let nextUnaccountedCodepoint = 0x0000;
+
+  for (let range of ranges) {
+    if (range.start > nextUnaccountedCodepoint) {
+      // Need to create a range BEFORE the next start of ranges
+      denseRanges.push({
+        start: nextUnaccountedCodepoint,
+        end: range.start - 1,
+        // If it's unassigned in the file, it should be 'Other'.
+        property: 'Other',
+      });
+    }
+
+    denseRanges.push(range);
+    nextUnaccountedCodepoint = range.end + 1;
+  }
+
+  // Create the last range (till the end)
+  if (nextUnaccountedCodepoint < MAX_CODE_POINT) {
+    denseRanges.push({
+      start: nextUnaccountedCodepoint,
+      end: MAX_CODE_POINT,
+      property: 'Other',
+    })
+  }
+
+  return denseRanges;
+}

common/models/wordbreakers/tools/data-compiler/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "extends": "../../../tsconfig.kmw-worker-base.json",
+
+  "compilerOptions": {
+    "baseUrl": "./",
+    "outDir": "../../build/tools/data-compiler/obj",
+    "tsBuildInfoFile": "../../build/tools/data-compiler/obj/tsconfig.tsbuildinfo",
+    "rootDir": "./",
+    "module": "node16",
+    "moduleResolution": "node16"
+  },
+  "references": [
+    { "path": "../../../types" }
+  ],
+  "include": [
+	  "./**/*"
+  ],
+  "exclude": [
+	  "node_modules"
+  ]
+}

common/models/wordbreakers/tsconfig.json

@@ -3,18 +3,17 @@
 
   "compilerOptions": {
     "baseUrl": "./",
-    "outDir": "build/obj",
-    "tsBuildInfoFile": "build/obj/tsconfig.tsbuildinfo",
-    "rootDir": "./src"
+    "outDir": "./build/main/obj",
+    "tsBuildInfoFile": "./build/main/obj/tsconfig.tsbuildinfo",
+    "rootDir": "./src/main"
   },
   "references": [
     { "path": "../types" }
   ],
   "include": [
-	"src/**/*"
+	  "./src/main/**/*"
   ],
   "exclude": [
-	"node_modules",
-	"test/**/*.ts"
+	  "node_modules"
   ]
-}
+}

package-lock.json

@@ -404,7 +404,7 @@
         "eventemitter3": "^5.0.0",
         "restructure": "^3.0.1",
         "sax": ">=0.6.0",
-        "semver": "^7.5.2",
+        "semver": "^7.5.4",
         "xmlbuilder": "~11.0.0"
       },
       "devDependencies": {
@@ -1153,7 +1153,7 @@
         "@keymanapp/keyman-version": "*",
         "@keymanapp/kmc-kmn": "*",
         "@keymanapp/ldml-keyboard-constants": "*",
-        "semver": "^7.5.2"
+        "semver": "^7.5.4"
       },
       "devDependencies": {
         "@keymanapp/developer-test-helpers": "*",
@@ -1942,7 +1942,7 @@
         "open": "^8.4.0",
         "restructure": "^3.0.1",
         "sax": ">=0.6.0",
-        "semver": "^7.5.2",
+        "semver": "^7.5.4",
         "ws": "^8.17.1",
         "xmlbuilder": "~11.0.0"
       },
@@ -14630,7 +14630,7 @@
       "devDependencies": {
         "@types/semver": "^7.1.0",
         "@types/yargs": "^17.0.26",
-        "semver": "^7.5.2"
+        "semver": "^7.5.4"
       }
     },
     "resources/build/version/node_modules/ansi-regex": {

resources/standards-data/unicode-character-database/download.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+## START STANDARD BUILD SCRIPT INCLUDE
+# adjust relative paths as necessary
+THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")"
+. "$(dirname "$THIS_SCRIPT")/../../../resources/build/builder.inc.sh"
+## END STANDARD BUILD SCRIPT INCLUDE
+
+. "$KEYMAN_ROOT/resources/build/minimum-versions.inc.sh"
+
+################################ Main script ################################
+
+builder_describe \
+  "Downloads Unicode data files, version $KEYMAN_VERSION_UNICODE (see minimum-versions.inc.sh), to be committed to repo." \
+  download+
+
+builder_describe_outputs \
+  download     /resources/standards-data/unicode-character-database/UnicodeData.txt
+
+builder_parse "$@"
+
+# Used by Developer
+BLOCKS_SRC_HREF="https://www.unicode.org/Public/$KEYMAN_VERSION_UNICODE/ucd/Blocks.txt"
+BLOCKS_SRC_LOCAL="./Blocks.txt"
+
+UNICODE_DATA_SRC_HREF="https://www.unicode.org/Public/$KEYMAN_VERSION_UNICODE/ucd/UnicodeData.txt"
+UNICODE_DATA_SRC_LOCAL="./UnicodeData.txt"
+
+# Used by common/models/wordbreakers for the default Unicode wordbreaker.
+WORDBREAK_PROP_SRC_HREF="https://www.unicode.org/Public/$KEYMAN_VERSION_UNICODE/ucd/auxiliary/WordBreakProperty.txt"
+WORDBREAK_PROP_SRC_LOCAL="./WordBreakProperty.txt"
+
+EMOJI_DATA_SRC_HREF="https://www.unicode.org/Public/$KEYMAN_VERSION_UNICODE/ucd/emoji/emoji-data.txt"
+EMOJI_DATA_SRC_LOCAL="./emoji-data.txt"
+
+function downloadPropertyFile() {
+  local SRC="$1"
+  local DEST="$2"
+
+  local RETRY=5       # Curl retries this number of times before giving up
+  local RETRY_DELAY=5 # Make curl sleep this amount of time before each retry when a transfer has failed
+
+  echo "Downloading ${SRC} - ${RETRY} attempts"
+  # local URL_DOWNLOAD_FILE=`curl --retry "$RETRY" --retry-delay "$RETRY_DELAY" --silent "${SRC}" | "$JQ" -r .txt`
+  curl --fail --retry "$RETRY" --retry-delay "$RETRY_DELAY" --silent "$SRC" --output "$DEST" || {
+      builder_die "Downloading $SRC failed with error $?"
+  }
+}
+
+do_download() {
+  downloadPropertyFile "${BLOCKS_SRC_HREF}"          "${BLOCKS_SRC_LOCAL}"
+  downloadPropertyFile "${UNICODE_DATA_SRC_HREF}"    "${UNICODE_DATA_SRC_LOCAL}"
+
+  downloadPropertyFile "${WORDBREAK_PROP_SRC_HREF}"  "${WORDBREAK_PROP_SRC_LOCAL}"
+  downloadPropertyFile "${EMOJI_DATA_SRC_HREF}"      "${EMOJI_DATA_SRC_LOCAL}"
+}
+
+builder_run_action download  do_download