Merge pull request #12949 from keymanapp/feat/developer/12912-rewrite-kvk-font-data-in-package-compile

feat(developer): rewrite font data in .kvk from package metadata

developer/src/common/web/utils/src/font-family.ts

@@ -24,5 +24,32 @@ export async function getFontFamily(source: Uint8Array) {
     return null;
   }
 
+  return font.meta.property.find(prop => prop.name == 'font-family')?.text ?? null;
+}
+
+/**
+ * Extracts the font-family from an in-memory TTF or WOFF blob in `source`
+ * parameter.
+ *
+ * @param source   In-memory TTF or WOFF font blob
+ *
+ * @throws Uncaught exceptions from ttfMeta.promise if the font file is invalid.
+ *
+ * @returns If the file is invalid or cannot be parsed, returns `null`,
+ *          otherwise returns the font family as a string.
+ */
+export function getFontFamilySync(source: Uint8Array) {
+  /* c8 ignore next 3 */
+  if(!source) {
+    return null;
+  }
+
+  const buffer = Buffer.from(source);
+  const font = ttfMeta.ttfInfoSync(buffer);
+  /* c8 ignore next 3 */
+  if(!font) {
+    return null;
+  }
+
   return font.meta.property.find(prop => prop.name == 'font-family')?.text ?? null;
 }

developer/src/common/web/utils/src/index.ts

@@ -68,4 +68,6 @@ export * as SourceFilenamePatterns from './source-filename-patterns.js';
 export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader } from './xml-utils.js';
 
 export * as GitHubUrls from './github-urls.js';
-export * as CloudUrls from './cloud-urls.js';
+export * as CloudUrls from './cloud-urls.js';
+
+export { getFontFamily, getFontFamilySync } from './font-family.js';

developer/src/common/web/utils/src/ttfmeta/lib/index.js

@@ -31,6 +31,20 @@ export function ttfInfo(pathOrData, callback) {
   }
 }
 
+/**
+ * @namespace
+ * @param {string | number | Buffer | URL} pathOrData
+ * @returns {typeof ttf.result}
+ */
+export function ttfInfoSync(pathOrData) {
+  if (pathOrData instanceof Buffer) {
+    return ttf.ttfInfoSync(ttf.view(pathOrData));
+  } else {
+    const data = fs.readFileSync(pathOrData);
+    return ttf.ttfInfoSync(ttf.view(data));
+  }
+}
+
 /**
  * @param {string | number | Buffer | URL} pathOrData
  * @returns {Promise<typeof ttf.result>}
@@ -48,5 +62,5 @@ export function promise(pathOrData) {
 }
 
 /** @namespace */
-export const ttfMeta = { ttfInfo, promise };
+export const ttfMeta = { ttfInfo, ttfInfoSync, promise };
 export default ttfMeta;

developer/src/common/web/utils/src/ttfmeta/lib/main.js

@@ -300,6 +300,16 @@ export function ttfInfo(data, callback) {
   }
 }
 
+/**
+ * @namespace
+ * @param {*} data
+ * @returns {typeof result}
+ */
+export function ttfInfoSync(data) {
+  resultTables(data);
+  return result;
+}
+
 /**
  * @param {string | number | Buffer | URL | DataView} pathOrData
  * @returns {Promise<typeof result>}

developer/src/common/web/utils/tsconfig.json

@@ -5,8 +5,12 @@
       "rootDir": ".",
       "outDir": "./build/",
       "baseUrl": ".",
+
+      /** for ttfmeta */
+      "allowJs": true,
     },
   "include": [
     "src/**/*.ts",
+    "src/ttfmeta/lib/*.js",
   ],
 }

developer/src/kmc-keyboard-info/src/keyboard-info-compiler.ts

@@ -12,7 +12,7 @@ import { CompilerCallbacks, KeymanCompiler, CompilerOptions, KeymanCompilerResul
 import { KmpCompiler } from "@keymanapp/kmc-package";
 
 import { SchemaValidators } from "@keymanapp/common-types";
-import { getFontFamily } from "./font-family.js";
+import { getFontFamily } from "@keymanapp/developer-utils";
 
 const regionNames = new Intl.DisplayNames(['en'], { type: "region" });
 const scriptNames = new Intl.DisplayNames(['en'], { type: "script" });

developer/src/kmc-keyboard-info/test/font-family.tests.ts

@@ -2,7 +2,7 @@ import * as fs from 'fs';
 import { assert } from 'chai';
 import 'mocha';
 import { makePathToFixture } from './helpers/index.js';
-import { getFontFamily } from '../src/font-family.js';
+import { getFontFamily } from '@keymanapp/developer-utils';
 
 const AFGHAN_TURKMEN_DISPLAY_FONT = makePathToFixture('afghan_turkmen', "Lateef-Regular.ttf");
 const AFGHAN_TURKMEN_OSK_FONT = makePathToFixture('afghan_turkmen', "Lateef-Bold.ttf");

developer/src/kmc-keyboard-info/tsconfig.json

@@ -13,7 +13,6 @@
   },
   "include": [
     "src/**/*.ts",
-    "src/ttfmeta/lib/*.js",
     "src/imports/langtags.js",
   ],
   "references": [

developer/src/kmc-package/src/compiler/kmp-compiler.ts

@@ -18,6 +18,8 @@ import { PackageKeyboardTargetValidator } from './package-keyboard-target-valida
 import { PackageMetadataUpdater } from './package-metadata-updater.js';
 import { markdownToHTML } from './markdown.js';
 import { PackageValidation } from './package-validation.js';
+import { getFontFamilySync } from "@keymanapp/developer-utils";
+import { setKvkFontData } from './kvk-font-facename.js';
 
 const KMP_JSON_FILENAME = 'kmp.json';
 const KMP_INF_FILENAME = 'kmp.inf';
@@ -483,36 +485,36 @@ export class KmpCompiler implements KeymanCompiler {
         this.callbacks.reportMessage(PackageCompilerMessages.Warn_AbsolutePath({filename: filename}));
       }
 
+      let memberFileData = this.getMemberFileData(kpsFilename, filename);
+      if(!memberFileData) {
+        failed = true;
+        return;
+      }
       filename = this.callbacks.resolveFilename(kpsFilename, filename);
       const basename = this.callbacks.path.basename(filename);
 
-      if(!this.callbacks.fs.existsSync(filename)) {
-        this.callbacks.reportMessage(PackageCompilerMessages.Error_FileDoesNotExist({filename: filename}));
-        failed = true;
-        return;
+      if(KeymanFileTypes.filenameIs(filename, KeymanFileTypes.Binary.VisualKeyboard)) {
+        if(!this.isKvkFileBinary(memberFileData)) {
+          // warn the few users who are still doing this -- non-binary .kvk should not be included in package!
+          this.callbacks.reportMessage(PackageCompilerMessages.Warn_FileIsNotABinaryKvkFile({filename: filename}));
+        } else {
+          memberFileData = this.setKvkFontData(kpsFilename, data, filename, memberFileData);
+        }
       }
 
-      let memberFileData;
-      try {
-        memberFileData = this.callbacks.loadFile(filename);
-      } catch(e) {
-        this.callbacks.reportMessage(PackageCompilerMessages.Error_FileCouldNotBeRead({filename: filename, e: e}));
-        failed = true;
-        return;
-      }
-
-      this.warnIfKvkFileIsNotBinary(filename, memberFileData);
-
       zip.file(basename, memberFileData);
-
-      // Remove path data from files before JSON save
-      value.name = basename;
     });
 
     if(failed) {
       return null;
     }
 
+    data.files.forEach((value) => {
+      const basename = this.callbacks.path.basename(value.name);
+      // Remove path data from files before JSON save
+      value.name = basename;
+    });
+
     // TODO #9477: transform .md to .htm
 
     // Remove path data from file references in options
@@ -557,26 +559,87 @@ export class KmpCompiler implements KeymanCompiler {
     return transcodeToCP1252(s);
   }
 
+  private getMemberFileData(kpsFilename: string, filename: string) {
+    filename = this.callbacks.resolveFilename(kpsFilename, filename);
+
+    if(!this.callbacks.fs.existsSync(filename)) {
+      this.callbacks.reportMessage(PackageCompilerMessages.Error_FileDoesNotExist({filename: filename}));
+      return null;
+    }
+
+    try {
+      return this.callbacks.loadFile(filename);
+    } catch(e) {
+      this.callbacks.reportMessage(PackageCompilerMessages.Error_FileCouldNotBeRead({filename: filename, e: e}));
+      return null;
+    }
+  }
+
+  private setKvkFontData(kpsFilename: string, data: KmpJsonFile.KmpJsonFile, filename: string, kvk: Uint8Array) {
+    // find the appropriate font to set
+    const kvkId = this.callbacks.path.basename(filename.toLowerCase(), '.kvk');
+    const kbd = data.keyboards.find(k => k.id == kvkId);
+    if(!kbd) {
+      // cannot find a matching keyboard
+      this.callbacks.reportMessage(PackageCompilerMessages.Warn_CannotFindMatchingKeyboardForVisualKeyboard({filename}));
+      return kvk;
+    }
+    const fontFilename = kbd.oskFont || kbd.displayFont;
+    if(!fontFilename) {
+      // no font data to set
+      return kvk;
+    }
+
+    // Look up the full font filename
+    const fontFile = data.files.find(file => this.callbacks.path.basename(file.name.toLowerCase()) == fontFilename.toLowerCase());
+    if(!fontFile) {
+      // font cannot be found
+      this.callbacks.reportMessage(PackageCompilerMessages.Warn_CannotFindFontForKeyboard({id: kbd.id, fontFilename}));
+      return kvk;
+    }
+
+    // the font is a filename, included in the .kps
+    const fontData = this.getMemberFileData(kpsFilename, fontFile.name);
+    if(!fontData) {
+      // cannot find source font
+      this.callbacks.reportMessage(PackageCompilerMessages.Warn_CannotFindFontForKeyboard({id: kbd.id, fontFilename: fontFile.name}));
+      return kvk;
+    }
+
+    const fontFacename = getFontFamilySync(fontData);
+    if(!fontFacename) {
+      // Font facename could not be extracted from ttf
+      this.callbacks.reportMessage(PackageCompilerMessages.Warn_CannotReadFont({fontFilename: fontFile.name}));
+      return kvk;
+    }
+
+    const result = setKvkFontData(kvk, fontFacename);
+    if(!result) {
+      // KVK is invalid
+      this.callbacks.reportMessage(PackageCompilerMessages.Warn_VisualKeyboardFileIsInvalid({filename}));
+      return kvk;
+    }
+
+    return result;
+  }
+
   /**
    * Legacy .kmp compiler would transform xml-format .kvk files into a binary .kvk file; now
-   * we want that to remain the responsibility of the keyboard compiler, so we'll warn the
-   * few users who are still doing this
+   * we want that to remain the responsibility of the keyboard compiler.
    */
-  private warnIfKvkFileIsNotBinary(filename: string, data: Uint8Array) {
-    if(!KeymanFileTypes.filenameIs(filename, KeymanFileTypes.Binary.VisualKeyboard)) {
-      return;
-    }
-
+  private isKvkFileBinary(data: Uint8Array) {
     if(data.byteLength < 4) {
       // TODO: Not a valid .kvk file; should we be reporting this?
-      return;
+      return false;
     }
 
+    // Must start with 'KVKF'
     if(data[0] != KvkFile.KVK_HEADER_IDENTIFIER_BYTES[0] ||
       data[1] != KvkFile.KVK_HEADER_IDENTIFIER_BYTES[1] ||
       data[2] != KvkFile.KVK_HEADER_IDENTIFIER_BYTES[2] ||
       data[3] != KvkFile.KVK_HEADER_IDENTIFIER_BYTES[3]) {
-      this.callbacks.reportMessage(PackageCompilerMessages.Warn_FileIsNotABinaryKvkFile({filename: filename}));
+      return false;
     }
+    return true;
   }
 }

developer/src/kmc-package/src/compiler/kvk-font-facename.ts

@@ -0,0 +1,16 @@
+import { KvkFileReader, KvkFileWriter } from "@keymanapp/common-types";
+
+export function setKvkFontData(fileData: Uint8Array, fontFacename: string) {
+  const reader = new KvkFileReader();
+  const kvk = reader.read(fileData);
+  if(!kvk) {
+    // kvk is invalid
+    return null;
+  }
+
+  kvk.header.ansiFont.name = fontFacename;
+  kvk.header.unicodeFont.name = fontFacename;
+
+  const writer = new KvkFileWriter();
+  return writer.write(kvk);
+}

developer/src/kmc-package/src/compiler/package-compiler-messages.ts

@@ -164,5 +164,25 @@ export class PackageCompilerMessages {
   static Error_RequiredParameterMissing = (o:{param: string}) => m(
     this.ERROR_RequiredParameterMissing, `Source parameter '${def(o.param)}' is required.`
   );
+
+  static WARN_CannotFindMatchingKeyboardForVisualKeyboard = SevWarn | 0x0023;
+  static Warn_CannotFindMatchingKeyboardForVisualKeyboard = (o:{filename: string}) => m(
+    this.WARN_CannotFindMatchingKeyboardForVisualKeyboard, `Cannot find matching keyboard for visual keyboard file '${def(o.filename)}'.`
+  );
+
+  static WARN_CannotFindFontForKeyboard = SevWarn | 0x0024;
+  static Warn_CannotFindFontForKeyboard = (o:{id: string, fontFilename: string}) => m(
+    this.WARN_CannotFindFontForKeyboard, `Cannot find font file '${def(o.fontFilename)}' for keyboard '${def(o.id)}'.`
+  );
+
+  static WARN_CannotReadFont = SevWarn | 0x0025;
+  static Warn_CannotReadFont = (o:{fontFilename: string}) => m(
+    this.WARN_CannotReadFont, `Cannot parse font file '${def(o.fontFilename)}' to get facename.`
+  );
+
+  static WARN_VisualKeyboardFileIsInvalid = SevWarn | 0x0026;
+  static Warn_VisualKeyboardFileIsInvalid = (o:{filename: string}) => m(
+    this.WARN_VisualKeyboardFileIsInvalid, `Visual keyboard file '${def(o.filename)}' is invalid.`
+  );
 }
 

developer/src/kmc-package/test/fixtures/binary_kvk_file/source/binary_kvk_file.kps

@@ -27,7 +27,7 @@
       <FileType>.kvk</FileType>
     </File>
     <File>
-      <Name>../../invalid/basic.kmx</Name>
+      <Name>binary_kvk_file.kmx</Name>
       <Description>Keyboard Basic</Description>
       <CopyLocation>0</CopyLocation>
       <FileType>.kmx</FileType>
@@ -36,7 +36,7 @@
   <Keyboards>
     <Keyboard>
       <Name>Basic</Name>
-      <ID>basic</ID>
+      <ID>binary_kvk_file</ID>
       <Version>1.0</Version>
       <Languages>
         <Language ID="KM">Khmer</Language>