From 45a6291a3e34c43b51bd0563bdac8233c74200f9 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 20 Jan 2025 09:19:47 +0700 Subject: [PATCH 1/4] refactor(developer): move ttfInfo to developer-utils We will need ttfInfo in kmc-package, so move it to the shared module. --- .../web/utils}/src/font-family.ts | 27 +++++++++++++++++++ developer/src/common/web/utils/src/index.ts | 4 ++- .../web/utils}/src/ttfmeta/LICENSE | 0 .../web/utils}/src/ttfmeta/README.md | 0 .../web/utils}/src/ttfmeta/lib/index.js | 16 ++++++++++- .../web/utils}/src/ttfmeta/lib/main.js | 10 +++++++ .../web/utils}/src/ttfmeta/lib/meta.js | 0 .../web/utils}/src/ttfmeta/test/local.mjs | 0 .../utils}/src/ttfmeta/test/mocha-index.mjs | 0 developer/src/common/web/utils/tsconfig.json | 4 +++ .../src/keyboard-info-compiler.ts | 2 +- .../test/font-family.tests.ts | 2 +- developer/src/kmc-keyboard-info/tsconfig.json | 1 - 13 files changed, 61 insertions(+), 5 deletions(-) rename developer/src/{kmc-keyboard-info => common/web/utils}/src/font-family.ts (51%) rename developer/src/{kmc-keyboard-info => common/web/utils}/src/ttfmeta/LICENSE (100%) rename developer/src/{kmc-keyboard-info => common/web/utils}/src/ttfmeta/README.md (100%) rename developer/src/{kmc-keyboard-info => common/web/utils}/src/ttfmeta/lib/index.js (73%) rename developer/src/{kmc-keyboard-info => common/web/utils}/src/ttfmeta/lib/main.js (97%) rename developer/src/{kmc-keyboard-info => common/web/utils}/src/ttfmeta/lib/meta.js (100%) rename developer/src/{kmc-keyboard-info => common/web/utils}/src/ttfmeta/test/local.mjs (100%) rename developer/src/{kmc-keyboard-info => common/web/utils}/src/ttfmeta/test/mocha-index.mjs (100%) diff --git a/developer/src/kmc-keyboard-info/src/font-family.ts b/developer/src/common/web/utils/src/font-family.ts similarity index 51% rename from developer/src/kmc-keyboard-info/src/font-family.ts rename to developer/src/common/web/utils/src/font-family.ts index a78e341de7..d8fad53e37 100644 --- a/developer/src/kmc-keyboard-info/src/font-family.ts +++ b/developer/src/common/web/utils/src/font-family.ts @@ -24,5 +24,32 @@ export async function getFontFamily(source: Uint8Array) { return null; } + return font.meta.property.find(prop => prop.name == 'font-family')?.text ?? null; +} + +/** + * Extracts the font-family from an in-memory TTF or WOFF blob in `source` + * parameter. + * + * @param source In-memory TTF or WOFF font blob + * + * @throws Uncaught exceptions from ttfMeta.promise if the font file is invalid. + * + * @returns If the file is invalid or cannot be parsed, returns `null`, + * otherwise returns the font family as a string. + */ +export function getFontFamilySync(source: Uint8Array) { + /* c8 ignore next 3 */ + if(!source) { + return null; + } + + const buffer = Buffer.from(source); + const font = ttfMeta.ttfInfoSync(buffer); + /* c8 ignore next 3 */ + if(!font) { + return null; + } + return font.meta.property.find(prop => prop.name == 'font-family')?.text ?? null; } \ No newline at end of file diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index 290efd5fc7..19845f0e8f 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -68,4 +68,6 @@ export * as SourceFilenamePatterns from './source-filename-patterns.js'; export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader } from './xml-utils.js'; export * as GitHubUrls from './github-urls.js'; -export * as CloudUrls from './cloud-urls.js'; \ No newline at end of file +export * as CloudUrls from './cloud-urls.js'; + +export { getFontFamily, getFontFamilySync } from './font-family.js'; \ No newline at end of file diff --git a/developer/src/kmc-keyboard-info/src/ttfmeta/LICENSE b/developer/src/common/web/utils/src/ttfmeta/LICENSE similarity index 100% rename from developer/src/kmc-keyboard-info/src/ttfmeta/LICENSE rename to developer/src/common/web/utils/src/ttfmeta/LICENSE diff --git a/developer/src/kmc-keyboard-info/src/ttfmeta/README.md b/developer/src/common/web/utils/src/ttfmeta/README.md similarity index 100% rename from developer/src/kmc-keyboard-info/src/ttfmeta/README.md rename to developer/src/common/web/utils/src/ttfmeta/README.md diff --git a/developer/src/kmc-keyboard-info/src/ttfmeta/lib/index.js b/developer/src/common/web/utils/src/ttfmeta/lib/index.js similarity index 73% rename from developer/src/kmc-keyboard-info/src/ttfmeta/lib/index.js rename to developer/src/common/web/utils/src/ttfmeta/lib/index.js index eb94d53c1f..53972297f1 100644 --- a/developer/src/kmc-keyboard-info/src/ttfmeta/lib/index.js +++ b/developer/src/common/web/utils/src/ttfmeta/lib/index.js @@ -31,6 +31,20 @@ export function ttfInfo(pathOrData, callback) { } } +/** + * @namespace + * @param {string | number | Buffer | URL} pathOrData + * @returns {typeof ttf.result} + */ +export function ttfInfoSync(pathOrData) { + if (pathOrData instanceof Buffer) { + return ttf.ttfInfoSync(ttf.view(pathOrData)); + } else { + const data = fs.readFileSync(pathOrData); + return ttf.ttfInfoSync(ttf.view(data)); + } +} + /** * @param {string | number | Buffer | URL} pathOrData * @returns {Promise} @@ -48,5 +62,5 @@ export function promise(pathOrData) { } /** @namespace */ -export const ttfMeta = { ttfInfo, promise }; +export const ttfMeta = { ttfInfo, ttfInfoSync, promise }; export default ttfMeta; diff --git a/developer/src/kmc-keyboard-info/src/ttfmeta/lib/main.js b/developer/src/common/web/utils/src/ttfmeta/lib/main.js similarity index 97% rename from developer/src/kmc-keyboard-info/src/ttfmeta/lib/main.js rename to developer/src/common/web/utils/src/ttfmeta/lib/main.js index 239a6244dc..a952512f26 100644 --- a/developer/src/kmc-keyboard-info/src/ttfmeta/lib/main.js +++ b/developer/src/common/web/utils/src/ttfmeta/lib/main.js @@ -300,6 +300,16 @@ export function ttfInfo(data, callback) { } } +/** + * @namespace + * @param {*} data + * @returns {typeof result} + */ +export function ttfInfoSync(data) { + resultTables(data); + return result; +} + /** * @param {string | number | Buffer | URL | DataView} pathOrData * @returns {Promise} diff --git a/developer/src/kmc-keyboard-info/src/ttfmeta/lib/meta.js b/developer/src/common/web/utils/src/ttfmeta/lib/meta.js similarity index 100% rename from developer/src/kmc-keyboard-info/src/ttfmeta/lib/meta.js rename to developer/src/common/web/utils/src/ttfmeta/lib/meta.js diff --git a/developer/src/kmc-keyboard-info/src/ttfmeta/test/local.mjs b/developer/src/common/web/utils/src/ttfmeta/test/local.mjs similarity index 100% rename from developer/src/kmc-keyboard-info/src/ttfmeta/test/local.mjs rename to developer/src/common/web/utils/src/ttfmeta/test/local.mjs diff --git a/developer/src/kmc-keyboard-info/src/ttfmeta/test/mocha-index.mjs b/developer/src/common/web/utils/src/ttfmeta/test/mocha-index.mjs similarity index 100% rename from developer/src/kmc-keyboard-info/src/ttfmeta/test/mocha-index.mjs rename to developer/src/common/web/utils/src/ttfmeta/test/mocha-index.mjs diff --git a/developer/src/common/web/utils/tsconfig.json b/developer/src/common/web/utils/tsconfig.json index 6f96f75ee7..362ec3a972 100644 --- a/developer/src/common/web/utils/tsconfig.json +++ b/developer/src/common/web/utils/tsconfig.json @@ -5,8 +5,12 @@ "rootDir": ".", "outDir": "./build/", "baseUrl": ".", + + /** for ttfmeta */ + "allowJs": true, }, "include": [ "src/**/*.ts", + "src/ttfmeta/lib/*.js", ], } diff --git a/developer/src/kmc-keyboard-info/src/keyboard-info-compiler.ts b/developer/src/kmc-keyboard-info/src/keyboard-info-compiler.ts index ee784429b9..3e1db45aed 100644 --- a/developer/src/kmc-keyboard-info/src/keyboard-info-compiler.ts +++ b/developer/src/kmc-keyboard-info/src/keyboard-info-compiler.ts @@ -12,7 +12,7 @@ import { CompilerCallbacks, KeymanCompiler, CompilerOptions, KeymanCompilerResul import { KmpCompiler } from "@keymanapp/kmc-package"; import { SchemaValidators } from "@keymanapp/common-types"; -import { getFontFamily } from "./font-family.js"; +import { getFontFamily } from "@keymanapp/developer-utils"; const regionNames = new Intl.DisplayNames(['en'], { type: "region" }); const scriptNames = new Intl.DisplayNames(['en'], { type: "script" }); diff --git a/developer/src/kmc-keyboard-info/test/font-family.tests.ts b/developer/src/kmc-keyboard-info/test/font-family.tests.ts index b27fad8d74..d926e75c65 100644 --- a/developer/src/kmc-keyboard-info/test/font-family.tests.ts +++ b/developer/src/kmc-keyboard-info/test/font-family.tests.ts @@ -2,7 +2,7 @@ import * as fs from 'fs'; import { assert } from 'chai'; import 'mocha'; import { makePathToFixture } from './helpers/index.js'; -import { getFontFamily } from '../src/font-family.js'; +import { getFontFamily } from '@keymanapp/developer-utils'; const AFGHAN_TURKMEN_DISPLAY_FONT = makePathToFixture('afghan_turkmen', "Lateef-Regular.ttf"); const AFGHAN_TURKMEN_OSK_FONT = makePathToFixture('afghan_turkmen', "Lateef-Bold.ttf"); diff --git a/developer/src/kmc-keyboard-info/tsconfig.json b/developer/src/kmc-keyboard-info/tsconfig.json index d09b25c560..124c812ecf 100644 --- a/developer/src/kmc-keyboard-info/tsconfig.json +++ b/developer/src/kmc-keyboard-info/tsconfig.json @@ -13,7 +13,6 @@ }, "include": [ "src/**/*.ts", - "src/ttfmeta/lib/*.js", "src/imports/langtags.js", ], "references": [ From 0b749a5df2b9b3b7aca1d493cf93017ee0ca8af5 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 20 Jan 2025 09:21:50 +0700 Subject: [PATCH 2/4] feat(developer): rewrite font data in .kvk from package metadata When compiling a package, rewrite the font data in embedded .kvk files to use the facename from the font referenced in OSKFont or DisplayFont. This also does some safety checking for cross-references of the .kvk, and the font data. Fixes: #12912 --- .../kmc-package/src/compiler/kmp-compiler.ts | 113 ++++++++++++++---- .../src/compiler/kvk-font-facename.ts | 16 +++ .../src/compiler/package-compiler-messages.ts | 20 ++++ 3 files changed, 124 insertions(+), 25 deletions(-) create mode 100644 developer/src/kmc-package/src/compiler/kvk-font-facename.ts diff --git a/developer/src/kmc-package/src/compiler/kmp-compiler.ts b/developer/src/kmc-package/src/compiler/kmp-compiler.ts index 607823c7dd..ca15cae14d 100644 --- a/developer/src/kmc-package/src/compiler/kmp-compiler.ts +++ b/developer/src/kmc-package/src/compiler/kmp-compiler.ts @@ -18,6 +18,8 @@ import { PackageKeyboardTargetValidator } from './package-keyboard-target-valida import { PackageMetadataUpdater } from './package-metadata-updater.js'; import { markdownToHTML } from './markdown.js'; import { PackageValidation } from './package-validation.js'; +import { getFontFamilySync } from "@keymanapp/developer-utils"; +import { setKvkFontData } from './kvk-font-facename.js'; const KMP_JSON_FILENAME = 'kmp.json'; const KMP_INF_FILENAME = 'kmp.inf'; @@ -483,36 +485,35 @@ export class KmpCompiler implements KeymanCompiler { this.callbacks.reportMessage(PackageCompilerMessages.Warn_AbsolutePath({filename: filename})); } + let memberFileData = this.getMemberFileData(kpsFilename, filename); + if(!memberFileData) { + failed = true; + return; + } filename = this.callbacks.resolveFilename(kpsFilename, filename); const basename = this.callbacks.path.basename(filename); - if(!this.callbacks.fs.existsSync(filename)) { - this.callbacks.reportMessage(PackageCompilerMessages.Error_FileDoesNotExist({filename: filename})); - failed = true; - return; + if(KeymanFileTypes.filenameIs(filename, KeymanFileTypes.Binary.VisualKeyboard)) { + if(!this.isKvkFileBinary(memberFileData)) { + this.callbacks.reportMessage(PackageCompilerMessages.Warn_FileIsNotABinaryKvkFile({filename: filename})); + } else { + memberFileData = this.setKvkFontData(kpsFilename, data, filename, memberFileData); + } } - let memberFileData; - try { - memberFileData = this.callbacks.loadFile(filename); - } catch(e) { - this.callbacks.reportMessage(PackageCompilerMessages.Error_FileCouldNotBeRead({filename: filename, e: e})); - failed = true; - return; - } - - this.warnIfKvkFileIsNotBinary(filename, memberFileData); - zip.file(basename, memberFileData); - - // Remove path data from files before JSON save - value.name = basename; }); if(failed) { return null; } + data.files.forEach((value) => { + const basename = this.callbacks.path.basename(value.name); + // Remove path data from files before JSON save + value.name = basename; + }); + // TODO #9477: transform .md to .htm // Remove path data from file references in options @@ -557,26 +558,88 @@ export class KmpCompiler implements KeymanCompiler { return transcodeToCP1252(s); } + private getMemberFileData(kpsFilename: string, filename: string) { + filename = this.callbacks.resolveFilename(kpsFilename, filename); + + if(!this.callbacks.fs.existsSync(filename)) { + this.callbacks.reportMessage(PackageCompilerMessages.Error_FileDoesNotExist({filename: filename})); + return null; + } + + try { + return this.callbacks.loadFile(filename); + } catch(e) { + this.callbacks.reportMessage(PackageCompilerMessages.Error_FileCouldNotBeRead({filename: filename, e: e})); + return null; + } + } + + private setKvkFontData(kpsFilename: string, data: KmpJsonFile.KmpJsonFile, filename: string, kvk: Uint8Array) { + // find the appropriate font to set + const kvkId = this.callbacks.path.basename(filename.toLowerCase(), '.kvk'); + const kbd = data.keyboards.find(k => k.id == kvkId); + if(!kbd) { + // cannot find a matching keyboard + this.callbacks.reportMessage(PackageCompilerMessages.Warn_CannotFindMatchingKeyboardForVisualKeyboard({filename})); + return kvk; + } + const fontFilename = kbd.oskFont || kbd.displayFont; + if(!fontFilename) { + // no font data to set + return kvk; + } + + // Look up the full font filename + const fontFile = data.files.find(file => this.callbacks.path.basename(file.name.toLowerCase()) == fontFilename.toLowerCase()); + if(!fontFile) { + // font cannot be found + this.callbacks.reportMessage(PackageCompilerMessages.Warn_CannotFindFontForKeyboard({id: kbd.id, fontFilename})); + return kvk; + } + + // the font is a filename, included in the .kps + const fontData = this.getMemberFileData(kpsFilename, fontFile.name); + if(!fontData) { + // cannot find source font + this.callbacks.reportMessage(PackageCompilerMessages.Warn_CannotFindFontForKeyboard({id: kbd.id, fontFilename: fontFile.name})); + return kvk; + } + + const fontFacename = getFontFamilySync(fontData); + if(!fontFacename) { + // Font facename could not be extracted from ttf + this.callbacks.reportMessage(PackageCompilerMessages.Warn_CannotReadFont({fontFilename: fontFile.name})); + return kvk; + } + + const result = setKvkFontData(kvk, fontFacename); + if(!result) { + // KVK is invalid + this.callbacks.reportMessage(PackageCompilerMessages.Warn_VisualKeyboardFileIsInvalid({filename})); + return kvk; + } + + return result; + } + /** * Legacy .kmp compiler would transform xml-format .kvk files into a binary .kvk file; now * we want that to remain the responsibility of the keyboard compiler, so we'll warn the * few users who are still doing this */ - private warnIfKvkFileIsNotBinary(filename: string, data: Uint8Array) { - if(!KeymanFileTypes.filenameIs(filename, KeymanFileTypes.Binary.VisualKeyboard)) { - return; - } - + private isKvkFileBinary(data: Uint8Array) { if(data.byteLength < 4) { // TODO: Not a valid .kvk file; should we be reporting this? - return; + return false; } + // Must start with 'KVKF' if(data[0] != KvkFile.KVK_HEADER_IDENTIFIER_BYTES[0] || data[1] != KvkFile.KVK_HEADER_IDENTIFIER_BYTES[1] || data[2] != KvkFile.KVK_HEADER_IDENTIFIER_BYTES[2] || data[3] != KvkFile.KVK_HEADER_IDENTIFIER_BYTES[3]) { - this.callbacks.reportMessage(PackageCompilerMessages.Warn_FileIsNotABinaryKvkFile({filename: filename})); + return false; } + return true; } } diff --git a/developer/src/kmc-package/src/compiler/kvk-font-facename.ts b/developer/src/kmc-package/src/compiler/kvk-font-facename.ts new file mode 100644 index 0000000000..b7c69da8e8 --- /dev/null +++ b/developer/src/kmc-package/src/compiler/kvk-font-facename.ts @@ -0,0 +1,16 @@ +import { KvkFileReader, KvkFileWriter } from "@keymanapp/common-types"; + +export function setKvkFontData(fileData: Uint8Array, fontFacename: string) { + const reader = new KvkFileReader(); + const kvk = reader.read(fileData); + if(!kvk) { + // kvk is invalid + return null; + } + + kvk.header.ansiFont.name = fontFacename; + kvk.header.unicodeFont.name = fontFacename; + + const writer = new KvkFileWriter(); + return writer.write(kvk); +} \ No newline at end of file diff --git a/developer/src/kmc-package/src/compiler/package-compiler-messages.ts b/developer/src/kmc-package/src/compiler/package-compiler-messages.ts index 9a593a3a11..6545b7d963 100644 --- a/developer/src/kmc-package/src/compiler/package-compiler-messages.ts +++ b/developer/src/kmc-package/src/compiler/package-compiler-messages.ts @@ -164,5 +164,25 @@ export class PackageCompilerMessages { static Error_RequiredParameterMissing = (o:{param: string}) => m( this.ERROR_RequiredParameterMissing, `Source parameter '${def(o.param)}' is required.` ); + + static WARN_CannotFindMatchingKeyboardForVisualKeyboard = SevWarn | 0x0023; + static Warn_CannotFindMatchingKeyboardForVisualKeyboard = (o:{filename: string}) => m( + this.WARN_CannotFindMatchingKeyboardForVisualKeyboard, `Cannot find matching keyboard for visual keyboard file '${def(o.filename)}'.` + ); + + static WARN_CannotFindFontForKeyboard = SevWarn | 0x0024; + static Warn_CannotFindFontForKeyboard = (o:{id: string, fontFilename: string}) => m( + this.WARN_CannotFindFontForKeyboard, `Cannot find font file '${def(o.fontFilename)}' for keyboard '${def(o.id)}'.` + ); + + static WARN_CannotReadFont = SevWarn | 0x0025; + static Warn_CannotReadFont = (o:{fontFilename: string}) => m( + this.WARN_CannotReadFont, `Cannot parse font file '${def(o.fontFilename)}' to get facename.` + ); + + static WARN_VisualKeyboardFileIsInvalid = SevWarn | 0x0026; + static Warn_VisualKeyboardFileIsInvalid = (o:{filename: string}) => m( + this.WARN_VisualKeyboardFileIsInvalid, `Visual keyboard file '${def(o.filename)}' is invalid.` + ); } From d24f3477bd6fab8021b0ae613c32456cf43e0b4f Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 20 Jan 2025 10:06:25 +0700 Subject: [PATCH 3/4] chore(developer): update binary_kvk_file test to make .kps more valid The test .kps for binary_kvk_file tests was not very coherent, which meant that it failed with the changes and additional consistency checking included in this PR. --- .../binary_kvk_file/source/binary_kvk_file.kmx | Bin 0 -> 352 bytes .../binary_kvk_file/source/binary_kvk_file.kps | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 developer/src/kmc-package/test/fixtures/binary_kvk_file/source/binary_kvk_file.kmx diff --git a/developer/src/kmc-package/test/fixtures/binary_kvk_file/source/binary_kvk_file.kmx b/developer/src/kmc-package/test/fixtures/binary_kvk_file/source/binary_kvk_file.kmx new file mode 100644 index 0000000000000000000000000000000000000000..bd6b832cab772d03a45460e6c982d79772a9ab6e GIT binary patch literal 352 zcmZvYyAAdPzftU-(5_7TAk3e<2#yh{6J%lpJ>i; zp8cfbGTcz0i3ZwI$I5E289asBOpaP`wxny)?MPoMtp7qQ?)E9)e`0JAA>To*E2jVe literal 0 HcmV?d00001 diff --git a/developer/src/kmc-package/test/fixtures/binary_kvk_file/source/binary_kvk_file.kps b/developer/src/kmc-package/test/fixtures/binary_kvk_file/source/binary_kvk_file.kps index de17212a84..1a02984beb 100644 --- a/developer/src/kmc-package/test/fixtures/binary_kvk_file/source/binary_kvk_file.kps +++ b/developer/src/kmc-package/test/fixtures/binary_kvk_file/source/binary_kvk_file.kps @@ -27,7 +27,7 @@ .kvk - ../../invalid/basic.kmx + binary_kvk_file.kmx Keyboard Basic 0 .kmx @@ -36,7 +36,7 @@ Basic - basic + binary_kvk_file 1.0 Khmer From 11d92be941125981e79477b4fdc85b2f6af4f5b8 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 20 Jan 2025 10:25:56 +0700 Subject: [PATCH 4/4] chore: address review comments --- developer/src/kmc-package/src/compiler/kmp-compiler.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/developer/src/kmc-package/src/compiler/kmp-compiler.ts b/developer/src/kmc-package/src/compiler/kmp-compiler.ts index ca15cae14d..cd6cf6df36 100644 --- a/developer/src/kmc-package/src/compiler/kmp-compiler.ts +++ b/developer/src/kmc-package/src/compiler/kmp-compiler.ts @@ -495,6 +495,7 @@ export class KmpCompiler implements KeymanCompiler { if(KeymanFileTypes.filenameIs(filename, KeymanFileTypes.Binary.VisualKeyboard)) { if(!this.isKvkFileBinary(memberFileData)) { + // warn the few users who are still doing this -- non-binary .kvk should not be included in package! this.callbacks.reportMessage(PackageCompilerMessages.Warn_FileIsNotABinaryKvkFile({filename: filename})); } else { memberFileData = this.setKvkFontData(kpsFilename, data, filename, memberFileData); @@ -624,8 +625,7 @@ export class KmpCompiler implements KeymanCompiler { /** * Legacy .kmp compiler would transform xml-format .kvk files into a binary .kvk file; now - * we want that to remain the responsibility of the keyboard compiler, so we'll warn the - * few users who are still doing this + * we want that to remain the responsibility of the keyboard compiler. */ private isKvkFileBinary(data: Uint8Array) { if(data.byteLength < 4) {