From 790d76a9251316d5301a3bc71021f8f83434071d Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Thu, 15 May 2025 10:37:31 +0700 Subject: [PATCH 01/73] maint(ios): Use zip.inc.sh to prep release artifacts --- ios/exclude.in | 2 ++ ios/tools/prepRelease.sh | 10 ++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 ios/exclude.in diff --git a/ios/exclude.in b/ios/exclude.in new file mode 100644 index 0000000000..d6c495d66c --- /dev/null +++ b/ios/exclude.in @@ -0,0 +1,2 @@ +samples/*/build.sh + diff --git a/ios/tools/prepRelease.sh b/ios/tools/prepRelease.sh index 22e580555e..7bcb9aced3 100755 --- a/ios/tools/prepRelease.sh +++ b/ios/tools/prepRelease.sh @@ -15,6 +15,8 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" . "${THIS_SCRIPT%/*}/../../resources/build/build-utils.sh" ## END STANDARD BUILD SCRIPT INCLUDE +. "$KEYMAN_ROOT/resources/build/zip.inc.sh" + # This script runs from its parent's folder cd "$(dirname "$THIS_SCRIPT")/.." @@ -71,13 +73,13 @@ echo "engine dest: $KMEI_DST" echo "Zipping ${FRAMEWORK} => ${UPLOAD_DIR}/${KMEI_DST}..." cd "${KMEI_FRAMEWORK_BASE}" -zip -qrX "${KMEI_DST}" ${FRAMEWORK} +zip_files "${KMEI_DST}" "../../../../exclude.in" "-qrX" "${FRAMEWORK}" cd "$WORK_DIR" echo "Copying Keyman Engine samples into ${UPLOAD_DIR}/${KMEI_DST_NAME}..." cp -rf "${KEYMAN_SAMPLES}" "${UPLOAD_DIR}/samples" cd "${UPLOAD_DIR}" -zip -qr "${KMEI_DST_NAME}" "samples" +zip_files "${KMEI_DST_NAME}" "../../exclude.in" "-qr" "samples" rm -rf "samples" cd "$WORK_DIR" @@ -97,7 +99,7 @@ KEYMANAPP_SIM_APP_DST="keyman-ios-simulator-${BUILD_NUMBER}.app.zip" echo "Zipping Keyman simulator artifact ${KEYMANAPP_SIM_APP} => ${UPLOAD_DIR}/${KEYMANAPP_SIM_APP_DST}..." cd "${KEYMANAPP_SIM_FOLDER}" -zip -qrX "${WORK_DIR}/${UPLOAD_DIR}/${KEYMANAPP_SIM_APP_DST}" "Keyman.app" +zip_files "${WORK_DIR}/${UPLOAD_DIR}/${KEYMANAPP_SIM_APP_DST}" "../../../../exclude.in" "-qrX" "Keyman.app" echo "${WORK_DIR}/${UPLOAD_DIR}/${KEYMANAPP_SIM_APP_DST}" cd "$WORK_DIR" @@ -118,7 +120,7 @@ if [ "${RELEASE_OEM_FIRSTVOICES}" = true ]; then echo "Zipping FirstVoices simulator artifact ${FIRSTVOICESAPP_SIM_APP} => ${UPLOAD_DIR}/${KEYMANAPP_SIM_APP_DST}..." cd "${FIRSTVOICESAPP_SIM_FOLDER}" - zip -qrX "${WORK_DIR}/${UPLOAD_DIR}/${FIRSTVOICESAPP_SIM_APP_DST}" "FirstVoices.app" + zip_files "${WORK_DIR}/${UPLOAD_DIR}/${FIRSTVOICESAPP_SIM_APP_DST}" "../../../../../../../ios/exclude.in" "-qrX" "FirstVoices.app" cd "$WORK_DIR" fi From 38be1d9b233be39e6aabf1e1da561229d40c945d Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Sun, 18 May 2025 06:48:19 +0700 Subject: [PATCH 02/73] maint(ios): Update zip_files calls --- ios/tools/prepRelease.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ios/tools/prepRelease.sh b/ios/tools/prepRelease.sh index 7bcb9aced3..f7247eb995 100755 --- a/ios/tools/prepRelease.sh +++ b/ios/tools/prepRelease.sh @@ -73,13 +73,13 @@ echo "engine dest: $KMEI_DST" echo "Zipping ${FRAMEWORK} => ${UPLOAD_DIR}/${KMEI_DST}..." cd "${KMEI_FRAMEWORK_BASE}" -zip_files "${KMEI_DST}" "../../../../exclude.in" "-qrX" "${FRAMEWORK}" +zip_files "${KMEI_DST}" "-qrX" "${FRAMEWORK}" cd "$WORK_DIR" echo "Copying Keyman Engine samples into ${UPLOAD_DIR}/${KMEI_DST_NAME}..." cp -rf "${KEYMAN_SAMPLES}" "${UPLOAD_DIR}/samples" cd "${UPLOAD_DIR}" -zip_files "${KMEI_DST_NAME}" "../../exclude.in" "-qr" "samples" +zip_files "${KMEI_DST_NAME}" "-x@../../exclude.in" "-qr" "samples" rm -rf "samples" cd "$WORK_DIR" @@ -99,7 +99,7 @@ KEYMANAPP_SIM_APP_DST="keyman-ios-simulator-${BUILD_NUMBER}.app.zip" echo "Zipping Keyman simulator artifact ${KEYMANAPP_SIM_APP} => ${UPLOAD_DIR}/${KEYMANAPP_SIM_APP_DST}..." cd "${KEYMANAPP_SIM_FOLDER}" -zip_files "${WORK_DIR}/${UPLOAD_DIR}/${KEYMANAPP_SIM_APP_DST}" "../../../../exclude.in" "-qrX" "Keyman.app" +zip_files "${WORK_DIR}/${UPLOAD_DIR}/${KEYMANAPP_SIM_APP_DST}" "-qrX" "Keyman.app" echo "${WORK_DIR}/${UPLOAD_DIR}/${KEYMANAPP_SIM_APP_DST}" cd "$WORK_DIR" @@ -120,7 +120,7 @@ if [ "${RELEASE_OEM_FIRSTVOICES}" = true ]; then echo "Zipping FirstVoices simulator artifact ${FIRSTVOICESAPP_SIM_APP} => ${UPLOAD_DIR}/${KEYMANAPP_SIM_APP_DST}..." cd "${FIRSTVOICESAPP_SIM_FOLDER}" - zip_files "${WORK_DIR}/${UPLOAD_DIR}/${FIRSTVOICESAPP_SIM_APP_DST}" "../../../../../../../ios/exclude.in" "-qrX" "FirstVoices.app" + zip_files "${WORK_DIR}/${UPLOAD_DIR}/${FIRSTVOICESAPP_SIM_APP_DST}" "-qrX" "FirstVoices.app" cd "$WORK_DIR" fi From 6e8f96174983fcf7f99c921dfa015b9e8c113cb9 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Mon, 19 May 2025 09:39:03 +0700 Subject: [PATCH 03/73] maint(ios): Convert some zip flags for 7z --- ios/tools/prepRelease.sh | 9 ++++---- resources/build/zip.inc.sh | 46 +++++++++++++++++++++++++++++--------- 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/ios/tools/prepRelease.sh b/ios/tools/prepRelease.sh index f7247eb995..af67632965 100755 --- a/ios/tools/prepRelease.sh +++ b/ios/tools/prepRelease.sh @@ -70,16 +70,17 @@ FRAMEWORK="KeymanEngine.xcframework" KEYMAN_SAMPLES="samples" echo "engine dest: $KMEI_DST" +ZIP_FLAGS=("-q" "-r" "-X") # quiet, recursive, no-extra echo "Zipping ${FRAMEWORK} => ${UPLOAD_DIR}/${KMEI_DST}..." cd "${KMEI_FRAMEWORK_BASE}" -zip_files "${KMEI_DST}" "-qrX" "${FRAMEWORK}" +zip_files "${KMEI_DST}" "${ZIP_FLAGS[@]}" "${FRAMEWORK}" cd "$WORK_DIR" echo "Copying Keyman Engine samples into ${UPLOAD_DIR}/${KMEI_DST_NAME}..." cp -rf "${KEYMAN_SAMPLES}" "${UPLOAD_DIR}/samples" cd "${UPLOAD_DIR}" -zip_files "${KMEI_DST_NAME}" "-x@../../exclude.in" "-qr" "samples" +zip_files "${KMEI_DST_NAME}" "-x@../../exclude.in" "${ZIP_FLAGS[@]}" "samples" rm -rf "samples" cd "$WORK_DIR" @@ -99,7 +100,7 @@ KEYMANAPP_SIM_APP_DST="keyman-ios-simulator-${BUILD_NUMBER}.app.zip" echo "Zipping Keyman simulator artifact ${KEYMANAPP_SIM_APP} => ${UPLOAD_DIR}/${KEYMANAPP_SIM_APP_DST}..." cd "${KEYMANAPP_SIM_FOLDER}" -zip_files "${WORK_DIR}/${UPLOAD_DIR}/${KEYMANAPP_SIM_APP_DST}" "-qrX" "Keyman.app" +zip_files "${WORK_DIR}/${UPLOAD_DIR}/${KEYMANAPP_SIM_APP_DST}" "${ZIP_FLAGS[@]}" "Keyman.app" echo "${WORK_DIR}/${UPLOAD_DIR}/${KEYMANAPP_SIM_APP_DST}" cd "$WORK_DIR" @@ -120,7 +121,7 @@ if [ "${RELEASE_OEM_FIRSTVOICES}" = true ]; then echo "Zipping FirstVoices simulator artifact ${FIRSTVOICESAPP_SIM_APP} => ${UPLOAD_DIR}/${KEYMANAPP_SIM_APP_DST}..." cd "${FIRSTVOICESAPP_SIM_FOLDER}" - zip_files "${WORK_DIR}/${UPLOAD_DIR}/${FIRSTVOICESAPP_SIM_APP_DST}" "-qrX" "FirstVoices.app" + zip_files "${WORK_DIR}/${UPLOAD_DIR}/${FIRSTVOICESAPP_SIM_APP_DST}" "${ZIP_FLAGS[@]}" "FirstVoices.app" cd "$WORK_DIR" fi diff --git a/resources/build/zip.inc.sh b/resources/build/zip.inc.sh index 2b09685779..6b247705b3 100644 --- a/resources/build/zip.inc.sh +++ b/resources/build/zip.inc.sh @@ -23,13 +23,34 @@ function zip_files() { shift # Parse rest of parameters - FLAGS=() - INCLUDE=() + local ZIP_FLAGS=() + local SEVENZ_FLAGS=('a') # 7z requires a command + local INCLUDE=() while [[ $# -gt 0 ]] ; do case "$1" in + -r) + # Common flags to zip and 7z + ZIP_FLAGS+=($1) + SEVENZ_FLAGS+=($1) + shift + ;; + -1) + # Compression level where -1 indicates fastest compression speed + ZIP_FLAGS+=($1) + SEVENZ_FLAGS+=("-mx1") + echo "SEVENZ_FLAGS: ${SEVENZ_FLAGS[@]}" + shift + ;; + -9) + # Compression level where -9 indicates the slowest compression speed + ZIP_FLAGS+=($1) + SEVENZ_FLAGS+=("-mx9") + echo "SEVENZ_FLAGS: ${SEVENZ_FLAGS[@]}" + shift + ;; -*) - # zip/7z flags - FLAGS+=($1) + # Rest of zip flags. + ZIP_FLAGS+=($1) shift ;; *) @@ -40,7 +61,7 @@ function zip_files() { esac done - COMPRESS_CMD=zip + local COMPRESS_CMD=zip SEVENZ_CMD= if ! command -v zip 2>&1 > /dev/null; then # Fallback to 7z @@ -53,15 +74,18 @@ function zip_files() { SEVENZ=7z ;; esac - - # 7z command to add files - COMPRESS_CMD="${SEVENZ}" - SEVENZ_CMD="a" fi + + # 7z command to add files so clear zip flags + COMPRESS_CMD="${SEVENZ}" + ZIP_FLAGS=() + else + # Using zip so clear 7z flags + SEVENZ_FLAGS=() fi # Create archive - builder_echo_debug "${COMPRESS_CMD} ${SEVENZ_CMD} ${FLAGS[@]} ${ZIP_FILE} ${INCLUDE[@]}" - "${COMPRESS_CMD}" ${SEVENZ_CMD} ${FLAGS[@]} ${ZIP_FILE} ${INCLUDE[@]} + builder_echo_debug "${COMPRESS_CMD} ${SEVENZ_FLAGS[@]} ${ZIP_FLAGS[@]} ${ZIP_FILE} ${INCLUDE[@]}" + "${COMPRESS_CMD}" ${SEVENZ_FLAGS[@]} ${ZIP_FLAGS[@]} ${ZIP_FILE} ${INCLUDE[@]} } From af6faddd5d93a440c758618bb07394e0d268de27 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Mon, 19 May 2025 09:42:32 +0700 Subject: [PATCH 04/73] maint(ios): Cleanup debug echos --- resources/build/zip.inc.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/resources/build/zip.inc.sh b/resources/build/zip.inc.sh index 6b247705b3..2b4f8f16b7 100644 --- a/resources/build/zip.inc.sh +++ b/resources/build/zip.inc.sh @@ -38,14 +38,12 @@ function zip_files() { # Compression level where -1 indicates fastest compression speed ZIP_FLAGS+=($1) SEVENZ_FLAGS+=("-mx1") - echo "SEVENZ_FLAGS: ${SEVENZ_FLAGS[@]}" shift ;; -9) # Compression level where -9 indicates the slowest compression speed ZIP_FLAGS+=($1) SEVENZ_FLAGS+=("-mx9") - echo "SEVENZ_FLAGS: ${SEVENZ_FLAGS[@]}" shift ;; -*) From bf27e65d557d39cb045b8b58fbef3dae1d330401 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Mon, 19 May 2025 14:37:15 +0700 Subject: [PATCH 05/73] maint(common): Convert quiet mode flags for 7z --- resources/build/zip.inc.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/resources/build/zip.inc.sh b/resources/build/zip.inc.sh index 2b4f8f16b7..3bac48ca55 100644 --- a/resources/build/zip.inc.sh +++ b/resources/build/zip.inc.sh @@ -34,6 +34,13 @@ function zip_files() { SEVENZ_FLAGS+=($1) shift ;; + -q) + # quiet mode -> disable progress indicator, set output log level 0 + ZIP_FLAGS+=($1) + SEVENZ_FLAGS+=("-bd") + SEVENZ_FLAGS+=("-bb0") + shift + ;; -1) # Compression level where -1 indicates fastest compression speed ZIP_FLAGS+=($1) @@ -60,7 +67,6 @@ function zip_files() { done local COMPRESS_CMD=zip - SEVENZ_CMD= if ! command -v zip 2>&1 > /dev/null; then # Fallback to 7z if [[ -z "${SEVENZ+x}" ]]; then From 4a4277ff5a413ff46f6887053cd07a66490072de Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Wed, 21 May 2025 08:23:08 +0700 Subject: [PATCH 06/73] refactor(common): Add regex for compression level --- resources/build/zip.inc.sh | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/resources/build/zip.inc.sh b/resources/build/zip.inc.sh index 3bac48ca55..bef8616024 100644 --- a/resources/build/zip.inc.sh +++ b/resources/build/zip.inc.sh @@ -41,17 +41,16 @@ function zip_files() { SEVENZ_FLAGS+=("-bb0") shift ;; - -1) - # Compression level where -1 indicates fastest compression speed + -[0123456789]) + # Compression level where + # -0 indicates no compression + # -1 indicates low compression (fastest) + # -9 indicates ultra compression (slowest) ZIP_FLAGS+=($1) - SEVENZ_FLAGS+=("-mx1") - shift - ;; - -9) - # Compression level where -9 indicates the slowest compression speed - ZIP_FLAGS+=($1) - SEVENZ_FLAGS+=("-mx9") - shift + if [[ $1 =~ -([0-9]) ]] then + SEVENZ_FLAGS+=("-mx${BASH_REMATCH[1]}") + fi + shift; ;; -*) # Rest of zip flags. From 3cb9fe2a65450d1cee8e3d0d8d61e43ff6387519 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Thu, 22 May 2025 16:20:18 +0200 Subject: [PATCH 07/73] fix(linux): fix variable replacements This fixes the version numbers in `km-config` which broke with #13854. This also fixes opening the download page in `km-config` because that uses a version number in the URL. Fixes: #14031 Follow-up-of: #13854 --- linux/keyman-config/build.sh | 16 ++++++++-------- linux/keyman-config/keyman_config/version.py.in | 16 ++++++++-------- linux/scripts/reconf.sh | 16 ++++++++-------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/linux/keyman-config/build.sh b/linux/keyman-config/build.sh index 0eccb04b13..8da21f35ce 100755 --- a/linux/keyman-config/build.sh +++ b/linux/keyman-config/build.sh @@ -62,14 +62,14 @@ build_action() { builder_echo "Create version.py" pushd keyman_config sed \ - -e "s/_KEYMAN_VERSION_/${KEYMAN_VERSION}/g" \ - -e "s/_KEYMAN_VERSION_WITH_TAG_/${KEYMAN_VERSION_WITH_TAG}/g" \ - -e "s/_KEYMAN_VERSION_GIT_TAG_/${KEYMAN_VERSION_GIT_TAG}/g" \ - -e "s/_KEYMAN_VERSION_MAJOR_/${KEYMAN_VERSION_MAJOR}/g" \ - -e "s/_KEYMAN_VERSION_RELEASE_/${KEYMAN_VERSION_RELEASE}/g" \ - -e "s/_KEYMAN_TIER_/${KEYMAN_TIER}/g" \ - -e "s/_KEYMAN_VERSION_ENVIRONMENT_/${KEYMAN_VERSION_ENVIRONMENT}/g" \ - -e "s/_UPLOAD_SENTRY_/${UPLOAD_SENTRY}/g" \ + -e "s/__KEYMAN_VERSION__/${KEYMAN_VERSION}/g" \ + -e "s/__KEYMAN_VERSION_WITH_TAG__/${KEYMAN_VERSION_WITH_TAG}/g" \ + -e "s/__KEYMAN_VERSION_GIT_TAG__/${KEYMAN_VERSION_GIT_TAG}/g" \ + -e "s/__KEYMAN_VERSION_MAJOR__/${KEYMAN_VERSION_MAJOR}/g" \ + -e "s/__KEYMAN_VERSION_RELEASE__/${KEYMAN_VERSION_RELEASE}/g" \ + -e "s/__KEYMAN_TIER__/${KEYMAN_TIER}/g" \ + -e "s/__KEYMAN_VERSION_ENVIRONMENT__/${KEYMAN_VERSION_ENVIRONMENT}/g" \ + -e "s/__UPLOAD_SENTRY__/${UPLOAD_SENTRY}/g" \ version.py.in > version.py popd pushd buildtools diff --git a/linux/keyman-config/keyman_config/version.py.in b/linux/keyman-config/keyman_config/version.py.in index 3221a6ddff..4c3c02c9a4 100644 --- a/linux/keyman-config/keyman_config/version.py.in +++ b/linux/keyman-config/keyman_config/version.py.in @@ -4,12 +4,12 @@ # 1) we don't load dependencies by storing it in __init__.py # 2) we can import it in setup.py for the same reason # 3) we can import it into your module module -__version__ = "_KEYMAN_VERSION_" -__versionwithtag__ = "_KEYMAN_VERSION_WITH_TAG_" -__majorversion__ = "_KEYMAN_VERSION_MAJOR_" -__releaseversion__ = "_KEYMAN_VERSION_RELEASE_" -__tier__ = "_KEYMAN_TIER_" +__version__ = "__KEYMAN_VERSION__" +__versionwithtag__ = "__KEYMAN_VERSION_WITH_TAG__" +__majorversion__ = "__KEYMAN_VERSION_MAJOR__" +__releaseversion__ = "__KEYMAN_VERSION_RELEASE__" +__tier__ = "__KEYMAN_TIER__" __pkgversion__ = "(local)" -__environment__ = "_KEYMAN_VERSION_ENVIRONMENT_" -__uploadsentry__ = "_UPLOAD_SENTRY_" -__versiongittag__ = "_KEYMAN_VERSION_GIT_TAG_" +__environment__ = "__KEYMAN_VERSION_ENVIRONMENT__" +__uploadsentry__ = "__UPLOAD_SENTRY__" +__versiongittag__ = "__KEYMAN_VERSION_GIT_TAG__" diff --git a/linux/scripts/reconf.sh b/linux/scripts/reconf.sh index da1610545b..d6d2c3891c 100755 --- a/linux/scripts/reconf.sh +++ b/linux/scripts/reconf.sh @@ -32,14 +32,14 @@ quilt new version_py.diff quilt add "version.py" sed \ - -e "s/_KEYMAN_VERSION_/${KEYMAN_VERSION}/g" \ - -e "s/_KEYMAN_VERSION_WITH_TAG_/${KEYMAN_VERSION_WITH_TAG}/g" \ - -e "s/_KEYMAN_VERSION_GIT_TAG_/${KEYMAN_VERSION_GIT_TAG}/g" \ - -e "s/_KEYMAN_VERSION_MAJOR_/${KEYMAN_VERSION_MAJOR}/g" \ - -e "s/_KEYMAN_VERSION_RELEASE_/${KEYMAN_VERSION_RELEASE}/g" \ - -e "s/_KEYMAN_TIER_/${KEYMAN_TIER}/g" \ - -e "s/_KEYMAN_VERSION_ENVIRONMENT_/${KEYMAN_VERSION_ENVIRONMENT}/g" \ - -e "s/_UPLOAD_SENTRY_/${UPLOAD_SENTRY}/g" \ + -e "s/__KEYMAN_VERSION__/${KEYMAN_VERSION}/g" \ + -e "s/__KEYMAN_VERSION_WITH_TAG__/${KEYMAN_VERSION_WITH_TAG}/g" \ + -e "s/__KEYMAN_VERSION_GIT_TAG__/${KEYMAN_VERSION_GIT_TAG}/g" \ + -e "s/__KEYMAN_VERSION_MAJOR__/${KEYMAN_VERSION_MAJOR}/g" \ + -e "s/__KEYMAN_VERSION_RELEASE__/${KEYMAN_VERSION_RELEASE}/g" \ + -e "s/__KEYMAN_TIER__/${KEYMAN_TIER}/g" \ + -e "s/__KEYMAN_VERSION_ENVIRONMENT__/${KEYMAN_VERSION_ENVIRONMENT}/g" \ + -e "s/__UPLOAD_SENTRY__/${UPLOAD_SENTRY}/g" \ version.py.in > version.py quilt refresh quilt pop -a From a3e919f4ac9eb37421ee91eb28e0eb735dde7bf9 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Tue, 14 Jan 2025 11:27:52 +0100 Subject: [PATCH 08/73] chore(web): cleanup async methods in UI components Cherry-pick of changes in #12291. Addresses code review comment. Test-bot: skip --- web/src/app/ui/kmwuibutton.ts | 6 +++--- web/src/app/ui/kmwuifloat.ts | 6 +++--- web/src/app/ui/kmwuitoggle.ts | 18 +++++++++--------- web/src/app/ui/kmwuitoolbar.ts | 8 ++++---- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/web/src/app/ui/kmwuibutton.ts b/web/src/app/ui/kmwuibutton.ts index e004e3f0cd..c8717e5dda 100644 --- a/web/src/app/ui/kmwuibutton.ts +++ b/web/src/app/ui/kmwuibutton.ts @@ -28,7 +28,7 @@ if(!keymanweb) { /** * Do not enclose in an anonymous function, as the compiler may create - * global scope variables to replace true, false, null, whcih can then collide + * global scope variables to replace true, false, null, which can then collide * with other variables. * Instead, use the --output-wrapper command during optimization, which will * add the anonymous function to enclose all code, including those optimized @@ -106,7 +106,7 @@ if(!keymanweb) { * @param {Event} _id keyboard selection event * @return {boolean} */ - readonly _SelectKeyboard = (_id: Event) => { + readonly _SelectKeyboard = async (_id: Event): Promise => { let id: string = ''; if(typeof(_id) == 'object') { let t: HTMLElement = null; @@ -135,7 +135,7 @@ if(!keymanweb) { _k.className='selected'; } this._KMWSel = _k; - keymanweb.setActiveKeyboard(_name,_lgc); + await keymanweb.setActiveKeyboard(_name,_lgc); } else { _name=null; } diff --git a/web/src/app/ui/kmwuifloat.ts b/web/src/app/ui/kmwuifloat.ts index 4e6b8df103..614119ed4e 100644 --- a/web/src/app/ui/kmwuifloat.ts +++ b/web/src/app/ui/kmwuifloat.ts @@ -450,15 +450,15 @@ if(!keymanweb) { * @param {Object} e event * Description Change active keyboard in response to user selection event */ - readonly SelectKeyboardChange = (e: Event) => { + readonly SelectKeyboardChange = async (e: Event) => { keymanweb.activatingUI(true); if(this.KeyboardSelector.value != '-') { const i=this.KeyboardSelector.selectedIndex; const t=this.KeyboardSelector.options[i].value.split(':'); - keymanweb.setActiveKeyboard(t[0],t[1]); + await keymanweb.setActiveKeyboard(t[0],t[1]); } else { - keymanweb.setActiveKeyboard(''); + await keymanweb.setActiveKeyboard(''); } //if(osk['show']) osk['show'](osk['isEnabled']()); handled by keyboard change event??? diff --git a/web/src/app/ui/kmwuitoggle.ts b/web/src/app/ui/kmwuitoggle.ts index fe1a878825..29d9a802f7 100644 --- a/web/src/app/ui/kmwuitoggle.ts +++ b/web/src/app/ui/kmwuitoggle.ts @@ -187,7 +187,7 @@ if(!keymanweb) { /** * Toggle a single keyboard on or off - KMW button control event **/ - readonly switchSingleKbd = () => { + readonly switchSingleKbd = async () => { const _v = keymanweb.getActiveKeyboard() == ''; let nLastKbd=0, kbdName='', lgCode=''; @@ -202,10 +202,10 @@ if(!keymanweb) { kbdName = this.keyboards[nLastKbd]._InternalName; lgCode = this.keyboards[nLastKbd]._LanguageCode; - keymanweb.setActiveKeyboard(kbdName,lgCode); + await keymanweb.setActiveKeyboard(kbdName,lgCode); this.lastActiveKeyboard = nLastKbd; } else { - keymanweb.setActiveKeyboard(''); + await keymanweb.setActiveKeyboard(''); } if(this.kbdButton) { @@ -216,7 +216,7 @@ if(!keymanweb) { /** * Switch to the next keyboard in the list - KMW button control event **/ - readonly switchNextKbd = () => { + readonly switchNextKbd = async () => { let _v = (keymanweb.getActiveKeyboard() == ''); let kbdName='', lgCode=''; @@ -227,16 +227,16 @@ if(!keymanweb) { kbdName = this.keyboards[0]._InternalName; lgCode = this.keyboards[0]._LanguageCode; - keymanweb.setActiveKeyboard(kbdName,lgCode); + await keymanweb.setActiveKeyboard(kbdName,lgCode); this.lastActiveKeyboard = 0; } else { if(this.lastActiveKeyboard == this.keyboards.length-1) { - keymanweb.setActiveKeyboard(''); + await keymanweb.setActiveKeyboard(''); _v = false; } else { kbdName = this.keyboards[++this.lastActiveKeyboard]._InternalName; lgCode = this.keyboards[this.lastActiveKeyboard]._LanguageCode; - keymanweb.setActiveKeyboard(kbdName,lgCode); + await keymanweb.setActiveKeyboard(kbdName,lgCode); _v = true; } } @@ -558,7 +558,7 @@ if(!keymanweb) { * @param {number} _kbd * Description Select a keyboard from the drop down menu **/ - selectKbd(_kbd: number) { + async selectKbd(_kbd: number): Promise { let _name,_lgCode; if(_kbd < 0) { _name = ''; @@ -568,7 +568,7 @@ if(!keymanweb) { _lgCode = this.keyboards[_kbd]._LanguageCode; } - keymanweb.setActiveKeyboard(_name,_lgCode); + await keymanweb.setActiveKeyboard(_name,_lgCode); keymanweb.focusLastActiveElement(); this.kbdButton._setSelected(_name != ''); if(_kbd >= 0) { diff --git a/web/src/app/ui/kmwuitoolbar.ts b/web/src/app/ui/kmwuitoolbar.ts index b1e142287b..fc9f17a3f1 100644 --- a/web/src/app/ui/kmwuitoolbar.ts +++ b/web/src/app/ui/kmwuitoolbar.ts @@ -820,7 +820,7 @@ if(!keymanweb) { * @param {boolean} updateKeyman * @return {boolean} **/ - selectKeyboard(event: Event, lang: LanguageEntry, kbd: KeyboardDetail, updateKeyman: boolean) { + async selectKeyboard(event: Event, lang: LanguageEntry, kbd: KeyboardDetail, updateKeyman: boolean) { keymanweb.activatingUI(true); if(this.selectedLanguage) { @@ -841,7 +841,7 @@ if(!keymanweb) { // Return focus to input area and activate the selected keyboard this.addKeyboardToList(lang, kbd); if(updateKeyman) { - keymanweb.setActiveKeyboard(kbd.InternalName, kbd.LanguageCode).then(() => { + await keymanweb.setActiveKeyboard(kbd.InternalName, kbd.LanguageCode).then(() => { // Restore focus _after_ the keyboard finishes loading. this.setLastFocus(); }); @@ -943,7 +943,7 @@ if(!keymanweb) { * @return {boolean} * Description Update the UI when all keyboards disabled by user **/ - readonly offButtonClickEvent = (event: Event) => { + readonly offButtonClickEvent = async (event: Event) => { if(this.toolbarNode.className != 'kmw_controls_disabled') { this.hideKeyboardsForLanguage(null); if(this.selectedLanguage) { @@ -959,7 +959,7 @@ if(!keymanweb) { // Return the focus to the input area and set the active keyboard to nothing this.setLastFocus(); - keymanweb.setActiveKeyboard('',''); + await keymanweb.setActiveKeyboard('',''); //Save current state when deselecting a keyboard (may not be needed) this.saveCookie(); From 870dec3da36ecb42041c11b972fa878a69d94915 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 22 May 2025 15:54:08 -0500 Subject: [PATCH 09/73] feat(developer): add percentage (permille) of messages without line numbers --- developer/src/kmc-ldml/test/messages.tests.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer/src/kmc-ldml/test/messages.tests.ts b/developer/src/kmc-ldml/test/messages.tests.ts index 8b5593d569..a29644936e 100644 --- a/developer/src/kmc-ldml/test/messages.tests.ts +++ b/developer/src/kmc-ldml/test/messages.tests.ts @@ -51,7 +51,7 @@ describe('LdmlCompilerMessages', function () { // Once this goes to zero, make it an error if it goes up! // Oh, and while you're here, once this is zero, uncomment the code in testCompilationCases // that asserts that all messages are actually generated with an offset. - console.warn(`TODO-LDML (#10622) ${noLines}/${total} messages did not have detectable offset (line number) reporting.`); + console.warn(`TODO-LDML (#10622) ${noLines}/${total} ${Number((noLines/total)*1000).toFixed(0)}‰ of message(s) did not have detectable offset (line number) reporting.`); } }); }); From 00b4de5f53d3ef57e73ed133af4fc30e47e97414 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 22 May 2025 17:03:40 -0500 Subject: [PATCH 10/73] feat(developer): add line numbers to tran compiler - move ObjectWithMetadata into symbol utils - don't validate transforms twice (tran + bksp) This is for #13937 but does not resolve all items --- developer/src/common/web/utils/src/index.ts | 2 +- .../src/common/web/utils/src/symbol-utils.ts | 12 +++- .../src/compiler/ldml-compiler-messages.ts | 48 ++++++------- developer/src/kmc-ldml/src/compiler/tran.ts | 69 ++++++++++--------- developer/src/kmc-ldml/src/util/util.ts | 18 ++--- developer/src/kmc-ldml/test/tran.tests.ts | 2 +- 6 files changed, 83 insertions(+), 68 deletions(-) diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index 937cd98642..ef6b489b67 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -68,7 +68,7 @@ export { UrlSubpathCompilerCallback } from './utils/UrlSubpathCompilerCallback.j export { DeveloperUtilsMessages } from './developer-utils-messages.js'; export * as SourceFilenamePatterns from './source-filename-patterns.js'; export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, KeymanXMLMetadata, XML_FILENAME_SYMBOL } from './xml-utils.js'; -export { SymbolUtils } from './symbol-utils.js'; +export { SymbolUtils, ObjectWithMetadata } from './symbol-utils.js'; export * as LineUtils from './line-utils.js'; export * as GitHubUrls from './github-urls.js'; export * as CloudUrls from './cloud-urls.js'; diff --git a/developer/src/common/web/utils/src/symbol-utils.ts b/developer/src/common/web/utils/src/symbol-utils.ts index 92cf07dd02..9404b404d8 100644 --- a/developer/src/common/web/utils/src/symbol-utils.ts +++ b/developer/src/common/web/utils/src/symbol-utils.ts @@ -6,6 +6,14 @@ * Utilities for manipulating Symbol properties */ + +/** + * Any object with metadata, for example for line number errs. + * Defined as 'any' here to reduce noise on the client side. + * @see {@link KeymanXMLReader.getMetaData()} + */ +export type ObjectWithMetadata = any; + export class SymbolUtils { /** * Copy symbols shallowly from 'from' onto 'onto' @@ -13,8 +21,8 @@ export class SymbolUtils { * @param from source for symbols * @returns the onto object */ - public static copySymbols(onto: T, from: any): T { - const o = onto as any; + public static copySymbols(onto: T, from: ObjectWithMetadata): T { + const o = onto as ObjectWithMetadata; for (const sym of Object.getOwnPropertySymbols(from)) { o[sym] = from[sym]; } diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index f597126719..a5636104f2 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -1,5 +1,5 @@ import { util } from "@keymanapp/common-types"; -import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_FILENAME_SYMBOL, CompilerEvent, KeymanXMLReader } from '@keymanapp/developer-utils'; +import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_FILENAME_SYMBOL, CompilerEvent, KeymanXMLReader, ObjectWithMetadata } from '@keymanapp/developer-utils'; // const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.LdmlKeyboardCompiler; @@ -9,14 +9,6 @@ const SevError = CompilerErrorSeverity.Error | CompilerErrorNamespace.LdmlKeyboa // sub-numberspace for transform errors const SevErrorTransform = SevError | 0xF00; -/** - * Any object with metadata, for line number errs. - * Could be for example an LKKeys or KMXPlus.KeysKeys object. - * Defined as 'any' here to reduce noise on the client side. - * @see {@link KeymanXMLReader.getMetaData()} - */ -type ObjectWithMetadata = any; - /** * Convenience function for constructing CompilerEvents with line numbers. * Use it as below: (abbreviated as mx()) @@ -202,19 +194,23 @@ export class LdmlCompilerMessages { // Not hit due to XML parsing static ERROR_InvalidTransformsType = SevError | 0x0018; - static Error_InvalidTransformsType = (o:{types: string[]}) => - m(this.ERROR_InvalidTransformsType, `Invalid transforms types: '${def(o.types?.join(','))}'`); + static Error_InvalidTransformsType = (o:{type: string}, x?: ObjectWithMetadata) => mx( + this.ERROR_InvalidTransformsType, x, + `Invalid transforms type: '${def(o.type)}'`, + ); static ERROR_DuplicateTransformsType = SevError | 0x0019; - static Error_DuplicateTransformsType = (o:{types: string[]}) => - m(this.ERROR_DuplicateTransformsType, `Duplicate transforms types: '${def(o.types?.join(','))}'`); + static Error_DuplicateTransformsType = (o:{type: string}, x?: ObjectWithMetadata) => mx( + this.ERROR_DuplicateTransformsType, x, + `Duplicate transforms type: '${def(o.type)}'`, + ); static ERROR_MixedTransformGroup = SevError | 0x001A; - static Error_MixedTransformGroup = () => + static Error_MixedTransformGroup = (x?: ObjectWithMetadata) => m(this.ERROR_MixedTransformGroup, `transformGroup cannot contain both reorder and transform elements`); static ERROR_EmptyTransformGroup = SevError | 0x001B; - static Error_EmptyTransformGroup = () => + static Error_EmptyTransformGroup = (x?: ObjectWithMetadata) => m(this.ERROR_EmptyTransformGroup, `transformGroup must have either reorder or transform elements`); static ERROR_MissingStringVariable = SevError | 0x001C; @@ -264,12 +260,16 @@ export class LdmlCompilerMessages { m(this.ERROR_IllegalCharacters, `File contains ${def(o.count)} illegal character(s), including ${util.describeCodepoint(o.lowestCh)}`); static HINT_CharClassImplicitDenorm = SevHint | 0x0026; - static Hint_CharClassImplicitDenorm = (o: { lowestCh: number }) => - m(this.HINT_CharClassImplicitDenorm, `File has character classes which span non-NFD character(s), including ${util.describeCodepoint(o.lowestCh)}. These will not match any text.`); + static Hint_CharClassImplicitDenorm = (o: { lowestCh: number }, x?: ObjectWithMetadata) => mx( + this.HINT_CharClassImplicitDenorm, x, + `File has character classes which span non-NFD character(s), including ${util.describeCodepoint(o.lowestCh)}. These will not match any text.`, + ); static WARN_CharClassExplicitDenorm = SevWarn | 0x0027; - static Warn_CharClassExplicitDenorm = (o: { lowestCh: number }) => - m(this.WARN_CharClassExplicitDenorm, `File has character classes which include non-NFD characters(s), including ${util.describeCodepoint(o.lowestCh)}. These will not match any text.`); + static Warn_CharClassExplicitDenorm = (o: { lowestCh: number }, x?: ObjectWithMetadata) => mx( + this.WARN_CharClassExplicitDenorm, x, + `File has character classes which include non-NFD characters(s), including ${util.describeCodepoint(o.lowestCh)}. These will not match any text.`, + ); static ERROR_UnparseableReorderSet = SevError | 0x0028; static Error_UnparseableReorderSet = (o: { from: string, set: string }) => @@ -294,8 +294,10 @@ export class LdmlCompilerMessages { // Available: 0x02C-0x2F static ERROR_InvalidQuadEscape = SevError | 0x0030; - static Error_InvalidQuadEscape = (o: { cp: number }) => - m(this.ERROR_InvalidQuadEscape, `Invalid escape "\\u${util.hexQuad(o?.cp || 0)}". Hint: Use "\\u{${def(o?.cp?.toString(16))}}"`); + static Error_InvalidQuadEscape = (o: { cp: number }, x?: ObjectWithMetadata) => mx( + this.ERROR_InvalidQuadEscape, x, + `Invalid escape "\\u${util.hexQuad(o?.cp || 0)}". Hint: Use "\\u{${def(o?.cp?.toString(16))}}"`, + ); // // Transform syntax errors begin at ...F00 (SevErrorTransform) @@ -319,8 +321,8 @@ export class LdmlCompilerMessages { `); static ERROR_TransformFromMatchesNothing = SevErrorTransform | 0x02; - static Error_TransformFromMatchesNothing = (o: { from: string }) => m( - this.ERROR_TransformFromMatchesNothing, + static Error_TransformFromMatchesNothing = (o: { from: string }, x?: ObjectWithMetadata) => mx( + this.ERROR_TransformFromMatchesNothing, x, `Invalid transfom from="${def(o.from)}": Matches an empty string.` ); diff --git a/developer/src/kmc-ldml/src/compiler/tran.ts b/developer/src/kmc-ldml/src/compiler/tran.ts index 59750eed98..2db96ab7cc 100644 --- a/developer/src/kmc-ldml/src/compiler/tran.ts +++ b/developer/src/kmc-ldml/src/compiler/tran.ts @@ -55,37 +55,34 @@ export abstract class TransformCompiler type); + if (transforms && this.id == "tran") { + // Only run this part in the main "tran" compiler. + // we use this map to get back to a context object for the error message + const typeToObject = new Map(); + transforms.forEach(t => typeToObject.set(t.type, t)); + const types = transforms.map(({type}) => type); if (!verifyValidAndUnique(types, - types => reportMessage(LdmlCompilerMessages.Error_DuplicateTransformsType({ types })), - new Set(['simple', 'backspace']), - types => reportMessage(LdmlCompilerMessages.Error_InvalidTransformsType({ types })))) { + types => types.forEach(type => + reportMessage(LdmlCompilerMessages.Error_DuplicateTransformsType({ type }, typeToObject.get(type)))), + new Set(ALLOWED_TYPES), + types => types.forEach(type => + reportMessage(LdmlCompilerMessages.Error_InvalidTransformsType({ type }, typeToObject.get(type)))))) { valid = false; } - // check for mixed groups - let mixed = false; - let empty = false; - transforms.forEach(({transformGroup}) => transformGroup.forEach((transformGroup) => { + transforms.forEach(({ type, transformGroup }) => transformGroup.forEach((transformGroup) => { + if (this.type != type) return; // only validate bskp in bksp, tran in tran, etc. if (transformGroup.reorder?.length && transformGroup.transform?.length) { - mixed = true; - } - if (!transformGroup.reorder?.length && !transformGroup.transform?.length) { - empty = true; + valid = false; + reportMessage(LdmlCompilerMessages.Error_MixedTransformGroup(transformGroup)); + } else if (!transformGroup.reorder?.length && !transformGroup.transform?.length) { + valid = false; + reportMessage(LdmlCompilerMessages.Error_EmptyTransformGroup(transformGroup)); } })); - if (mixed) { - valid = false; - reportMessage(LdmlCompilerMessages.Error_MixedTransformGroup()); // report this once - } - if (empty) { - valid = false; - reportMessage(LdmlCompilerMessages.Error_EmptyTransformGroup()); // report this once - } // TODO-LDML: linting here should check for identical from, but this involves a double-parse which is ugly // TODO-LDML: unicodesets means that either we fully parse them and verify conflicting rules or the linting is imperfect @@ -96,7 +93,7 @@ export abstract class TransformCompiler void, - allowed?: Set, - onInvalid?: (invalids: string[]) => void) +export function verifyValidAndUnique( + values: T[], + onDuplicate: (duplicates: T[]) => void, + allowed?: Set, + onInvalid?: (invalids: T[]) => void) : boolean { - const dups: string[] = []; - const invalids: string[] = []; - const seen = new Set(); + const dups: T[] = []; + const invalids: T[] = []; + const seen = new Set(); for (const value of values) { if (allowed && !allowed.has(value)) { invalids.push(value); @@ -140,7 +140,7 @@ export function verifyValidAndUnique( } } - function dedupedSortedArray(values: string[]) : string[] { + function dedupedSortedArray(values: T[]) : T[] { return Array.from(new Set(values)).sort(); } diff --git a/developer/src/kmc-ldml/test/tran.tests.ts b/developer/src/kmc-ldml/test/tran.tests.ts index d64e26525d..aaeec16d6d 100644 --- a/developer/src/kmc-ldml/test/tran.tests.ts +++ b/developer/src/kmc-ldml/test/tran.tests.ts @@ -129,7 +129,7 @@ describe('tran', function () { { subpath: 'sections/tran/fail-duplicate-type.xml', errors: [ - LdmlCompilerMessages.Error_DuplicateTransformsType({types: ['simple']}) + LdmlCompilerMessages.Error_DuplicateTransformsType({type: 'simple'}) ] }, { From acc4aba32cd999c54436d8fcb430acd5ca2f49a8 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Fri, 23 May 2025 05:57:25 +0700 Subject: [PATCH 11/73] fix(ios): Apply review comments * Change zip_files to add_zip_files * Clarify flags used for zip vs 7z * Use `zip-excludes` instead of `excludes.in` for the list of file to exclude from the archive --- ios/tools/prepRelease.sh | 8 ++++---- ios/{exclude.in => zip-excludes} | 0 resources/build/zip.inc.sh | 19 ++++++++++++------- 3 files changed, 16 insertions(+), 11 deletions(-) rename ios/{exclude.in => zip-excludes} (100%) diff --git a/ios/tools/prepRelease.sh b/ios/tools/prepRelease.sh index af67632965..cd82520662 100755 --- a/ios/tools/prepRelease.sh +++ b/ios/tools/prepRelease.sh @@ -74,13 +74,13 @@ ZIP_FLAGS=("-q" "-r" "-X") # quiet, recursive, no-extra echo "Zipping ${FRAMEWORK} => ${UPLOAD_DIR}/${KMEI_DST}..." cd "${KMEI_FRAMEWORK_BASE}" -zip_files "${KMEI_DST}" "${ZIP_FLAGS[@]}" "${FRAMEWORK}" +add_zip_files "${KMEI_DST}" "${ZIP_FLAGS[@]}" "${FRAMEWORK}" cd "$WORK_DIR" echo "Copying Keyman Engine samples into ${UPLOAD_DIR}/${KMEI_DST_NAME}..." cp -rf "${KEYMAN_SAMPLES}" "${UPLOAD_DIR}/samples" cd "${UPLOAD_DIR}" -zip_files "${KMEI_DST_NAME}" "-x@../../exclude.in" "${ZIP_FLAGS[@]}" "samples" +add_zip_files "${KMEI_DST_NAME}" "-x@../../zip-excludes" "${ZIP_FLAGS[@]}" "samples" rm -rf "samples" cd "$WORK_DIR" @@ -100,7 +100,7 @@ KEYMANAPP_SIM_APP_DST="keyman-ios-simulator-${BUILD_NUMBER}.app.zip" echo "Zipping Keyman simulator artifact ${KEYMANAPP_SIM_APP} => ${UPLOAD_DIR}/${KEYMANAPP_SIM_APP_DST}..." cd "${KEYMANAPP_SIM_FOLDER}" -zip_files "${WORK_DIR}/${UPLOAD_DIR}/${KEYMANAPP_SIM_APP_DST}" "${ZIP_FLAGS[@]}" "Keyman.app" +add_zip_files "${WORK_DIR}/${UPLOAD_DIR}/${KEYMANAPP_SIM_APP_DST}" "${ZIP_FLAGS[@]}" "Keyman.app" echo "${WORK_DIR}/${UPLOAD_DIR}/${KEYMANAPP_SIM_APP_DST}" cd "$WORK_DIR" @@ -121,7 +121,7 @@ if [ "${RELEASE_OEM_FIRSTVOICES}" = true ]; then echo "Zipping FirstVoices simulator artifact ${FIRSTVOICESAPP_SIM_APP} => ${UPLOAD_DIR}/${KEYMANAPP_SIM_APP_DST}..." cd "${FIRSTVOICESAPP_SIM_FOLDER}" - zip_files "${WORK_DIR}/${UPLOAD_DIR}/${FIRSTVOICESAPP_SIM_APP_DST}" "${ZIP_FLAGS[@]}" "FirstVoices.app" + add_zip_files "${WORK_DIR}/${UPLOAD_DIR}/${FIRSTVOICESAPP_SIM_APP_DST}" "${ZIP_FLAGS[@]}" "FirstVoices.app" cd "$WORK_DIR" fi diff --git a/ios/exclude.in b/ios/zip-excludes similarity index 100% rename from ios/exclude.in rename to ios/zip-excludes diff --git a/resources/build/zip.inc.sh b/resources/build/zip.inc.sh index bef8616024..3dc6a390c2 100644 --- a/resources/build/zip.inc.sh +++ b/resources/build/zip.inc.sh @@ -8,13 +8,14 @@ # # TODO: refactor with /resources/build/win/zip.inc.sh -# zip/7z to create an archive with the following parameters (in order) +# Add files to create a zip/7z archive with the following parameters (in order) # [zip filename] -# [list of flags to pass to zip command. Flags start with a single-dash +# [list of flags to pass to zip command] Flags start with a single-dash # -x@filename for a file containing list of files to exclude from the archive -# -* all other flags] +# -* all other flags +# Flags passed in are treated as zip parameters, and internally converterted to 7z flags as applicable # [list of files to include in zip] -function zip_files() { +function add_zip_files() { # Parse parameters @@ -29,11 +30,13 @@ function zip_files() { while [[ $# -gt 0 ]] ; do case "$1" in -r) - # Common flags to zip and 7z + # recursive paths - Identical flag to zip and 7z ZIP_FLAGS+=($1) SEVENZ_FLAGS+=($1) shift ;; + + # Zip flags that have a corresponding 7z flag -q) # quiet mode -> disable progress indicator, set output log level 0 ZIP_FLAGS+=($1) @@ -52,11 +55,13 @@ function zip_files() { fi shift; ;; + -*) - # Rest of zip flags. + # Remaining zip flags that don't apply to 7z ZIP_FLAGS+=($1) shift ;; + *) # files to include in the archive INCLUDE+=($1) @@ -88,7 +93,7 @@ function zip_files() { fi # Create archive - builder_echo_debug "${COMPRESS_CMD} ${SEVENZ_FLAGS[@]} ${ZIP_FLAGS[@]} ${ZIP_FILE} ${INCLUDE[@]}" + # builder_echo_debug "${COMPRESS_CMD} ${SEVENZ_FLAGS[@]} ${ZIP_FLAGS[@]} ${ZIP_FILE} ${INCLUDE[@]}" "${COMPRESS_CMD}" ${SEVENZ_FLAGS[@]} ${ZIP_FLAGS[@]} ${ZIP_FILE} ${INCLUDE[@]} } From dba5108989e81853a78cf71c703e7db4a85f2dd4 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Fri, 23 May 2025 17:42:45 +0200 Subject: [PATCH 12/73] chore(web): mark some private methods as being private --- web/src/app/ui/kmwuibutton.ts | 4 ++-- web/src/app/ui/kmwuifloat.ts | 2 +- web/src/app/ui/kmwuitoggle.ts | 2 +- web/src/app/ui/kmwuitoolbar.ts | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/web/src/app/ui/kmwuibutton.ts b/web/src/app/ui/kmwuibutton.ts index c8717e5dda..16377d4f3d 100644 --- a/web/src/app/ui/kmwuibutton.ts +++ b/web/src/app/ui/kmwuibutton.ts @@ -106,7 +106,7 @@ if(!keymanweb) { * @param {Event} _id keyboard selection event * @return {boolean} */ - readonly _SelectKeyboard = async (_id: Event): Promise => { + private readonly _SelectKeyboard = async (_id: Event): Promise => { let id: string = ''; if(typeof(_id) == 'object') { let t: HTMLElement = null; @@ -252,7 +252,7 @@ if(!keymanweb) { * * @param {Event} e event */ - readonly _SelectorMouseOut = (e: MouseEvent) => { + private readonly _SelectorMouseOut = (e: MouseEvent) => { if(keymanweb.activatingUI) { keymanweb.activatingUI(0); } diff --git a/web/src/app/ui/kmwuifloat.ts b/web/src/app/ui/kmwuifloat.ts index 614119ed4e..38fabb5bd7 100644 --- a/web/src/app/ui/kmwuifloat.ts +++ b/web/src/app/ui/kmwuifloat.ts @@ -450,7 +450,7 @@ if(!keymanweb) { * @param {Object} e event * Description Change active keyboard in response to user selection event */ - readonly SelectKeyboardChange = async (e: Event) => { + private readonly SelectKeyboardChange = async (e: Event) => { keymanweb.activatingUI(true); if(this.KeyboardSelector.value != '-') { diff --git a/web/src/app/ui/kmwuitoggle.ts b/web/src/app/ui/kmwuitoggle.ts index 29d9a802f7..1aa4561465 100644 --- a/web/src/app/ui/kmwuitoggle.ts +++ b/web/src/app/ui/kmwuitoggle.ts @@ -558,7 +558,7 @@ if(!keymanweb) { * @param {number} _kbd * Description Select a keyboard from the drop down menu **/ - async selectKbd(_kbd: number): Promise { + private async selectKbd(_kbd: number): Promise { let _name,_lgCode; if(_kbd < 0) { _name = ''; diff --git a/web/src/app/ui/kmwuitoolbar.ts b/web/src/app/ui/kmwuitoolbar.ts index fc9f17a3f1..daf1adf8ca 100644 --- a/web/src/app/ui/kmwuitoolbar.ts +++ b/web/src/app/ui/kmwuitoolbar.ts @@ -820,7 +820,7 @@ if(!keymanweb) { * @param {boolean} updateKeyman * @return {boolean} **/ - async selectKeyboard(event: Event, lang: LanguageEntry, kbd: KeyboardDetail, updateKeyman: boolean) { + private async selectKeyboard(event: Event, lang: LanguageEntry, kbd: KeyboardDetail, updateKeyman: boolean) { keymanweb.activatingUI(true); if(this.selectedLanguage) { @@ -943,7 +943,7 @@ if(!keymanweb) { * @return {boolean} * Description Update the UI when all keyboards disabled by user **/ - readonly offButtonClickEvent = async (event: Event) => { + private readonly offButtonClickEvent = async (event: Event) => { if(this.toolbarNode.className != 'kmw_controls_disabled') { this.hideKeyboardsForLanguage(null); if(this.selectedLanguage) { From 2c52a733ec48dbb1c2c881d3356b549cbf374564 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Sun, 25 May 2025 06:03:37 +0700 Subject: [PATCH 13/73] maint: add stable-19.0 triggers This are a little early but good to have in place. Per release process wiki, this should have been done at time of 18.0 stable release but was missed (by me). --- resources/build/trigger-definitions.inc.sh | 66 +++++----------------- 1 file changed, 14 insertions(+), 52 deletions(-) diff --git a/resources/build/trigger-definitions.inc.sh b/resources/build/trigger-definitions.inc.sh index 90f58f97d8..e4ed49d90d 100644 --- a/resources/build/trigger-definitions.inc.sh +++ b/resources/build/trigger-definitions.inc.sh @@ -62,7 +62,7 @@ bc_test_common_linux=(Keyman_Test_Common_Linux) vcs_test=HttpsGithubComKeymanappKeymanPRs -# Master (Alpha) Build Configurations; these diverge when we need new build configurations for new versions +# Master (Alpha) Build Configurations; these may diverge when we need new build configurations for new versions bc_master_android=(KeymanAndroid_Build) bc_master_ios=(Keyman_iOS_Master) @@ -86,57 +86,7 @@ bc_beta_developer=(Keyman_Developer_Release) vcs_beta=HttpsGithubComKeymanappKeyman -# Stable 14.0 Build Configurations - -bc_stable_14_0_android=(KeymanAndroid_Build) -bc_stable_14_0_ios=(Keyman_iOS_Master) -bc_stable_14_0_linux=(KeymanLinux_Master deb-release-packaging_GitHub) -bc_stable_14_0_mac=(KeymanMac_Master) -bc_stable_14_0_windows=(Keyman_Build) -bc_stable_14_0_web=(Keymanweb_Build) - -# 16.0+ Keyman Developer is split from Windows, but TC configuration will not fail -# on older versions, just no-op -bc_stable_14_0_developer=(Keyman_Developer_Release) - -vcs_stable_14_0=HttpsGithubComKeymanappKeyman - -# Stable 15.0 Build Configurations - -bc_stable_15_0_android=(KeymanAndroid_Build) -bc_stable_15_0_ios=(Keyman_iOS_Master) -bc_stable_15_0_linux=(KeymanLinux_Master deb-release-packaging_GitHub) -bc_stable_15_0_mac=(KeymanMac_Master) -bc_stable_15_0_windows=(Keyman_Build) -bc_stable_15_0_web=(Keymanweb_Build) - -vcs_stable_15_0=HttpsGithubComKeymanappKeyman - -# Stable 16.0 Build Configurations - -bc_stable_16_0_android=(KeymanAndroid_Build) -bc_stable_16_0_ios=(Keyman_iOS_Master) -bc_stable_16_0_linux=(KeymanLinux_Master deb-release-packaging_GitHub) -bc_stable_16_0_mac=(KeymanMac_Master) -bc_stable_16_0_windows=(Keyman_Build) -bc_stable_16_0_web=(Keymanweb_Build) -bc_stable_16_0_developer=(Keyman_Developer_Release) - -vcs_stable_16_0=HttpsGithubComKeymanappKeyman - -# Stable 17.0 Build Configurations - -bc_stable_17_0_android=(KeymanAndroid_Build) -bc_stable_17_0_ios=(Keyman_iOS_Master) -bc_stable_17_0_linux=(KeymanLinux_Master deb-release-packaging_GitHub) -bc_stable_17_0_mac=(KeymanMac_Master) -bc_stable_17_0_windows=(Keyman_Build) -bc_stable_17_0_web=(Keymanweb_Build) -bc_stable_17_0_developer=(Keyman_Developer_Release) - -vcs_stable_17_0=HttpsGithubComKeymanappKeyman - -# Stable 17.0 Build Configurations +# Stable 18.0 Build Configurations bc_stable_18_0_android=(KeymanAndroid_Build) bc_stable_18_0_ios=(Keyman_iOS_Master) @@ -147,3 +97,15 @@ bc_stable_18_0_web=(Keymanweb_Build) bc_stable_18_0_developer=(Keyman_Developer_Release) vcs_stable_18_0=HttpsGithubComKeymanappKeyman + +# Stable 19.0 Build Configurations + +bc_stable_19_0_android=(KeymanAndroid_Build) +bc_stable_19_0_ios=(Keyman_iOS_Master) +bc_stable_19_0_linux=(KeymanLinux_Master deb-release-packaging_GitHub) +bc_stable_19_0_mac=(KeymanMac_Master) +bc_stable_19_0_windows=(Keyman_Build) +bc_stable_19_0_web=(Keymanweb_Build) +bc_stable_19_0_developer=(Keyman_Developer_Release) + +vcs_stable_19_0=HttpsGithubComKeymanappKeyman From 9f15e7c7ec0251eded4f86824d9d828296ab4859 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Sun, 25 May 2025 06:35:28 +0700 Subject: [PATCH 14/73] maint: add empty 'scripts' tag to package.json to silence warning Also removes './' from front of gosh.js to address another npm cleanup (from `npm pkg fix`). Fixes: #13833 --- common/test/resources/package.json | 1 + common/web/keyman-version/package.json | 1 + core/include/ldml/package.json | 1 + core/tools/api-header-extractor/package.json | 1 + developer/src/tike/xml/app/package.json | 1 + resources/gosh/package.json | 3 ++- resources/tools/check-markdown/package.json | 1 + 7 files changed, 8 insertions(+), 1 deletion(-) diff --git a/common/test/resources/package.json b/common/test/resources/package.json index 9295190f2d..69a4a7a1b6 100644 --- a/common/test/resources/package.json +++ b/common/test/resources/package.json @@ -3,6 +3,7 @@ "description": "Common test resources for Keyman's JS & TS-based components & products", "type": "module", "license": "MIT", + "scripts": {}, "devDependencies": { "@keymanapp/resources-gosh": "*", "typescript": "^5.4.5" diff --git a/common/web/keyman-version/package.json b/common/web/keyman-version/package.json index bdd2e9bc84..74cce896bd 100644 --- a/common/web/keyman-version/package.json +++ b/common/web/keyman-version/package.json @@ -11,6 +11,7 @@ "/build/" ], "license": "MIT", + "scripts": {}, "type": "module", "devDependencies": { "typescript": "^5.4.5" diff --git a/core/include/ldml/package.json b/core/include/ldml/package.json index 1e9a26c1c0..ba9acf063e 100644 --- a/core/include/ldml/package.json +++ b/core/include/ldml/package.json @@ -13,6 +13,7 @@ "license": "MIT", "type": "module", "main": "build/keyman_core_ldml.js", + "scripts": {}, "repository": { "type": "git", "url": "git+https://github.com/keymanapp/keyman.git" diff --git a/core/tools/api-header-extractor/package.json b/core/tools/api-header-extractor/package.json index 73b7a77bf3..f8fb512a3a 100644 --- a/core/tools/api-header-extractor/package.json +++ b/core/tools/api-header-extractor/package.json @@ -2,5 +2,6 @@ "name": "@keymanapp/api-header-extractor", "private": true, "type": "module", + "scripts": {}, "main": "build/src/index.js" } diff --git a/developer/src/tike/xml/app/package.json b/developer/src/tike/xml/app/package.json index d0696c3986..ecb8d845a3 100644 --- a/developer/src/tike/xml/app/package.json +++ b/developer/src/tike/xml/app/package.json @@ -6,6 +6,7 @@ "url": "https://github.com/keymanapp/keyman/issues" }, "homepage": "https://keyman.com/", + "scripts": {}, "dependencies": { "monaco-editor": "0.15.6" } diff --git a/resources/gosh/package.json b/resources/gosh/package.json index bc70a30cc0..f87cfcf7db 100644 --- a/resources/gosh/package.json +++ b/resources/gosh/package.json @@ -8,7 +8,7 @@ "homepage": "https://github.com/keymanapp/keyman", "license": "MIT", "bin": { - "gosh": "./gosh.js" + "gosh": "gosh.js" }, "private": true, "files": [ @@ -18,6 +18,7 @@ "type": "git", "url": "git+https://github.com/keymanapp/keyman.git" }, + "scripts": {}, "bugs": { "url": "https://github.com/keymanapp/keyman/issues" } diff --git a/resources/tools/check-markdown/package.json b/resources/tools/check-markdown/package.json index 16c85f65a4..a75e6e3568 100644 --- a/resources/tools/check-markdown/package.json +++ b/resources/tools/check-markdown/package.json @@ -4,6 +4,7 @@ "type": "module", "main": "build/index.js", "license": "MIT", + "scripts": {}, "devDependencies": { "marked": "^14.1.2", "chalk": "^2.4.2" From efafd22b2f15240a781266013713c3d827c612b7 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Mon, 26 May 2025 16:31:21 +0200 Subject: [PATCH 15/73] maint(common): update build coordinator for `resources/teamcity` changes This modifies the build triggers so that changes in `resources/teamcity` don't trigger builds on all platforms anymore. If a change happened in `resources/teamcity/$platform` or `resources/teamcity/includes` it will trigger a build for `$platform`. Since bash doesn't support negative look-aheads we use `grep` with the Perl regex option. Part-of: #13943 Build-bot: skip --- resources/build/run-required-test-builds.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/build/run-required-test-builds.sh b/resources/build/run-required-test-builds.sh index eada3e90ca..e108bc7717 100755 --- a/resources/build/run-required-test-builds.sh +++ b/resources/build/run-required-test-builds.sh @@ -166,8 +166,8 @@ while IFS= read -r line; do # Which platform are we watching? eval watch='$'watch_$platform # Add common patterns to the watch list - watch="^($platform|(oem/[^/]+/$platform)|resources|$watch)" - if [[ "$line" =~ $watch ]]; then + watch="^(${platform}|(oem/[^/]+/${platform})|resources/((?!teamcity)|teamcity/(${platform}|includes))|$watch)" + if echo "${line}" | grep --quiet --perl-regexp "${watch}"; then build_platforms+=($platform) fi fi From 577b439721cd19ef3fa831d0de26eed1cd5bea78 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 26 May 2025 10:44:12 -0500 Subject: [PATCH 16/73] feat(developer): add line numbers to disp compiler Fixes #13938 --- developer/src/kmc-ldml/src/compiler/disp.ts | 9 +++++---- .../kmc-ldml/src/compiler/ldml-compiler-messages.ts | 12 ++++++++---- developer/src/kmc-ldml/test/disp.tests.ts | 10 +++++----- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/developer/src/kmc-ldml/src/compiler/disp.ts b/developer/src/kmc-ldml/src/compiler/disp.ts index 65119118d5..5b63a99a8f 100644 --- a/developer/src/kmc-ldml/src/compiler/disp.ts +++ b/developer/src/kmc-ldml/src/compiler/disp.ts @@ -31,20 +31,21 @@ export class DispCompiler extends SectionCompiler { const ids = new Set(); if (this.keyboard3.displays?.display) { - for (const { output, keyId } of this.keyboard3.displays?.display) { + for (const display of this.keyboard3.displays?.display) { + const { output, keyId } = display; if ((output && keyId) || (!output && !keyId)) { - this.callbacks.reportMessage(LdmlCompilerMessages.Error_DisplayNeedsToOrId({ output, keyId })); + this.callbacks.reportMessage(LdmlCompilerMessages.Error_DisplayNeedsToOrId({ output, keyId }, display)); return false; } else if (output) { if (tos.has(output)) { - this.callbacks.reportMessage(LdmlCompilerMessages.Error_DisplayIsRepeated({ output })); + this.callbacks.reportMessage(LdmlCompilerMessages.Error_DisplayIsRepeated({ output }, display)); return false; } else { tos.add(output); } } else if (keyId) { if (ids.has(keyId)) { - this.callbacks.reportMessage(LdmlCompilerMessages.Error_DisplayIsRepeated({ keyId })); + this.callbacks.reportMessage(LdmlCompilerMessages.Error_DisplayIsRepeated({ keyId }, display)); return false; } else { ids.add(keyId); diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index a5636104f2..98dad153d9 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -147,8 +147,10 @@ export class LdmlCompilerMessages { } static ERROR_DisplayIsRepeated = SevError | 0x0010; - static Error_DisplayIsRepeated = (o:{output?: string, keyId?: string}) => - m(this.ERROR_DisplayIsRepeated, `display ${LdmlCompilerMessages.outputOrKeyId(o)} has more than one display entry.`); + static Error_DisplayIsRepeated = (o:{output?: string, keyId?: string}, x?: ObjectWithMetadata) => mx( + this.ERROR_DisplayIsRepeated, x, + `display ${LdmlCompilerMessages.outputOrKeyId(o)} has more than one display entry.`, + ); static ERROR_KeyMissingToGapOrSwitch = SevError | 0x0011; static Error_KeyMissingToGapOrSwitch = (o:{keyId: string}, x?: ObjectWithMetadata) => mx( @@ -244,8 +246,10 @@ export class LdmlCompilerMessages { m(this.ERROR_MissingMarkers, `Markers used for matching but not defined: ${def(o.ids?.join(','))}`); static ERROR_DisplayNeedsToOrId = SevError | 0x0022; - static Error_DisplayNeedsToOrId = (o:{output?: string, keyId?: string}) => - m(this.ERROR_DisplayNeedsToOrId, `display ${LdmlCompilerMessages.outputOrKeyId(o)} needs output= or keyId=, but not both`); + static Error_DisplayNeedsToOrId = (o:{output?: string, keyId?: string}, x?: ObjectWithMetadata) => mx( + this.ERROR_DisplayNeedsToOrId, x, + `display ${LdmlCompilerMessages.outputOrKeyId(o)} needs output= or keyId=, but not both`, + ); static HINT_PUACharacters = SevHint | 0x0023; static Hint_PUACharacters = (o: { count: number, lowestCh: number }) => diff --git a/developer/src/kmc-ldml/test/disp.tests.ts b/developer/src/kmc-ldml/test/disp.tests.ts index 80b416bf94..032e4829bf 100644 --- a/developer/src/kmc-ldml/test/disp.tests.ts +++ b/developer/src/kmc-ldml/test/disp.tests.ts @@ -1,7 +1,7 @@ import 'mocha'; import {assert} from 'chai'; import { DispCompiler } from '../src/compiler/disp.js'; -import { compilerTestCallbacks, loadSectionFixture, testCompilationCases } from './helpers/index.js'; +import { compilerTestCallbacks, loadSectionFixture, testCompilationCases, withOffset } from './helpers/index.js'; import { KMXPlus } from '@keymanapp/common-types'; import { LdmlCompilerMessages } from '../src/compiler/ldml-compiler-messages.js'; @@ -67,25 +67,25 @@ describe('disp', function () { const disp = await loadSectionFixture(DispCompiler, 'sections/disp/invalid-dupto.xml', compilerTestCallbacks) as Disp; assert.isNull(disp); assert.equal(compilerTestCallbacks.messages.length, 1); - assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_DisplayIsRepeated({ output: 'e' })); + assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_DisplayIsRepeated({ output: 'e' }, withOffset(330))); }); it('should reject duplicate ids', async function() { const disp = await loadSectionFixture(DispCompiler, 'sections/disp/invalid-dupid.xml', compilerTestCallbacks) as Disp;1 assert.isNull(disp); assert.equal(compilerTestCallbacks.messages.length, 1); - assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_DisplayIsRepeated({ keyId: 'e' })); + assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_DisplayIsRepeated({ keyId: 'e' }, withOffset(329))); }); it('should reject if neither to nor id', async function() { const disp = await loadSectionFixture(DispCompiler, 'sections/disp/invalid-none.xml', compilerTestCallbacks) as Disp; assert.isNull(disp); assert.equal(compilerTestCallbacks.messages.length, 1); - assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_DisplayNeedsToOrId({})); + assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_DisplayNeedsToOrId({}, withOffset(182))); }); it('should reject if both to and id', async function() { const disp = await loadSectionFixture(DispCompiler, 'sections/disp/invalid-both.xml', compilerTestCallbacks) as Disp; assert.isNull(disp); assert.equal(compilerTestCallbacks.messages.length, 1); - assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_DisplayNeedsToOrId({ output: 'e', keyId: 'e' })); + assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_DisplayNeedsToOrId({ output: 'e', keyId: 'e' }, withOffset(182))); }); testCompilationCases(DispCompiler, [ { From d2af8cf41a75e69858b9001b4be92596b30e14cb Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 26 May 2025 11:34:05 -0500 Subject: [PATCH 17/73] feat(developer): add line numbers to layr compiler Fixes #13939 --- developer/src/kmc-ldml/src/compiler/layr.ts | 4 ++-- .../kmc-ldml/src/compiler/ldml-compiler-messages.ts | 12 ++++++++---- .../sections/layr/invalid-missing-layer2.xml | 1 + developer/src/kmc-ldml/test/layr.tests.ts | 9 ++++++++- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/developer/src/kmc-ldml/src/compiler/layr.ts b/developer/src/kmc-ldml/src/compiler/layr.ts index 2c6a03ecaf..ca99d09883 100644 --- a/developer/src/kmc-ldml/src/compiler/layr.ts +++ b/developer/src/kmc-ldml/src/compiler/layr.ts @@ -33,7 +33,7 @@ export class LayrCompiler extends SectionCompiler { hardwareLayers++; if (hardwareLayers > 1) { valid = false; - this.callbacks.reportMessage(LdmlCompilerMessages.Error_ExcessHardware({formId})); + this.callbacks.reportMessage(LdmlCompilerMessages.Error_ExcessHardware({formId}, layers)); } } layers.layer.forEach((layer) => { @@ -48,7 +48,7 @@ export class LayrCompiler extends SectionCompiler { if (totalLayerCount === 0) { // TODO-LDML: does not validate touch layers yet // no layers seen anywhere valid = false; - this.callbacks.reportMessage(LdmlCompilerMessages.Error_MustBeAtLeastOneLayerElement()); + this.callbacks.reportMessage(LdmlCompilerMessages.Error_MustBeAtLeastOneLayerElement(this.keyboard3?.layers[0])); } return valid; } diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 98dad153d9..23066e4e40 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -126,8 +126,10 @@ export class LdmlCompilerMessages { ); static ERROR_MustBeAtLeastOneLayerElement = SevError | 0x000E; - static Error_MustBeAtLeastOneLayerElement = () => - m(this.ERROR_MustBeAtLeastOneLayerElement, `The source file must contain at least one layer element.`); + static Error_MustBeAtLeastOneLayerElement = (x?: ObjectWithMetadata) => mx( + this.ERROR_MustBeAtLeastOneLayerElement, x, + `The source file must contain at least one layer element.`, + ); static HINT_NoDisplayForSwitch = SevHint | 0x000F; static Hint_NoDisplayForSwitch = (o: { id: string }) => @@ -159,8 +161,10 @@ export class LdmlCompilerMessages { ); static ERROR_ExcessHardware = SevError | 0x0012; - static Error_ExcessHardware = (o:{formId: string}) => m(this.ERROR_ExcessHardware, - `layers formId=${def(o.formId)}: Can only have one non-'touch' element`); + static Error_ExcessHardware = (o:{formId: string}, x?: ObjectWithMetadata) => mx( + this.ERROR_ExcessHardware, x, + `layers formId=${def(o.formId)}: Can only have one non-'touch' element`, + ); static ERROR_InvalidHardware = SevError | 0x0013; static Error_InvalidHardware = (o: { formId: string }, x?: ObjectWithMetadata) => mx( diff --git a/developer/src/kmc-ldml/test/fixtures/sections/layr/invalid-missing-layer2.xml b/developer/src/kmc-ldml/test/fixtures/sections/layr/invalid-missing-layer2.xml index c8a38602ea..331cd7e9e7 100644 --- a/developer/src/kmc-ldml/test/fixtures/sections/layr/invalid-missing-layer2.xml +++ b/developer/src/kmc-ldml/test/fixtures/sections/layr/invalid-missing-layer2.xml @@ -5,4 +5,5 @@ + diff --git a/developer/src/kmc-ldml/test/layr.tests.ts b/developer/src/kmc-ldml/test/layr.tests.ts index 5d08dd621a..9d6c2112f5 100644 --- a/developer/src/kmc-ldml/test/layr.tests.ts +++ b/developer/src/kmc-ldml/test/layr.tests.ts @@ -2,7 +2,7 @@ import 'mocha'; import { assert } from 'chai'; import { LayrCompiler } from '../src/compiler/layr.js'; import { LdmlCompilerMessages } from '../src/compiler/ldml-compiler-messages.js'; -import { compilerTestCallbacks, testCompilationCases } from './helpers/index.js'; +import { compilerTestCallbacks, testCompilationCases, withOffset } from './helpers/index.js'; import { KMXPlus } from '@keymanapp/common-types'; import { constants } from '@keymanapp/ldml-keyboard-constants'; @@ -108,7 +108,14 @@ describe('layr', function () { { // missing layer element subpath: 'sections/layr/invalid-missing-layer.xml', + errors: [LdmlCompilerMessages.Error_MustBeAtLeastOneLayerElement(withOffset(258))], + retainOffsetInMessages: true, + }, + { + // missing layer element + subpath: 'sections/layr/invalid-missing-layer2.xml', errors: [LdmlCompilerMessages.Error_MustBeAtLeastOneLayerElement()], + retainOffsetInMessages: true, }, { // keep in sync with similar test in test-keys.ts From b681885eca27b4f48fc1036a1b9a30689d876014 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 26 May 2025 12:30:59 -0500 Subject: [PATCH 18/73] feat(developer): add more tests to layr compiler - check for duplicate minDeviceWidth - check for invalid minDeviceWidth Fixes #13939 --- core/include/ldml/keyman_core_ldml.h | 2 ++ core/include/ldml/keyman_core_ldml.ts | 10 +++++++ developer/src/kmc-ldml/src/compiler/layr.ts | 20 ++++++++++--- .../src/compiler/ldml-compiler-messages.ts | 16 +++++++++- .../sections/layr/error-bad-width-0.xml | 29 +++++++++++++++++++ .../sections/layr/error-bad-width-1024.xml | 29 +++++++++++++++++++ .../sections/layr/error-bad-width-1500.xml | 22 ++++++++++++++ .../sections/layr/error-bad-width-x.xml | 22 ++++++++++++++ .../sections/layr/error-dup-width.xml | 29 +++++++++++++++++++ developer/src/kmc-ldml/test/layr.tests.ts | 13 +++++++++ 10 files changed, 187 insertions(+), 5 deletions(-) create mode 100644 developer/src/kmc-ldml/test/fixtures/sections/layr/error-bad-width-0.xml create mode 100644 developer/src/kmc-ldml/test/fixtures/sections/layr/error-bad-width-1024.xml create mode 100644 developer/src/kmc-ldml/test/fixtures/sections/layr/error-bad-width-1500.xml create mode 100644 developer/src/kmc-ldml/test/fixtures/sections/layr/error-bad-width-x.xml create mode 100644 developer/src/kmc-ldml/test/fixtures/sections/layr/error-dup-width.xml diff --git a/core/include/ldml/keyman_core_ldml.h b/core/include/ldml/keyman_core_ldml.h index 189b37fc96..f521cb356f 100644 --- a/core/include/ldml/keyman_core_ldml.h +++ b/core/include/ldml/keyman_core_ldml.h @@ -46,6 +46,8 @@ #define LDML_KEYS_MOD_OTHER 0x10000 #define LDML_KEYS_MOD_SHIFT 0x10 #define LDML_LAYR_LIST_HARDWARE_TOUCH "touch" +#define LDML_LAYR_MAX_MINDEVICEWIDTH 0x3E7 +#define LDML_LAYR_MIN_MINDEVICEWIDTH 0x1 #define LDML_LENGTH_BKSP 0xC #define LDML_LENGTH_BKSP_ITEM 0x10 #define LDML_LENGTH_DISP 0x10 diff --git a/core/include/ldml/keyman_core_ldml.ts b/core/include/ldml/keyman_core_ldml.ts index 093ad839c0..c975c94b71 100644 --- a/core/include/ldml/keyman_core_ldml.ts +++ b/core/include/ldml/keyman_core_ldml.ts @@ -362,6 +362,16 @@ class Constants { */ readonly length_layr_key = 4; + /** + * Minimum allowed minDeviceWidth for a layer list + */ + readonly layr_min_minDeviceWidth = 1; + + /** + * Maximum allowed minDeviceWidth for a layer list + */ + readonly layr_max_minDeviceWidth = 999; + /* ------------------------------------------------------------------ * list section ------------------------------------------------------------------ */ diff --git a/developer/src/kmc-ldml/src/compiler/layr.ts b/developer/src/kmc-ldml/src/compiler/layr.ts index ca99d09883..ec96a6bddd 100644 --- a/developer/src/kmc-ldml/src/compiler/layr.ts +++ b/developer/src/kmc-ldml/src/compiler/layr.ts @@ -20,14 +20,26 @@ export class LayrCompiler extends SectionCompiler { let valid = true; let totalLayerCount = 0; let hardwareLayers = 0; - // let touchLayers = 0; + let touchLayers = 0; + const deviceWidths = new Set(); this.keyboard3.layers?.forEach((layers) => { const { formId } = layers; if (formId === 'touch') { - // touchLayers++; - // multiple touch layers are OK + touchLayers++; totalLayerCount += layers.layer?.length; - // TODO-LDML: check that widths are distinct + const { minDeviceWidth } = layers; + if (!minDeviceWidth || + minDeviceWidth < constants.layr_min_minDeviceWidth || + minDeviceWidth > constants.layr_max_minDeviceWidth || + Number.isNaN(Number(minDeviceWidth))) { + valid = false; + this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidLayerWidth({minDeviceWidth}, layers)); + } else if (deviceWidths.has(minDeviceWidth)) { + valid = false; + this.callbacks.reportMessage(LdmlCompilerMessages.Error_DuplicateLayerWidth({minDeviceWidth}, layers)); + } else { + deviceWidths.add(minDeviceWidth); + } } else { // hardware hardwareLayers++; diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 23066e4e40..32aa34e32c 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -299,7 +299,21 @@ export class LdmlCompilerMessages { static Warn_StringDenorm = (o: { s: string }) => m(this.WARN_StringDenorm, `File contains string "${def(o.s)}" that is neither NFC nor NFD.`); - // Available: 0x02C-0x2F + static ERROR_DuplicateLayerWidth = SevError | 0x002C; + static Error_DuplicateLayerWidth = (o: { minDeviceWidth: number }, x?: ObjectWithMetadata) => mx( + this.ERROR_DuplicateLayerWidth, x, + `Two or more layers have minDeviceWidth=${def(o?.minDeviceWidth)}`, + `Touch layers must have distinct widths.` + ); + + static ERROR_InvalidLayerWidth = SevError | 0x002D; + static Error_InvalidLayerWidth = (o: { minDeviceWidth: number }, x?: ObjectWithMetadata) => mx( + this.ERROR_InvalidLayerWidth, x, + `Invalid Layers minDeviceWidth=${def(o?.minDeviceWidth)}`, + `Width must be between 1-999 (millimeters), inclusive.` // sync with layr_max_minDeviceWidth / layr_max_maxDeviceWidth (from spec) + ); + + // Available: 0x02E-0x2F static ERROR_InvalidQuadEscape = SevError | 0x0030; static Error_InvalidQuadEscape = (o: { cp: number }, x?: ObjectWithMetadata) => mx( diff --git a/developer/src/kmc-ldml/test/fixtures/sections/layr/error-bad-width-0.xml b/developer/src/kmc-ldml/test/fixtures/sections/layr/error-bad-width-0.xml new file mode 100644 index 0000000000..3054afceb7 --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/sections/layr/error-bad-width-0.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/fixtures/sections/layr/error-bad-width-1024.xml b/developer/src/kmc-ldml/test/fixtures/sections/layr/error-bad-width-1024.xml new file mode 100644 index 0000000000..d1d34b598f --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/sections/layr/error-bad-width-1024.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/fixtures/sections/layr/error-bad-width-1500.xml b/developer/src/kmc-ldml/test/fixtures/sections/layr/error-bad-width-1500.xml new file mode 100644 index 0000000000..5000e27304 --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/sections/layr/error-bad-width-1500.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/fixtures/sections/layr/error-bad-width-x.xml b/developer/src/kmc-ldml/test/fixtures/sections/layr/error-bad-width-x.xml new file mode 100644 index 0000000000..fec6f8d26f --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/sections/layr/error-bad-width-x.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/fixtures/sections/layr/error-dup-width.xml b/developer/src/kmc-ldml/test/fixtures/sections/layr/error-dup-width.xml new file mode 100644 index 0000000000..695e2dbde0 --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/sections/layr/error-dup-width.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/layr.tests.ts b/developer/src/kmc-ldml/test/layr.tests.ts index 9d6c2112f5..610efff212 100644 --- a/developer/src/kmc-ldml/test/layr.tests.ts +++ b/developer/src/kmc-ldml/test/layr.tests.ts @@ -167,5 +167,18 @@ describe('layr', function () { } }, }, + { + subpath: 'sections/layr/error-dup-width.xml', + errors: [ + LdmlCompilerMessages.Error_DuplicateLayerWidth({ minDeviceWidth: 120}), + ] + }, + ...[0, 1024, 1500, `x` as unknown as number].map(minDeviceWidth => ({ + subpath: `sections/layr/error-bad-width-${minDeviceWidth}.xml`, + errors: [ + // + LdmlCompilerMessages.Error_InvalidLayerWidth({ minDeviceWidth }), + ] + })), ]); }); From aed42a8321962fcd2411c88f2d74604b69cdffdf Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Mon, 26 May 2025 14:04:07 -0400 Subject: [PATCH 19/73] auto: increment master version to 19.0.51 --- HISTORY.md | 4 ++++ VERSION.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index e0dd29f6d9..d359fa613a 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,9 @@ # Keyman Version History +## 19.0.50 alpha 2025-05-26 + +* chore(web): mark async methods in UI components (#14034) + ## 19.0.49 alpha 2025-05-25 * feat(developer): revamp line numbers per design doc (#14016) diff --git a/VERSION.md b/VERSION.md index a86baa2e63..88b0239e14 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -19.0.50 \ No newline at end of file +19.0.51 \ No newline at end of file From f9305f33f2d97e828ed3dc64194636444f5aa119 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 26 May 2025 13:12:40 -0500 Subject: [PATCH 20/73] feat(developer): add line numbers to loca compiler Fixes #13940 --- .../src/compiler/ldml-compiler-messages.ts | 17 ++++++++++----- developer/src/kmc-ldml/src/compiler/loca.ts | 21 ++++++++++++++----- developer/src/kmc-ldml/test/loca.tests.ts | 12 +++++------ 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 32aa34e32c..3d4d987436 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -56,7 +56,10 @@ export class LdmlCompilerMessages { ); static ERROR_InvalidLocale = SevError | 0x0002; - static Error_InvalidLocale = (o:{tag: string}) => m(this.ERROR_InvalidLocale, `Invalid BCP 47 locale form '${def(o.tag)}'`); + static Error_InvalidLocale = (o:{tag: string}, x?: ObjectWithMetadata) => mx( + this.ERROR_InvalidLocale, x, + `Invalid BCP 47 locale form '${def(o.tag)}'`, + ); static ERROR_HardwareLayerHasTooManyRows = SevError | 0x0003; static Error_HardwareLayerHasTooManyRows = (x?: ObjectWithMetadata) => mx( @@ -77,16 +80,20 @@ export class LdmlCompilerMessages { ); static HINT_OneOrMoreRepeatedLocales = SevHint | 0x0006; - static Hint_OneOrMoreRepeatedLocales = () => - m(this.HINT_OneOrMoreRepeatedLocales, `After minimization, one or more locales is repeated and has been removed`); + static Hint_OneOrMoreRepeatedLocales = (x?: ObjectWithMetadata) => mx( + this.HINT_OneOrMoreRepeatedLocales, x, + `After minimization, one or more locales is repeated and has been removed`, + ); static ERROR_InvalidFile = SevError | 0x0007; static Error_InvalidFile = (o:{errorText: string}) => m(this.ERROR_InvalidFile, `The source file has an invalid structure: ${def(o.errorText)}`); static HINT_LocaleIsNotMinimalAndClean = SevHint | 0x0008; - static Hint_LocaleIsNotMinimalAndClean = (o:{sourceLocale: string, locale: string}) => - m(this.HINT_LocaleIsNotMinimalAndClean, `Locale '${def(o.sourceLocale)}' is not minimal or correctly formatted and should be '${def(o.locale)}'`); + static Hint_LocaleIsNotMinimalAndClean = (o:{sourceLocale: string, locale: string}, x?: ObjectWithMetadata) => mx( + this.HINT_LocaleIsNotMinimalAndClean, x, + `Locale '${def(o.sourceLocale)}' is not minimal or correctly formatted and should be '${def(o.locale)}'`, + ); static ERROR_InvalidScanCode = SevError | 0x0009; static Error_InvalidScanCode = (o: { codes: string, id: string }, x?: ObjectWithMetadata) => mx( diff --git a/developer/src/kmc-ldml/src/compiler/loca.ts b/developer/src/kmc-ldml/src/compiler/loca.ts index b54e7d6eb0..48be2d032f 100644 --- a/developer/src/kmc-ldml/src/compiler/loca.ts +++ b/developer/src/kmc-ldml/src/compiler/loca.ts @@ -1,6 +1,6 @@ import { constants } from "@keymanapp/ldml-keyboard-constants"; import { KMXPlus } from '@keymanapp/common-types'; -import { LDMLKeyboard } from '@keymanapp/developer-utils'; +import { LDMLKeyboard, ObjectWithMetadata } from '@keymanapp/developer-utils'; import { SectionCompiler } from "./section-compiler.js"; import { LdmlCompilerMessages } from "./ldml-compiler-messages.js"; @@ -21,7 +21,17 @@ export class LocaCompiler extends SectionCompiler { */ private getLocales = (keyboard: LKKeyboard) => - [keyboard.locale].concat(Array.isArray(keyboard.locales?.locale) ? keyboard.locales.locale.map(v => v.id) : []) + [keyboard.locale].concat(Array.isArray(keyboard.locales?.locale) ? keyboard.locales.locale.map(v => v.id) : []); + + /** + * Attempt to locate context metadata for a locale, for error messages + * @param loc locale tag + * @returns context metadata or null + */ + private contextForLocale(loc: string) : ObjectWithMetadata { + if (loc == this.keyboard3.locale) return this.keyboard3; + return this.keyboard3?.locales; + } public validate(): boolean { let valid = true; @@ -31,7 +41,7 @@ export class LocaCompiler extends SectionCompiler { new Intl.Locale(tag); } catch(e) { if(e instanceof RangeError) { - this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidLocale({tag})); + this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidLocale({tag}, this.contextForLocale(tag))); valid = false; } else { /* c8 ignore next 2 */ @@ -51,7 +61,8 @@ export class LocaCompiler extends SectionCompiler { const locales = sourceLocales.map((sourceLocale: string) => { const locale = new Intl.Locale(sourceLocale).minimize().toString(); if(locale != sourceLocale) { - this.callbacks.reportMessage(LdmlCompilerMessages.Hint_LocaleIsNotMinimalAndClean({sourceLocale, locale})); + this.callbacks.reportMessage(LdmlCompilerMessages.Hint_LocaleIsNotMinimalAndClean({ sourceLocale, locale }, + this.contextForLocale(sourceLocale))); } return locale; }); @@ -63,7 +74,7 @@ export class LocaCompiler extends SectionCompiler { result.locales = canonicalLocales.map(locale => sections.strs.allocString(locale)); if(result.locales.length < locales.length) { - this.callbacks.reportMessage(LdmlCompilerMessages.Hint_OneOrMoreRepeatedLocales()); + this.callbacks.reportMessage(LdmlCompilerMessages.Hint_OneOrMoreRepeatedLocales(this.keyboard3?.locales)); } return result; diff --git a/developer/src/kmc-ldml/test/loca.tests.ts b/developer/src/kmc-ldml/test/loca.tests.ts index e8d035a30e..287ee62b00 100644 --- a/developer/src/kmc-ldml/test/loca.tests.ts +++ b/developer/src/kmc-ldml/test/loca.tests.ts @@ -1,7 +1,7 @@ import 'mocha'; import { assert } from 'chai'; import { LocaCompiler } from '../src/compiler/loca.js'; -import { compilerTestCallbacks, loadSectionFixture } from './helpers/index.js'; +import { compilerTestCallbacks, loadSectionFixture, withOffset } from './helpers/index.js'; import { KMXPlus } from '@keymanapp/common-types'; import { LdmlCompilerMessages } from '../src/compiler/ldml-compiler-messages.js'; @@ -26,10 +26,10 @@ describe('loca', function () { // Note: multiple.xml includes fr-FR twice, with differing case, which should be canonicalized assert.equal(compilerTestCallbacks.messages.length, 4); - assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Hint_LocaleIsNotMinimalAndClean({sourceLocale: 'fr-FR', locale: 'fr'})); - assert.deepEqual(compilerTestCallbacks.messages[1], LdmlCompilerMessages.Hint_LocaleIsNotMinimalAndClean({sourceLocale: 'km-khmr-kh', locale: 'km'})); - assert.deepEqual(compilerTestCallbacks.messages[2], LdmlCompilerMessages.Hint_LocaleIsNotMinimalAndClean({sourceLocale: 'fr-fr', locale: 'fr'})); - assert.deepEqual(compilerTestCallbacks.messages[3], LdmlCompilerMessages.Hint_OneOrMoreRepeatedLocales()); + assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Hint_LocaleIsNotMinimalAndClean({sourceLocale: 'fr-FR', locale: 'fr'}, withOffset(136))); + assert.deepEqual(compilerTestCallbacks.messages[1], LdmlCompilerMessages.Hint_LocaleIsNotMinimalAndClean({sourceLocale: 'km-khmr-kh', locale: 'km'}, withOffset(136))); + assert.deepEqual(compilerTestCallbacks.messages[2], LdmlCompilerMessages.Hint_LocaleIsNotMinimalAndClean({sourceLocale: 'fr-fr', locale: 'fr'}, withOffset(136))); + assert.deepEqual(compilerTestCallbacks.messages[3], LdmlCompilerMessages.Hint_OneOrMoreRepeatedLocales(withOffset(136))); // Original is 6 locales, now five minimized in the results assert.equal(loca.locales.length, 5); @@ -46,6 +46,6 @@ describe('loca', function () { assert.equal(compilerTestCallbacks.messages.length, 1); // We'll only test one invalid BCP 47 tag to verify that we are properly calling BCP 47 validation routines. // Furthermore, we are testing BCP 47 structure, not the validity of each subtag -- we must assume the author knows of new subtags! - assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_InvalidLocale({tag:'en-*'})); + assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_InvalidLocale({tag:'en-*'}, withOffset(40))); }) }); From 4d5c09aa36634300dab648efcd5fed8148765915 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Tue, 27 May 2025 12:28:56 +0200 Subject: [PATCH 21/73] maint(common): tweaks to labels Add label `$platform` for changes to `resources/teamcity/$platform` or `resources/teamcity/includes`. Also remove no longer existent `linux/legacy/` paths. Fixes: #13943 Test-bot: skip --- .github/labeler.yml | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index 6876d8298e..9db0dfb4ab 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -11,7 +11,10 @@ docs: docs/** # Add labels based on changed files using actions/labeler # -android/: android/** +android/: + - android/** + - resources/teamcity/includes/** + - resources/teamcity/android/** android/app/: android/KMAPro/** android/engine/: android/KMEA/** android/samples/: android/Samples/** @@ -24,30 +27,35 @@ core/: developer/: - developer/** - + - resources/teamcity/includes/** + - resources/teamcity/developer/** developer/compilers/: - developer/src/kmc/** - developer/src/kmcmplib/** - developer/src/kmc-*/** - developer/ide/: - developer/src/server/** - developer/src/tike/** -ios/: ios/** +ios/: + - ios/** + - resources/teamcity/includes/** + - resources/teamcity/ios/** ios/app/: ios/keyman/** ios/engine/: ios/engine/** ios/samples/: ios/samples/** -linux/: linux/** +linux/: + - linux/** + - resources/teamcity/includes/** + - resources/teamcity/linux/** linux/config/: linux/keyman-config/** -linux/engine/: - - linux/ibus-keyman/** - - linux/legacy/ibus-kmfl/** - - linux/legacy/kmflcomp/** - - linux/legacy/libkmfl/** +linux/engine/: linux/ibus-keyman/** -mac/: mac/** +mac/: + - mac/** + - resources/teamcity/includes/** + - resources/teamcity/mac/** # mac/config/: # mac/engine/: mac/** @@ -59,13 +67,19 @@ oem/fv/windows/: oem/firstvoices/windows/** resources/: resources/** -web/: web/** +web/: + - web/** + - resources/teamcity/includes/** + - resources/teamcity/web/** # web/bookmarklet/ web/engine/: web/source/** web/ui/: web/source/kmwui* web/samples/: web/samples/** web/predictive-text/: web/src/engine/predictive-text/** -windows/: windows/** +windows/: + - windows/** + - resources/teamcity/includes/** + - resources/teamcity/windows/** windows/config/: windows/src/desktop/** windows/engine/: windows/src/engine/** From 3a7663f7c1d8655da6ac676ea38b6efc8b18db11 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Tue, 27 May 2025 17:07:23 +0200 Subject: [PATCH 22/73] fix(common): properly use newly installed node version Previously, if we had to install a new node version, we activated it in the same subshell as the installation. This had the effect that `NVM_BIN` was set to the new version only in the subshell, but otherwise it still had the wrong value. This caused build failures. Subsequent builds would succeed because the node version was now available. This change fixes things by using an explicit `if` instead of the construct with parens which opened another subshell. Fixes: #14056 Test-bot: skip --- resources/build/_builder_nvm.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/resources/build/_builder_nvm.sh b/resources/build/_builder_nvm.sh index a1bb9a4cc1..063aec2404 100755 --- a/resources/build/_builder_nvm.sh +++ b/resources/build/_builder_nvm.sh @@ -41,8 +41,10 @@ type -t nvm >/dev/null || { } } -nvm use "$REQUIRED_NODE_VERSION" || \ - (nvm install "$REQUIRED_NODE_VERSION" && nvm use "$REQUIRED_NODE_VERSION") +if ! nvm use "${REQUIRED_NODE_VERSION}"; then + nvm install "${REQUIRED_NODE_VERSION}" + nvm use "${REQUIRED_NODE_VERSION}" +fi # Beware the hardcoded path below -- it should already be in the system PATH From 28b919bce87a2fc008a59f09fe726a58a74468a4 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Tue, 27 May 2025 10:46:16 -0500 Subject: [PATCH 23/73] feat(developer): update tran compiler per code review For: #13937 --- .../src/compiler/ldml-compiler-messages.ts | 14 +++++++++----- developer/src/kmc-ldml/src/compiler/tran.ts | 7 +++++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index a5636104f2..f5fd897be8 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -206,12 +206,16 @@ export class LdmlCompilerMessages { ); static ERROR_MixedTransformGroup = SevError | 0x001A; - static Error_MixedTransformGroup = (x?: ObjectWithMetadata) => - m(this.ERROR_MixedTransformGroup, `transformGroup cannot contain both reorder and transform elements`); + static Error_MixedTransformGroup = (x?: ObjectWithMetadata) => mx( + this.ERROR_MixedTransformGroup, x, + `transformGroup cannot contain both reorder and transform elements`, + ); static ERROR_EmptyTransformGroup = SevError | 0x001B; - static Error_EmptyTransformGroup = (x?: ObjectWithMetadata) => - m(this.ERROR_EmptyTransformGroup, `transformGroup must have either reorder or transform elements`); + static Error_EmptyTransformGroup = (x?: ObjectWithMetadata) => mx( + this.ERROR_EmptyTransformGroup, x, + `transformGroup must have either reorder or transform elements`, + ); static ERROR_MissingStringVariable = SevError | 0x001C; static Error_MissingStringVariable = (o:{id: string}, x?: ObjectWithMetadata) => mx( @@ -262,7 +266,7 @@ export class LdmlCompilerMessages { static HINT_CharClassImplicitDenorm = SevHint | 0x0026; static Hint_CharClassImplicitDenorm = (o: { lowestCh: number }, x?: ObjectWithMetadata) => mx( this.HINT_CharClassImplicitDenorm, x, - `File has character classes which span non-NFD character(s), including ${util.describeCodepoint(o.lowestCh)}. These will not match any text.`, + `File has character classes which span non-NFD character(s), including ${util.describeCodepoint(o.lowestCh)}. These will not match any text.`, ); static WARN_CharClassExplicitDenorm = SevWarn | 0x0027; diff --git a/developer/src/kmc-ldml/src/compiler/tran.ts b/developer/src/kmc-ldml/src/compiler/tran.ts index 2db96ab7cc..32ebbca3dd 100644 --- a/developer/src/kmc-ldml/src/compiler/tran.ts +++ b/developer/src/kmc-ldml/src/compiler/tran.ts @@ -66,10 +66,13 @@ export abstract class TransformCompiler type); if (!verifyValidAndUnique(types, types => types.forEach(type => - reportMessage(LdmlCompilerMessages.Error_DuplicateTransformsType({ type }, typeToObject.get(type)))), + reportMessage(LdmlCompilerMessages.Error_DuplicateTransformsType({ type }, typeToObject.get(type))) + ), new Set(ALLOWED_TYPES), types => types.forEach(type => - reportMessage(LdmlCompilerMessages.Error_InvalidTransformsType({ type }, typeToObject.get(type)))))) { + reportMessage(LdmlCompilerMessages.Error_InvalidTransformsType({ type }, typeToObject.get(type))) + ) + )) { valid = false; } From cb4e17e9fe39e47d94874d414e31868d20a70050 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Tue, 27 May 2025 14:01:16 -0400 Subject: [PATCH 24/73] auto: increment master version to 19.0.52 --- HISTORY.md | 6 ++++++ VERSION.md | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index d359fa613a..7dde3fcd47 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,11 @@ # Keyman Version History +## 19.0.51 alpha 2025-05-27 + +* maint: add stable-19.0 triggers (#14043) +* maint: add empty 'scripts' tag to package.json to silence warning (#14044) +* fix(linux): fix version variable references in kmconfig (#14032) + ## 19.0.50 alpha 2025-05-26 * chore(web): mark async methods in UI components (#14034) diff --git a/VERSION.md b/VERSION.md index 88b0239e14..4ce97ae864 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -19.0.51 \ No newline at end of file +19.0.52 \ No newline at end of file From e7cb8b0617fe9a266a4c921dbda97e49608d1f7b Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Wed, 28 May 2025 11:31:26 +0200 Subject: [PATCH 25/73] maint(common): tweak labels for common and core This change addresses code review comments. `common` and `core` also have TC build configs, so we add labels for them as well. Co-authored-by: Marc Durdin --- .github/labeler.yml | 7 ++++++- resources/build/run-required-test-builds.sh | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index 9db0dfb4ab..fc37bf64b7 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -19,11 +19,16 @@ android/app/: android/KMAPro/** android/engine/: android/KMEA/** android/samples/: android/Samples/** -common/: common/** +common/: + - common/** + - resources/teamcity/includes/** + - resources/teamcity/common/** common/web/: common/web/** core/: - core/** + - resources/teamcity/includes/** + - resources/teamcity/core/** developer/: - developer/** diff --git a/resources/build/run-required-test-builds.sh b/resources/build/run-required-test-builds.sh index e108bc7717..e5ff54ba7b 100755 --- a/resources/build/run-required-test-builds.sh +++ b/resources/build/run-required-test-builds.sh @@ -167,6 +167,7 @@ while IFS= read -r line; do eval watch='$'watch_$platform # Add common patterns to the watch list watch="^(${platform}|(oem/[^/]+/${platform})|resources/((?!teamcity)|teamcity/(${platform}|includes))|$watch)" + # Since bash doesn't support negative look-aheads we use grep to test if echo "${line}" | grep --quiet --perl-regexp "${watch}"; then build_platforms+=($platform) fi From 3e4d463fc1231decbf99e364748b5c14c5685e0e Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Wed, 28 May 2025 11:38:50 +0200 Subject: [PATCH 26/73] maint(linux): move TC build scripts to `resources/teamcity/linux` Test-bot: skip --- resources/teamcity/README.md | 2 +- resources/teamcity/{ => linux}/keyman-linux-release.sh | 4 ++-- .../teamcity/{ => linux}/keyman-linux-test-integration.sh | 2 +- resources/teamcity/{ => linux}/keyman-linux-test.sh | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) rename resources/teamcity/{ => linux}/keyman-linux-release.sh (97%) rename resources/teamcity/{ => linux}/keyman-linux-test-integration.sh (95%) rename resources/teamcity/{ => linux}/keyman-linux-test.sh (96%) diff --git a/resources/teamcity/README.md b/resources/teamcity/README.md index 9fe012a942..8a53868ca6 100644 --- a/resources/teamcity/README.md +++ b/resources/teamcity/README.md @@ -27,7 +27,7 @@ export DOCKER_RUNNING=1 and then run the build, e.g. ```bash -resources/teamcity/keyman-linux-test.sh configure,build,test +resources/teamcity/linux/keyman-linux-test.sh configure,build,test ``` NOTE: by default this will run the build as `root` in the container, diff --git a/resources/teamcity/keyman-linux-release.sh b/resources/teamcity/linux/keyman-linux-release.sh similarity index 97% rename from resources/teamcity/keyman-linux-release.sh rename to resources/teamcity/linux/keyman-linux-release.sh index eb518dac7c..38849fa9ba 100755 --- a/resources/teamcity/keyman-linux-release.sh +++ b/resources/teamcity/linux/keyman-linux-release.sh @@ -5,7 +5,7 @@ ## START STANDARD BUILD SCRIPT INCLUDE # adjust relative paths as necessary THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" -. "${THIS_SCRIPT%/*}/../../resources/build/builder.inc.sh" +. "${THIS_SCRIPT%/*}/../../../resources/build/builder.inc.sh" ## END STANDARD BUILD SCRIPT INCLUDE # shellcheck disable=SC2154 @@ -22,7 +22,7 @@ builder_describe \ "configure install dependencies" \ "build make a release build" \ "test run unit tests" \ - "publish make a source tarball" \ + "publish make a source tarball and publish to downloads and launchpad" \ "--gpgkey=GPGKEYGRIP GPG key for signing" \ "--gpgpw=GPGKEYPW GPG key passphrase" \ "--rsync-path=RSYNC_PATH rsync path on remote server" \ diff --git a/resources/teamcity/keyman-linux-test-integration.sh b/resources/teamcity/linux/keyman-linux-test-integration.sh similarity index 95% rename from resources/teamcity/keyman-linux-test-integration.sh rename to resources/teamcity/linux/keyman-linux-test-integration.sh index 8aa2c5d9eb..edb287f696 100755 --- a/resources/teamcity/keyman-linux-test-integration.sh +++ b/resources/teamcity/linux/keyman-linux-test-integration.sh @@ -6,7 +6,7 @@ ## START STANDARD BUILD SCRIPT INCLUDE # adjust relative paths as necessary THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" -. "${THIS_SCRIPT%/*}/../../resources/build/builder.inc.sh" +. "${THIS_SCRIPT%/*}/../../../resources/build/builder.inc.sh" ## END STANDARD BUILD SCRIPT INCLUDE # shellcheck disable=SC2154 diff --git a/resources/teamcity/keyman-linux-test.sh b/resources/teamcity/linux/keyman-linux-test.sh similarity index 96% rename from resources/teamcity/keyman-linux-test.sh rename to resources/teamcity/linux/keyman-linux-test.sh index 5477d32855..0d9505d351 100755 --- a/resources/teamcity/keyman-linux-test.sh +++ b/resources/teamcity/linux/keyman-linux-test.sh @@ -6,7 +6,7 @@ ## START STANDARD BUILD SCRIPT INCLUDE # adjust relative paths as necessary THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" -. "${THIS_SCRIPT%/*}/../../resources/build/builder.inc.sh" +. "${THIS_SCRIPT%/*}/../../../resources/build/builder.inc.sh" ## END STANDARD BUILD SCRIPT INCLUDE # shellcheck disable=SC2154 From bb06fe06c6640a971461e33cccdf0791286599eb Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Wed, 28 May 2025 14:01:39 -0400 Subject: [PATCH 27/73] auto: increment master version to 19.0.53 --- HISTORY.md | 9 +++++++++ VERSION.md | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 7dde3fcd47..428255c09e 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,14 @@ # Keyman Version History +## 19.0.52 alpha 2025-05-28 + +* feat(developer): line numbers for tran compiler (#14037) +* feat(developer): add line numbers to disp compiler (#14048) +* feat(developer): line numbers for layr compiler (#14050) +* feat(developer): line numbers for LocaCompiler (#14054) +* fix(common): properly use newly installed node version (#14057) +* maint(common): update build coordinator and labels for `resources/teamcity` changes (#14047) + ## 19.0.51 alpha 2025-05-27 * maint: add stable-19.0 triggers (#14043) diff --git a/VERSION.md b/VERSION.md index 4ce97ae864..13de026a3e 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -19.0.52 \ No newline at end of file +19.0.53 \ No newline at end of file From 1ba77cc5b2042a91fe3e760a65200af25be85b0b Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Wed, 28 May 2025 13:08:09 +0200 Subject: [PATCH 28/73] maint(linux): improve output of install action --- resources/teamcity/includes/tc-actions.inc.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/resources/teamcity/includes/tc-actions.inc.sh b/resources/teamcity/includes/tc-actions.inc.sh index f6973418b6..4b817a6f21 100644 --- a/resources/teamcity/includes/tc-actions.inc.sh +++ b/resources/teamcity/includes/tc-actions.inc.sh @@ -9,9 +9,10 @@ linux_clean_action() { # Install required dependencies for building Keyman on Linux. linux_install_dependencies_action() { - builder_heading "Installing dependencies" + builder_echo start "install dependencies" "Installing dependencies" . "${KEYMAN_ROOT}/linux/scripts/package-build.inc.sh" checkAndInstallRequirements + builder_echo end "install dependencies" success "Finished installing dependencies" } # Install additional dependencies required for determining test coverage. From c70464e23f59505b409989e7d46349a7a47d08a8 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 28 May 2025 20:05:14 -0500 Subject: [PATCH 29/73] fix(developer): ldml: make sure .run() calls compile() with postValidation - .run() doesn't call validate() and then compile(), so it needs to call compile() with postValidation=true Fixes: #14067 --- developer/src/kmc-ldml/src/compiler/compiler.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/developer/src/kmc-ldml/src/compiler/compiler.ts b/developer/src/kmc-ldml/src/compiler/compiler.ts index b69e984728..d66631e0b5 100644 --- a/developer/src/kmc-ldml/src/compiler/compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/compiler.ts @@ -136,7 +136,7 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { if (!source) { return null; } - const kmx = await this.compile(source); + const kmx = await this.compile(source, true); if (!kmx) { return null; } @@ -352,6 +352,8 @@ export class LdmlKeyboardCompiler implements KeymanCompiler { * Transforms in-memory LDML keyboard xml file to an intermediate * representation of a .kmx file. * @param source - in-memory representation of LDML keyboard xml file + * @param postValidate - pass true if sections should run a 'validate' phase at the very end. + * Set this to true if you aren't calling validate() separately. * @returns KMXPlusFile intermediate file */ public async compile(source: LDMLKeyboardXMLSourceFile, postValidate?: boolean): Promise { From 633f9551c751722cb5d1376e786c20fcf119fddc Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 28 May 2025 17:09:18 -0500 Subject: [PATCH 30/73] feat(developer): update strs compiler to collect context - attempt to find an object with a context For: #13932 --- common/web/types/src/kmx/kmx-plus/kmx-plus.ts | 16 ++++++++++++ .../kmc-ldml/src/compiler/empty-compiler.ts | 26 +++++++++++++++---- .../src/compiler/ldml-compiler-messages.ts | 24 +++++++++++------ 3 files changed, 53 insertions(+), 13 deletions(-) diff --git a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts index 8d6cd57003..310c566f5b 100644 --- a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts +++ b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts @@ -135,6 +135,18 @@ export class StrsItem { isEqual(a: StrsItem): boolean { return a.value === this.value && a.char === this.char; } + + private _context : any = null; + + /** add any context from the options to this strsitem */ + setContext(opts?: StrsOptions) { + // At present, there's only a single piece of context available + this._context = this._context || opts?.x; + } + + get context() : any { + return this._context; + } }; /** @@ -161,6 +173,8 @@ export interface StrsOptions { nfd?: boolean; /** string can be stored as a single CharStrsItem, not in strs table. */ singleOk?: boolean; + /** optional context */ + x?: any; }; export class Strs extends Section { @@ -192,6 +206,8 @@ export class Strs extends Section { result = new StrsItem(s); this.strings.push(result); } + // give an option to set the context + result.setContext(opts); return result; } diff --git a/developer/src/kmc-ldml/src/compiler/empty-compiler.ts b/developer/src/kmc-ldml/src/compiler/empty-compiler.ts index 4b9aec7613..c0fe1d1ef6 100644 --- a/developer/src/kmc-ldml/src/compiler/empty-compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/empty-compiler.ts @@ -1,7 +1,7 @@ import { SectionIdent, constants } from '@keymanapp/ldml-keyboard-constants'; import { SectionCompiler } from "./section-compiler.js"; import { util, KMXPlus, LdmlKeyboardTypes } from "@keymanapp/common-types"; -import { CompilerCallbacks, LDMLKeyboard } from "@keymanapp/developer-utils"; +import { CompilerCallbacks, LDMLKeyboard, ObjectWithMetadata } from "@keymanapp/developer-utils"; import { VarsCompiler } from './vars.js'; import { LdmlCompilerMessages } from './ldml-compiler-messages.js'; @@ -33,13 +33,26 @@ export class StrsCompiler extends EmptyCompiler { public postValidate(section?: KMXPlus.Section): boolean { const strs = section; + /** attempt to find a context object for the string */ + function findContextForString(s: string): ObjectWithMetadata { + // try exact match + for(const str of strs.strings) { + if (str.value == s) return str.context; + } + // try substring match + for(const str of strs.strings) { + if (str.value.includes(s)) return str.context; + } + return null; + } + if (strs) { const badStringAnalyzer = new util.BadStringAnalyzer(); const CONTAINS_MARKER_REGEX = new RegExp(LdmlKeyboardTypes.MarkerParser.ANY_MARKER_MATCH); for (let s of strs.allProcessedStrings.values()) { // stop at the first denormalized string if (!util.isNormalized(s)) { - this.callbacks.reportMessage(LdmlCompilerMessages.Warn_StringDenorm({s})); + this.callbacks.reportMessage(LdmlCompilerMessages.Warn_StringDenorm({s}, findContextForString(s))); } // replace all \\uXXXX with the actual code point. // this lets us analyze whether there are PUA, unassigned, etc. @@ -63,16 +76,19 @@ export class StrsCompiler extends EmptyCompiler { const illegals = m.get(util.BadStringType.illegal); if (puas) { const [count, lowestCh] = [puas.size, Array.from(puas.values()).sort((a, b) => a - b)[0]]; - this.callbacks.reportMessage(LdmlCompilerMessages.Hint_PUACharacters({ count, lowestCh })) + this.callbacks.reportMessage(LdmlCompilerMessages.Hint_PUACharacters({ count, lowestCh }, + findContextForString(String.fromCodePoint(lowestCh)))); } if (unassigneds) { const [count, lowestCh] = [unassigneds.size, Array.from(unassigneds.values()).sort((a, b) => a - b)[0]]; - this.callbacks.reportMessage(LdmlCompilerMessages.Warn_UnassignedCharacters({ count, lowestCh })) + this.callbacks.reportMessage(LdmlCompilerMessages.Warn_UnassignedCharacters({ count, lowestCh }, + findContextForString(String.fromCodePoint(lowestCh)))); } if (illegals) { // do this last, because we will return false. const [count, lowestCh] = [illegals.size, Array.from(illegals.values()).sort((a, b) => a - b)[0]]; - this.callbacks.reportMessage(LdmlCompilerMessages.Error_IllegalCharacters({ count, lowestCh })) + this.callbacks.reportMessage(LdmlCompilerMessages.Error_IllegalCharacters({ count, lowestCh }, + findContextForString(String.fromCodePoint(lowestCh)))); return false; } } diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 2955e1a95e..4e65138912 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -267,16 +267,22 @@ export class LdmlCompilerMessages { ); static HINT_PUACharacters = SevHint | 0x0023; - static Hint_PUACharacters = (o: { count: number, lowestCh: number }) => - m(this.HINT_PUACharacters, `File contains ${def(o.count)} PUA character(s), including ${util.describeCodepoint(o.lowestCh)}`); + static Hint_PUACharacters = (o: { count: number, lowestCh: number }, x?: ObjectWithMetadata) => mx( + this.HINT_PUACharacters, x, + `File contains ${def(o.count)} PUA character(s), including ${util.describeCodepoint(o.lowestCh)}`, + ); static WARN_UnassignedCharacters = SevWarn | 0x0024; - static Warn_UnassignedCharacters = (o: { count: number, lowestCh: number }) => - m(this.WARN_UnassignedCharacters, `File contains ${def(o.count)} unassigned character(s), including ${util.describeCodepoint(o.lowestCh)}`); + static Warn_UnassignedCharacters = (o: { count: number, lowestCh: number }, x?: ObjectWithMetadata) => mx( + this.WARN_UnassignedCharacters, x, + `File contains ${def(o.count)} unassigned character(s), including ${util.describeCodepoint(o.lowestCh)}`, + ); static ERROR_IllegalCharacters = SevError | 0x0025; - static Error_IllegalCharacters = (o: { count: number, lowestCh: number }) => - m(this.ERROR_IllegalCharacters, `File contains ${def(o.count)} illegal character(s), including ${util.describeCodepoint(o.lowestCh)}`); + static Error_IllegalCharacters = (o: { count: number, lowestCh: number }, x?: ObjectWithMetadata) => mx( + this.ERROR_IllegalCharacters, x, + `File contains ${def(o.count)} illegal character(s), including ${ util.describeCodepoint(o.lowestCh) }`, + ); static HINT_CharClassImplicitDenorm = SevHint | 0x0026; static Hint_CharClassImplicitDenorm = (o: { lowestCh: number }, x?: ObjectWithMetadata) => mx( @@ -307,8 +313,10 @@ export class LdmlCompilerMessages { ); static WARN_StringDenorm = SevWarn | 0x002B; - static Warn_StringDenorm = (o: { s: string }) => - m(this.WARN_StringDenorm, `File contains string "${def(o.s)}" that is neither NFC nor NFD.`); + static Warn_StringDenorm = (o: { s: string }, x?: ObjectWithMetadata) => mx( + this.WARN_StringDenorm, x, + `File contains string "${def(o.s)}" that is neither NFC nor NFD.`, + ); static ERROR_DuplicateLayerWidth = SevError | 0x002C; static Error_DuplicateLayerWidth = (o: { minDeviceWidth: number }, x?: ObjectWithMetadata) => mx( From db24750bd3e4cdbd31e6e380c0b12ac938d58de5 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Wed, 28 May 2025 19:08:48 -0500 Subject: [PATCH 31/73] feat(developer): update strs compiler to collect context - default state needs to be null not undefined For: #13932 --- common/web/types/src/kmx/kmx-plus/kmx-plus.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts index 310c566f5b..767a221f9c 100644 --- a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts +++ b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts @@ -136,7 +136,7 @@ export class StrsItem { return a.value === this.value && a.char === this.char; } - private _context : any = null; + private _context: any = undefined; /** add any context from the options to this strsitem */ setContext(opts?: StrsOptions) { From e33322fe8a095e591991cfc45df7dcb6b571172b Mon Sep 17 00:00:00 2001 From: rc-swag <58423624+rc-swag@users.noreply.github.com> Date: Thu, 29 May 2025 13:32:33 +1000 Subject: [PATCH 32/73] fix(windows): change from error to breadcrumb and log As this will also be seen when a user declines elevation it is not really an error just breadcrumb incase there is some actual error as a result. fixes: #14066 --- .../main/Keyman.System.UpdateStateMachine.pas | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/windows/src/desktop/kmshell/main/Keyman.System.UpdateStateMachine.pas b/windows/src/desktop/kmshell/main/Keyman.System.UpdateStateMachine.pas index b34fbf4fd4..99a0985545 100644 --- a/windows/src/desktop/kmshell/main/Keyman.System.UpdateStateMachine.pas +++ b/windows/src/desktop/kmshell/main/Keyman.System.UpdateStateMachine.pas @@ -987,10 +987,11 @@ begin executeResult := WaitForElevatedConfiguration(0, '-ikp'); if (executeResult <> 0) then begin - TKeymanSentryClient.Client.MessageEvent - (Sentry.Client.SENTRY_LEVEL_ERROR, - 'Executing kmshell process to install keyboard packages failed:"' + - IntToStr(Ord(executeResult)) + '"'); + TKeymanSentryClient.Breadcrumb('error', + 'Executing kmshell process to install keyboard packages failed"' + + IntToStr(Ord(executeResult)) + '"', 'update'); + KL.Log('InstallingState.DoInstallKeyman failed executing kmshell ' + + 'or the update was not found in the cache'); ChangeState(IdleState); end; end @@ -1042,9 +1043,10 @@ begin if not FResult then begin bucStateContext.RemoveCachedFiles; - TKeymanSentryClient.Client.MessageEvent(Sentry.Client.SENTRY_LEVEL_ERROR, - 'Executing kmshell process to install failed:"' + - IntToStr(Ord(FResult)) + '"'); + TKeymanSentryClient.Breadcrumb('error', + 'InstallingState.DoInstallKeyman: failed executing kmshell or file not found', 'update'); + KL.Log('InstallingState.DoInstallKeyman failed executing kmshell ' + + 'or the update was not found in the cache'); ChangeState(IdleState); end; From c3705bf9eff94509dcdcbce3a6b3c5844c361394 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 29 May 2025 09:09:58 -0500 Subject: [PATCH 33/73] feat(developer): ldml: strscompiler: add context to more allocString call sites - disp, keys, layr, tran For: #13932 --- developer/src/kmc-ldml/src/compiler/disp.ts | 6 ++++-- developer/src/kmc-ldml/src/compiler/keys.ts | 14 ++++++++------ developer/src/kmc-ldml/src/compiler/layr.ts | 6 +++--- developer/src/kmc-ldml/src/compiler/tran.ts | 13 +++++++++---- 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/developer/src/kmc-ldml/src/compiler/disp.ts b/developer/src/kmc-ldml/src/compiler/disp.ts index 5b63a99a8f..8121888fcf 100644 --- a/developer/src/kmc-ldml/src/compiler/disp.ts +++ b/developer/src/kmc-ldml/src/compiler/disp.ts @@ -61,7 +61,7 @@ export class DispCompiler extends SectionCompiler { const result = new Disp(); // displayOptions - result.baseCharacter = sections.strs.allocString(this.keyboard3.displays?.displayOptions?.baseCharacter, {unescape: true}); + result.baseCharacter = sections.strs.allocString(this.keyboard3.displays?.displayOptions?.baseCharacter, { unescape: true, x: this.keyboard3?.displays?.displayOptions }); // displays result.disps = this.keyboard3.displays?.display.map(display => ({ @@ -69,11 +69,13 @@ export class DispCompiler extends SectionCompiler { stringVariables: true, markers: true, unescape: true, + x: display, }, sections), - id: sections.strs.allocString(display.keyId), // not escaped, not substituted + id: sections.strs.allocString(display.keyId, { x: display }), // not escaped, not substituted display: sections.strs.allocString(display.display, { stringVariables: true, unescape: true, + x: display, }, sections), })) || []; // TODO-LDML: need coverage for the [] diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index 5ca61ceb48..01c4ffc675 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -324,12 +324,13 @@ export class KeysCompiler extends SectionCompiler { // allocate the in-memory const flicks: KeysFlicks = new KeysFlicks( - sections.strs.allocString(flickId) + sections.strs.allocString(flickId, { x: flick }) ); // add data from each segment - for (const { keyId, directions } of flick.flickSegment) { - const keyIdStr = sections.strs.allocString(keyId); + for (const flickSegment of flick.flickSegment) { + const { keyId, directions } = flickSegment; + const keyIdStr = sections.strs.allocString(keyId, { x: flickSegment }); const directionsList: ListItem = sections.list.allocListFromSpaces( directions, { }, @@ -369,20 +370,20 @@ export class KeysCompiler extends SectionCompiler { if (!!gap) { flags |= constants.keys_key_flags_gap; } - const id = sections.strs.allocString(key.id); + const id = sections.strs.allocString(key.id, { x: key }); const longPress: ListItem = sections.list.allocListFromSpaces( longPressKeyIds, {}, sections); const longPressDefault = sections.strs.allocString(longPressDefaultKeyId, - {}, + { x: key }, sections); const multiTap: ListItem = sections.list.allocListFromSpaces( multiTapKeyIds, {}, sections); - const keySwitch = sections.strs.allocString(layerId); // 'switch' is a reserved word + const keySwitch = sections.strs.allocString(layerId, { x: key }); // 'switch' is a reserved word const toRaw = output; @@ -395,6 +396,7 @@ export class KeysCompiler extends SectionCompiler { unescape: true, singleOk: true, nfd: true, + x: key, }, sections); if (!to.isOneChar) { diff --git a/developer/src/kmc-ldml/src/compiler/layr.ts b/developer/src/kmc-ldml/src/compiler/layr.ts index ec96a6bddd..b0c615f41b 100644 --- a/developer/src/kmc-ldml/src/compiler/layr.ts +++ b/developer/src/kmc-ldml/src/compiler/layr.ts @@ -69,13 +69,13 @@ export class LayrCompiler extends SectionCompiler { const sect = new Layr(); sect.lists = this.keyboard3.layers.map((layers) => { - const hardware = sections.strs.allocString(layers.formId); + const hardware = sections.strs.allocString(layers.formId, {x:layers}); // Already validated in validate const layerEntries = []; for (const layer of layers.layer) { const rows = layer.row.map((row) => { const erow: LayrRow = { - keys: row.keys.trim().split(/[ \t]+/).map((id) => sections.strs.allocString(id)), + keys: row.keys.trim().split(/[ \t]+/).map((id) => sections.strs.allocString(id, { x: row })), }; // include linenumber info for row return SectionCompiler.copySymbols(erow, row); @@ -84,7 +84,7 @@ export class LayrCompiler extends SectionCompiler { // push a layer entry for each modifier set for (const mod of mods) { layerEntries.push({ - id: sections.strs.allocString(layer.id), + id: sections.strs.allocString(layer.id, {x:layer}), mod, rows, }); diff --git a/developer/src/kmc-ldml/src/compiler/tran.ts b/developer/src/kmc-ldml/src/compiler/tran.ts index 32ebbca3dd..875da13406 100644 --- a/developer/src/kmc-ldml/src/compiler/tran.ts +++ b/developer/src/kmc-ldml/src/compiler/tran.ts @@ -18,6 +18,7 @@ import { verifyValidAndUnique } from "../util/util.js"; import { LdmlCompilerMessages } from "./ldml-compiler-messages.js"; import { Substitutions, SubstitutionUse } from "./substitution-tracker.js"; import { transform_from_parse, transform_to_parse } from "../util/abnf/abnf.js"; +import { StrsOptions } from "../../../../../common/web/types/src/kmx/kmx-plus/kmx-plus.js"; type TransformCompilerType = 'simple' | 'backspace'; @@ -138,6 +139,8 @@ export abstract class TransformCompiler Date: Thu, 29 May 2025 14:00:34 -0400 Subject: [PATCH 34/73] auto: increment master version to 19.0.54 --- HISTORY.md | 4 ++++ VERSION.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 428255c09e..0a01c6c253 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,9 @@ # Keyman Version History +## 19.0.53 alpha 2025-05-29 + +* fix(developer): ensure that kmc doesn't skip validate phase of kmc-ldml (#14068) + ## 19.0.52 alpha 2025-05-28 * feat(developer): line numbers for tran compiler (#14037) diff --git a/VERSION.md b/VERSION.md index 13de026a3e..c16038ebd9 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -19.0.53 \ No newline at end of file +19.0.54 \ No newline at end of file From 70ced9847a643a7e925f542b30afab94c2c2429f Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 29 May 2025 13:53:23 -0500 Subject: [PATCH 35/73] chore(developer): ldml: unit test for .run() validation - re-run invalid-illegal.xml through compiler.run() - ensure it fails Fixes: #14067 --- .../src/kmc-ldml/test/compiler-e2e.tests.ts | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/developer/src/kmc-ldml/test/compiler-e2e.tests.ts b/developer/src/kmc-ldml/test/compiler-e2e.tests.ts index d9da6c3f34..e7756226bd 100644 --- a/developer/src/kmc-ldml/test/compiler-e2e.tests.ts +++ b/developer/src/kmc-ldml/test/compiler-e2e.tests.ts @@ -6,6 +6,7 @@ import { compareXml } from './helpers/compareXml.js'; import { LdmlKeyboardCompiler } from '../src/compiler/compiler.js'; import { kmxToXml } from '../src/util/serialize.js'; import { writeFileSync } from 'node:fs'; +import { LdmlCompilerMessages } from '../src/main.js'; /** Overall compiler tests */ describe('compiler-tests', function() { @@ -42,6 +43,25 @@ describe('compiler-tests', function() { assert.isNotNull(kvk?.data); }); + it('should-validate-on-run compiling sections/strs/invalid-illegal.xml', async function() { + this.timeout(4000); + const inputFilename = makePathToFixture('sections/strs/invalid-illegal.xml'); + + // should fail validation + const k = new LdmlKeyboardCompiler(); + await k.init(compilerTestCallbacks, { ...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false }); + + const runOutput = await k.run(inputFilename, "invalid-illegal.kmx"); // need the exact name passed to build-fixtures + assert.isNull(runOutput, "Expect invalid-illegal to fail to run()"); + assert.sameDeepMembers(compilerTestCallbacks.messages, [ + // copied from strs.tests.ts + // validation messages + LdmlCompilerMessages.Error_IllegalCharacters({ count: 5, lowestCh: 0xFDD0 }), + LdmlCompilerMessages.Hint_PUACharacters({ count: 2, lowestCh: 0xE010 }), + ]); + }); + + it('should-serialize-kmx', async function() { this.timeout(4000); // Let's build basic.xml From f0bbf53ce4d54d430a6b279590d5e3c1864e2165 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Thu, 29 May 2025 21:24:34 -0400 Subject: [PATCH 36/73] auto: increment stable-18.0 version to 18.0.238 (history cherry-pick to master) --- HISTORY.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 0a01c6c253..ad4a6f6786 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -325,6 +325,25 @@ * refactor(windows): rename `TKeymanMutex.MutexOwned` to `TakeOwnership` and add `ReleaseOwnership` (#13168) * chore: increment to alpha 19.0 (#13187) +## 18.0.237 stable 2025-05-30 + +* maint(android): Update FirstVoices launcher icon and red theme (#13917) +* fix(web): empty layout fontSize property should default to 1em (#13960) +* fix(linux): use correct header guard define in OrderedOutputDevice.h (#13963) +* chore(linux): Update debian changelog (#13888) +* fix(mac): handle PackageInfo section in kmp.inf file (#13957) +* feat(android): log active keyboard, model IDs with reported errors (#14003) +* chore(ios): FirstVoices certificate update (#14005) +* feat(ios): log active keyboard, model IDs with reported errors (#14006) +* fix(linux): configurable path in `.service` file when using basu (#13999) +* maint(linux): add `--no-werror` option to not report warnings as errors (#14000) +* fix(windows): use json file name against cache folder and ename Enter and Exit methods for TState Object (#14010) +* chore(ios): update FirstVoices AppIcon (#14011) +* maint: change builder_completion script to a .inc.sh (#14008) +* fix(windows): only breadcrumb unexpected state on first run (#14026) +* fix(windows): skip sending handle kmshell events to upgrade state machine for keyboard install modes (#14028) +* fix(windows): fix sentry error message reporting (#14029) + ## 18.0.236 stable 2025-05-12 * chore(linux): Update debian changelog (#13737) From 10e21cb4333ad281c31cfc2e0319931e6af833b5 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Fri, 30 May 2025 11:02:46 +0200 Subject: [PATCH 37/73] =?UTF-8?q?maint(linux):=20`sudo`=20not=20required?= =?UTF-8?q?=20for=20removing=20temporary=20files=20=F0=9F=8D=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since we have write access to the directory we don't need `sudo` for removing the temporary dependency files if we pass `-f` to `rm`. Cherry-pick-of: commit 4f27f6e256995e4513c700eba9fa7c7b3cec0a33 Cherry-pick-of: #14085 Test-bot: skip --- linux/scripts/package-build.inc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linux/scripts/package-build.inc.sh b/linux/scripts/package-build.inc.sh index 44ac603a7a..c53b73c653 100644 --- a/linux/scripts/package-build.inc.sh +++ b/linux/scripts/package-build.inc.sh @@ -78,5 +78,5 @@ function checkAndInstallRequirements() sudo mk-build-deps debian/control wait_for_apt_deb && sudo DEBIAN_FRONTEND="noninteractive" \ apt-get -qy --allow-downgrades install ./keyman-build-deps_*.deb - sudo rm -f keyman-build-deps_* + rm -f keyman-build-deps_* } From 097c03d2c340518430016b835ab38b16532003e1 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 30 May 2025 09:26:39 -0500 Subject: [PATCH 38/73] feat(developer): ldml: strscompiler: exclude line numbers from some tests assertions - remove line number information for some "simpler" uses of the compileKeyboard() test helper For: #13932 --- developer/src/kmc-ldml/test/helpers/index.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/developer/src/kmc-ldml/test/helpers/index.ts b/developer/src/kmc-ldml/test/helpers/index.ts index 7d714bb2b6..ce7dc8233f 100644 --- a/developer/src/kmc-ldml/test/helpers/index.ts +++ b/developer/src/kmc-ldml/test/helpers/index.ts @@ -143,6 +143,7 @@ export async function compileKeyboard(inputFilename: string, options: LdmlCompil assert.isNotNull(source, 'k.load should not have returned null'); const valid = await k.validate(source); + zapMessageMetadata(); if (validateMessages) { assert.sameDeepMembers(compilerTestCallbacks.messages, validateMessages, "validation messages mismatch"); assert.notEqual(valid, expectFailValidate, 'validation failure'); @@ -154,6 +155,7 @@ export async function compileKeyboard(inputFilename: string, options: LdmlCompil if (!valid) return null; // get out, if the above asserts didn't get us out. const kmx = await k.compile(source); + zapMessageMetadata(); if (compileMessages) { assert.sameDeepMembers(compilerTestCallbacks.messages, compileMessages, "compiler messages mismatch"); } else { @@ -172,6 +174,16 @@ export function checkMessages() { assert.isEmpty(compilerTestCallbacks.messages, compilerEventFormat(compilerTestCallbacks.messages)); } +/** These tests aren't prepared for line number information in messages. Remove it so that comparisons pass. */ +function zapMessageMetadata() { + for(const i in compilerTestCallbacks.messages) { + delete compilerTestCallbacks.messages[i].column; + delete compilerTestCallbacks.messages[i].filename; + delete compilerTestCallbacks.messages[i].line; + delete compilerTestCallbacks.messages[i].offset; + } +} + /** * Like CompilerEvent, but supports regex matching. */ From 96f0f783f583335dc249616ecb5a08fdef4b2637 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 30 May 2025 11:34:57 -0500 Subject: [PATCH 39/73] feat(developer): ldml: strscompiler: remaining allocString() calls This concludes all of the allocString() calls that aren't part of lists, sets, elements etc. Fixes: #13932 --- developer/src/kmc-ldml/src/compiler/loca.ts | 2 +- developer/src/kmc-ldml/src/compiler/meta.ts | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/developer/src/kmc-ldml/src/compiler/loca.ts b/developer/src/kmc-ldml/src/compiler/loca.ts index 48be2d032f..3cc222e613 100644 --- a/developer/src/kmc-ldml/src/compiler/loca.ts +++ b/developer/src/kmc-ldml/src/compiler/loca.ts @@ -71,7 +71,7 @@ export class LocaCompiler extends SectionCompiler { // yet include `getCanonicalLocales` but node 16 does include it so we can // safely use it. Also well supported in modern browsers. const canonicalLocales = (Intl as any).getCanonicalLocales(locales) as string[]; - result.locales = canonicalLocales.map(locale => sections.strs.allocString(locale)); + result.locales = canonicalLocales.map(locale => sections.strs.allocString(locale, {x: this.contextForLocale(locale)})); if(result.locales.length < locales.length) { this.callbacks.reportMessage(LdmlCompilerMessages.Hint_OneOrMoreRepeatedLocales(this.keyboard3?.locales)); diff --git a/developer/src/kmc-ldml/src/compiler/meta.ts b/developer/src/kmc-ldml/src/compiler/meta.ts index 57f0cb36c0..e6cd6371ee 100644 --- a/developer/src/kmc-ldml/src/compiler/meta.ts +++ b/developer/src/kmc-ldml/src/compiler/meta.ts @@ -53,12 +53,18 @@ export class MetaCompiler extends SectionCompiler { public compile(sections: DependencySections): Meta { const result = new Meta(); - result.author = sections.strs.allocString(this.keyboard3.info?.author); - result.conform = sections.strs.allocString(this.keyboard3.conformsTo); - result.layout = sections.strs.allocString(this.keyboard3.info?.layout); - result.name = sections.strs.allocString(this.keyboard3.info?.name); - result.indicator = sections.strs.allocString(this.keyboard3.info?.indicator); - result.version = sections.strs.allocString(this.keyboard3.version?.number ?? "0.0.0"); + result.author = sections.strs.allocString(this.keyboard3.info?.author, + {x: this.keyboard3.info}); + result.conform = sections.strs.allocString(this.keyboard3.conformsTo, + {x: this.keyboard3}); + result.layout = sections.strs.allocString(this.keyboard3.info?.layout, + {x: this.keyboard3.info}); + result.name = sections.strs.allocString(this.keyboard3.info?.name, + {x: this.keyboard3.info}); + result.indicator = sections.strs.allocString(this.keyboard3.info?.indicator, + {x: this.keyboard3.info}); + result.version = sections.strs.allocString(this.keyboard3.version?.number ?? "0.0.0", + {x: this.keyboard3.version}); result.settings = (this.keyboard3.settings?.normalization == "disabled" ? KeyboardSettings.normalizationDisabled : 0); return result; From 8662acc46dd8505344a5cfe3ebe8706c90c8fc4f Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 30 May 2025 12:55:57 -0500 Subject: [PATCH 40/73] feat(developer): ldml: list compiler: add context to lists add contexts to lists and elements and uset Fixes: #13935 --- .../types/src/kmx/kmx-plus/element-string.ts | 12 ++++---- common/web/types/src/kmx/kmx-plus/kmx-plus.ts | 30 ++++++++++--------- developer/src/kmc-ldml/src/compiler/keys.ts | 6 ++-- developer/src/kmc-ldml/src/compiler/tran.ts | 4 +-- developer/src/kmc-ldml/src/compiler/vars.ts | 9 +++++- 5 files changed, 35 insertions(+), 26 deletions(-) diff --git a/common/web/types/src/kmx/kmx-plus/element-string.ts b/common/web/types/src/kmx/kmx-plus/element-string.ts index 49af2e1f49..9d3b518f04 100644 --- a/common/web/types/src/kmx/kmx-plus/element-string.ts +++ b/common/web/types/src/kmx/kmx-plus/element-string.ts @@ -1,5 +1,5 @@ import { constants } from '@keymanapp/ldml-keyboard-constants'; -import { DependencySections, StrsItem, UsetItem } from './kmx-plus.js'; +import { DependencySections, StrsItem, StrsOptions, UsetItem } from './kmx-plus.js'; import { ElementParser, ElementSegment, ElementType } from '../../ldml-keyboard/pattern-parser.js'; import * as util from '../../util/util.js'; import MATCH_HEX_ESCAPE = util.MATCH_HEX_ESCAPE; @@ -33,7 +33,7 @@ export class ElementString extends Array { * @param source if a string array, does not get reinterpreted as UnicodeSet. This is used with vars, etc. Or pass `["str"]` for an explicit 1-element elem. * If it is a string, will be interpreted per reorder element rules. */ - static fromStrings(sections: DependencySections, source: string | string[], order?: string, tertiary?: string, tertiary_base?: string, prebase?: string) : ElementString { + static fromStrings(sections: DependencySections, options: StrsOptions, source: string | string[], order?: string, tertiary?: string, tertiary_base?: string, prebase?: string) : ElementString { // the returned array const array = new ElementString(); if(!source) { @@ -85,17 +85,17 @@ export class ElementString extends Array { if (!uset) { return null; // UnicodeSet error already thrown } - elem.uset = sections.uset.allocUset(uset, sections); - elem.value = sections.strs.allocString('', {singleOk: true}); // no string + elem.uset = sections.uset.allocUset(uset, sections, options?.x); + elem.value = sections.strs.allocString('', {...options, singleOk: true}); // no string } else if (item.type === ElementType.codepoint || item.type === ElementType.escaped || item.type === ElementType.string) { // some kind of a string let str = item.segment; if (item.type === ElementType.escaped && !MATCH_HEX_ESCAPE.test(str)) { str = unescapeOneQuadString(str); // TODO-LDML: any other escape forms here? - elem.value = sections.strs.allocString(str, { singleOk: true }); + elem.value = sections.strs.allocString(str, { ...options, singleOk: true }); } else { - elem.value = sections.strs.allocString(str, { unescape: true, singleOk: true }); + elem.value = sections.strs.allocString(str, { ...options, unescape: true, singleOk: true }); } // Now did we end up with one char or no? if (elem.value.isOneChar) { diff --git a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts index 767a221f9c..5bc3bc5a92 100644 --- a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts +++ b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts @@ -41,14 +41,14 @@ export class Elem extends Section { strings: ElementString[] = []; constructor(sections: DependencySections) { super(); - this.strings.push(ElementString.fromStrings(sections, '')); // C7043: null element string + this.strings.push(ElementString.fromStrings(sections, {}, '')); // C7043: null element string } /** * @param source if a string array, does not get reinterpreted as UnicodeSet. This is used with vars, etc. Or pass `["str"]` for an explicit 1-element elem. * If it is a string, will be interpreted per reorder element ruls. */ - allocElementString(sections: DependencySections, source: string | string[], order?: string, tertiary?: string, tertiary_base?: string, prebase?: string): ElementString { - const s = ElementString.fromStrings(sections, source, order, tertiary, tertiary_base, prebase); + allocElementString(sections: DependencySections, options: StrsOptions, source: string | string[], order?: string, tertiary?: string, tertiary_base?: string, prebase?: string): ElementString { + const s = ElementString.fromStrings(sections, options, source, order, tertiary, tertiary_base, prebase); if (!s) return s; let result = this.strings.find(item => item.isEqual(s)); if(result === undefined) { @@ -374,11 +374,13 @@ export class Vars extends Section { export class VarsItem extends Section { id: StrsItem; value: StrsItem; + x?: any; - constructor(id: string, value: string, sections: DependencySections) { + constructor(id: string, value: string, sections: DependencySections, x?: any) { super(); this.id = sections.strs.allocString(id); this.value = sections.strs.allocString(value, {unescape: true}); + this.x = x; } valid() : boolean { @@ -387,8 +389,8 @@ export class VarsItem extends Section { }; export class UnicodeSetItem extends VarsItem { - constructor(id: string, value: string, sections: DependencySections, usetparser: UnicodeSetParser) { - super(id, value, sections); + constructor(id: string, value: string, sections: DependencySections, usetparser: UnicodeSetParser, x?: any) { + super(id, value, sections, x); const needRanges = sections.usetparser.sizeUnicodeSet(value); if (needRanges >= 0) { this.unicodeSet = sections.usetparser.parseUnicodeSet(value, needRanges); @@ -401,9 +403,9 @@ export class UnicodeSetItem extends VarsItem { }; export class SetVarItem extends VarsItem { - constructor(id: string, value: string[], sections: DependencySections) { - super(id, value.join(' '), sections); - this.items = sections.elem.allocElementString(sections, value); + constructor(id: string, value: string[], sections: DependencySections, x?: any) { + super(id, value.join(' '), sections, x); + this.items = sections.elem.allocElementString(sections, {x}, value); } items: ElementString; // element string array valid() : boolean { @@ -412,8 +414,8 @@ export class SetVarItem extends VarsItem { }; export class StringVarItem extends VarsItem { - constructor(id: string, value: string, sections: DependencySections) { - super(id, value, sections); + constructor(id: string, value: string, sections: DependencySections, x?: any) { + super(id, value, sections, x); } // no added fields }; @@ -451,7 +453,7 @@ export class Tran extends Section { }; export class UsetItem { - constructor(public uset: UnicodeSet, public str: StrsItem) { + constructor(public uset: UnicodeSet, public str: StrsItem, public x?: any) { } compareTo(other: UsetItem) : number { return this.str.compareTo(other.str); @@ -460,11 +462,11 @@ export class UsetItem { export class Uset extends Section { usets: UsetItem[] = []; - allocUset(set: UnicodeSet, sections: DependencySections) : UsetItem { + allocUset(set: UnicodeSet, sections: DependencySections, x?: any) : UsetItem { // match the same pattern let result = this.usets.find(s => set.pattern == s.uset.pattern); if (result === undefined) { - result = new UsetItem(set, sections.strs.allocString(set.pattern)); + result = new UsetItem(set, sections.strs.allocString(set.pattern), x); this.usets.push(result); } return result; diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index 01c4ffc675..829c148334 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -333,7 +333,7 @@ export class KeysCompiler extends SectionCompiler { const keyIdStr = sections.strs.allocString(keyId, { x: flickSegment }); const directionsList: ListItem = sections.list.allocListFromSpaces( directions, - { }, + { x: flickSegment }, sections); flicks.flicks.push({ directions: directionsList, @@ -372,7 +372,7 @@ export class KeysCompiler extends SectionCompiler { } const id = sections.strs.allocString(key.id, { x: key }); const longPress: ListItem = sections.list.allocListFromSpaces( - longPressKeyIds, {}, + longPressKeyIds, { x: key }, sections); const longPressDefault = sections.strs.allocString(longPressDefaultKeyId, @@ -381,7 +381,7 @@ export class KeysCompiler extends SectionCompiler { const multiTap: ListItem = sections.list.allocListFromSpaces( multiTapKeyIds, - {}, + { x: key }, sections); const keySwitch = sections.strs.allocString(layerId, { x: key }); // 'switch' is a reserved word diff --git a/developer/src/kmc-ldml/src/compiler/tran.ts b/developer/src/kmc-ldml/src/compiler/tran.ts index 875da13406..da47ed46a9 100644 --- a/developer/src/kmc-ldml/src/compiler/tran.ts +++ b/developer/src/kmc-ldml/src/compiler/tran.ts @@ -305,8 +305,8 @@ export abstract class TransformCompiler m !== LdmlKeyboardTypes.MarkerParser.ANY_MARKER_ID).sort(); - result.markers = sections.list.allocList(allMarkers, {}, sections); + result.markers = sections.list.allocList(allMarkers, { + // x: - we don't have an easy way to get context at this point + // however, this is the string with the marker *id*. So, from the StrsCompiler + // point of view, the value of this context would be if the marker ID had, say, + // invalid Unicode in it for ERROR_IllegalCharacters - but, that wouldn't be a + // valid Marker ID either. + // So, it's OK to not pass context with this particular list. + }, sections); // sets need to be added late, because they can refer to markers variables?.set?.forEach((e) => From b27d236a39792774b226572ee6c76adc43950a08 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Fri, 30 May 2025 14:00:42 -0400 Subject: [PATCH 41/73] auto: increment master version to 19.0.55 --- HISTORY.md | 4 ++++ VERSION.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index ad4a6f6786..b455685d26 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,9 @@ # Keyman Version History +## 19.0.54 alpha 2025-05-30 + +* maint(linux): move TC build scripts to `resources/teamcity/linux` (#14061) + ## 19.0.53 alpha 2025-05-29 * fix(developer): ensure that kmc doesn't skip validate phase of kmc-ldml (#14068) diff --git a/VERSION.md b/VERSION.md index c16038ebd9..0c1dd988d0 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -19.0.54 \ No newline at end of file +19.0.55 \ No newline at end of file From f74cbde633abe5d26c0ddc97f8a1fd751ef0f2d5 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Fri, 30 May 2025 21:18:16 +0200 Subject: [PATCH 42/73] maint(linux): update to new version of gha-ubuntu-packaging Version 2.0 of gha-ubuntu-packaging retries 5 times to install the packages, so hopefully this will improve the reliability. Test-bot: skip --- .github/actions/build-binary-packages/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build-binary-packages/action.yml b/.github/actions/build-binary-packages/action.yml index f3948119b8..09e23e3f7a 100644 --- a/.github/actions/build-binary-packages/action.yml +++ b/.github/actions/build-binary-packages/action.yml @@ -31,7 +31,7 @@ runs: path: artifacts/keyman-srcpkg - name: Build - uses: sillsdev/gha-ubuntu-packaging@b619077451b27c16dc6fd699bc1daf8d5ce07659 # v1.2 + uses: sillsdev/gha-ubuntu-packaging@1ab4a5967afbadab82a480936b9e53d7190acdf2 # v2.0 with: dist: "${{ inputs.dist }}" platform: "${{ inputs.arch }}" From 9d0d34bf09e9341bad57322d212d2397cba5c0f7 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 30 May 2025 14:34:40 -0500 Subject: [PATCH 43/73] feat(developer): ldml: list compiler: add context to lists. vars, markers, usets... - update several messages and the substitution tracker to take context Fixes: #13935 --- common/web/types/src/kmx/kmx-plus/kmx-plus.ts | 2 +- developer/src/kmc-ldml/src/compiler/disp.ts | 7 +- developer/src/kmc-ldml/src/compiler/keys.ts | 2 +- .../src/compiler/ldml-compiler-messages.ts | 26 +++++--- .../src/compiler/substitution-tracker.ts | 60 ++++++++--------- developer/src/kmc-ldml/src/compiler/tran.ts | 16 +++-- developer/src/kmc-ldml/src/compiler/vars.ts | 64 +++++++++++-------- developer/src/kmc-ldml/test/tran.tests.ts | 2 +- developer/src/kmc-ldml/test/vars.tests.ts | 2 +- 9 files changed, 100 insertions(+), 81 deletions(-) diff --git a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts index 5bc3bc5a92..373cc57b27 100644 --- a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts +++ b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts @@ -216,7 +216,7 @@ export class Strs extends Section { s = s ?? ''; // type check everything else if (typeof s !== 'string') { - throw new Error('alloc_string: s must be a string, undefined, or null.'); + throw new Error(`Internal Error: processString: s must be a string, undefined, or null, not ${typeof s} ${s}`); } // substitute variables if (opts?.stringVariables) { diff --git a/developer/src/kmc-ldml/src/compiler/disp.ts b/developer/src/kmc-ldml/src/compiler/disp.ts index 8121888fcf..da732cbc5d 100644 --- a/developer/src/kmc-ldml/src/compiler/disp.ts +++ b/developer/src/kmc-ldml/src/compiler/disp.ts @@ -12,9 +12,10 @@ import { SubstitutionUse, Substitutions } from "./substitution-tracker.js"; export class DispCompiler extends SectionCompiler { static validateSubstitutions(keyboard: LDMLKeyboard.LKKeyboard, st : Substitutions): boolean { - keyboard.displays?.display?.forEach(({ display, output }) => { - st.addStringAndMarkerSubstitution(SubstitutionUse.match, output); - st.addStringSubstitution(SubstitutionUse.emit, display); + keyboard.displays?.display?.forEach((e) => { + const { display, output } = e; + st.addStringAndMarkerSubstitution(SubstitutionUse.match, output, e); + st.addStringSubstitution(SubstitutionUse.emit, display, e); }); // no marker references in 'id' return true; diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index 829c148334..c896467bdb 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -37,7 +37,7 @@ export class KeysCompiler extends SectionCompiler { for (const keyId of usedKeys.values()) { const key = keyBag.get(keyId); if (!key) continue; // key not found is handled elsewhere. - st.addStringAndMarkerSubstitution(SubstitutionUse.emit, key.output); + st.addStringAndMarkerSubstitution(SubstitutionUse.emit, key.output, key); } return true; } diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 4e65138912..38d508aea9 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -237,12 +237,16 @@ export class LdmlCompilerMessages { ); static ERROR_MissingSetVariable = SevError | 0x001D; - static Error_MissingSetVariable = (o:{id: string}) => - m(this.ERROR_MissingSetVariable, `Reference to undefined set variable: \$[${def(o.id)}]`); + static Error_MissingSetVariable = (o:{id: string}, x?: ObjectWithMetadata) => mx( + this.ERROR_MissingSetVariable, x, + `Reference to undefined set variable: \$[${def(o.id)}]`, + ); static ERROR_MissingUnicodeSetVariable = SevError | 0x001E; - static Error_MissingUnicodeSetVariable = (o:{id: string}) => - m(this.ERROR_MissingUnicodeSetVariable, `Reference to undefined UnicodeSet variable: \$[${def(o.id)}]`); + static Error_MissingUnicodeSetVariable = (o:{id: string}, x?: ObjectWithMetadata) => mx( + this.ERROR_MissingUnicodeSetVariable, x, + `Reference to undefined UnicodeSet variable: \$[${def(o.id)}]`, + ); static ERROR_NeedSpacesBetweenSetVariables = SevError | 0x001F; static Error_NeedSpacesBetweenSetVariables = (o:{item: string}, x?: ObjectWithMetadata) => mx( @@ -257,8 +261,10 @@ export class LdmlCompilerMessages { ); static ERROR_MissingMarkers = SevError | 0x0021; - static Error_MissingMarkers = (o: { ids: string[] }) => - m(this.ERROR_MissingMarkers, `Markers used for matching but not defined: ${def(o.ids?.join(','))}`); + static Error_MissingMarkers = (o: { ids: string }, x?: ObjectWithMetadata) => mx( + this.ERROR_MissingMarkers, x, + `Markers used for matching but not defined: ${def(o.ids)}` + ); static ERROR_DisplayNeedsToOrId = SevError | 0x0022; static Error_DisplayNeedsToOrId = (o:{output?: string, keyId?: string}, x?: ObjectWithMetadata) => mx( @@ -301,14 +307,14 @@ export class LdmlCompilerMessages { m(this.ERROR_UnparseableReorderSet, `Illegal UnicodeSet "${def(o.set)}" in reorder "${def(o.from)}`); static ERROR_InvalidVariableIdentifier = SevError | 0x0029; - static Error_InvalidVariableIdentifier = (o: { id: string }) => m( - this.ERROR_InvalidVariableIdentifier, + static Error_InvalidVariableIdentifier = (o: { id: string }, x?: ObjectWithMetadata) => mx( + this.ERROR_InvalidVariableIdentifier, x, `Invalid variable identifier "${def(o.id)}". Identifiers must be between 1 and 32 characters, and can use A-Z, a-z, 0-9, and _.`, ); static ERROR_InvalidMarkerIdentifier = SevError | 0x002A; - static Error_InvalidMarkerIdentifier = (o: { id: string }) => m( - this.ERROR_InvalidMarkerIdentifier, + static Error_InvalidMarkerIdentifier = (o: { id: string }, x?: ObjectWithMetadata) => mx( + this.ERROR_InvalidMarkerIdentifier, x, `Invalid marker identifier "\m{${def(o.id)}}". Identifiers must be between 1 and 32 characters, and can use A-Z, a-z, 0-9, and _.`, ); diff --git a/developer/src/kmc-ldml/src/compiler/substitution-tracker.ts b/developer/src/kmc-ldml/src/compiler/substitution-tracker.ts index 6f07475490..61daefa0cf 100644 --- a/developer/src/kmc-ldml/src/compiler/substitution-tracker.ts +++ b/developer/src/kmc-ldml/src/compiler/substitution-tracker.ts @@ -1,4 +1,5 @@ import { LdmlKeyboardTypes } from "@keymanapp/common-types"; +import { ObjectWithMetadata } from "@keymanapp/developer-utils"; /** * Verb for SubstitutionTracker.add() @@ -14,7 +15,7 @@ export enum SubstitutionUse { variable, } -type SubstitutionSet = Set; +type SubstitutionSet = Map; /** Tracks usage of markers */ export class SubstitutionTracker { @@ -28,10 +29,10 @@ export class SubstitutionTracker { all: SubstitutionSet; constructor() { - this.emitted = new Set(); - this.consumed = new Set(); - this.matched = new Set(); - this.all = new Set(); + this.emitted = new Map(); + this.consumed = new Map(); + this.matched = new Map(); + this.all = new Map(); } /** @@ -39,32 +40,33 @@ export class SubstitutionTracker { * @param verb what kind of use we are adding * @param markers list of substitutions to add */ - add(verb: SubstitutionUse, markers: string[]) { + add(verb: SubstitutionUse, markers: string[], x?: ObjectWithMetadata) { if (!markers.length) { return; // skip if empty } + x = x || {}; // need at least an empty object if (verb == SubstitutionUse.emit) { markers.forEach((m) => { - this.emitted.add(m); - this.all.add(m); + this.emitted.set(m, x); + this.all.set(m,x); }); } else if (verb == SubstitutionUse.consume) { markers.forEach((m) => { - this.consumed.add(m); - this.all.add(m); + this.consumed.set(m,x); + this.all.set(m,x); }); } else if (verb == SubstitutionUse.match) { markers.forEach((m) => { - this.matched.add(m); - this.all.add(m); + this.matched.set(m,x); + this.all.set(m,x); }); } else if (verb == SubstitutionUse.variable) { markers.forEach((m) => { // we don't know, so add it to all three - this.matched.add(m); - this.emitted.add(m); - this.consumed.add(m); - this.all.add(m); + this.matched.set(m,x); + this.emitted.set(m,x); + this.consumed.set(m,x); + this.all.set(m,x); }); /* c8 skip next 3 */ } else { @@ -75,27 +77,27 @@ export class SubstitutionTracker { /** rollup of several substitution types */ export class Substitutions { - addSetAndStringSubtitution(verb: SubstitutionUse, str?: string) { - this.set.add(verb, LdmlKeyboardTypes.VariableParser.allSetReferences(str)); - this.addStringAndMarkerSubstitution(verb, str); + addSetAndStringSubtitution(verb: SubstitutionUse, str?: string, x?: ObjectWithMetadata) { + this.set.add(verb, LdmlKeyboardTypes.VariableParser.allSetReferences(str), x); + this.addStringAndMarkerSubstitution(verb, str, x); } /** add a string that can have string var substitutions or markers */ - addStringAndMarkerSubstitution(verb: SubstitutionUse, str?: string) { - this.addMarkers(verb, str); - this.addStringSubstitution(verb, str); + addStringAndMarkerSubstitution(verb: SubstitutionUse, str?: string, x?: ObjectWithMetadata) { + this.addMarkers(verb, str, x); + this.addStringSubstitution(verb, str, x); } - addStringSubstitution(verb: SubstitutionUse, str?: string) { - this.string.add(verb, LdmlKeyboardTypes.VariableParser.allStringReferences(str)); + addStringSubstitution(verb: SubstitutionUse, str?: string, x?: ObjectWithMetadata) { + this.string.add(verb, LdmlKeyboardTypes.VariableParser.allStringReferences(str), x); } /** add a string that's just markers */ - addMarkers(verb: SubstitutionUse, str?: string) { - this.markers.add(verb, LdmlKeyboardTypes.MarkerParser.allReferences(str)); - LdmlKeyboardTypes.MarkerParser.allBrokenReferences(str).forEach(m => this.badMarkers.add(m)); + addMarkers(verb: SubstitutionUse, str?: string, x?: ObjectWithMetadata) { + this.markers.add(verb, LdmlKeyboardTypes.MarkerParser.allReferences(str), x); + LdmlKeyboardTypes.MarkerParser.allBrokenReferences(str).forEach(m => this.badMarkers.set(m, x)); } // all valid markers markers: SubstitutionTracker; // all invalid markers - badMarkers: Set; + badMarkers: SubstitutionSet; // all valid set ids set: SubstitutionTracker; // all valid string ids @@ -108,6 +110,6 @@ export class Substitutions { this.set = new SubstitutionTracker(); this.string = new SubstitutionTracker(); this.uset = new SubstitutionTracker(); - this.badMarkers = new Set(); + this.badMarkers = new Map(); } } diff --git a/developer/src/kmc-ldml/src/compiler/tran.ts b/developer/src/kmc-ldml/src/compiler/tran.ts index da47ed46a9..f0db46336d 100644 --- a/developer/src/kmc-ldml/src/compiler/tran.ts +++ b/developer/src/kmc-ldml/src/compiler/tran.ts @@ -27,22 +27,24 @@ export abstract class TransformCompiler transforms.transformGroup.forEach(transformGroup => { - transformGroup.transform?.forEach(({ to, from }) => { - st.addSetAndStringSubtitution(SubstitutionUse.consume, from); - st.addSetAndStringSubtitution(SubstitutionUse.emit, to); + transformGroup.transform?.forEach((transform) => { + const { to, from } = transform; + st.addSetAndStringSubtitution(SubstitutionUse.consume, from, transform); + st.addSetAndStringSubtitution(SubstitutionUse.emit, to, transform); const mapFrom = LdmlKeyboardTypes.VariableParser.CAPTURE_SET_REFERENCE.exec(from); const mapTo = LdmlKeyboardTypes.VariableParser.MAPPED_SET_REFERENCE.exec(to || ''); if (mapFrom) { // add the 'from' as a match - st.set.add(SubstitutionUse.consume, [mapFrom[1]]); + st.set.add(SubstitutionUse.consume, [mapFrom[1]], transform); } if (mapTo) { // add the 'from' as a match - st.set.add(SubstitutionUse.emit, [mapTo[1]]); + st.set.add(SubstitutionUse.emit, [mapTo[1]], transform); } }); - transformGroup.reorder?.forEach(({ before }) => { - st.addStringSubstitution(SubstitutionUse.consume, before); + transformGroup.reorder?.forEach((reorder) => { + const { before } = reorder; + st.addStringSubstitution(SubstitutionUse.consume, before, reorder); }); })); return true; diff --git a/developer/src/kmc-ldml/src/compiler/vars.ts b/developer/src/kmc-ldml/src/compiler/vars.ts index c7cdc9cca2..2a0538732a 100644 --- a/developer/src/kmc-ldml/src/compiler/vars.ts +++ b/developer/src/kmc-ldml/src/compiler/vars.ts @@ -1,6 +1,6 @@ import { SectionIdent, constants } from "@keymanapp/ldml-keyboard-constants"; import { KMXPlus, LdmlKeyboardTypes } from '@keymanapp/common-types'; -import { LDMLKeyboard, CompilerCallbacks } from '@keymanapp/developer-utils'; +import { LDMLKeyboard, CompilerCallbacks, ObjectWithMetadata } from '@keymanapp/developer-utils'; import { SectionCompiler } from "./section-compiler.js"; import Vars = KMXPlus.Vars; import StringVarItem = KMXPlus.StringVarItem; @@ -40,9 +40,9 @@ export class VarsCompiler extends SectionCompiler { return valid; } - private validateIdentifier(id: string) { + private validateIdentifier(id: string, x?: ObjectWithMetadata) { if(!id.match(LdmlKeyboardTypes.VariableParser.ID)) { // From DTD - this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidVariableIdentifier({id})); + this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidVariableIdentifier({id}, x)); return false; } return true; @@ -77,7 +77,7 @@ export class VarsCompiler extends SectionCompiler { // Strings for (const e of variables.string) { const { id, value } = e; - if(!this.validateIdentifier(id)) { + if(!this.validateIdentifier(id, e)) { valid = false; continue; } @@ -90,9 +90,9 @@ export class VarsCompiler extends SectionCompiler { allStrings.add(ref); // avoids multiple reports of same missing variable } } - st.string.add(SubstitutionUse.variable, stringrefs); + st.string.add(SubstitutionUse.variable, stringrefs, e); allStrings.add(id); - st.addMarkers(SubstitutionUse.variable, value); + st.addMarkers(SubstitutionUse.variable, value, e); } // Sets for (const e of variables.set) { @@ -105,7 +105,7 @@ export class VarsCompiler extends SectionCompiler { allSets.add(id); // check for illegal references, here. const stringrefs = LdmlKeyboardTypes.VariableParser.allStringReferences(value); - st.string.add(SubstitutionUse.variable, stringrefs); + st.string.add(SubstitutionUse.variable, stringrefs, e); // Now split into spaces. const items: string[] = LdmlKeyboardTypes.VariableParser.setSplitter(value); @@ -116,7 +116,7 @@ export class VarsCompiler extends SectionCompiler { valid = false; this.callbacks.reportMessage(LdmlCompilerMessages.Error_NeedSpacesBetweenSetVariables({ item }, e)); } else { - st.set.add(SubstitutionUse.variable, setrefs); // the reference to a 'map' + st.set.add(SubstitutionUse.variable, setrefs, e); // the reference to a 'map' } // TODO-LDML: Are there other illegal cases here? what about "x$[set]"? } @@ -131,7 +131,7 @@ export class VarsCompiler extends SectionCompiler { addId(id, e); allUnicodeSets.add(id); const stringrefs = LdmlKeyboardTypes.VariableParser.allStringReferences(value); - st.string.add(SubstitutionUse.variable, stringrefs); + st.string.add(SubstitutionUse.variable, stringrefs, e); const setrefs = LdmlKeyboardTypes.VariableParser.allSetReferences(value); for (const id2 of setrefs) { if (!allUnicodeSets.has(id2)) { @@ -140,7 +140,7 @@ export class VarsCompiler extends SectionCompiler { // $[set] in a UnicodeSet must be another UnicodeSet. this.callbacks.reportMessage(LdmlCompilerMessages.Error_CantReferenceSetFromUnicodeSet({ id: id2 }, e)); } else { - st.uset.add(SubstitutionUse.variable, [id2]); + st.uset.add(SubstitutionUse.variable, [id2], e); } } } @@ -157,24 +157,24 @@ export class VarsCompiler extends SectionCompiler { } // check for any missing vars - for (const id of st.set.all) { + for (const [id,x] of st.set.all.entries()) { // note: we check the uset list also. we don't know until later which was // intended. collisions are handled separately. if (!allSets.has(id) && !allUnicodeSets.has(id)) { valid = false; - this.callbacks.reportMessage(LdmlCompilerMessages.Error_MissingSetVariable({ id })); + this.callbacks.reportMessage(LdmlCompilerMessages.Error_MissingSetVariable({ id }, x)); } } - for (const id of st.string.all) { + for (const [id,x] of st.string.all.entries()) { if (!allStrings.has(id)) { valid = false; - this.callbacks.reportMessage(LdmlCompilerMessages.Error_MissingStringVariable({ id })); + this.callbacks.reportMessage(LdmlCompilerMessages.Error_MissingStringVariable({ id }, x)); } } - for (const id of st.uset.all) { + for (const [id,x] of st.uset.all.entries()) { if (!allUnicodeSets.has(id)) { valid = false; - this.callbacks.reportMessage(LdmlCompilerMessages.Error_MissingUnicodeSetVariable({ id })); + this.callbacks.reportMessage(LdmlCompilerMessages.Error_MissingUnicodeSetVariable({ id }, x)); } } @@ -198,13 +198,13 @@ export class VarsCompiler extends SectionCompiler { // see if there are any matched-but-not-emitted markers const matchedNotEmitted : Set = new Set(); const mt = st.markers; - for (const m of mt.matched.values()) { + for (const m of mt.matched.keys()) { if (m === LdmlKeyboardTypes.MarkerParser.ANY_MARKER_ID) continue; // match-all marker if (!mt.emitted.has(m)) { matchedNotEmitted.add(m); } } - for (const m of mt.consumed.values()) { + for (const m of mt.consumed.keys()) { if (m === LdmlKeyboardTypes.MarkerParser.ANY_MARKER_ID) continue; // match-all marker if (!mt.emitted.has(m)) { matchedNotEmitted.add(m); @@ -213,7 +213,9 @@ export class VarsCompiler extends SectionCompiler { // report once if (matchedNotEmitted.size > 0) { - this.callbacks.reportMessage(LdmlCompilerMessages.Error_MissingMarkers({ ids: Array.from(matchedNotEmitted.values()).sort() })); + const ids = Array.from(matchedNotEmitted.values()).sort(); + const x = mt.all.get(ids[0]); // get the FIRST object for context + this.callbacks.reportMessage(LdmlCompilerMessages.Error_MissingMarkers({ ids: ids.join(',')}, x)); valid = false; } @@ -222,19 +224,24 @@ export class VarsCompiler extends SectionCompiler { if (!!st.badMarkers.size) { valid = false; - st.badMarkers.forEach(id => - this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidMarkerIdentifier({ id }))); + for (const [id, x] of st.badMarkers.entries()) { + this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidMarkerIdentifier({ id }, x)); + } } return valid; } validateSubstitutions(keyboard: LDMLKeyboard.LKKeyboard, st : Substitutions) : boolean { - keyboard?.variables?.string?.forEach(({value}) => - st.markers.add(SubstitutionUse.variable, LdmlKeyboardTypes.MarkerParser.allReferences(value))); + keyboard?.variables?.string?.forEach((string) => { + const {value} = string; + st.markers.add(SubstitutionUse.variable, LdmlKeyboardTypes.MarkerParser.allReferences(value), string); + }); // get markers mentioned in a set - keyboard?.variables?.set?.forEach(({ value }) => - LdmlKeyboardTypes.VariableParser.setSplitter(value).forEach(v => st.markers.add(SubstitutionUse.match, LdmlKeyboardTypes.MarkerParser.allReferences(v)))); + keyboard?.variables?.set?.forEach((set) => { + const { value } = set; + LdmlKeyboardTypes.VariableParser.setSplitter(value).forEach(v => st.markers.add(SubstitutionUse.match, LdmlKeyboardTypes.MarkerParser.allReferences(v), set)); + }); return true; } @@ -260,14 +267,15 @@ export class VarsCompiler extends SectionCompiler { const mt = st.markers; // collect all markers, excluding the match-all - const allMarkers : string[] = Array.from(mt.all).filter(m => m !== LdmlKeyboardTypes.MarkerParser.ANY_MARKER_ID).sort(); + const allMarkers : string[] = Array.from(mt.all.keys()).filter(m => m !== LdmlKeyboardTypes.MarkerParser.ANY_MARKER_ID).sort(); result.markers = sections.list.allocList(allMarkers, { - // x: - we don't have an easy way to get context at this point + // pass the first object in the listr + x: mt.all.get(allMarkers[0]||'') // however, this is the string with the marker *id*. So, from the StrsCompiler // point of view, the value of this context would be if the marker ID had, say, // invalid Unicode in it for ERROR_IllegalCharacters - but, that wouldn't be a // valid Marker ID either. - // So, it's OK to not pass context with this particular list. + // So it's not likely that this context will be used, but it's available. }, sections); // sets need to be added late, because they can refer to markers diff --git a/developer/src/kmc-ldml/test/tran.tests.ts b/developer/src/kmc-ldml/test/tran.tests.ts index aaeec16d6d..95004867c6 100644 --- a/developer/src/kmc-ldml/test/tran.tests.ts +++ b/developer/src/kmc-ldml/test/tran.tests.ts @@ -466,7 +466,7 @@ describe('bksp', function () { 'doesnt_exist_1', 'doesnt_exist_2', 'doesnt_exist_3', - ] + ].join(',') }), ], }, diff --git a/developer/src/kmc-ldml/test/vars.tests.ts b/developer/src/kmc-ldml/test/vars.tests.ts index 17c725366f..bb0ee5d389 100644 --- a/developer/src/kmc-ldml/test/vars.tests.ts +++ b/developer/src/kmc-ldml/test/vars.tests.ts @@ -238,7 +238,7 @@ describe('vars', function () { 'doesnt_exist_1', 'doesnt_exist_2', 'doesnt_exist_3', - ] + ].join(',') }), ], }, From 814c9b9ef946b50de0bf2f21a8ef81b87cbe08f6 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 30 May 2025 14:47:10 -0500 Subject: [PATCH 44/73] feat(developer): ldml: list compiler: add context to linter Fixes: #13935 --- .../src/compiler/ldml-compiler-messages.ts | 15 +++++++++------ .../src/kmc-ldml/src/compiler/linter-keycaps.ts | 4 ++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 38d508aea9..6cbfa8e161 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -108,15 +108,16 @@ export class LdmlCompilerMessages { ); static ERROR_GestureKeyNotFoundInKeyBag = SevError | 0x000B; - static Error_GestureKeyNotFoundInKeyBag = (o:{keyId: string, parentKeyId: string, attribute: string}, x?: ObjectWithMetadata) => - mx( + static Error_GestureKeyNotFoundInKeyBag = (o:{keyId: string, parentKeyId: string, attribute: string}, x?: ObjectWithMetadata) => mx( this.ERROR_GestureKeyNotFoundInKeyBag, x, `Key '${def(o.keyId)}' not found in key bag, referenced from other '${def(o.parentKeyId)}' in ${def(o.attribute)}`, ); static HINT_NoDisplayForMarker = SevHint | 0x000C; - static Hint_NoDisplayForMarker = (o: { id: string }) => - m(this.HINT_NoDisplayForMarker, `Key element with id "${def(o.id)}" has only marker output, but there is no matching display element by output or keyId. Keycap may be blank.`); + static Hint_NoDisplayForMarker = (o: { id: string }, x?: ObjectWithMetadata) => mx( + this.HINT_NoDisplayForMarker, x, + `Key element with id "${def(o.id)}" has only marker output, but there is no matching display element by output or keyId. Keycap may be blank.`, + ); static ERROR_InvalidVersion = SevError | 0x000D; static Error_InvalidVersion = (o: { version: string; }, x?: ObjectWithMetadata) => mx( @@ -139,8 +140,10 @@ export class LdmlCompilerMessages { ); static HINT_NoDisplayForSwitch = SevHint | 0x000F; - static Hint_NoDisplayForSwitch = (o: { id: string }) => - m(this.HINT_NoDisplayForSwitch, `Key element with id "${def(o.id)}" is a layer switch key, but there is no matching display element by keyId. Keycap may be blank.`); + static Hint_NoDisplayForSwitch = (o: { id: string }, x?: ObjectWithMetadata) => mx( + this.HINT_NoDisplayForSwitch, x, + `Key element with id "${def(o.id)}" is a layer switch key, but there is no matching display element by keyId. Keycap may be blank.`, + ); /** annotate the to= or id= entry */ private static outputOrKeyId(o:{output?: string, keyId?: string}) { diff --git a/developer/src/kmc-ldml/src/compiler/linter-keycaps.ts b/developer/src/kmc-ldml/src/compiler/linter-keycaps.ts index 8f1006c54a..9612190fdf 100644 --- a/developer/src/kmc-ldml/src/compiler/linter-keycaps.ts +++ b/developer/src/kmc-ldml/src/compiler/linter-keycaps.ts @@ -35,13 +35,13 @@ export class LinterKeycaps extends Linter { const disp = this.findDisp(id.value, to.value); if (!disp) { - this.callbacks.reportMessage(LdmlCompilerMessages.Hint_NoDisplayForSwitch({ id: id.value })); + this.callbacks.reportMessage(LdmlCompilerMessages.Hint_NoDisplayForSwitch({ id: id.value }, key)); } } else if (to.value !== '' && nonMarkerOutput === '') { // has output, but only markers const disp = this.findDisp(id.value, to.value); if (!disp) { - this.callbacks.reportMessage(LdmlCompilerMessages.Hint_NoDisplayForMarker({ id: id.value })); + this.callbacks.reportMessage(LdmlCompilerMessages.Hint_NoDisplayForMarker({ id: id.value }, key)); } } } From bc3a8ff36d19e95d8cf8c674c9cb448cdbde694f Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 30 May 2025 15:41:16 -0500 Subject: [PATCH 45/73] feat(developer): ldml: tran compiler: all relevant messages in ldml have line numbers! Fixes: #13937 --- .../src/compiler/ldml-compiler-messages.ts | 33 +++++++++++-------- developer/src/kmc-ldml/src/compiler/tran.ts | 22 ++++++------- developer/src/kmc-ldml/test/messages.tests.ts | 17 ++++------ 3 files changed, 37 insertions(+), 35 deletions(-) diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 6cbfa8e161..749d44ed67 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -85,6 +85,7 @@ export class LdmlCompilerMessages { `After minimization, one or more locales is repeated and has been removed`, ); + // This is the only allowed use of m() vs mx() in this file, all the others take context. static ERROR_InvalidFile = SevError | 0x0007; static Error_InvalidFile = (o:{errorText: string}) => m(this.ERROR_InvalidFile, `The source file has an invalid structure: ${def(o.errorText)}`); @@ -306,8 +307,10 @@ export class LdmlCompilerMessages { ); static ERROR_UnparseableReorderSet = SevError | 0x0028; - static Error_UnparseableReorderSet = (o: { from: string, set: string }) => - m(this.ERROR_UnparseableReorderSet, `Illegal UnicodeSet "${def(o.set)}" in reorder "${def(o.from)}`); + static Error_UnparseableReorderSet = (o: { from: string, set: string }, x?: ObjectWithMetadata) => mx( + this.ERROR_UnparseableReorderSet, x, + `Illegal UnicodeSet "${def(o.set)}" in reorder "${def(o.from)}`, + ); static ERROR_InvalidVariableIdentifier = SevError | 0x0029; static Error_InvalidVariableIdentifier = (o: { id: string }, x?: ObjectWithMetadata) => mx( @@ -354,16 +357,18 @@ export class LdmlCompilerMessages { // This is a bit of a catch-all and represents messages bubbling up from the underlying regex engine static ERROR_UnparseableTransformFrom = SevErrorTransform | 0x00; - static Error_UnparseableTransformFrom = (o: { from: string, message: string }) => - m(this.ERROR_UnparseableTransformFrom, `Invalid transform from="${def(o.from)}": "${def(o.message)}"`); + static Error_UnparseableTransformFrom = (o: { from: string, message: string }, x?: ObjectWithMetadata) => mx( + this.ERROR_UnparseableTransformFrom, x, + `Invalid transform from="${def(o.from)}": "${def(o.message)}"`, + ); //------------------------------------------------------------------------------| // max length of detail message lines (checked by verifyCompilerMessagesObject) | //------------------------------------------------------------------------------| static ERROR_IllegalTransformDollarsign = SevErrorTransform | 0x01; - static Error_IllegalTransformDollarsign = (o: { from: string }) => m( - this.ERROR_IllegalTransformDollarsign, + static Error_IllegalTransformDollarsign = (o: { from: string }, x?: ObjectWithMetadata) => mx( + this.ERROR_IllegalTransformDollarsign, x, `Invalid transform from="${def(o.from)}": Unescaped dollar-sign ($) is not valid transform syntax.`, ` **Hint**: Use \`\\$\` to match a literal dollar-sign. If this precedes a variable name, the variable name may not be valid (A-Z, a-z, 0-9, _, 32 @@ -377,30 +382,30 @@ export class LdmlCompilerMessages { ); static ERROR_IllegalTransformPlus = SevErrorTransform | 0x03; - static Error_IllegalTransformPlus = (o: { from: string }) => m( - this.ERROR_IllegalTransformPlus, + static Error_IllegalTransformPlus = (o: { from: string }, x?: ObjectWithMetadata) => mx( + this.ERROR_IllegalTransformPlus, x, `Invalid transform from="${def(o.from)}": Unescaped plus (+) is not valid transform syntax.`, ` **Hint**: Use \`\\+\` to match a literal plus. `); static ERROR_IllegalTransformAsterisk = SevErrorTransform | 0x04; - static Error_IllegalTransformAsterisk = (o: { from: string }) =>m( - this.ERROR_IllegalTransformAsterisk, + static Error_IllegalTransformAsterisk = (o: { from: string }, x?: ObjectWithMetadata) =>mx( + this.ERROR_IllegalTransformAsterisk, x, `Invalid transform from="${def(o.from)}": Unescaped asterisk (*) is not valid transform syntax.`, ` **Hint**: Use \`\\*\` to match a literal asterisk. `); static ERROR_IllegalTransformToUset = SevErrorTransform | 0x05; - static Error_IllegalTransformToUset = (o: { to: string }) => m( - this.ERROR_IllegalTransformToUset, + static Error_IllegalTransformToUset = (o: { to: string }, x?: ObjectWithMetadata) => mx( + this.ERROR_IllegalTransformToUset, x, `Invalid transform to="${def(o.to)}": Set variable (\\$[…]) cannot be used in 'to=' unless part of a map.`, ` **Hint**: If a map was meant, must use the form \`\`. `); static ERROR_UnparseableTransformTo = SevErrorTransform | 0x06; - static Error_UnparseableTransformTo = (o: {to: string, message: string}) => m( - this.ERROR_UnparseableTransformTo, + static Error_UnparseableTransformTo = (o: {to: string, message: string}, x?: ObjectWithMetadata) => mx( + this.ERROR_UnparseableTransformTo, x, `Invalid transform to="${def(o.to)}": "${def(o.message)}"`, ); diff --git a/developer/src/kmc-ldml/src/compiler/tran.ts b/developer/src/kmc-ldml/src/compiler/tran.ts index f0db46336d..c6236c1be7 100644 --- a/developer/src/kmc-ldml/src/compiler/tran.ts +++ b/developer/src/kmc-ldml/src/compiler/tran.ts @@ -1,6 +1,6 @@ import { constants, SectionIdent } from "@keymanapp/ldml-keyboard-constants"; import { KMXPlus, LdmlKeyboardTypes, util } from '@keymanapp/common-types'; -import { CompilerCallbacks, LDMLKeyboard } from "@keymanapp/developer-utils"; +import { CompilerCallbacks, LDMLKeyboard, ObjectWithMetadata } from "@keymanapp/developer-utils"; import { SectionCompiler } from "./section-compiler.js"; import Bksp = KMXPlus.Bksp; @@ -204,7 +204,7 @@ export abstract class TransformCompiler(); /** takes line numbers */ let lines = 0; const fakeOffsetNumber = 1234; const fakeOffsetObject = withOffset(fakeOffsetNumber); for(const key of keys) { - if(typeof m[key] == 'function') { + // exclude this one, does not need line numbers + if (key == 'Error_InvalidFile') continue; + if (typeof m[key] == 'function') { total++; const f = m[key] as Function; // console.log(`${f.name}: ${f.length}`); if (f.length === 0) { // Error_foo() - noLines++; + noLines.add(key); continue; } // now try to call it @@ -42,16 +44,11 @@ describe('LdmlCompilerMessages', function () { expect(resp.offset).to.equal(fakeOffsetNumber, `Offset number round trip for error ${f.name} did not work, check the message function`); } else { // did not get a column number back - noLines++; + noLines.add(key); } } } expect(lines).to.not.be.equal(0, `None of ${total} messages had offset reporting.`); - if (noLines > 0) { - // Once this goes to zero, make it an error if it goes up! - // Oh, and while you're here, once this is zero, uncomment the code in testCompilationCases - // that asserts that all messages are actually generated with an offset. - console.warn(`TODO-LDML (#10622) ${noLines}/${total} ${Number((noLines/total)*1000).toFixed(0)}‰ of message(s) did not have detectable offset (line number) reporting.`); - } + expect(Array.from(noLines.values())).to.deep.equal([], `${noLines.size}/${total} ${Number((noLines.size/total)*100).toFixed(0)}% of message(s) did not have detectable offset (line number) reporting`); }); }); From e1638ad4589592124687ce884bfa14e3582e2082 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Sat, 31 May 2025 07:46:11 -0500 Subject: [PATCH 46/73] feat(comon): ldml: test fix for element strings with context Fixes: #13937 --- .../kmx/kmx-plus/element-string.tests.ts | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/common/web/types/tests/kmx/kmx-plus/element-string.tests.ts b/common/web/types/tests/kmx/kmx-plus/element-string.tests.ts index 49a4c2c954..3818f7022b 100644 --- a/common/web/types/tests/kmx/kmx-plus/element-string.tests.ts +++ b/common/web/types/tests/kmx/kmx-plus/element-string.tests.ts @@ -91,12 +91,12 @@ describe('Test of ElementString file', () => { }); describe('Test of fromStrings()', () => { it('returns an empty ElementString if source is null', () => { - const es = ElementString.fromStrings({}, null); + const es = ElementString.fromStrings({}, {}, null); assert.deepEqual(es, new ElementString()); }); it('can create an ElementString from a string array', () => { sections.strs.allocString = stubStrsAllocString_Char; - const actual = ElementString.fromStrings(sections, ["𐌰", "𐌱", "𐌲"]); + const actual = ElementString.fromStrings(sections, {}, ["𐌰", "𐌱", "𐌲"]); const expected = [ initElemElement(GOTHIC_A), initElemElement(GOTHIC_B), @@ -106,7 +106,7 @@ describe('Test of ElementString file', () => { }); it('can create an ElementString from a string', () => { sections.strs.allocString = stubStrsAllocString_Char; - const actual = ElementString.fromStrings(sections, "𐌰𐌱𐌲"); + const actual = ElementString.fromStrings(sections, {}, "𐌰𐌱𐌲"); const expected = [ initElemElement(GOTHIC_A), initElemElement(GOTHIC_B), @@ -118,6 +118,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", "1 2 3", ); @@ -132,6 +133,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", "1", ); @@ -146,6 +148,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", "1 2", ); @@ -160,6 +163,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", "1 A 3", ); @@ -174,6 +178,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, "1 2 3", @@ -189,6 +194,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, "1", @@ -204,6 +210,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, "1 2", @@ -219,6 +226,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, "1 A 3", @@ -234,6 +242,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, null, @@ -250,6 +259,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, null, @@ -266,6 +276,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, null, @@ -282,6 +293,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, null, @@ -299,6 +311,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, null, @@ -316,6 +329,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, null, @@ -333,7 +347,7 @@ describe('Test of ElementString file', () => { ElementParser.segment = stubElementParserSegment_Uset; sections.strs.allocString = stubStrsAllocString_Str; sections.uset.allocUset = stubUsetAllocUset; - const actual = ElementString.fromStrings(sections, "[𐌰-𐍊]"); + const actual = ElementString.fromStrings(sections, {}, "[𐌰-𐍊]"); const expected = [ initElemElement( new StrsItem(''), @@ -348,17 +362,17 @@ describe('Test of ElementString file', () => { it('returns null for an invalid unicode set size', () => { ElementParser.segment = stubElementParserSegment_Uset; sections.usetparser.sizeUnicodeSet = (pattern: string) : number => -1; - assert.isNull(ElementString.fromStrings(sections, "[𐌰-𐍊]")); + assert.isNull(ElementString.fromStrings(sections, {}, "[𐌰-𐍊]")); }); it('returns null if it cannot parse the unicode set', () => { ElementParser.segment = stubElementParserSegment_Uset; sections.usetparser.parseUnicodeSet = (pattern: string, rangeCount: number) : UnicodeSet | null => null; - assert.isNull(ElementString.fromStrings(sections, "[𐌰-𐍊]")); + assert.isNull(ElementString.fromStrings(sections, {}, "[𐌰-𐍊]")); }); it('can handle quad strings', () => { sections.strs.allocString = stubStrsAllocString_Char; ElementParser.segment = stubElementParserSegment_Escaped; - const actual = ElementString.fromStrings(sections, "\\ud800\\udf30"); + const actual = ElementString.fromStrings(sections, {}, "\\ud800\\udf30"); const expected = [ initElemElement(HI_GOTHIC_A), initElemElement(LO_GOTHIC_A), @@ -367,7 +381,7 @@ describe('Test of ElementString file', () => { }); it('can handle ElemElement of string type', () => { sections.strs.allocString = stubStrsAllocString_Str; - const actual = ElementString.fromStrings(sections, ["𐌰𐌱𐌲",]); + const actual = ElementString.fromStrings(sections, {}, ["𐌰𐌱𐌲",]); const expected = [ initElemElement( new StrsItem("𐌰𐌱𐌲"), From 8a69f108e2e08b92304cfc54b6d95c7b4eca8a15 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Sat, 31 May 2025 14:00:40 -0400 Subject: [PATCH 47/73] auto: increment master version to 19.0.56 --- HISTORY.md | 5 +++++ VERSION.md | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index b455685d26..9048b14d87 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,10 @@ # Keyman Version History +## 19.0.55 alpha 2025-05-31 + +* maint(linux): `sudo` not required for removing temporary files (#14086) +* maint(linux): update to new version of gha-ubuntu-packaging (#14089) + ## 19.0.54 alpha 2025-05-30 * maint(linux): move TC build scripts to `resources/teamcity/linux` (#14061) diff --git a/VERSION.md b/VERSION.md index 0c1dd988d0..4073c70600 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -19.0.55 \ No newline at end of file +19.0.56 \ No newline at end of file From 86061d0523e08b7478ef01a028bd9609ddb53983 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Mon, 19 May 2025 14:37:47 +0700 Subject: [PATCH 48/73] maint(android): Add archive step to prep release artifacts --- android/.gitignore | 3 +++ android/build.sh | 46 +++++++++++++++++++++++++++++++++++++++++++++- android/exclude.in | 1 + 3 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 android/exclude.in diff --git a/android/.gitignore b/android/.gitignore index a5720bccc7..e822a25c2b 100644 --- a/android/.gitignore +++ b/android/.gitignore @@ -57,6 +57,9 @@ KMAPro/**/assets/*.kmp # Play Store release notes (copied from whatsnew.md) KMAPro/kMAPro/src/main/play/release-notes/**/*.txt +# Release artifacts +upload/ + # Eclipse files .project .settings/ diff --git a/android/build.sh b/android/build.sh index 27ba447b4c..e98d38d98d 100755 --- a/android/build.sh +++ b/android/build.sh @@ -15,6 +15,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" . "$KEYMAN_ROOT/resources/build/build-utils-ci.inc.sh" +. "$KEYMAN_ROOT/resources/build/zip.inc.sh" ################################ Main script ################################ @@ -37,6 +38,7 @@ builder_describe \ build \ test \ "publish Publishes symbols to Sentry and the APKs to the Play Store." \ + "archive Copy release artifacts to upload/ and rsync to downloads.keyman" \ "--ci+ Deprecated build option. Remove in 20.0" \ --upload-sentry+ \ ":engine=KMEA Keyman Engine for Android" \ @@ -70,4 +72,46 @@ function do_test_help() { builder_run_action test:help do_test_help -builder_run_child_actions publish \ No newline at end of file +builder_run_child_actions publish + +# Copy release artifacts to upload/ and rsync to downloads.keyman.com +if builder_start_action archive; then + UPLOAD_PATH="$KEYMAN_ROOT/android/upload/${KEYMAN_VERSION}" + KEYMAN_ENGINE_ANDROID_ZIP="keyman-engine-android-${KEYMAN_VERSION}.zip" + KEYMAN_APK="keyman-${KEYMAN_VERSION}.apk" + FIRSTVOICES_APK="firstvoices-${KEYMAN_VERSION}.apk" + + mkdir -p "${UPLOAD_PATH}" + + # Create Keyman Engine for Android archive + # zip/7z flags to: (disable progress), output log level 0 + cp "${KEYMAN_ROOT}/android/KMAPro/kMAPro/libs/keyman-engine.aar" ./ + if [ "$BUILDER_OS" == "win" ]; then + # win uses 7zip flags + ZIP_FILE="${UPLOAD_PATH}/${KEYMAN_ENGINE_ANDROID_ZIP}" + zip_files "${ZIP_FILE[@]}" "-bd" "-bb0" "-x@exclude.in" "keyman-engine.aar" "Samples/" # need a? + else + ZIP_FILE="${UPLOAD_PATH}/${KEYMAN_ENGINE_ANDROID_ZIP}" + zip_files "${ZIP_FILE}" "-r" "-q" "-x@exclude.in" "keyman-engine.aar" "Samples/" + fi + + # Copy release APKs + cp "${KEYMAN_ROOT}/android/KMAPro/kMAPro/build/outputs/apk/release/${KEYMAN_APK}" \ + "${UPLOAD_PATH}/${KEYMAN_APK}" + cp "${KEYMAN_ROOT}/oem/firstvoices/android/app/build/outputs/apk/release/${FIRSTVOICES_APK}" \ + "${UPLOAD_PATH}/${FIRSTVOICES_APK}" + + # + # Write download info files + # + + cd "${UPLOAD_PATH}" + write_download_info "Keyman Engine for Android" "${KEYMAN_ENGINE_ANDROID_ZIP}" "${KEYMAN_VERSION}" "${KEYMAN_TIER}" "android" + write_download_info "Keyman for Android" "${KEYMAN_APK}" "${KEYMAN_VERSION}" "${KEYMAN_TIER}" "android" + + if [ "${RELEASE_OEM_FIRSTVOICES-false}" = true ]; then + write_download_info "FirstVoices Keyboards" "${FIRSTVOICES_APK}" "${KEYMAN_VERSION}" "${KEYMAN_TIER}" "android" + fi + + builder_finish_action success archive +fi \ No newline at end of file diff --git a/android/exclude.in b/android/exclude.in new file mode 100644 index 0000000000..1c2b62205d --- /dev/null +++ b/android/exclude.in @@ -0,0 +1 @@ +Samples/*/build.sh From 6e5e25b5ea21af67daa89a78d80aeab40bdd833b Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Mon, 2 Jun 2025 11:21:34 +0700 Subject: [PATCH 49/73] fix(android): Updates to handle zip-excludes --- android/build.sh | 34 +++++++++++++++++----------- android/{exclude.in => zip-excludes} | 0 resources/build/zip.inc.sh | 6 +++++ 3 files changed, 27 insertions(+), 13 deletions(-) rename android/{exclude.in => zip-excludes} (100%) diff --git a/android/build.sh b/android/build.sh index e98d38d98d..866a75da09 100755 --- a/android/build.sh +++ b/android/build.sh @@ -84,22 +84,30 @@ if builder_start_action archive; then mkdir -p "${UPLOAD_PATH}" # Create Keyman Engine for Android archive - # zip/7z flags to: (disable progress), output log level 0 - cp "${KEYMAN_ROOT}/android/KMAPro/kMAPro/libs/keyman-engine.aar" ./ - if [ "$BUILDER_OS" == "win" ]; then - # win uses 7zip flags - ZIP_FILE="${UPLOAD_PATH}/${KEYMAN_ENGINE_ANDROID_ZIP}" - zip_files "${ZIP_FILE[@]}" "-bd" "-bb0" "-x@exclude.in" "keyman-engine.aar" "Samples/" # need a? - else - ZIP_FILE="${UPLOAD_PATH}/${KEYMAN_ENGINE_ANDROID_ZIP}" - zip_files "${ZIP_FILE}" "-r" "-q" "-x@exclude.in" "keyman-engine.aar" "Samples/" - fi + ZIP_FILE="${UPLOAD_PATH}/${KEYMAN_ENGINE_ANDROID_ZIP}" + ZIP_FLAGS=("-q" "-r") # quiet, recursive - # Copy release APKs + builder_echo "Copying Keyman Engine for Android into ${UPLOAD_PATH}..." + cd "${UPLOAD_PATH}" + cp "${KEYMAN_ROOT}/android/KMAPro/kMAPro/libs/keyman-engine.aar" ./ + add_zip_files "${ZIP_FILE}" "${ZIP_FLAGS[@]}" "keyman-engine.aar" + rm -f "keyman-engine.aar" + + builder_echo "Copying Keyman Engine for Android Sample projects into ${UPLOAD_PATH}..." + cp -rf "${KEYMAN_ROOT}/android/Samples" ./ + add_zip_files "${ZIP_FILE}" "-x@../../zip-excludes" "${ZIP_FLAGS[@]}" "Samples" + rm -rf "Samples" + + # Copy release APK cp "${KEYMAN_ROOT}/android/KMAPro/kMAPro/build/outputs/apk/release/${KEYMAN_APK}" \ "${UPLOAD_PATH}/${KEYMAN_APK}" - cp "${KEYMAN_ROOT}/oem/firstvoices/android/app/build/outputs/apk/release/${FIRSTVOICES_APK}" \ - "${UPLOAD_PATH}/${FIRSTVOICES_APK}" + + # FirstVoices app + + if [ "${RELEASE_OEM_FIRSTVOICES-false}" = true ]; then + cp "${KEYMAN_ROOT}/oem/firstvoices/android/app/build/outputs/apk/release/${FIRSTVOICES_APK}" \ + "${UPLOAD_PATH}/${FIRSTVOICES_APK}" + fi # # Write download info files diff --git a/android/exclude.in b/android/zip-excludes similarity index 100% rename from android/exclude.in rename to android/zip-excludes diff --git a/resources/build/zip.inc.sh b/resources/build/zip.inc.sh index 3dc6a390c2..01818d57e8 100644 --- a/resources/build/zip.inc.sh +++ b/resources/build/zip.inc.sh @@ -35,6 +35,12 @@ function add_zip_files() { SEVENZ_FLAGS+=($1) shift ;; + -x@*) + # Filename for a file containing list of files to exclude from the archive - Identical flag to zip and 7z + ZIP_FLAGS+=($1) + SEVENZ_FLAGS+=($1) + shift + ;; # Zip flags that have a corresponding 7z flag -q) From ceb7f741ae7794a9a03aa6abb6f7b465d326dee1 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Mon, 2 Jun 2025 13:35:17 +0700 Subject: [PATCH 50/73] fix(android): Fix typos --- android/build.sh | 2 +- resources/build/zip.inc.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/android/build.sh b/android/build.sh index 866a75da09..2133012ef3 100755 --- a/android/build.sh +++ b/android/build.sh @@ -122,4 +122,4 @@ if builder_start_action archive; then fi builder_finish_action success archive -fi \ No newline at end of file +fi diff --git a/resources/build/zip.inc.sh b/resources/build/zip.inc.sh index 01818d57e8..1f80a71edb 100644 --- a/resources/build/zip.inc.sh +++ b/resources/build/zip.inc.sh @@ -56,7 +56,7 @@ function add_zip_files() { # -1 indicates low compression (fastest) # -9 indicates ultra compression (slowest) ZIP_FLAGS+=($1) - if [[ $1 =~ -([0-9]) ]] then + if [[ $1 =~ -([0-9]) ]]; then SEVENZ_FLAGS+=("-mx${BASH_REMATCH[1]}") fi shift; From ea6bd736cac80a65d0647bd376c67ae62801c354 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Mon, 2 Jun 2025 09:19:25 +0200 Subject: [PATCH 51/73] maint(linux): update to new version of gha-ubuntu-packaging The previous version 2.0 of gha-ubuntu-packaging was missing `--` to separate the command to be called from `retry`, so the arguments for that command were considered arguments for `retry`. The new version 2.0.1 of gha-ubuntu-packaging fixes this. Follow-up-of: #14089 Test-bot: skip --- .github/actions/build-binary-packages/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build-binary-packages/action.yml b/.github/actions/build-binary-packages/action.yml index 09e23e3f7a..e764a26ac7 100644 --- a/.github/actions/build-binary-packages/action.yml +++ b/.github/actions/build-binary-packages/action.yml @@ -31,7 +31,7 @@ runs: path: artifacts/keyman-srcpkg - name: Build - uses: sillsdev/gha-ubuntu-packaging@1ab4a5967afbadab82a480936b9e53d7190acdf2 # v2.0 + uses: sillsdev/gha-ubuntu-packaging@556b268762be9bea0f39861a7392587211ef6a58 # v2.0.1 with: dist: "${{ inputs.dist }}" platform: "${{ inputs.arch }}" From 004c40606a5c7e9b87c46c7abed544c91122c420 Mon Sep 17 00:00:00 2001 From: rc-swag <58423624+rc-swag@users.noreply.github.com> Date: Mon, 2 Jun 2025 20:54:07 +1000 Subject: [PATCH 52/73] fix(windows): review comment fix log msg Co-authored-by: Eberhard Beilharz --- .../desktop/kmshell/main/Keyman.System.UpdateStateMachine.pas | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/windows/src/desktop/kmshell/main/Keyman.System.UpdateStateMachine.pas b/windows/src/desktop/kmshell/main/Keyman.System.UpdateStateMachine.pas index 99a0985545..9f1b4b73d7 100644 --- a/windows/src/desktop/kmshell/main/Keyman.System.UpdateStateMachine.pas +++ b/windows/src/desktop/kmshell/main/Keyman.System.UpdateStateMachine.pas @@ -990,8 +990,8 @@ begin TKeymanSentryClient.Breadcrumb('error', 'Executing kmshell process to install keyboard packages failed"' + IntToStr(Ord(executeResult)) + '"', 'update'); - KL.Log('InstallingState.DoInstallKeyman failed executing kmshell ' + - 'or the update was not found in the cache'); + KL.Log('InstallingState.LaunchInstallPackageProcess failed executing kmshell ' + + 'process to install keyboard packages: "' + IntToStr(Ord(executeResult)) + '"'); ChangeState(IdleState); end; end From 581e4e03ef078175a2a61fc195e93d044da35ec7 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Wed, 28 May 2025 18:00:53 +0200 Subject: [PATCH 53/73] chore(linux): add TC test runner This adds a modified version of JetBrains' TeamCity test runner that makes use of the `parent` property in the service messages and thus allows to properly indent and fold the block that contains the test output. Test-bot: skip --- linux/keyman-config/run-tests.sh | 12 +++- linux/tools/teamcity_testrunner/README.md | 12 ++++ linux/tools/teamcity_testrunner/unittestpy.py | 61 +++++++++++++++++++ resources/builder.inc.sh | 24 +++++--- resources/teamcity/includes/tc-actions.inc.sh | 5 +- 5 files changed, 103 insertions(+), 11 deletions(-) create mode 100644 linux/tools/teamcity_testrunner/README.md create mode 100644 linux/tools/teamcity_testrunner/unittestpy.py diff --git a/linux/keyman-config/run-tests.sh b/linux/keyman-config/run-tests.sh index 3fa49f081d..baa90a41f8 100755 --- a/linux/keyman-config/run-tests.sh +++ b/linux/keyman-config/run-tests.sh @@ -1,5 +1,5 @@ #!/bin/bash -PYTHONPATH=.:${PYTHONPATH} +export PYTHONPATH=.:${PYTHONPATH} XDG_CONFIG_HOME=$(mktemp --directory) export XDG_CONFIG_HOME @@ -13,6 +13,8 @@ if [[ "$1" == "--coverage" ]]; then fi if [[ -n "${TEAMCITY_GIT_PATH}" ]]; then + PYTHONPATH=$(dirname "$0")/../tools:${PYTHONPATH} + if ! pip3 list --format=columns | grep -q teamcity-messages; then if [[ -n "${TEAMCITY_PLATFORM}" ]] || [[ -n "${DOCKER_RUNNING}" ]]; then # Ubuntu 24.04+ prevents mixing pip and system packages and wants us @@ -29,7 +31,9 @@ if [[ -n "${TEAMCITY_GIT_PATH}" ]]; then # shellcheck disable=SC2086 pip3 install --user ${PIP_ARGS:-} teamcity-messages fi - test_module=teamcity.unittestpy + test_module=teamcity_testrunner.unittestpy + echo "##teamcity[testStarted name='|[keyman-config|] Running unit tests']" + echo "##teamcity[flowStarted flowId='unit_tests']" else test_module=unittest extra_opts=-v @@ -38,4 +42,8 @@ fi # shellcheck disable=SC2086 python3 ${coverage:-} -m "${test_module:-}" discover ${extra_opts:-} -s tests/ -p "*_tests.py" +if [[ -n "${TEAMCITY_GIT_PATH}" ]]; then + echo "##teamcity[flowFinished flowId='unit_tests']" + echo "##teamcity[testFinished name='|[keyman-config|] Finished running unit tests']" +fi rm -rf "${XDG_CONFIG_HOME}" diff --git a/linux/tools/teamcity_testrunner/README.md b/linux/tools/teamcity_testrunner/README.md new file mode 100644 index 0000000000..0872c0e7fa --- /dev/null +++ b/linux/tools/teamcity_testrunner/README.md @@ -0,0 +1,12 @@ +# Teamcity Test Runner + +This implements an improved version of JetBrains' unittest runner for +Teamcity. It makes the output of the tests indented and collapsible under +the test block by specifying a parent flow. + +Requires the `teamcity-messages` package. + +## References + +- +- diff --git a/linux/tools/teamcity_testrunner/unittestpy.py b/linux/tools/teamcity_testrunner/unittestpy.py new file mode 100644 index 0000000000..543277cce1 --- /dev/null +++ b/linux/tools/teamcity_testrunner/unittestpy.py @@ -0,0 +1,61 @@ +import sys +from teamcity.messages import TeamcityServiceMessages +from teamcity.unittestpy import TeamcityTestResult, TeamcityTestRunner +from unittest import main + + +class KeymanTeamcityServiceMessages(TeamcityServiceMessages): + def __init__(self, parent_flow='unit_tests'): + self.parent_flow = parent_flow + super().__init__() + + def testStarted(self, testName, captureStandardOutput=None, flowId=None, metainfo=None): + self.message('testStarted', name=testName, metainfo=metainfo) + self.message('flowStarted', flowId=flowId, parent=self.parent_flow) + + def testFinished(self, testName, testDuration=None, flowId=None): + self.message('flowFinished', flowId=flowId, parent=self.parent_flow) + if testDuration is not None: + duration_ms = testDuration.days * 86400000 + \ + testDuration.seconds * 1000 + \ + int(testDuration.microseconds / 1000) + self.message('testFinished', name=testName, duration=str(duration_ms)) + else: + self.message('testFinished', name=testName) + + def testIgnored(self, testName, message='', flowId=None): + self.message('flowFinished', flowId=flowId, parent=self.parent_flow) + self.message('testIgnored', name=testName, message=message) + + def testFailed(self, testName, message='', details='', flowId=None, comparison_failure=None): + self.message('flowFinished', flowId=flowId, parent=self.parent_flow) + if not comparison_failure: + self.message('testFailed', name=testName, message=message, details=details) + else: + diff_message = u"\n{0} != {1}\n".format(comparison_failure.actual, comparison_failure.expected) + self.message( + 'testFailed', + name=testName, + message=str(message) + diff_message, + details=details, + type="comparisonFailure", + actual=comparison_failure.actual, + expected=comparison_failure.expected, + ) + + +class KeymanTestResult(TeamcityTestResult): + def __init__(self, stream=sys.stdout, descriptions=None, verbosity=None): + super().__init__(stream, descriptions, verbosity) + self.messages = KeymanTeamcityServiceMessages() + + +class KeymanTestRunner(TeamcityTestRunner): + resultclass = KeymanTestResult + + def run(self, test): + return super().run(test) + + +if __name__ == '__main__': + main(module=None, testRunner=KeymanTestRunner()) diff --git a/resources/builder.inc.sh b/resources/builder.inc.sh index 24683543b8..25c9b5994c 100755 --- a/resources/builder.inc.sh +++ b/resources/builder.inc.sh @@ -181,28 +181,30 @@ function builder_heading() { builder_echo() { - local color=white message= mark= block= action= do_output=true + local color=white message= mark= block= action= do_output=true test= local echo_target=echo if [[ $# -gt 1 ]]; then if [[ $1 =~ ^(white|grey|green|success|blue|heading|yellow|warning|red|error|purple|brightwhite|teal|debug|setmark)$ ]]; then color="$1" shift - elif [[ $1 == "start" ]]; then + elif [[ $1 == "start" ]] || [[ $1 == "startTest" ]]; then # builder_echo start block message - action="$1" + test="$1" block="$2" shift 2 + action="start" color="heading" if ! builder_is_running_on_teamcity && builder_is_child_build; then do_output=${_builder_debug_internal:-false} fi - elif [[ $1 == "end" ]]; then + elif [[ $1 == "end" ]] || [[ $1 == "endTest" ]]; then # builder_echo end block status message - action="$1" + test="$1" block="$2" color="$3" shift 3 + action="end" if [[ "${color}" != "error" ]] && ! builder_is_running_on_teamcity && builder_is_child_build; then do_output=${_builder_debug_internal:-false} fi @@ -211,7 +213,11 @@ builder_echo() { message="$*" if [[ "${action}" == "start" ]] && builder_is_running_on_teamcity; then - $echo_target -e "##teamcity[blockOpened name='|[${THIS_SCRIPT_IDENTIFIER}|] ${block}']" + if [[ "${test}" == "startTest" ]]; then + $echo_target -e "##teamcity[testSuiteStarted name='|[${THIS_SCRIPT_IDENTIFIER}|] ${block}']" + else + $echo_target -e "##teamcity[blockOpened name='|[${THIS_SCRIPT_IDENTIFIER}|] ${block}']" + fi fi if ${do_output}; then @@ -241,7 +247,11 @@ builder_echo() { fi if [[ "${action}" == "end" ]] && builder_is_running_on_teamcity; then - $echo_target -e "##teamcity[blockClosed name='|[${THIS_SCRIPT_IDENTIFIER}|] ${block}']" + if [[ "${test}" == "endTest" ]]; then + $echo_target -e "##teamcity[testSuiteFinished name='|[${THIS_SCRIPT_IDENTIFIER}|] ${block}']" + else + $echo_target -e "##teamcity[blockClosed name='|[${THIS_SCRIPT_IDENTIFIER}|] ${block}']" + fi fi } diff --git a/resources/teamcity/includes/tc-actions.inc.sh b/resources/teamcity/includes/tc-actions.inc.sh index 4b817a6f21..df008894c2 100644 --- a/resources/teamcity/includes/tc-actions.inc.sh +++ b/resources/teamcity/includes/tc-actions.inc.sh @@ -47,7 +47,8 @@ linux_build_action() { # Run unit tests for Keyman for Linux. linux_unit_tests_action() { - builder_echo start unit_tests "Running unit tests" + builder_echo startTest unit_tests "Running unit tests" + rm -f /tmp/ibus-engine-keyman.log rm -f /tmp/ibus-daemon.log # symlink might point to wrong location, so delete it - will be re-created during tests @@ -56,6 +57,6 @@ linux_unit_tests_action() { export NO_AT_BRIDGE=1 # shellcheck disable=SC2068 "${KEYMAN_ROOT}/linux/build.sh" test $@ - builder_echo end unit_tests success "Finished running unit tests" + builder_echo endTest unit_tests success "Finished running unit tests" } From 3d006659f9b6e758ac6f274ff74b4d307b06110c Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Fri, 30 May 2025 11:29:08 +0200 Subject: [PATCH 54/73] chore(linux): clarify that it's a Python test runner Rename the directory to make it clearer that it's test runner for Python tests. Addresses code review comments. --- linux/keyman-config/run-tests.sh | 2 +- .../{teamcity_testrunner => teamcity_pytestrunner}/README.md | 4 ++-- .../unittestpy.py | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename linux/tools/{teamcity_testrunner => teamcity_pytestrunner}/README.md (74%) rename linux/tools/{teamcity_testrunner => teamcity_pytestrunner}/unittestpy.py (100%) diff --git a/linux/keyman-config/run-tests.sh b/linux/keyman-config/run-tests.sh index baa90a41f8..57b6b47aa7 100755 --- a/linux/keyman-config/run-tests.sh +++ b/linux/keyman-config/run-tests.sh @@ -31,7 +31,7 @@ if [[ -n "${TEAMCITY_GIT_PATH}" ]]; then # shellcheck disable=SC2086 pip3 install --user ${PIP_ARGS:-} teamcity-messages fi - test_module=teamcity_testrunner.unittestpy + test_module=teamcity_pytestrunner.unittestpy echo "##teamcity[testStarted name='|[keyman-config|] Running unit tests']" echo "##teamcity[flowStarted flowId='unit_tests']" else diff --git a/linux/tools/teamcity_testrunner/README.md b/linux/tools/teamcity_pytestrunner/README.md similarity index 74% rename from linux/tools/teamcity_testrunner/README.md rename to linux/tools/teamcity_pytestrunner/README.md index 0872c0e7fa..00122b59ec 100644 --- a/linux/tools/teamcity_testrunner/README.md +++ b/linux/tools/teamcity_pytestrunner/README.md @@ -1,6 +1,6 @@ -# Teamcity Test Runner +# Teamcity Python Test Runner -This implements an improved version of JetBrains' unittest runner for +This implements an improved version of JetBrains' python unittest runner for Teamcity. It makes the output of the tests indented and collapsible under the test block by specifying a parent flow. diff --git a/linux/tools/teamcity_testrunner/unittestpy.py b/linux/tools/teamcity_pytestrunner/unittestpy.py similarity index 100% rename from linux/tools/teamcity_testrunner/unittestpy.py rename to linux/tools/teamcity_pytestrunner/unittestpy.py From 5012839df6845fb9ec7a78991fa5c9d6c9839aae Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Sat, 31 May 2025 07:46:11 -0500 Subject: [PATCH 55/73] feat(comon): ldml: test fix for element strings with context Fixes: #13934 (cherry picked from commit e1638ad4589592124687ce884bfa14e3582e2082) --- .../kmx/kmx-plus/element-string.tests.ts | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/common/web/types/tests/kmx/kmx-plus/element-string.tests.ts b/common/web/types/tests/kmx/kmx-plus/element-string.tests.ts index 49a4c2c954..3818f7022b 100644 --- a/common/web/types/tests/kmx/kmx-plus/element-string.tests.ts +++ b/common/web/types/tests/kmx/kmx-plus/element-string.tests.ts @@ -91,12 +91,12 @@ describe('Test of ElementString file', () => { }); describe('Test of fromStrings()', () => { it('returns an empty ElementString if source is null', () => { - const es = ElementString.fromStrings({}, null); + const es = ElementString.fromStrings({}, {}, null); assert.deepEqual(es, new ElementString()); }); it('can create an ElementString from a string array', () => { sections.strs.allocString = stubStrsAllocString_Char; - const actual = ElementString.fromStrings(sections, ["𐌰", "𐌱", "𐌲"]); + const actual = ElementString.fromStrings(sections, {}, ["𐌰", "𐌱", "𐌲"]); const expected = [ initElemElement(GOTHIC_A), initElemElement(GOTHIC_B), @@ -106,7 +106,7 @@ describe('Test of ElementString file', () => { }); it('can create an ElementString from a string', () => { sections.strs.allocString = stubStrsAllocString_Char; - const actual = ElementString.fromStrings(sections, "𐌰𐌱𐌲"); + const actual = ElementString.fromStrings(sections, {}, "𐌰𐌱𐌲"); const expected = [ initElemElement(GOTHIC_A), initElemElement(GOTHIC_B), @@ -118,6 +118,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", "1 2 3", ); @@ -132,6 +133,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", "1", ); @@ -146,6 +148,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", "1 2", ); @@ -160,6 +163,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", "1 A 3", ); @@ -174,6 +178,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, "1 2 3", @@ -189,6 +194,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, "1", @@ -204,6 +210,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, "1 2", @@ -219,6 +226,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, "1 A 3", @@ -234,6 +242,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, null, @@ -250,6 +259,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, null, @@ -266,6 +276,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, null, @@ -282,6 +293,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, null, @@ -299,6 +311,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, null, @@ -316,6 +329,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, null, @@ -333,7 +347,7 @@ describe('Test of ElementString file', () => { ElementParser.segment = stubElementParserSegment_Uset; sections.strs.allocString = stubStrsAllocString_Str; sections.uset.allocUset = stubUsetAllocUset; - const actual = ElementString.fromStrings(sections, "[𐌰-𐍊]"); + const actual = ElementString.fromStrings(sections, {}, "[𐌰-𐍊]"); const expected = [ initElemElement( new StrsItem(''), @@ -348,17 +362,17 @@ describe('Test of ElementString file', () => { it('returns null for an invalid unicode set size', () => { ElementParser.segment = stubElementParserSegment_Uset; sections.usetparser.sizeUnicodeSet = (pattern: string) : number => -1; - assert.isNull(ElementString.fromStrings(sections, "[𐌰-𐍊]")); + assert.isNull(ElementString.fromStrings(sections, {}, "[𐌰-𐍊]")); }); it('returns null if it cannot parse the unicode set', () => { ElementParser.segment = stubElementParserSegment_Uset; sections.usetparser.parseUnicodeSet = (pattern: string, rangeCount: number) : UnicodeSet | null => null; - assert.isNull(ElementString.fromStrings(sections, "[𐌰-𐍊]")); + assert.isNull(ElementString.fromStrings(sections, {}, "[𐌰-𐍊]")); }); it('can handle quad strings', () => { sections.strs.allocString = stubStrsAllocString_Char; ElementParser.segment = stubElementParserSegment_Escaped; - const actual = ElementString.fromStrings(sections, "\\ud800\\udf30"); + const actual = ElementString.fromStrings(sections, {}, "\\ud800\\udf30"); const expected = [ initElemElement(HI_GOTHIC_A), initElemElement(LO_GOTHIC_A), @@ -367,7 +381,7 @@ describe('Test of ElementString file', () => { }); it('can handle ElemElement of string type', () => { sections.strs.allocString = stubStrsAllocString_Str; - const actual = ElementString.fromStrings(sections, ["𐌰𐌱𐌲",]); + const actual = ElementString.fromStrings(sections, {}, ["𐌰𐌱𐌲",]); const expected = [ initElemElement( new StrsItem("𐌰𐌱𐌲"), From fab29aad7748f21f896c4b74c5151bba50a019a7 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 2 Jun 2025 08:52:00 -0500 Subject: [PATCH 56/73] feat(developer): ldml: update errors for unparseable reorder Fixes: #14091 --- .../types/src/kmx/kmx-plus/element-string.ts | 4 +- .../ldml-keyboard/unicodeset-parser-api.ts | 4 +- .../web/utils/src/compiler-interfaces.ts | 57 ++++++++++++++++++ developer/src/common/web/utils/src/index.ts | 2 +- .../src/kmc-kmn/src/compiler/compiler.ts | 23 ++++---- .../src/compiler/kmn-compiler-messages.ts | 22 +++---- .../src/compiler/ldml-compiler-messages.ts | 59 +------------------ 7 files changed, 87 insertions(+), 84 deletions(-) diff --git a/common/web/types/src/kmx/kmx-plus/element-string.ts b/common/web/types/src/kmx/kmx-plus/element-string.ts index 9d3b518f04..e65d996f34 100644 --- a/common/web/types/src/kmx/kmx-plus/element-string.ts +++ b/common/web/types/src/kmx/kmx-plus/element-string.ts @@ -81,9 +81,9 @@ export class ElementString extends Array { // error. So we can just exit here. return null; // UnicodeSet error } - const uset = sections.usetparser.parseUnicodeSet(item.segment, needRanges); + const uset = sections.usetparser.parseUnicodeSet(item.segment, needRanges, options?.x); if (!uset) { - return null; // UnicodeSet error already thrown + return null; // UnicodeSet error already added to callback } elem.uset = sections.uset.allocUset(uset, sections, options?.x); elem.value = sections.strs.allocString('', {...options, singleOk: true}); // no string diff --git a/common/web/types/src/ldml-keyboard/unicodeset-parser-api.ts b/common/web/types/src/ldml-keyboard/unicodeset-parser-api.ts index 0c890aa95a..797a5415b9 100644 --- a/common/web/types/src/ldml-keyboard/unicodeset-parser-api.ts +++ b/common/web/types/src/ldml-keyboard/unicodeset-parser-api.ts @@ -8,13 +8,13 @@ export interface UnicodeSetParser { * @param pattern string to parse such as `[a-z]` * @param rangeCount number of ranges to allow for */ - parseUnicodeSet(pattern: string, rangeCount: number) : UnicodeSet | null; + parseUnicodeSet(pattern: string, rangeCount: number, x?: any) : UnicodeSet | null; /** * Calculate the number of ranges in a UnicodeSet * @param pattern string to parse such as `[a-z]` * @returns number of ranges, or -1 (with callback-reported err) on err */ - sizeUnicodeSet(pattern: string) : number; + sizeUnicodeSet(pattern: string, x?: any) : number; } /** diff --git a/developer/src/common/web/utils/src/compiler-interfaces.ts b/developer/src/common/web/utils/src/compiler-interfaces.ts index aaeffc8d8c..3cce4e75ef 100644 --- a/developer/src/common/web/utils/src/compiler-interfaces.ts +++ b/developer/src/common/web/utils/src/compiler-interfaces.ts @@ -1,4 +1,6 @@ import { CompilerCallbacks } from "./compiler-callbacks.js"; +import { ObjectWithMetadata } from "./symbol-utils.js"; +import { KeymanXMLReader, XML_FILENAME_SYMBOL } from "./xml-utils.js"; /** * Abstract interface for compiler error and warning messages @@ -206,6 +208,27 @@ export class CompilerError { } return null; } + + /** + * Get an offset from o and set event's offset field + * @param event a compiler event, such as from functions in this class + * @param x any object parsed from XML or with the XML_META_DATA_SYMBOL symbol copied over + * @returns modified event object + */ + public static setFromMetadata(event: CompilerEvent, x?: ObjectWithMetadata): CompilerEvent { + if(x) { + const metadata = KeymanXMLReader.getMetaData(x) || {}; + const offset = metadata?.startIndex; + if (offset) { + event.offset = offset; + } + const filename = event.filename || metadata[XML_FILENAME_SYMBOL]; + if (filename) { + event.filename = filename; + } + } + return event; + } }; /** @deprecated use `CompilerError.severity` instead */ @@ -423,6 +446,40 @@ export function dedentCompilerMessageDetail(event: CompilerEvent) { return (event.detail ?? '').replace(/^[ ]+/gm, ''); } +/** + * Convenience function for constructing CompilerEvents with line numbers. + * Use it as below: (abbreviated as mx()) + * + * ```js + * // Note: Indentation makes "InvalidScanCode" line up thrice + * static ERROR_InvalidScanCode = SevError | 0x0009; + * // Note: + * // 1. All parameters are passed in 'o', the context object is only used for context even if + * // it contains redundant info. + * // 2. No code execution within the arrow function other than the 'mx' call, string interpolation, + * // with `${def(o.property)}` as the max complexity of interpolation. + * static Error_InvalidScanCode = (o:{id: string, invalidCodeList: string}, x: ObjectWithMetadata) => mx( + * this.ERROR_InvalidScanCode, x, + * `Form '${def(o.id)}' has invalid/unknown scancodes '${def(o.codes)}'`, + * // Note: If detail is omitted, leave the trailing comma on the prior line to leave room for it + * `…additional markdown detail…` + * ); + * ``` + * + * @param code Unique numeric value of the event + * @param message A short description of the error presented to the user + * @param context Object to be used as a source for line number information + * @param detail Detailed Markdown-formatted description of the error + * including references to documentation, remediation options. + * @see CompilerMessageSpec + * @returns the event + */ +export function CompilerMessageObjectSpec(code: number, context: ObjectWithMetadata, message: string, detail?: string): CompilerEvent { + let evt = CompilerMessageSpec(code, message, detail); // constructs raw message + evt = CompilerError.setFromMetadata(evt, context); // updates with offset from context + return evt; +}; + export const CompilerMessageDef = (param: any) => String(param ?? ``); export const CompilerMessageSpecWithException = (code: number, message: string, exceptionVar: any, detail?: string) : CompilerEvent => ({ diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index ef6b489b67..c39afa8bff 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -48,7 +48,7 @@ export { export { defaultCompilerOptions, CompilerBaseOptions, CompilerOptions, CompilerEvent, CompilerErrorNamespace, CompilerErrorSeverity, CompilerCallbackOptions, - CompilerError, CompilerMessageSpec, CompilerMessageSpecWithException, compilerErrorSeverity, CompilerErrorMask, compilerErrorSeverityName, + CompilerError, CompilerMessageSpec, CompilerMessageObjectSpec, CompilerMessageSpecWithException, compilerErrorSeverity, CompilerErrorMask, compilerErrorSeverityName, compilerErrorFormatCode, CompilerMessageDef, compilerLogLevelToSeverity, CompilerLogLevel, compilerEventFormat, ALL_COMPILER_LOG_LEVELS, ALL_COMPILER_LOG_FORMATS, CompilerLogFormat, diff --git a/developer/src/kmc-kmn/src/compiler/compiler.ts b/developer/src/kmc-kmn/src/compiler/compiler.ts index d251bc2c5b..21985ad09a 100644 --- a/developer/src/kmc-kmn/src/compiler/compiler.ts +++ b/developer/src/kmc-kmn/src/compiler/compiler.ts @@ -9,7 +9,8 @@ TODO: implement additional interfaces: import { VisualKeyboard, KvkFileReader, LdmlKeyboardTypes, KeymanFileTypes, KvkFileWriter } from '@keymanapp/common-types'; import { CompilerCallbacks, CompilerEvent, CompilerOptions, KeymanCompiler, KeymanCompilerArtifacts, - KeymanCompilerArtifactOptional, KeymanCompilerResult, KeymanCompilerArtifact, KvksFileReader + KeymanCompilerArtifactOptional, KeymanCompilerResult, KeymanCompilerArtifact, KvksFileReader, + CompilerError } from '@keymanapp/developer-utils'; import * as Osk from './osk.js'; import loadWasmHost from '../import/kmcmplib/wasm-host.js'; @@ -529,7 +530,7 @@ export class KmnCompiler implements KeymanCompiler, LdmlKeyboardTypes.UnicodeSet * @param rangeCount - number of ranges to allocate * @returns UnicodeSet accessor object, or null on failure */ - public parseUnicodeSet(pattern: string, rangeCount: number) : LdmlKeyboardTypes.UnicodeSet | null { + public parseUnicodeSet(pattern: string, rangeCount: number, x?: any) : LdmlKeyboardTypes.UnicodeSet | null { if(!this.verifyInitialized()) { /* c8 ignore next 2 */ // verifyInitialized will set a callback if needed @@ -562,7 +563,7 @@ export class KmnCompiler implements KeymanCompiler, LdmlKeyboardTypes.UnicodeSet // rc is negative: it's an error code. this.wasmExports.free(buf); // translate error code into callback - this.callbacks.reportMessage(getUnicodeSetError(rc)); + this.callbacks.reportMessage(getUnicodeSetError(rc, x)); return null; } } @@ -570,7 +571,7 @@ export class KmnCompiler implements KeymanCompiler, LdmlKeyboardTypes.UnicodeSet /** * @internal */ - public sizeUnicodeSet(pattern: string) : number { + public sizeUnicodeSet(pattern: string, x?: any) : number { if(!this.verifyInitialized()) { /* c8 ignore next 2 */ return null; @@ -582,7 +583,7 @@ export class KmnCompiler implements KeymanCompiler, LdmlKeyboardTypes.UnicodeSet if (rc >= 0) { return rc; } else { - this.callbacks.reportMessage(getUnicodeSetError(rc)); + this.callbacks.reportMessage(getUnicodeSetError(rc, x)); return -1; } } @@ -593,7 +594,7 @@ export class KmnCompiler implements KeymanCompiler, LdmlKeyboardTypes.UnicodeSet * @param rc parseUnicodeSet error code * @returns the compiler event */ -function getUnicodeSetError(rc: number) : CompilerEvent { +function getUnicodeSetError(rc: number, x?: any) : CompilerEvent { // from kmcmplib.h const KMCMP_ERROR_SYNTAX_ERR = -1; const KMCMP_ERROR_HAS_STRINGS = -2; @@ -601,16 +602,16 @@ function getUnicodeSetError(rc: number) : CompilerEvent { const KMCMP_FATAL_OUT_OF_RANGE = -4; switch(rc) { case KMCMP_ERROR_SYNTAX_ERR: - return KmnCompilerMessages.Error_UnicodeSetSyntaxError(); + return KmnCompilerMessages.Error_UnicodeSetSyntaxError(x); case KMCMP_ERROR_HAS_STRINGS: - return KmnCompilerMessages.Error_UnicodeSetHasStrings(); + return KmnCompilerMessages.Error_UnicodeSetHasStrings(x); case KMCMP_ERROR_UNSUPPORTED_PROPERTY: - return KmnCompilerMessages.Error_UnicodeSetHasProperties(); + return KmnCompilerMessages.Error_UnicodeSetHasProperties(x); case KMCMP_FATAL_OUT_OF_RANGE: - return KmnCompilerMessages.Fatal_UnicodeSetOutOfRange(); + return KmnCompilerMessages.Fatal_UnicodeSetOutOfRange(x); default: /* c8 ignore next */ - return KmnCompilerMessages.Fatal_UnexpectedException({e: `Unexpected UnicodeSet error code ${rc}`}); + return CompilerError.setFromMetadata(KmnCompilerMessages.Fatal_UnexpectedException({e: `Unexpected UnicodeSet error code ${rc}`}), x); } } diff --git a/developer/src/kmc-kmn/src/compiler/kmn-compiler-messages.ts b/developer/src/kmc-kmn/src/compiler/kmn-compiler-messages.ts index 591083a85c..3274a62ca1 100644 --- a/developer/src/kmc-kmn/src/compiler/kmn-compiler-messages.ts +++ b/developer/src/kmc-kmn/src/compiler/kmn-compiler-messages.ts @@ -1,6 +1,6 @@ import { KeyAddress } from "../kmw-compiler/validate-layout-file.js"; import { kmnfile } from "../kmw-compiler/compiler-globals.js"; -import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerEvent, CompilerMessageSpec as m, CompilerMessageDef as def, CompilerMessageSpecWithException, KeymanUrls } from "@keymanapp/developer-utils"; +import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerEvent, CompilerMessageSpec as m, CompilerMessageObjectSpec as mx, CompilerMessageDef as def, CompilerMessageSpecWithException, KeymanUrls, ObjectWithMetadata } from "@keymanapp/developer-utils"; const Namespace = CompilerErrorNamespace.KmnCompiler; const SevInfo = CompilerErrorSeverity.Info | Namespace; @@ -12,6 +12,9 @@ const SevFatal = CompilerErrorSeverity.Fatal | Namespace; // For messages from the KeymanWeb compiler, we need to construct our messages // slightly differently. This could be refactored in the future, as it is not // obvious which messages should use which function. +// Specifically, this could perhaps use CompilerFileCallbacks which provides a +// default filename. +// Then, there could be a CompilerMessageSpecWithLine const mw = (code: number, message: string, o?: {filename?: string, line?: number}) : CompilerEvent => ({ ...m(code, message), filename: o?.filename ?? kmnfile, @@ -104,9 +107,8 @@ export class KmnCompilerMessages { ); static FATAL_UnicodeSetOutOfRange = SevFatal | 0x904; - static Fatal_UnicodeSetOutOfRange = () => CompilerMessageSpecWithException( - this.FATAL_UnicodeSetOutOfRange, - null, + static Fatal_UnicodeSetOutOfRange = (x?: ObjectWithMetadata) => mx( + this.FATAL_UnicodeSetOutOfRange, x, `UnicodeSet buffer was too small`, `Raised when caller to UnicodeSet functions provides an invalid buffer. If you experience this error, it should be reported to the Keyman team for @@ -116,8 +118,8 @@ export class KmnCompilerMessages { // TODO: rename the following functions to Error_UsetHasStrings etc static ERROR_UnicodeSetHasStrings = SevError | 0x905; - static Error_UnicodeSetHasStrings = () => m( - this.ERROR_UnicodeSetHasStrings, + static Error_UnicodeSetHasStrings = (x?: ObjectWithMetadata) => mx( + this.ERROR_UnicodeSetHasStrings, x, `uset contains strings, not allowed`, `The provided uset uses multi-character strings, (\`{}\` notation, e.g. \`[żġħ{ie}{għ}]\`. ). Although full UnicodeSets support strings, LDML @@ -128,8 +130,8 @@ export class KmnCompilerMessages { ); static ERROR_UnicodeSetHasProperties = SevError | 0x906; - static Error_UnicodeSetHasProperties = () => m( - this.ERROR_UnicodeSetHasProperties, + static Error_UnicodeSetHasProperties = (x?: ObjectWithMetadata) => mx( + this.ERROR_UnicodeSetHasProperties, x, `uset contains properties, not allowed`, `The provided uset uses property notation (\`\\p{…}\` or \`[:…:]\`). LDML keyboards do not support Unicode properties in usets, because that would @@ -140,8 +142,8 @@ export class KmnCompilerMessages { ); static ERROR_UnicodeSetSyntaxError = SevError | 0x907; - static Error_UnicodeSetSyntaxError = () => m( - this.ERROR_UnicodeSetSyntaxError, + static Error_UnicodeSetSyntaxError = (x?: ObjectWithMetadata) => mx( + this.ERROR_UnicodeSetSyntaxError, x, `uset had a Syntax Error while parsing`, `The provided uset has a syntax error and could not be parsed. Verify the format of the uset against the specification. diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 749d44ed67..c268bbfd00 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -1,5 +1,5 @@ import { util } from "@keymanapp/common-types"; -import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_FILENAME_SYMBOL, CompilerEvent, KeymanXMLReader, ObjectWithMetadata } from '@keymanapp/developer-utils'; +import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageObjectSpec as mx, CompilerMessageSpec as m, CompilerMessageDef as def, ObjectWithMetadata } from '@keymanapp/developer-utils'; // const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.LdmlKeyboardCompiler; @@ -9,42 +9,6 @@ const SevError = CompilerErrorSeverity.Error | CompilerErrorNamespace.LdmlKeyboa // sub-numberspace for transform errors const SevErrorTransform = SevError | 0xF00; -/** - * Convenience function for constructing CompilerEvents with line numbers. - * Use it as below: (abbreviated as mx()) - * - * ```js - * // Note: Indentation makes "InvalidScanCode" line up thrice - * static ERROR_InvalidScanCode = SevError | 0x0009; - * // Note: - * // 1. All parameters are passed in 'o', the context object is only used for context even if - * // it contains redundant info. - * // 2. No code execution within the arrow function other than the 'mx' call, string interpolation, - * // with `${def(o.property)}` as the max complexity of interpolation. - * static Error_InvalidScanCode = (o:{id: string, invalidCodeList: string}, x: ObjectWithMetadata) => mx( - * this.ERROR_InvalidScanCode, x, - * `Form '${def(o.id)}' has invalid/unknown scancodes '${def(o.codes)}'`, - * // Note: If detail is omitted, leave the trailing comma on the prior line to leave room for it - * `…additional markdown detail…` - * ); - * ``` - * - * @param code Unique numeric value of the event - * @param message A short description of the error presented to the user - * @param context Object to be used as a source for line number information - * @param detail Detailed Markdown-formatted description of the error - * including references to documentation, remediation options. - * @see CompilerMessageSpec - * @returns - */ -function CompilerMessageObjectSpec(code: number, context: ObjectWithMetadata, message: string, detail?: string): CompilerEvent { - let evt = m(code, message, detail); // constructs raw message - evt = LdmlCompilerMessages.offset(evt, context); // updates with offset from context - return evt; -}; - -const mx = CompilerMessageObjectSpec; - /** * @internal */ @@ -408,25 +372,4 @@ export class LdmlCompilerMessages { this.ERROR_UnparseableTransformTo, x, `Invalid transform to="${def(o.to)}": "${def(o.message)}"`, ); - - /** - * Get an offset from o and set e's offset field - * @param event a compiler event, such as from functions in this class - * @param x any object parsed from XML or with the XML_META_DATA_SYMBOL symbol copied over - * @returns modified event object - */ - static offset(event: CompilerEvent, x?: any): CompilerEvent { - if(x) { - const metadata = KeymanXMLReader.getMetaData(x) || {}; - const offset = metadata?.startIndex; - if (offset) { - event.offset = offset; - } - const filename = event.filename || metadata[XML_FILENAME_SYMBOL]; - if (filename) { - event.filename = filename; - } - } - return event; - } } From 77abdd6469052c625deced8187072ab1c272cb96 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 2 Jun 2025 08:59:50 -0500 Subject: [PATCH 57/73] feat(developer): ldml: update errors for unparseable reorder - remove the unparseable reorder message - instead, the unicodeset messages will be thrown. Fixes: #14091 --- .../src/kmc-ldml/src/compiler/ldml-compiler-messages.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index c268bbfd00..b29802761c 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -270,11 +270,7 @@ export class LdmlCompilerMessages { `File has character classes which include non-NFD characters(s), including ${util.describeCodepoint(o.lowestCh)}. These will not match any text.`, ); - static ERROR_UnparseableReorderSet = SevError | 0x0028; - static Error_UnparseableReorderSet = (o: { from: string, set: string }, x?: ObjectWithMetadata) => mx( - this.ERROR_UnparseableReorderSet, x, - `Illegal UnicodeSet "${def(o.set)}" in reorder "${def(o.from)}`, - ); + // Available: 0x0028 static ERROR_InvalidVariableIdentifier = SevError | 0x0029; static Error_InvalidVariableIdentifier = (o: { id: string }, x?: ObjectWithMetadata) => mx( From 536cdc3c457b40a2bdd336ca60315eda020fc638 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 2 Jun 2025 09:27:46 -0500 Subject: [PATCH 58/73] feat(developer): ldml: fix for test case Fixes: #13932 --- .../src/kmc-ldml/test/compiler-e2e.tests.ts | 4 +-- developer/src/kmc-ldml/test/helpers/index.ts | 30 +++++++++++++------ 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/developer/src/kmc-ldml/test/compiler-e2e.tests.ts b/developer/src/kmc-ldml/test/compiler-e2e.tests.ts index e7756226bd..69273c894c 100644 --- a/developer/src/kmc-ldml/test/compiler-e2e.tests.ts +++ b/developer/src/kmc-ldml/test/compiler-e2e.tests.ts @@ -1,7 +1,7 @@ import 'mocha'; import {assert} from 'chai'; import hextobin from '@keymanapp/hextobin'; -import {compileKeyboard, compilerTestCallbacks, compilerTestOptions, makePathToFixture} from './helpers/index.js'; +import {compileKeyboard, compilerTestCallbacks, compilerTestOptions, makePathToFixture, scrubContextFromMessages} from './helpers/index.js'; import { compareXml } from './helpers/compareXml.js'; import { LdmlKeyboardCompiler } from '../src/compiler/compiler.js'; import { kmxToXml } from '../src/util/serialize.js'; @@ -53,7 +53,7 @@ describe('compiler-tests', function() { const runOutput = await k.run(inputFilename, "invalid-illegal.kmx"); // need the exact name passed to build-fixtures assert.isNull(runOutput, "Expect invalid-illegal to fail to run()"); - assert.sameDeepMembers(compilerTestCallbacks.messages, [ + assert.sameDeepMembers( scrubContextFromMessages(compilerTestCallbacks.messages), [ // copied from strs.tests.ts // validation messages LdmlCompilerMessages.Error_IllegalCharacters({ count: 5, lowestCh: 0xFDD0 }), diff --git a/developer/src/kmc-ldml/test/helpers/index.ts b/developer/src/kmc-ldml/test/helpers/index.ts index ce7dc8233f..acce3a6183 100644 --- a/developer/src/kmc-ldml/test/helpers/index.ts +++ b/developer/src/kmc-ldml/test/helpers/index.ts @@ -271,6 +271,26 @@ export interface CompilationCase { retainOffsetInMessages?: boolean; } +/** + * Scrub 'context' from messages. to simplify unit tests + * @param messages input array of messages + * @returns copy of messages + */ +export function scrubContextFromMessages(messages: CompilerEvent[]): CompilerEvent[] { + return messages.map(m => { + const scrubbed = Object.assign({}, m); + // Turn this on once all messages have offsets, see messages.tests.ts + // if (!scrubbed.offset) { + // throw Error(`Error, no offset detected in message ${CompilerError.formatEvent(m)}`); + // } + delete scrubbed.offset; + delete scrubbed.line; + delete scrubbed.filename; + delete scrubbed.column; + return scrubbed; + }); +} + /** * Run a bunch of cases * @param cases cases to run @@ -295,15 +315,7 @@ export function testCompilationCases(compiler: SectionCompilerNew, cases : Compi let messagesToCheck = callbacks.messages; // scrub offsets from messages to reduce churn in the test casws if (!testcase.retainOffsetInMessages && callbacks.messages) { - messagesToCheck = callbacks.messages.map(m => { - const scrubbed = Object.assign({}, m); - // Turn this on once all messages have offsets, see messages.tests.ts - // if (!scrubbed.offset) { - // throw Error(`Error, no offset detected in message ${CompilerError.formatEvent(m)}`); - // } - delete scrubbed.offset; - return scrubbed; - }); + messagesToCheck = scrubContextFromMessages(callbacks.messages); } const testcaseErrors = matchCompilerEventsOrBoolean(messagesToCheck, testcase.errors); const testcaseWarnings = matchCompilerEvents(messagesToCheck, testcase.warnings); From 4884d7ceb5461a36cd10618b04538e45f4675529 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 2 Jun 2025 10:41:19 -0500 Subject: [PATCH 59/73] chore(developer): ldml: remove dup zapMessageMetadata() Fixes: #14099 --- developer/src/kmc-ldml/test/helpers/index.ts | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/developer/src/kmc-ldml/test/helpers/index.ts b/developer/src/kmc-ldml/test/helpers/index.ts index acce3a6183..967287cf87 100644 --- a/developer/src/kmc-ldml/test/helpers/index.ts +++ b/developer/src/kmc-ldml/test/helpers/index.ts @@ -143,9 +143,8 @@ export async function compileKeyboard(inputFilename: string, options: LdmlCompil assert.isNotNull(source, 'k.load should not have returned null'); const valid = await k.validate(source); - zapMessageMetadata(); if (validateMessages) { - assert.sameDeepMembers(compilerTestCallbacks.messages, validateMessages, "validation messages mismatch"); + assert.sameDeepMembers(scrubContextFromMessages(compilerTestCallbacks.messages), validateMessages, "validation messages mismatch"); assert.notEqual(valid, expectFailValidate, 'validation failure'); } else { checkMessages(); @@ -155,9 +154,8 @@ export async function compileKeyboard(inputFilename: string, options: LdmlCompil if (!valid) return null; // get out, if the above asserts didn't get us out. const kmx = await k.compile(source); - zapMessageMetadata(); if (compileMessages) { - assert.sameDeepMembers(compilerTestCallbacks.messages, compileMessages, "compiler messages mismatch"); + assert.sameDeepMembers(scrubContextFromMessages(compilerTestCallbacks.messages), compileMessages, "compiler messages mismatch"); } else { checkMessages(); } @@ -174,16 +172,6 @@ export function checkMessages() { assert.isEmpty(compilerTestCallbacks.messages, compilerEventFormat(compilerTestCallbacks.messages)); } -/** These tests aren't prepared for line number information in messages. Remove it so that comparisons pass. */ -function zapMessageMetadata() { - for(const i in compilerTestCallbacks.messages) { - delete compilerTestCallbacks.messages[i].column; - delete compilerTestCallbacks.messages[i].filename; - delete compilerTestCallbacks.messages[i].line; - delete compilerTestCallbacks.messages[i].offset; - } -} - /** * Like CompilerEvent, but supports regex matching. */ From 01777fc897521af2c50340fafe20cd1db0348fa5 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 2 Jun 2025 10:58:56 -0500 Subject: [PATCH 60/73] chore(developer): ldml: bring disp compiler messages in line with design - simplify some messages to use as the key, so there's not code running in the message function Fixes: #14099 --- developer/src/kmc-ldml/src/compiler/disp.ts | 6 +++--- .../src/compiler/ldml-compiler-messages.ts | 21 ++++--------------- developer/src/kmc-ldml/test/disp.tests.ts | 8 +++---- 3 files changed, 11 insertions(+), 24 deletions(-) diff --git a/developer/src/kmc-ldml/src/compiler/disp.ts b/developer/src/kmc-ldml/src/compiler/disp.ts index da732cbc5d..fd15c8ace7 100644 --- a/developer/src/kmc-ldml/src/compiler/disp.ts +++ b/developer/src/kmc-ldml/src/compiler/disp.ts @@ -35,18 +35,18 @@ export class DispCompiler extends SectionCompiler { for (const display of this.keyboard3.displays?.display) { const { output, keyId } = display; if ((output && keyId) || (!output && !keyId)) { - this.callbacks.reportMessage(LdmlCompilerMessages.Error_DisplayNeedsToOrId({ output, keyId }, display)); + this.callbacks.reportMessage(LdmlCompilerMessages.Error_DisplayNeedsToOrId({ display: display.display }, display)); return false; } else if (output) { if (tos.has(output)) { - this.callbacks.reportMessage(LdmlCompilerMessages.Error_DisplayIsRepeated({ output }, display)); + this.callbacks.reportMessage(LdmlCompilerMessages.Error_DisplayIsRepeated({ display: display.display }, display)); return false; } else { tos.add(output); } } else if (keyId) { if (ids.has(keyId)) { - this.callbacks.reportMessage(LdmlCompilerMessages.Error_DisplayIsRepeated({ keyId }, display)); + this.callbacks.reportMessage(LdmlCompilerMessages.Error_DisplayIsRepeated({ display: display.display }, display)); return false; } else { ids.add(keyId); diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index b29802761c..ba1bb6231c 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -110,23 +110,10 @@ export class LdmlCompilerMessages { `Key element with id "${def(o.id)}" is a layer switch key, but there is no matching display element by keyId. Keycap may be blank.`, ); - /** annotate the to= or id= entry */ - private static outputOrKeyId(o:{output?: string, keyId?: string}) { - if (o.output && o.keyId) { - return `output='${o.output}' keyId='${o.keyId}'`; - } else if(o.keyId) { - return `keyId='${o.keyId}'`; - } else if (o.output) { - return `output='${o.output}'`; - } else { - return ''; - } - } - static ERROR_DisplayIsRepeated = SevError | 0x0010; - static Error_DisplayIsRepeated = (o:{output?: string, keyId?: string}, x?: ObjectWithMetadata) => mx( + static Error_DisplayIsRepeated = (o:{display?: string}, x?: ObjectWithMetadata) => mx( this.ERROR_DisplayIsRepeated, x, - `display ${LdmlCompilerMessages.outputOrKeyId(o)} has more than one display entry.`, + `display display='${def(o.display)}' refers to the same keyId or output as another entry.`, ); static ERROR_KeyMissingToGapOrSwitch = SevError | 0x0011; @@ -235,9 +222,9 @@ export class LdmlCompilerMessages { ); static ERROR_DisplayNeedsToOrId = SevError | 0x0022; - static Error_DisplayNeedsToOrId = (o:{output?: string, keyId?: string}, x?: ObjectWithMetadata) => mx( + static Error_DisplayNeedsToOrId = (o:{display?: string}, x?: ObjectWithMetadata) => mx( this.ERROR_DisplayNeedsToOrId, x, - `display ${LdmlCompilerMessages.outputOrKeyId(o)} needs output= or keyId=, but not both`, + `display display='${def(o.display)}' needs output= or keyId=, but not both`, ); static HINT_PUACharacters = SevHint | 0x0023; diff --git a/developer/src/kmc-ldml/test/disp.tests.ts b/developer/src/kmc-ldml/test/disp.tests.ts index 032e4829bf..3bad95e339 100644 --- a/developer/src/kmc-ldml/test/disp.tests.ts +++ b/developer/src/kmc-ldml/test/disp.tests.ts @@ -67,25 +67,25 @@ describe('disp', function () { const disp = await loadSectionFixture(DispCompiler, 'sections/disp/invalid-dupto.xml', compilerTestCallbacks) as Disp; assert.isNull(disp); assert.equal(compilerTestCallbacks.messages.length, 1); - assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_DisplayIsRepeated({ output: 'e' }, withOffset(330))); + assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_DisplayIsRepeated({ display: '(e)' }, withOffset(330))); }); it('should reject duplicate ids', async function() { const disp = await loadSectionFixture(DispCompiler, 'sections/disp/invalid-dupid.xml', compilerTestCallbacks) as Disp;1 assert.isNull(disp); assert.equal(compilerTestCallbacks.messages.length, 1); - assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_DisplayIsRepeated({ keyId: 'e' }, withOffset(329))); + assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_DisplayIsRepeated({ display: '(e)' }, withOffset(329))); }); it('should reject if neither to nor id', async function() { const disp = await loadSectionFixture(DispCompiler, 'sections/disp/invalid-none.xml', compilerTestCallbacks) as Disp; assert.isNull(disp); assert.equal(compilerTestCallbacks.messages.length, 1); - assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_DisplayNeedsToOrId({}, withOffset(182))); + assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_DisplayNeedsToOrId({ display: '(f)'}, withOffset(182))); }); it('should reject if both to and id', async function() { const disp = await loadSectionFixture(DispCompiler, 'sections/disp/invalid-both.xml', compilerTestCallbacks) as Disp; assert.isNull(disp); assert.equal(compilerTestCallbacks.messages.length, 1); - assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_DisplayNeedsToOrId({ output: 'e', keyId: 'e' }, withOffset(182))); + assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_DisplayNeedsToOrId({ display: '(e)' }, withOffset(182))); }); testCompilationCases(DispCompiler, [ { From dc0023098d2e8e010655ac750604a4c1ce1dde48 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 2 Jun 2025 11:15:23 -0500 Subject: [PATCH 61/73] chore(developer): ldml: bring layr/keys compiler messages in line with design Fixes: #14099 --- developer/src/kmc-ldml/src/compiler/keys.ts | 2 +- developer/src/kmc-ldml/src/compiler/layr.ts | 2 +- .../kmc-ldml/src/compiler/ldml-compiler-messages.ts | 12 ++---------- developer/src/kmc-ldml/test/keys.tests.ts | 2 +- developer/src/kmc-ldml/test/layr.tests.ts | 5 ++--- 5 files changed, 7 insertions(+), 16 deletions(-) diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index c896467bdb..46b5381b4b 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -463,7 +463,7 @@ export class KeysCompiler extends SectionCompiler { const { modifiers } = layer; if (!validModifier(modifiers)) { this.callbacks.reportMessage( - LdmlCompilerMessages.Error_InvalidModifier({ modifiers, id: layer.id }, layer) + LdmlCompilerMessages.Error_InvalidModifier({ modifiers }, layer) ); valid = false; } diff --git a/developer/src/kmc-ldml/src/compiler/layr.ts b/developer/src/kmc-ldml/src/compiler/layr.ts index b0c615f41b..9dff22a5bf 100644 --- a/developer/src/kmc-ldml/src/compiler/layr.ts +++ b/developer/src/kmc-ldml/src/compiler/layr.ts @@ -52,7 +52,7 @@ export class LayrCompiler extends SectionCompiler { const { modifiers } = layer; totalLayerCount++; if (!validModifier(modifiers)) { - this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidModifier({ modifiers, id: layer.id }, layer)); + this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidModifier({ modifiers }, layer)); valid = false; } }); diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index ba1bb6231c..c1c4a6f9c5 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -134,18 +134,10 @@ export class LdmlCompilerMessages { `layers has invalid value formId=${def(o.formId)}`, ); - private static layerIdOrEmpty(layer : string) { - if (layer) { - return ` on layer id=${def(layer)}`; - } else { - return ''; - } - } - static ERROR_InvalidModifier = SevError | 0x0014; - static Error_InvalidModifier = (o:{modifiers: string, id: string}, x?: ObjectWithMetadata) => mx( + static Error_InvalidModifier = (o:{modifiers: string}, x?: ObjectWithMetadata) => mx( this.ERROR_InvalidModifier, x, - `layer has invalid modifiers='${def(o.modifiers)}'` + LdmlCompilerMessages.layerIdOrEmpty(o.id), + `layer has invalid modifiers='${def(o.modifiers)}'`, ); static ERROR_MissingFlicks = SevError | 0x0015; diff --git a/developer/src/kmc-ldml/test/keys.tests.ts b/developer/src/kmc-ldml/test/keys.tests.ts index 75b3fabd3e..3c3c842810 100644 --- a/developer/src/kmc-ldml/test/keys.tests.ts +++ b/developer/src/kmc-ldml/test/keys.tests.ts @@ -312,7 +312,7 @@ describe('keys.kmap', function () { { subpath: 'sections/keys/invalid-bad-modifier.xml', errors: [ - LdmlCompilerMessages.Error_InvalidModifier({id:'base',modifiers:'altR-shift'}), + LdmlCompilerMessages.Error_InvalidModifier({ modifiers: 'altR-shift' }), ] }, { diff --git a/developer/src/kmc-ldml/test/layr.tests.ts b/developer/src/kmc-ldml/test/layr.tests.ts index 610efff212..dc0334c0b1 100644 --- a/developer/src/kmc-ldml/test/layr.tests.ts +++ b/developer/src/kmc-ldml/test/layr.tests.ts @@ -96,8 +96,7 @@ describe('layr', function () { subpath: 'sections/keys/invalid-bad-modifier.xml', errors: [ LdmlCompilerMessages.Error_InvalidModifier({ - id: 'base', - modifiers: 'altR-shift' + modifiers: 'altR-shift', }), ], }, @@ -139,7 +138,7 @@ describe('layr', function () { { subpath: 'sections/layr/error-bogus-modifiers.xml', errors: [ - LdmlCompilerMessages.Error_InvalidModifier({ id: '', modifiers: 'caps bogus'}), + LdmlCompilerMessages.Error_InvalidModifier({ modifiers: 'caps bogus' }), ] }, { From 19d808c6e5963edeafd694d25bd48720719ad6cf Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Mon, 2 Jun 2025 18:37:22 +0200 Subject: [PATCH 62/73] refactor(web): clarify API of KeymanSentryManager This change marks the functions of `KeymanSentryManager` as public or private, depending on whether or not they are used outside of the module. Test-bot: skip --- web/src/engine/sentry-manager/src/index.ts | 26 +++++++++++----------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/web/src/engine/sentry-manager/src/index.ts b/web/src/engine/sentry-manager/src/index.ts index b8129766d1..92085e7869 100644 --- a/web/src/engine/sentry-manager/src/index.ts +++ b/web/src/engine/sentry-manager/src/index.ts @@ -12,8 +12,8 @@ type Options = { }; export class KeymanSentryManager { - keymanPlatform: string; - _enabled: boolean = true; + private keymanPlatform: string; + private _enabled: boolean = true; static STANDARD_ALIASABLE_FILES = { 'keymanweb.js': 'keymanweb.js', @@ -34,7 +34,7 @@ export class KeymanSentryManager { } // If we've recognized one of our source files, - aliasFilename(filename: string): string|null { + private aliasFilename(filename: string): string|null { if(!this.mayAlias(filename)) { return null; } @@ -49,14 +49,14 @@ export class KeymanSentryManager { } } - mayAlias(filename: string): boolean { + private mayAlias(filename: string): boolean { return !!KeymanSentryManager.STANDARD_ALIASABLE_FILES[filename]; } // Filters all expected but unnecessary path prefixes, affixes, and suffixes reported by Sentry from our products. // This allows us to mask all different sorts of installations with a single uploaded path. // Modifies original object. - pathFilter(event: any) { + private pathFilter(event: any) { // Get the underlying JS error. let exception = event.exception; if(!exception) { @@ -90,7 +90,7 @@ export class KeymanSentryManager { } // Attaches some useful debugging information to the specified object, pass-by-reference style. - attachEventMetadata(event: any) { + private attachEventMetadata(event: any) { // Ensure that the 'extra' object exists. (May not exist for synthetic/custom Errors.) event.extra = event.extra || {}; //@ts-ignore @@ -100,7 +100,7 @@ export class KeymanSentryManager { // Sanitizes the event object (in-place) to remove sensitive information // from the breadcrumbs and url (for embedded KeymanWeb) - sanitizeEvent(event: any) { + private sanitizeEvent(event: any) { if (event && event.breadcrumbs) { event.breadcrumbs.forEach((b: any) => { if (b.category == 'navigation') { @@ -123,7 +123,7 @@ export class KeymanSentryManager { * Also will sanitize the Sentry event. * @param event A Sentry-generated event */ - prepareEvent(event: any): boolean { + private prepareEvent(event: any): boolean { this.pathFilter(event); this.attachEventMetadata(event); this.sanitizeEvent(event); @@ -148,7 +148,7 @@ export class KeymanSentryManager { * if we want to prevent the event from being sent to the server. * @param event */ - prepareEventDebugWrapper(event: any) { + public prepareEventDebugWrapper(event: any) { if(DEBUG) { try { if(this.prepareEvent(event)) { @@ -178,7 +178,7 @@ export class KeymanSentryManager { * * https://stackoverflow.com/a/53214615/1836776 */ - initConsole() { + private initConsole() { // creating function declarations for better stacktraces (otherwise they'd be anonymous function expressions) let oldConsoleError = console.error; let _this = this; @@ -223,7 +223,7 @@ export class KeymanSentryManager { } } - init() { + public init() { // Do the actual Sentry initialization. //@ts-ignore Sentry.init({ @@ -238,11 +238,11 @@ export class KeymanSentryManager { this.initConsole(); } - get enabled(): boolean { + public get enabled(): boolean { return this._enabled; } - set enabled(value: boolean) { + public set enabled(value: boolean) { this._enabled = value; } } From 41cf045d3f82b56475377a497e1139d78ef6abcb Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 2 Jun 2025 12:20:39 -0500 Subject: [PATCH 63/73] chore(developer): ldml: add a test for complex interpolation in messages - and fix a case in keys Fixes: #14099 --- developer/src/kmc-ldml/src/compiler/keys.ts | 2 +- .../src/compiler/ldml-compiler-messages.ts | 6 +++--- developer/src/kmc-ldml/test/messages.tests.ts | 19 +++++++++++++++++++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index 46b5381b4b..3c890a5067 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -484,7 +484,7 @@ export class KeysCompiler extends SectionCompiler { LdmlCompilerMessages.Error_RowOnHardwareLayerHasTooManyKeys({ row: y + 1, hardware: layers.formId, - modifiers, + modifiers: modifiers || 'none', }, row) ); valid = false; diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index c1c4a6f9c5..f60ecebb82 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -34,7 +34,7 @@ export class LdmlCompilerMessages { static ERROR_RowOnHardwareLayerHasTooManyKeys = SevError | 0x0004; static Error_RowOnHardwareLayerHasTooManyKeys = (o: { row: number, hardware: string, modifiers: string }, x?: ObjectWithMetadata) => mx( this.ERROR_RowOnHardwareLayerHasTooManyKeys, x, - `Row #${def(o.row)} on 'hardware' ${def(o.hardware)} layer for modifier ${o.modifiers || 'none'} has too many keys`, + `Row #${def(o.row)} on 'hardware' ${def(o.hardware)} layer for modifier ${def(o.modifiers)} has too many keys`, ); static ERROR_KeyNotFoundInKeyBag = SevError | 0x0005; @@ -272,14 +272,14 @@ export class LdmlCompilerMessages { static ERROR_DuplicateLayerWidth = SevError | 0x002C; static Error_DuplicateLayerWidth = (o: { minDeviceWidth: number }, x?: ObjectWithMetadata) => mx( this.ERROR_DuplicateLayerWidth, x, - `Two or more layers have minDeviceWidth=${def(o?.minDeviceWidth)}`, + `Two or more layers have minDeviceWidth=${def(o.minDeviceWidth)}`, `Touch layers must have distinct widths.` ); static ERROR_InvalidLayerWidth = SevError | 0x002D; static Error_InvalidLayerWidth = (o: { minDeviceWidth: number }, x?: ObjectWithMetadata) => mx( this.ERROR_InvalidLayerWidth, x, - `Invalid Layers minDeviceWidth=${def(o?.minDeviceWidth)}`, + `Invalid Layers minDeviceWidth=${def(o.minDeviceWidth)}`, `Width must be between 1-999 (millimeters), inclusive.` // sync with layr_max_minDeviceWidth / layr_max_maxDeviceWidth (from spec) ); diff --git a/developer/src/kmc-ldml/test/messages.tests.ts b/developer/src/kmc-ldml/test/messages.tests.ts index e71891f3ef..55b908a956 100644 --- a/developer/src/kmc-ldml/test/messages.tests.ts +++ b/developer/src/kmc-ldml/test/messages.tests.ts @@ -51,4 +51,23 @@ describe('LdmlCompilerMessages', function () { expect(lines).to.not.be.equal(0, `None of ${total} messages had offset reporting.`); expect(Array.from(noLines.values())).to.deep.equal([], `${noLines.size}/${total} ${Number((noLines.size/total)*100).toFixed(0)}% of message(s) did not have detectable offset (line number) reporting`); }); + it('Should only have simple ${def(o...)} interpolation', () => { + const m = LdmlCompilerMessages as Record; + const keys = Object.keys(LdmlCompilerMessages); + const complexInterpolation : string[] = []; + for(const key of keys) { + // exclude this one, does not need line numbers + if (key == 'Error_InvalidFile') continue; + if (typeof m[key] == 'function') { + const f = m[key] as Function; + const s = f.toString(); + // scan for interpolation + // Interpolation.. A non-escaped ${ that's NOT followed by `def(o.` triggers this error. + if (/.*`.*(? Date: Mon, 2 Jun 2025 13:00:56 -0500 Subject: [PATCH 64/73] chore(developer): ldml: bring strs/tran compiler messages in line with design - fix some case pertaining to strings and escapes Fixes: #14099 --- .../kmc-ldml/src/compiler/empty-compiler.ts | 6 ++--- .../src/compiler/ldml-compiler-messages.ts | 22 +++++++++---------- developer/src/kmc-ldml/src/compiler/tran.ts | 9 ++++++-- .../src/kmc-ldml/test/compiler-e2e.tests.ts | 5 +++-- developer/src/kmc-ldml/test/messages.tests.ts | 2 +- developer/src/kmc-ldml/test/strs.tests.ts | 20 +++++++++-------- developer/src/kmc-ldml/test/tran.tests.ts | 7 +++--- 7 files changed, 40 insertions(+), 31 deletions(-) diff --git a/developer/src/kmc-ldml/src/compiler/empty-compiler.ts b/developer/src/kmc-ldml/src/compiler/empty-compiler.ts index c0fe1d1ef6..79015dca6b 100644 --- a/developer/src/kmc-ldml/src/compiler/empty-compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/empty-compiler.ts @@ -76,18 +76,18 @@ export class StrsCompiler extends EmptyCompiler { const illegals = m.get(util.BadStringType.illegal); if (puas) { const [count, lowestCh] = [puas.size, Array.from(puas.values()).sort((a, b) => a - b)[0]]; - this.callbacks.reportMessage(LdmlCompilerMessages.Hint_PUACharacters({ count, lowestCh }, + this.callbacks.reportMessage(LdmlCompilerMessages.Hint_PUACharacters({ count, lowestCh: util.describeCodepoint(lowestCh) }, findContextForString(String.fromCodePoint(lowestCh)))); } if (unassigneds) { const [count, lowestCh] = [unassigneds.size, Array.from(unassigneds.values()).sort((a, b) => a - b)[0]]; - this.callbacks.reportMessage(LdmlCompilerMessages.Warn_UnassignedCharacters({ count, lowestCh }, + this.callbacks.reportMessage(LdmlCompilerMessages.Warn_UnassignedCharacters({ count, lowestCh: util.describeCodepoint(lowestCh) }, findContextForString(String.fromCodePoint(lowestCh)))); } if (illegals) { // do this last, because we will return false. const [count, lowestCh] = [illegals.size, Array.from(illegals.values()).sort((a, b) => a - b)[0]]; - this.callbacks.reportMessage(LdmlCompilerMessages.Error_IllegalCharacters({ count, lowestCh }, + this.callbacks.reportMessage(LdmlCompilerMessages.Error_IllegalCharacters({ count, lowestCh: util.describeCodepoint(lowestCh) }, findContextForString(String.fromCodePoint(lowestCh)))); return false; } diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index f60ecebb82..f76e0d3b3a 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -1,4 +1,3 @@ -import { util } from "@keymanapp/common-types"; import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageObjectSpec as mx, CompilerMessageSpec as m, CompilerMessageDef as def, ObjectWithMetadata } from '@keymanapp/developer-utils'; // const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler; @@ -220,33 +219,33 @@ export class LdmlCompilerMessages { ); static HINT_PUACharacters = SevHint | 0x0023; - static Hint_PUACharacters = (o: { count: number, lowestCh: number }, x?: ObjectWithMetadata) => mx( + static Hint_PUACharacters = (o: { count: number, lowestCh: string }, x?: ObjectWithMetadata) => mx( this.HINT_PUACharacters, x, - `File contains ${def(o.count)} PUA character(s), including ${util.describeCodepoint(o.lowestCh)}`, + `File contains ${def(o.count)} PUA character(s), including ${def(o.lowestCh)}`, ); static WARN_UnassignedCharacters = SevWarn | 0x0024; - static Warn_UnassignedCharacters = (o: { count: number, lowestCh: number }, x?: ObjectWithMetadata) => mx( + static Warn_UnassignedCharacters = (o: { count: number, lowestCh: string }, x?: ObjectWithMetadata) => mx( this.WARN_UnassignedCharacters, x, - `File contains ${def(o.count)} unassigned character(s), including ${util.describeCodepoint(o.lowestCh)}`, + `File contains ${def(o.count)} unassigned character(s), including ${def(o.lowestCh)}`, ); static ERROR_IllegalCharacters = SevError | 0x0025; - static Error_IllegalCharacters = (o: { count: number, lowestCh: number }, x?: ObjectWithMetadata) => mx( + static Error_IllegalCharacters = (o: { count: number, lowestCh: string }, x?: ObjectWithMetadata) => mx( this.ERROR_IllegalCharacters, x, - `File contains ${def(o.count)} illegal character(s), including ${ util.describeCodepoint(o.lowestCh) }`, + `File contains ${def(o.count)} illegal character(s), including ${def(o.lowestCh) }`, ); static HINT_CharClassImplicitDenorm = SevHint | 0x0026; static Hint_CharClassImplicitDenorm = (o: { lowestCh: number }, x?: ObjectWithMetadata) => mx( this.HINT_CharClassImplicitDenorm, x, - `File has character classes which span non-NFD character(s), including ${util.describeCodepoint(o.lowestCh)}. These will not match any text.`, + `File has character classes which span non-NFD character(s), including ${def(o.lowestCh)}. These will not match any text.`, ); static WARN_CharClassExplicitDenorm = SevWarn | 0x0027; static Warn_CharClassExplicitDenorm = (o: { lowestCh: number }, x?: ObjectWithMetadata) => mx( this.WARN_CharClassExplicitDenorm, x, - `File has character classes which include non-NFD characters(s), including ${util.describeCodepoint(o.lowestCh)}. These will not match any text.`, + `File has character classes which include non-NFD characters(s), including ${def(o.lowestCh)}. These will not match any text.`, ); // Available: 0x0028 @@ -286,9 +285,10 @@ export class LdmlCompilerMessages { // Available: 0x02E-0x2F static ERROR_InvalidQuadEscape = SevError | 0x0030; - static Error_InvalidQuadEscape = (o: { cp: number }, x?: ObjectWithMetadata) => mx( + static Error_InvalidQuadEscape = (o: { cp: string, recommended: string }, x?: ObjectWithMetadata) => mx( this.ERROR_InvalidQuadEscape, x, - `Invalid escape "\\u${util.hexQuad(o?.cp || 0)}". Hint: Use "\\u{${def(o?.cp?.toString(16))}}"`, + `Invalid escape "${def(o.cp)}"`, + `**Hint**: Use "${def(o.recommended)}"`, ); // diff --git a/developer/src/kmc-ldml/src/compiler/tran.ts b/developer/src/kmc-ldml/src/compiler/tran.ts index c6236c1be7..b9eb9b4642 100644 --- a/developer/src/kmc-ldml/src/compiler/tran.ts +++ b/developer/src/kmc-ldml/src/compiler/tran.ts @@ -350,10 +350,15 @@ export abstract class TransformCompiler Date: Mon, 2 Jun 2025 14:02:37 -0400 Subject: [PATCH 65/73] auto: increment master version to 19.0.57 --- HISTORY.md | 6 ++++++ VERSION.md | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 9048b14d87..ecabec500e 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,11 @@ # Keyman Version History +## 19.0.56 alpha 2025-06-02 + +* maint(ios): Use zip.inc.sh to prep release artifacts (#13919) +* maint(linux): update to new version of gha-ubuntu-packaging (#14094) +* chore(linux): add Python TC test runner (#14062) + ## 19.0.55 alpha 2025-05-31 * maint(linux): `sudo` not required for removing temporary files (#14086) diff --git a/VERSION.md b/VERSION.md index 4073c70600..4223e2bb7b 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -19.0.56 \ No newline at end of file +19.0.57 \ No newline at end of file From 25ddb324f2f05e85dc3b2c29369337e01ab41242 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 2 Jun 2025 13:33:00 -0500 Subject: [PATCH 66/73] chore(developer): ldml: bring strs/tran compiler messages in line with design - fix a case where context wasn't being passed Fixes: #14099 --- developer/src/kmc-ldml/src/compiler/tran.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/developer/src/kmc-ldml/src/compiler/tran.ts b/developer/src/kmc-ldml/src/compiler/tran.ts index b9eb9b4642..1cc9bd0289 100644 --- a/developer/src/kmc-ldml/src/compiler/tran.ts +++ b/developer/src/kmc-ldml/src/compiler/tran.ts @@ -151,7 +151,7 @@ export abstract class TransformCompiler Date: Tue, 3 Jun 2025 05:58:48 +0700 Subject: [PATCH 67/73] maint(mac): make make-km-dmg.sh less quiet When build.sh was converted to a builder-style script, the -quiet flag was then passed into make-km-dmg.sh, which meant that it no longer gave valuable diagnostic output. This returns the script to default levels of output (there is also a -verbose or -debug level we could use). The variable QUIET_FLAG appears to have been used only in that one place. --- mac/build.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/mac/build.sh b/mac/build.sh index db61eed053..ea9822f909 100755 --- a/mac/build.sh +++ b/mac/build.sh @@ -69,16 +69,12 @@ PREPRELEASE=false UPDATE_VERSION_IN_PLIST=true DO_CODESIGN=true CODESIGNING_SUPPRESSION="CODE_SIGN_IDENTITY=\"\" CODE_SIGNING_REQUIRED=NO" - -QUIET=true +QUIET=false if builder_verbose; then BUILD_OPTIONS="" - QUIET_FLAG= - QUIET=false else BUILD_OPTIONS="-quiet" - QUIET_FLAG="-quiet" fi BUILD_ACTIONS="build" @@ -290,7 +286,7 @@ do_publish() { builder_heading "Preparing files for release deployment..." ./setup/build.sh - "$KM4MIM_BASE_PATH/make-km-dmg.sh" $QUIET_FLAG + "$KM4MIM_BASE_PATH/make-km-dmg.sh" # Create download info "$KM4MIM_BASE_PATH/write-download_info.sh" From d9ed728970d499e47c8fff864ee32edd25513e53 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Tue, 3 Jun 2025 07:59:55 +0700 Subject: [PATCH 68/73] Apply suggestions from code review Co-authored-by: Eberhard Beilharz --- android/build.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/android/build.sh b/android/build.sh index 2133012ef3..12017e2c6b 100755 --- a/android/build.sh +++ b/android/build.sh @@ -80,13 +80,12 @@ if builder_start_action archive; then KEYMAN_ENGINE_ANDROID_ZIP="keyman-engine-android-${KEYMAN_VERSION}.zip" KEYMAN_APK="keyman-${KEYMAN_VERSION}.apk" FIRSTVOICES_APK="firstvoices-${KEYMAN_VERSION}.apk" + ZIP_FILE="${UPLOAD_PATH}/${KEYMAN_ENGINE_ANDROID_ZIP}" + ZIP_FLAGS=("-q" "-r") # quiet, recursive mkdir -p "${UPLOAD_PATH}" # Create Keyman Engine for Android archive - ZIP_FILE="${UPLOAD_PATH}/${KEYMAN_ENGINE_ANDROID_ZIP}" - ZIP_FLAGS=("-q" "-r") # quiet, recursive - builder_echo "Copying Keyman Engine for Android into ${UPLOAD_PATH}..." cd "${UPLOAD_PATH}" cp "${KEYMAN_ROOT}/android/KMAPro/kMAPro/libs/keyman-engine.aar" ./ @@ -99,14 +98,12 @@ if builder_start_action archive; then rm -rf "Samples" # Copy release APK - cp "${KEYMAN_ROOT}/android/KMAPro/kMAPro/build/outputs/apk/release/${KEYMAN_APK}" \ - "${UPLOAD_PATH}/${KEYMAN_APK}" + cp "${KEYMAN_ROOT}/android/KMAPro/kMAPro/build/outputs/apk/release/${KEYMAN_APK}" ./ # FirstVoices app if [ "${RELEASE_OEM_FIRSTVOICES-false}" = true ]; then - cp "${KEYMAN_ROOT}/oem/firstvoices/android/app/build/outputs/apk/release/${FIRSTVOICES_APK}" \ - "${UPLOAD_PATH}/${FIRSTVOICES_APK}" + cp "${KEYMAN_ROOT}/oem/firstvoices/android/app/build/outputs/apk/release/${FIRSTVOICES_APK}" ./ fi # From edfa7ed2f93cc7e0389e54be38bcef1f7e95532a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Jun 2025 07:03:57 +0000 Subject: [PATCH 69/73] chore(deps-dev): bump tar-fs from 3.0.8 to 3.0.9 Bumps [tar-fs](https://github.com/mafintosh/tar-fs) from 3.0.8 to 3.0.9. - [Commits](https://github.com/mafintosh/tar-fs/compare/v3.0.8...v3.0.9) --- updated-dependencies: - dependency-name: tar-fs dependency-version: 3.0.9 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 268c33bf87..7dbf6acdad 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12738,9 +12738,9 @@ } }, "node_modules/tar-fs": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.8.tgz", - "integrity": "sha512-ZoROL70jptorGAlgAYiLoBLItEKw/fUxg9BSYK/dF/GAGYFJOJJJMvjPAKDJraCXFwadD456FCuvLWgfhMsPwg==", + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.9.tgz", + "integrity": "sha512-XF4w9Xp+ZQgifKakjZYmFdkLoSWd34VGKcsTCwlNWM7QG3ZbaxnTsaBwnjFZqHRf/rROxaR8rXnbtwdvaDI+lA==", "dev": true, "license": "MIT", "dependencies": { From 737839de571d4c670e685519e33f17e3209f9c0c Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Tue, 3 Jun 2025 14:01:37 -0400 Subject: [PATCH 70/73] auto: increment master version to 19.0.58 --- HISTORY.md | 5 +++++ VERSION.md | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index ecabec500e..8925fe401c 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,10 @@ # Keyman Version History +## 19.0.57 alpha 2025-06-03 + +* fix(windows): change from error to breadcrumb and log (#14069) +* maint(mac): make make-km-dmg.sh less quiet (#14105) + ## 19.0.56 alpha 2025-06-02 * maint(ios): Use zip.inc.sh to prep release artifacts (#13919) diff --git a/VERSION.md b/VERSION.md index 4223e2bb7b..e44218e2c5 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -19.0.57 \ No newline at end of file +19.0.58 \ No newline at end of file From dabb63ed81592f337af3b2cfa7e6973166686c42 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Wed, 4 Jun 2025 11:17:08 +0200 Subject: [PATCH 71/73] docs(linux): clarify requirement of patched ibus for Keyman < 18 --- linux/docs/help/common/index.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/linux/docs/help/common/index.md b/linux/docs/help/common/index.md index a7f9e786bf..6b80489211 100644 --- a/linux/docs/help/common/index.md +++ b/linux/docs/help/common/index.md @@ -15,12 +15,14 @@ sudo apt-get upgrade sudo apt-get install keyman onboard-keyman ``` -An older version of Keyman for Linux packages is also available in any current Debian at: +A possibly older version of Keyman for Linux packages is also available in +any current Debian at: [https://packages.debian.org/keyman](https://packages.debian.org/keyman) -**Note** in order to fully work you'll need a patched version of ibus -which is available in the above launchpad repo. Patched versions of ibus -will have the string `sil` in the package version number. +**Note** Versions before Keyman 18 required a patched version of ibus in order +to fully work. Patched versions of ibus were available in the above mentioned +launchpad repository and had the string `sil` in the package version number. +Keyman 18+ no longer requires a patched version of ibus. ## Q. How do I install a Keyman keyboard? From 792e9d64141fbbb0c108b44ae30ba38535c924d1 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Wed, 4 Jun 2025 08:10:58 -0400 Subject: [PATCH 72/73] auto: increment stable-18.0 version to 18.0.239 (history cherry-pick to master) --- HISTORY.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 8925fe401c..5376db1c37 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -345,6 +345,13 @@ * refactor(windows): rename `TKeymanMutex.MutexOwned` to `TakeOwnership` and add `ReleaseOwnership` (#13168) * chore: increment to alpha 19.0 (#13187) +## 18.0.238 stable 2025-06-04 + +* fix(developer): ldml: make sure .run() calls compile() for validation (#14071) +* maint(linux): fix typo of temporary dependency package (#14085) +* maint(linux): update to new version of gha-ubuntu-packaging (#14096) +* maint(mac): make make-km-dmg.sh less quiet (#14106) + ## 18.0.237 stable 2025-05-30 * maint(android): Update FirstVoices launcher icon and red theme (#13917) From dc67fd7d28322ec3d72b449794a19d40d2cf0c39 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Wed, 4 Jun 2025 16:39:56 +0200 Subject: [PATCH 73/73] maint(web): address code review comments - update and fix file header - renamed env variables for path to s.keyman.com/help.keyman.com to match what is used elsewhere --- resources/teamcity/web/keyman-web-release.sh | 16 +++++++--------- resources/teamcity/web/keyman-web-test.sh | 6 ++---- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/resources/teamcity/web/keyman-web-release.sh b/resources/teamcity/web/keyman-web-release.sh index 10015e3710..894bc68e6d 100755 --- a/resources/teamcity/web/keyman-web-release.sh +++ b/resources/teamcity/web/keyman-web-release.sh @@ -1,7 +1,5 @@ -#!/bin/bash -# Copyright (C) 2025 SIL International. All rights reserved. -# Distributed under the MIT License. See LICENSE.md file in the project -# root for full license information. +#!/usr/bin/env bash +# Keyman is copyright (C) SIL Global. MIT License. # # TC build script to build release of KeymanWeb. @@ -24,8 +22,8 @@ builder_describe \ "all run all actions" \ "build build Web + embedded" \ "publish publish release" \ - "--s.keyman.com=S_KEYMAN_COM_PATH path to s.keyman.com repository" \ - "--help.keyman.com=HELP_KEYMAN_COM_PATH path to help.keyman.com repository" + "--s.keyman.com=S_KEYMAN_COM path to s.keyman.com repository" \ + "--help.keyman.com=HELP_KEYMAN_COM path to help.keyman.com repository" builder_parse "$@" @@ -44,10 +42,10 @@ function _push_release_to_skeymancom() { # downloads.keyman.com so we can ensure files are available) builder_echo start publish "Publishing release to skeyman.com" - cd "${S_KEYMAN_COM_PATH:=${KEYMAN_ROOT}/../s.keyman.com}" + cd "${S_KEYMAN_COM:=${KEYMAN_ROOT}/../s.keyman.com}" git pull https://github.com/keymanapp/s.keyman.com.git master cd ../keyman/web - "${KEYMAN_ROOT}/web/ci.sh" prepare:s.keyman.com --s.keyman.com "${S_KEYMAN_COM_PATH}" + "${KEYMAN_ROOT}/web/ci.sh" prepare:s.keyman.com --s.keyman.com "${S_KEYMAN_COM}" builder_echo end publish success "Finished publishing release to skeyman.com" } @@ -70,7 +68,7 @@ function _zip_and_upload_artifacts() { function _upload_help() { builder_echo start "upload help" "Uploading new Keyman for Web help to help.keyman.com" - export HELP_KEYMAN_COM="${HELP_KEYMAN_COM_PATH:-${KEYMAN_ROOT}/../help.keyman.com}" + export HELP_KEYMAN_COM="${HELP_KEYMAN_COM:-${KEYMAN_ROOT}/../help.keyman.com}" cd "${KEYMAN_ROOT}/resources/build" node ../gosh/gosh.js ./help-keyman-com.sh web cd "${KEYMAN_ROOT}/web" diff --git a/resources/teamcity/web/keyman-web-test.sh b/resources/teamcity/web/keyman-web-test.sh index bb6368415f..c6436edfef 100755 --- a/resources/teamcity/web/keyman-web-test.sh +++ b/resources/teamcity/web/keyman-web-test.sh @@ -1,7 +1,5 @@ -#!/bin/bash -# Copyright (C) 2025 SIL International. All rights reserved. -# Distributed under the MIT License. See LICENSE.md file in the project -# root for full license information. +#!/usr/bin/env bash +# Keyman is copyright (C) SIL Global. MIT License. # # TC build script for Keyman Web/Test