From 61a114e937c6f739773733547483faf93f874652 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 30 Apr 2025 10:30:01 +0700 Subject: [PATCH 1/2] maint(common): test imported resource versions Examines imported resources such as standards data for version information, and compares against our published minimum-versions.inc.sh data. This test will be run in CI in common/tests. Fixes: #13582 Test-bot: skip --- docs/minimum-versions.md | 2 +- resources/build/minimum-versions.inc.sh | 8 +- resources/build/minimum-versions.tests.inc.sh | 88 +++++++++++++++++++ resources/build/publish-minimum-versions.sh | 13 +-- 4 files changed, 98 insertions(+), 13 deletions(-) create mode 100644 resources/build/minimum-versions.tests.inc.sh diff --git a/docs/minimum-versions.md b/docs/minimum-versions.md index 7ec93e317f..0bd31eebac 100644 --- a/docs/minimum-versions.md +++ b/docs/minimum-versions.md @@ -66,7 +66,7 @@ https://help.keyman.com/developer/engine/android/latest-version/ | KEYMAN_MIN_VERSION_NODE_MAJOR | 20 | | KEYMAN_MIN_VERSION_NPM | 10.5.1 | | KEYMAN_MIN_VERSION_VISUAL_STUDIO | 2019 | -| KEYMAN_VERSION_CLDR | 45 | +| KEYMAN_VERSION_CLDR | 46 | | KEYMAN_VERSION_GRADLE | 7.6.4 | | KEYMAN_VERSION_ICU | 73.1 | | KEYMAN_VERSION_ISO639_3 | 2024-05-22 | diff --git a/resources/build/minimum-versions.inc.sh b/resources/build/minimum-versions.inc.sh index a8ad69be05..eadd2442cf 100644 --- a/resources/build/minimum-versions.inc.sh +++ b/resources/build/minimum-versions.inc.sh @@ -24,7 +24,7 @@ KEYMAN_MIN_TARGET_VERSION_WEB_FIREFOX=79.0 # TBD KEYMAN_MIN_TARGET_VERSION_WEB_OPERA=47.0 # TBD KEYMAN_MIN_TARGET_VERSION_WEB_SAFARI=13.0 # iOS 13.0, macOS 10.13.6+ -# Dependency versions +# Dependency minimum versions KEYMAN_MIN_VERSION_NODE_MAJOR=20 # node version source of truth is /package.json:/engines/node; use KEYMAN_USE_NVM to automatically update KEYMAN_MIN_VERSION_NPM=10.5.1 # 10.5.0 has bug, discussed in #10350 KEYMAN_MIN_VERSION_EMSCRIPTEN=3.1.58 # Use KEYMAN_USE_EMSDK to automatically update to this version @@ -32,6 +32,8 @@ KEYMAN_MIN_VERSION_VISUAL_STUDIO=2019 KEYMAN_MIN_VERSION_MESON=1.0.0 KEYMAN_VERSION_GRADLE=7.6.4 # See /android/KMEA/gradle/wrapper/gradle-wrapper.properties + +# Dependencies with fixed versions KEYMAN_VERSION_ICU=73.1 # See /core/subprojects/icu-minimal.wrap # Language and runtime versions @@ -43,8 +45,8 @@ KEYMAN_MIN_VERSION_ANDROID_SDK=21 KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER=noble # Ubuntu 24.04 Noble # Data versions -- see resources/standards-data/readme.md -KEYMAN_VERSION_CLDR=45 # LDML Keyboards version +KEYMAN_VERSION_CLDR=46 # LDML Keyboards version KEYMAN_VERSION_ISO639_3=2024-05-22 # Date of last import -KEYMAN_VERSION_LANGTAGS=2025-02-18 # Date of last import +KEYMAN_VERSION_LANGTAGS=2025-02-18 # _version value KEYMAN_VERSION_LANGUAGE_SUBTAG_REGISTRY=2025-03-10 # Date from first line of language-subtag-registry KEYMAN_VERSION_UNICODE=16.0.0 # UCD + related data diff --git a/resources/build/minimum-versions.tests.inc.sh b/resources/build/minimum-versions.tests.inc.sh new file mode 100644 index 0000000000..8279a1061a --- /dev/null +++ b/resources/build/minimum-versions.tests.inc.sh @@ -0,0 +1,88 @@ +# +# Keyman is copyright (C) SIL Global. MIT License. +# +# Created by mcdurdin on 2025-04-30 +# +# Test that resources in our repository match the expected minimum versions, +# where possible. Uses a variety of mechanisms to extract the version +# + +. "${KEYMAN_ROOT}/resources/build/jq.inc.sh" + +# +# Tests that the referenced filename has the expected version +# found in the input variable. The actual version is extracted +# with a helper function named _minver_test_$variable, which +# parses the file or its metadata, as this may vary considerably +# +# Parameters +# 1: variable the name of the environment variable to test +# 2: filename the file to test the version against +# +_minver_test() { + local variable="$1" + local filename="$2" + + local actual_version="$(_minver_test_${variable} "$filename")" + local -n expected_version="${variable}" + + builder_echo "Testing \$$1, expecting '$expected_version'" + if [[ "$expected_version" != "$actual_version" ]]; then + builder_die "$filename: expected '$expected_version', actual version in repo is '$actual_version'" + fi +} + +_minver_test_KEYMAN_VERSION_LANGUAGE_SUBTAG_REGISTRY() { + # language-subtag-registry format: + # File-Date: 2025-03-10 + head "$1" -n 1 | cut -d" " -f 2 - +} + +_minver_test_KEYMAN_VERSION_UNICODE() { + # Blocks.txt format: + # # Blocks-16.0.0.txt + head "$1" -n 1 | cut -d- -f 2 - | cut -d. -f 1,2,3 - +} + +_minver_test_KEYMAN_VERSION_LANGTAGS() { + # entry with tag '_version' + "$JQ" -r -c '.[] | select(.tag == "_version") .date' "$1" +} + +_minver_test_KEYMAN_VERSION_CLDR() { + # finds the folder with the expected name + find "$1" -name '[0-9][0-9]' -type d -printf '%f' +} + +_minver_test_KEYMAN_VERSION_ISO639_3() { + # git commit log for last commit for the file + git log -1 --format="%cs" -- "$1" +} + +_minver_test_KEYMAN_VERSION_ICU() { + # source_filename = icu4c-73_1-src.tgz + grep "source_filename" "$1" | cut -d- -f 2 - | tr _ . +} + +_minver_test_KEYMAN_MIN_VERSION_NODE_MAJOR() { + # package.json, major.minor.patch, we use only major + jq -r '.engines.node' "$1" | cut -d. -f 1 - +} + +# +# Test that the minimum version of imported resources matches as expected +# +minver_test_all() { + local STANDARDS_DATA="$KEYMAN_ROOT/resources/standards-data" + + _minver_test KEYMAN_MIN_VERSION_NODE_MAJOR "$KEYMAN_ROOT/package.json" + + _minver_test KEYMAN_VERSION_ICU "$KEYMAN_ROOT/core/subprojects/icu-minimal.wrap" + + _minver_test KEYMAN_VERSION_LANGUAGE_SUBTAG_REGISTRY "$STANDARDS_DATA/language-subtag-registry/language-subtag-registry" + _minver_test KEYMAN_VERSION_UNICODE "$STANDARDS_DATA/unicode-character-database/Blocks.txt" + _minver_test KEYMAN_VERSION_UNICODE "$STANDARDS_DATA/unicode-character-database/WordBreakProperty.txt" + _minver_test KEYMAN_VERSION_LANGTAGS "$STANDARDS_DATA/langtags/langtags.json" + _minver_test KEYMAN_VERSION_CLDR "$STANDARDS_DATA/ldml-keyboards/" + _minver_test KEYMAN_VERSION_ISO639_3 "$STANDARDS_DATA/iso639-3/iso639-3.tab" +} \ No newline at end of file diff --git a/resources/build/publish-minimum-versions.sh b/resources/build/publish-minimum-versions.sh index 8c420acb6c..90ff5ca3d5 100755 --- a/resources/build/publish-minimum-versions.sh +++ b/resources/build/publish-minimum-versions.sh @@ -12,6 +12,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" ## END STANDARD BUILD SCRIPT INCLUDE . "${KEYMAN_ROOT}/resources/build/minimum-versions.inc.sh" +. "${KEYMAN_ROOT}/resources/build/minimum-versions.tests.inc.sh" builder_describe "Build and test minimum-versions" \ clean build test @@ -53,16 +54,10 @@ do_test() { builder_die "minimum-versions.md is not up to date. Please run 'publish-minimum-versions.sh build'" ) - rm "$new_table" + rm "$new_table" - # TODO: this should have a test for each of the known machine-extractable versions. - - # language-subtag-registry - - local NEW_KEYMAN_VERSION_LANGUAGE_SUBTAG_REGISTRY="$(head ../standards-data/language-subtag-registry/language-subtag-registry -n 1 | cut -d" " -f 2 -)" - if [[ "$NEW_KEYMAN_VERSION_LANGUAGE_SUBTAG_REGISTRY" != "$KEYMAN_VERSION_LANGUAGE_SUBTAG_REGISTRY" ]]; then - builder_die "language-subtag-registry is version '$NEW_KEYMAN_VERSION_LANGUAGE_SUBTAG_REGISTRY' but minimum-version.inc.sh has version '$KEYMAN_VERSION_LANGUAGE_SUBTAG_REGISTRY'" - fi + # From minimum-versions.tests.inc.sh + minver_test_all } builder_run_action clean rm -f "$KEYMAN_ROOT/docs/minimum-versions.md" From 1f3de1dc0a176076061750bdd86e76a7c08186c8 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 30 Apr 2025 15:01:27 +0700 Subject: [PATCH 2/2] chore(common): tweak tests for cross-platform compat --- resources/build/minimum-versions.tests.inc.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/resources/build/minimum-versions.tests.inc.sh b/resources/build/minimum-versions.tests.inc.sh index 8279a1061a..b961124081 100644 --- a/resources/build/minimum-versions.tests.inc.sh +++ b/resources/build/minimum-versions.tests.inc.sh @@ -51,7 +51,9 @@ _minver_test_KEYMAN_VERSION_LANGTAGS() { _minver_test_KEYMAN_VERSION_CLDR() { # finds the folder with the expected name - find "$1" -name '[0-9][0-9]' -type d -printf '%f' + ## find "$1" -name '[0-9][0-9]' -type d -printf '%f' + # note macos find does not support -printf, so went with this: + find "$1" -name '[0-9][0-9]' -type d -exec basename {} \; } _minver_test_KEYMAN_VERSION_ISO639_3() { @@ -66,7 +68,7 @@ _minver_test_KEYMAN_VERSION_ICU() { _minver_test_KEYMAN_MIN_VERSION_NODE_MAJOR() { # package.json, major.minor.patch, we use only major - jq -r '.engines.node' "$1" | cut -d. -f 1 - + "$JQ" -r '.engines.node' "$1" | cut -d. -f 1 - } #