From a2c9e7e514d25f7093938bbcfa277f38464c5b87 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 3 Oct 2022 10:59:00 +0700 Subject: [PATCH 01/28] chore(common/models): lmlayer tests now builder-based --- .../unit_tests/in_browser/base.conf.js | 2 - .../unit_tests/in_browser/browser-test.sh | 91 -------- common/predictive-text/unit_tests/test.sh | 208 ++++++++++-------- 3 files changed, 121 insertions(+), 180 deletions(-) delete mode 100755 common/predictive-text/unit_tests/in_browser/browser-test.sh diff --git a/common/predictive-text/unit_tests/in_browser/base.conf.js b/common/predictive-text/unit_tests/in_browser/base.conf.js index 9671ac9e0a..2c62af1854 100644 --- a/common/predictive-text/unit_tests/in_browser/base.conf.js +++ b/common/predictive-text/unit_tests/in_browser/base.conf.js @@ -41,7 +41,6 @@ module.exports = { // We don't have anything in these locations... yet. But they'll be useful for test resources. 'json/**/*.json', // Where pre-loaded JSON resides. {pattern: 'resources/**/*.*', watched: true, served: true, included: false}, // General testing resources. - {pattern: 'fixtures/**/*.html', watched: true} // HTML structures useful for testing. ], // list of files / patterns to exclude @@ -51,7 +50,6 @@ module.exports = { // preprocess matching files before serving them to the browser // available preprocessors: https://npmjs.org/browse/keyword/karma-preprocessor preprocessors: { - 'fixtures/**/*.html' : ['html2js'], 'json/**/*.json' : ['json_fixtures'] }, diff --git a/common/predictive-text/unit_tests/in_browser/browser-test.sh b/common/predictive-text/unit_tests/in_browser/browser-test.sh deleted file mode 100755 index 28c286ecf7..0000000000 --- a/common/predictive-text/unit_tests/in_browser/browser-test.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash - -# A testing script designed for automating in-browser testing. -# It's designed to be called by this folder's ./test.sh. - -display_usage ( ) { - echo "browser_test.sh os [-CI | -debug | -? | -h | -help] [-reporter ]" - echo - echo " os Should be one of the following: win, mac, linux" - echo "" - echo " -CI to run unit tests in CI mode on BrowserStack." - echo " This script requires your credentials to be set in environment variables - see " - echo " https://stackoverflow.com/questions/32450546/hiding-browserstack-key-in-karma" - echo "" - echo " -debug to establish a Karma server that facilitates unit test debugging" - echo " Not compatible with -CI." - echo "" - echo " -reporter sets the test engine to utilize the specified ." - echo " Valid options: BrowserStack, teamcity, dots, progress, mocha" - echo "" - echo " -? | -h | -help to display this help information" - echo "" - echo " Specifying no option will perform a simple, single run of the test cases on" - echo " the dominant set of browsers for the currently-detected active OS." - echo "" - exit 0 -} - -os_id=$1 - -get_browser_set_for_OS ( ) { - if [ $os_id = "mac" ]; then - BROWSERS="--browsers Firefox,Chrome,Safari" - elif [ $os_id = "win" ]; then - BROWSERS="--browsers Chrome" - else - BROWSERS="--browsers Firefox,Chrome" - fi -} - -# Defaults -get_browser_set_for_OS - -CONFIG=manual.conf.js # TODO - get/make OS-specific version -DEBUG=false -FLAGS= - -# Parse args -while [[ $# -gt 0 ]] ; do - key="$1" - case $key in - -CI) - CONFIG=CI.conf.js - ;; - -debug) - # Disables the default 'run once, then done' configuration needed for CI. - DEBUG=true - FLAGS="--no-single-run $FLAGS" - ;; - -reporter) - shift - FLAGS="--reporters $1 $FLAGS" - ;; - -h) - display_usage - ;; - -help) - display_usage - ;; - -\?) - display_usage - ;; - esac - shift # past argument -done - -if [ $DEBUG = true ] && [ $CONFIG = CI.conf.js ]; then - echo "-CI and -debug are not compatible!" - exit 1 -fi - -if [ $CONFIG = CI.conf.js ]; then - # If doing a CI run, use the file's default browser selection. - BROWSERS= -fi - -npm --no-color run karma -- start --log-level=debug $FLAGS $BROWSERS unit_tests/in_browser/$CONFIG - -CODE=$? - -exit $CODE diff --git a/common/predictive-text/unit_tests/test.sh b/common/predictive-text/unit_tests/test.sh index 8329e8c417..3ac7aba15c 100755 --- a/common/predictive-text/unit_tests/test.sh +++ b/common/predictive-text/unit_tests/test.sh @@ -9,31 +9,61 @@ THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BA . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" SCRIPT_ROOT="$(dirname "$THIS_SCRIPT")" -# A simple utility script to facilitate unit-testing for the LM Layer. -# It's rigged to be callable by NPM to facilitate testing during development when in other folders. +# This script runs from its own folder +cd "$SCRIPT_ROOT" -display_usage ( ) { - echo "test.sh [ -? | -h | -help]" - echo " -CI to perform continuous-integration friendly tests and reporting" - echo " -headless to disable the in-browser tests" - echo " -integrated to disable the 'headless' test suite" - echo " -skip-package-install (or -S) skips dependency updates" - echo " -? | -h | -help to display this help information" - echo "" - exit 0 -} +################################ Main script ################################ -init_dependencies ( ) { +# Defaults +FLAGS="--require ./unit_tests/helpers" + +builder_describe "Runs all tests for the language-modeling / predictive-text layer module" \ + "configure" \ + "test+" \ + ":libraries Runs unit tests for in-repo libraries used by this module"\ + ":headless Runs this module's headless user tests" \ + ":browser Runs this module's browser-based user tests" \ + "--ci Uses CI-based test configurations & emits CI-friendly test reports" \ + "--debug,-d Activates developer-friendly debug mode for unit tests where applicable" + +builder_parse "$@" + +do_configure() { # Ensure all testing dependencies are in place. - verify_npm_setup $fetch_deps + verify_npm_setup } -test-headless ( ) { - _FLAGS=$FLAGS - if (( CI_REPORTING )); then - _FLAGS="$_FLAGS --reporter mocha-teamcity-reporter" - fi +CONFIGURED= +if builder_start_action configure :libraries; then + # Ensure all testing dependencies are in place. + do_configure + CONFIGURED=configure:libraries + builder_finish_action success configure :libraries +fi + +if builder_start_action configure :headless; then + if [[ -n "$CONFIGURED" ]]; then + echo "Configuration already completed in ${BUILDER_TERM_START}${CONFIGURED}${BUILDER_TERM_END}; skipping." + else + do_configure + CONFIGURED=configure:headless + fi + builder_finish_action success configure :headless +fi + +if builder_start_action configure :browser; then + if [[ -n "$CONFIGURED" ]]; then + echo "Configuration already completed in ${BUILDER_TERM_START}${CONFIGURED}${BUILDER_TERM_END}; skipping." + else + do_configure + fi + builder_finish_action success configure :browser +fi + +if builder_start_action test :libraries; then + # Note: these do not yet provide TeamCity-friendly-formatted test reports. + # Sadly, appending them to the `npm run test` doesn't work; it'll require work in these modules. pushd "$KEYMAN_ROOT/common/models/wordbreakers" npm run test || fail "models/wordbreakers tests failed" popd @@ -43,80 +73,84 @@ test-headless ( ) { popd pushd "$KEYMAN_ROOT/common/models/types" + # Is not mocha-based. npm run test || fail "models/types tests failed" popd - npm run mocha -- --recursive $_FLAGS ./unit_tests/headless/*.js ./unit_tests/headless/**/*.js -} - -test-browsers ( ) { - _FLAGS=$FLAGS - if (( CI_REPORTING )); then - _FLAGS="$_FLAGS -CI -reporter teamcity,BrowserStack" - fi - - $SCRIPT_ROOT/in_browser/browser-test.sh $os_id $_FLAGS -} - -# Defaults -get_builder_OS # return: os_id="linux"|"mac"|"win" - -FLAGS="--require ./unit_tests/helpers" -CI_REPORTING=0 -RUN_HEADLESS=1 -RUN_BROWSERS=1 -fetch_deps=true - -# Parse args -while [[ $# -gt 0 ]] ; do - key="$1" - case $key in - -h|-help) - display_usage - exit - ;; - -CI) - CI_REPORTING=1 - ;; - -headless) - RUN_BROWSERS=0 - ;; - -integrated) - RUN_HEADLESS=0 - ;; - -skip-package-install|-S) - fetch_deps=false - ;; - esac - shift # past argument -done - -init_dependencies - -# Run headless (browserless) tests. -if (( RUN_HEADLESS )); then - test-headless || fail "DOMless tests failed!" + builder_finish_action success test :libraries fi -if (( RUN_BROWSERS )); then - if [[ $VERSION_ENVIRONMENT == test ]]; then - # If we are running a TeamCity test build, for now, only run BrowserStack - # tests when on a PR branch with a title including "(web)" or with the label - # test-browserstack. This is because the BrowserStack tests are currently - # unreliable, and the false positive failures are masking actual failures. - # - # We do not run BrowserStack tests on master, beta, or stable-x.y test - # builds. - RUN_BROWSERS=0 - if builder_pull_get_details; then - if [[ $builder_pull_title =~ \(web\) ]] || builder_pull_has_label test-browserstack; then - RUN_BROWSERS=1 - fi +if builder_start_action test :headless; then + MOCHA_FLAGS=$FLAGS + + if builder_has_option --ci; then + MOCHA_FLAGS="$MOCHA_FLAGS --reporter mocha-teamcity-reporter" + fi + + npm run mocha -- --recursive $MOCHA_FLAGS ./unit_tests/headless/*.js ./unit_tests/headless/**/*.js + + builder_finish_action success test :headless +fi + +# If we are running a TeamCity test build, for now, only run BrowserStack +# tests when on a PR branch with a title including "(web)" or with the label +# test-browserstack. This is because the BrowserStack tests are currently +# unreliable, and the false positive failures are masking actual failures. +# +# We do not run BrowserStack tests on master, beta, or stable-x.y test +# builds. +if [[ $VERSION_ENVIRONMENT == test ]] && builder_has_action test :browser; then + if builder_pull_get_details; then + if ! ([[ $builder_pull_title =~ \(web\) ]] || builder_pull_has_label test-browserstack); then + + echo "Auto-skipping ${BUILDER_TERM_START}test:browser${BUILDER_TERM_END} for unrelated CI test build" + exit 0 fi fi fi -# Run browser-based tests. -if (( RUN_BROWSERS )); then - test-browsers || fail "Browser-based tests failed!" -fi +get_browser_set_for_OS ( ) { + if [ $os_id = "mac" ]; then + BROWSERS="--browsers Firefox,Chrome,Safari" + elif [ $os_id = "win" ]; then + BROWSERS="--browsers Chrome" + else + BROWSERS="--browsers Firefox,Chrome" + fi +} + +if builder_start_action test :browser; then + KARMA_FLAGS=$FLAGS + KARMA_INFO_LEVEL="--log-level=warn" + + if builder_has_option --ci; then + KARMA_FLAGS="$KARMA_FLAGS --reporters teamcity,BrowserStack" + KARMA_CONFIG="CI.conf.js" + KARMA_INFO_LEVEL="--log-level=debug" + + if builder_has_option --debug; then + echo "${BUILDER_TERM_START}--ci${BUILDER_TERM_END} option set; ignoring ${BUILDER_TERM_START}--debug${BUILDER_TERM_END} option" + fi + else + KARMA_CONFIG="manual.conf.js" + if builder_has_option --debug; then + KARMA_FLAGS="$KARMA_FLAGS --no-single-run" + KARMA_CONFIG="manual.conf.js" + KARMA_INFO_LEVEL="--log-level=debug" + + echo + echo "${COLOR_YELLOW}You must manually terminate this mode (CTRL-C) for the script to exit.${COLOR_RESET}" + sleep 2 + fi + fi + + if [[ KARMA_CONFIG == "manual.conf.js" ]]; then + get_builder_OS # return: os_id="linux"|"mac"|"win" + get_browser_set_for_OS + else + BROWSERS= + fi + npm run karma -- start $KARMA_INFO_LEVEL $KARMA_FLAGS $BROWSERS unit_tests/in_browser/$KARMA_CONFIG + + builder_finish_action success test :browser +fi \ No newline at end of file From dd9d7737207133ea89e6fcedf822e2d7b3f8a1b9 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 3 Oct 2022 12:45:23 +0700 Subject: [PATCH 02/28] chore(common/models): lmlayer build.sh now builder-based --- common/predictive-text/build.sh | 206 +++++++++++------- .../unit_tests/headless/promise-store.js | 8 +- .../unit_tests/headless/top-level-lmlayer.js | 12 +- common/predictive-text/unit_tests/test.sh | 2 +- 4 files changed, 139 insertions(+), 89 deletions(-) diff --git a/common/predictive-text/build.sh b/common/predictive-text/build.sh index cbaef012d5..7897b1291f 100755 --- a/common/predictive-text/build.sh +++ b/common/predictive-text/build.sh @@ -20,92 +20,142 @@ THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BA # This script runs from its own folder cd "$(dirname "$THIS_SCRIPT")" -# Exit status on invalid usage. -EX_USAGE=64 - -LMLAYER_OUTPUT=build - -# Builds the top-level JavaScript file for use in browsers (the second stage of compilation) -build-browser () { - npm run tsc -- -b ./browser.tsconfig.json || fail "Could not build top-level browser-targeted JavaScript file." -} - -# Builds the top-level JavaScript file for use on Node (the second stage of compilation) -build-headless () { - npm run tsc -- -b ./tsconfig.json || fail "Could not build top-level node-targeted JavaScript file." -} - -# A nice, extensible method for -clean operations. Add to this as necessary. -clean ( ) { - if [ -d $LMLAYER_OUTPUT ]; then - rm -rf "$LMLAYER_OUTPUT" || fail "Failed to erase the prior build." - fi -} - -display_usage ( ) { - echo "Usage: $0 [-clean] [-skip-package-install | -S] [-test | -tdd]" - echo " $0 -help" - echo - echo " -clean to erase pre-existing build products before a re-build" - echo " -help displays this screen and exits" - echo " -skip-package-install (or -S) skips dependency updates" - echo " -tdd skips dependency updates, builds, then runs unit tests only" - echo " -test runs unit and integration tests after building" -} - ################################ Main script ################################ -run_tests=0 -fetch_deps=true -unit_tests_only=0 +builder_check_color "$@" -# Process command-line arguments -while [[ $# -gt 0 ]] ; do - key="$1" - case $key in - -clean) - clean - ;; - -help|-h) - display_usage - exit - ;; - -skip-package-install|-S) - fetch_deps=false - ;; - -test) - run_tests=1 - ;; - -tdd) - run_tests=1 - fetch_deps=false - unit_tests_only=1 - ;; - *) - echo "$0: invalid option: $key" - display_usage - exit $EX_USAGE - esac - shift # past the processed argument -done +builder_describe "Runs all tests for the gesture-recognizer module" \ + "clean" \ + "configure" \ + "build" \ + "test" \ + ":libraries Targets all in-repo libraries that this module is dependent upon" \ + ":headless A headless, Node-oriented version of the module useful for unit tests" \ + ":browser The standard version of the module for in-browser use" \ + "--ci Sets ${BUILDER_TERM_START}test${BUILDER_TERM_END} action to use CI-based test configurations & reporting" -# Check if Node.JS/npm is installed. -verify_npm_setup $fetch_deps +builder_parse "$@" -if $fetch_deps; then - # We need to build keyman-version and lm-worker with a script for now - "$KEYMAN_ROOT/common/web/keyman-version/build.sh" || fail "Could not build keyman-version" - "$KEYMAN_ROOT/common/web/lm-worker/build.sh" || fail "Could not build lm-worker" +# Exit status on invalid usage. +EX_USAGE=64 +LMLAYER_OUTPUT=build + +### CONFIGURE ACTIONS + +do_configure() { + # Check if Node.JS/npm is installed. + verify_npm_setup +} + +CONFIGURED= +if builder_start_action configure :libraries; then + do_configure + CONFIGURED=configure:libraries + + builder_finish_action success configure :libraries fi -build-browser || fail "Browser-oriented compilation failed." -build-headless || fail "Headless compilation failed." -echo "Typescript compilation successful." -if (( run_tests )); then - if (( unit_tests_only )); then - npm run test -- -headless || fail "Unit tests failed" +if builder_start_action configure :browser; then + if [ -n "$CONFIGURED" ]; then + echo "Configuration already completed in ${BUILDER_TERM_START}${CONFIGURED}${BUILDER_TERM_END}; skipping." else - npm test || fail "Tests failed" + do_configure + CONFIGURED=configure:browser fi + builder_finish_action success configure :browser fi + +if builder_start_action configure :headless; then + if [ -n "$CONFIGURED" ]; then + echo "Configuration already completed in ${BUILDER_TERM_START}${CONFIGURED}${BUILDER_TERM_END}; skipping." + else + do_configure + CONFIGURED=configure:headless + fi + + builder_finish_action success configure :headless +fi + +### CLEAN ACTIONS + +# A nice, extensible method for -clean operations. Add to this as necessary. +do_clean() { + if [ -d $LMLAYER_OUTPUT ]; then + rm -rf "$LMLAYER_OUTPUT" + fi +} + +CLEANED= +if builder_start_action clean :libraries; then + do_clean + CLEANED=clean:libraries + + builder_finish_action success clean :libraries +fi + +if builder_start_action clean :browser; then + if [ -n "$CLEANED" ]; then + echo "${BUILDER_TERM_START}clean${BUILDER_TERM_END} already completed as ${BUILDER_TERM_START}${CLEANED}${BUILDER_TERM_END}; skipping." + else + do_clean + CLEANED=clean:browser + fi + builder_finish_action success clean :browser +fi + +if builder_start_action clean :headless; then + if [ -n "$CLEANED" ]; then + echo "${BUILDER_TERM_START}clean${BUILDER_TERM_END} already completed as ${BUILDER_TERM_START}${CLEANED}${BUILDER_TERM_END}; skipping." + else + do_clean + CLEANED=clean:headless + fi + + builder_finish_action success clean :headless +fi + +### BUILD ACTIONS + +if builder_start_action build :libraries; then + "$KEYMAN_ROOT/common/web/keyman-version/build.sh" + "$KEYMAN_ROOT/common/web/lm-worker/build.sh" + + builder_finish_action success build :libraries +fi + +# Builds the top-level JavaScript file for use in browsers +if builder_start_action build :browser; then + npm run tsc -- -b ./browser.tsconfig.json + + builder_finish_action success build :browser +fi + +# Builds the top-level JavaScript file for use on Node +if builder_start_action build :headless; then + npm run tsc -- -b ./tsconfig.json || fail + + builder_finish_action success build :headless +fi + +### TEST ACTIONS +# Note - the actual test setup is done in a separate test script, but it's easy +# enough to route the calls through. + +if builder_start_action test :libraries; then + ./unit_tests/test.sh test:libraries + + builder_finish_action success test :libraries +fi + +if builder_start_action test :headless; then + ./unit_tests/test.sh test:headless + + builder_finish_action success test :headless +fi + +if builder_start_action test :browser; then + ./unit_tests/test.sh test:browser + + builder_finish_action success test :browser +fi \ No newline at end of file diff --git a/common/predictive-text/unit_tests/headless/promise-store.js b/common/predictive-text/unit_tests/headless/promise-store.js index 509ca09f50..f14b9c8555 100644 --- a/common/predictive-text/unit_tests/headless/promise-store.js +++ b/common/predictive-text/unit_tests/headless/promise-store.js @@ -1,7 +1,7 @@ var assert = require('chai').assert; var sinon = require('sinon'); -let PromiseStore = require('../../build').PromiseStore; +let PromiseStore = require('../../build/headless').PromiseStore; describe('PromiseStore', function () { describe('.make()', function () { @@ -19,13 +19,13 @@ describe('PromiseStore', function () { it('should reject the promise when a token is reused', function () { var promises = new PromiseStore(); - + var reusedToken = randomToken(); // These two fakes are to ensure the original is called. var originalResolve; var overwrittenResolve; - - + + // Add a promise, and an unrelated promise. new Promise(function (resolve, reject) { originalResolve = sinon.fake(resolve); diff --git a/common/predictive-text/unit_tests/headless/top-level-lmlayer.js b/common/predictive-text/unit_tests/headless/top-level-lmlayer.js index a4645e7e27..f4b891c8e0 100644 --- a/common/predictive-text/unit_tests/headless/top-level-lmlayer.js +++ b/common/predictive-text/unit_tests/headless/top-level-lmlayer.js @@ -1,7 +1,7 @@ var assert = require('chai').assert; var sinon = require('sinon'); -let LMLayer = require('../../build'); +let LMLayer = require('../../build/headless'); // Test the top-level LMLayer interface. // Note: these tests can only be run after BOTH stages of compilation are completed. @@ -54,7 +54,7 @@ describe('LMLayer', function() { assert.notProperty(data.source, 'code'); assert.property(data.source, 'file'); assert.isString(data.source.file); - + callAsynchronously(() => fakeWorker.onmessage({ data: { message: 'ready', @@ -113,7 +113,7 @@ describe('LMLayer', function() { * Returns an object implementing *enough* of the Worker * interface to fool the LMLayer into thinking it's * communicating with a bona fide Web Worker. - * + * * @returns {Worker} an object with sinon.fake() instances. */ function createFakeWorker(postMessage) { @@ -124,10 +124,10 @@ describe('LMLayer', function() { } /** - * Call a function in the future, i.e., later in the event loop. + * Call a function in the future, i.e., later in the event loop. * The call does NOT block the current execution. - * Use this to fake asynchronous callbacks. - * + * Use this to fake asynchronous callbacks. + * * @param {Function} fn function to call */ function callAsynchronously(fn) { diff --git a/common/predictive-text/unit_tests/test.sh b/common/predictive-text/unit_tests/test.sh index 3ac7aba15c..677a399a12 100755 --- a/common/predictive-text/unit_tests/test.sh +++ b/common/predictive-text/unit_tests/test.sh @@ -43,7 +43,7 @@ if builder_start_action configure :libraries; then fi if builder_start_action configure :headless; then - if [[ -n "$CONFIGURED" ]]; then + if [ -n "$CONFIGURED" ]; then echo "Configuration already completed in ${BUILDER_TERM_START}${CONFIGURED}${BUILDER_TERM_END}; skipping." else do_configure From 066eede2df935b9decf30d9a968e977e2b0beb31 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 3 Oct 2022 12:45:57 +0700 Subject: [PATCH 03/28] fix(common): fixes trap error passing for npm-emitted errors --- .../{build-utils.test.sh => tests/builder.inc.test.sh} | 8 ++++---- resources/builder.inc.sh | 7 +++++-- 2 files changed, 9 insertions(+), 6 deletions(-) rename resources/build/{build-utils.test.sh => tests/builder.inc.test.sh} (96%) mode change 100755 => 100644 diff --git a/resources/build/build-utils.test.sh b/resources/build/tests/builder.inc.test.sh old mode 100755 new mode 100644 similarity index 96% rename from resources/build/build-utils.test.sh rename to resources/build/tests/builder.inc.test.sh index b689fa0ff4..4fccb2a072 --- a/resources/build/build-utils.test.sh +++ b/resources/build/tests/builder.inc.test.sh @@ -5,7 +5,7 @@ set -eu ## START STANDARD BUILD SCRIPT INCLUDE # adjust relative paths as necessary THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")" -. "$(dirname "$THIS_SCRIPT")/build-utils.sh" +. "$(dirname "$THIS_SCRIPT")/../build-utils.sh" # END STANDARD BUILD SCRIPT INCLUDE . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" @@ -164,9 +164,9 @@ fi # Due to the nature of the build-utils-traps tests, only one may be # specified at a time; each ends with an `exit`. echo "Running separate tests" -$THIS_SCRIPT_PATH/tests/build-utils-traps.test.sh error -$THIS_SCRIPT_PATH/tests/build-utils-traps.test.sh error-in-function -$THIS_SCRIPT_PATH/tests/build-utils-traps.test.sh incomplete +$THIS_SCRIPT_PATH/build-utils-traps.test.sh error +$THIS_SCRIPT_PATH/build-utils-traps.test.sh error-in-function +$THIS_SCRIPT_PATH/build-utils-traps.test.sh incomplete echo "Fin" # Finally, run with --help so we can see what it looks like diff --git a/resources/builder.inc.sh b/resources/builder.inc.sh index 81c1d3cedd..d526b60c9a 100755 --- a/resources/builder.inc.sh +++ b/resources/builder.inc.sh @@ -7,9 +7,9 @@ # * THIS_SCRIPT_PATH defines the full path of the running script # * THIS_SCRIPT_NAME defines the basename of the running script # * THIS_SCRIPT_IDENTIFIER defines the repo-relative path of the running script -# * _builder_ functions and variables are internal use only for builder.inc.sh, and +# * _builder_ functions and variables are internal use only for builder.inc.sh, and # subject to change at any time. Do not use them in other scripts. -# * Note: the running script is the top-level script that includes either +# * Note: the running script is the top-level script that includes either # builder.inc.sh directly, or, just in the Keyman repo, via build-utils.sh. # @@ -188,6 +188,9 @@ _builder_failure_trap() { fi builder_finish_action failure $action $target + + # Ensure that the exit code propagates fully. + exit $trappedExitCode fi } From 4176605c376a76e8d43a31747a68e796964943ee Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 3 Oct 2022 14:04:48 +0700 Subject: [PATCH 04/28] chore(common/models): script doc tweak --- common/predictive-text/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/predictive-text/build.sh b/common/predictive-text/build.sh index 7897b1291f..3c7d521940 100755 --- a/common/predictive-text/build.sh +++ b/common/predictive-text/build.sh @@ -24,7 +24,7 @@ cd "$(dirname "$THIS_SCRIPT")" builder_check_color "$@" -builder_describe "Runs all tests for the gesture-recognizer module" \ +builder_describe "Builds the lm-layer module" \ "clean" \ "configure" \ "build" \ From 9f952cd6eb96ab515c8b0fbfe844a4bfc5982ba5 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 3 Oct 2022 15:03:13 +0700 Subject: [PATCH 05/28] chore(web): retools input-processor build, test scripts --- common/web/input-processor/build.sh | 171 ++++++++++++++++++++++++ common/web/input-processor/src/build.sh | 91 ------------- common/web/input-processor/test.sh | 90 ------------- web/unit_tests/test.sh | 2 +- 4 files changed, 172 insertions(+), 182 deletions(-) create mode 100755 common/web/input-processor/build.sh delete mode 100755 common/web/input-processor/src/build.sh delete mode 100755 common/web/input-processor/test.sh diff --git a/common/web/input-processor/build.sh b/common/web/input-processor/build.sh new file mode 100755 index 0000000000..cf280b0f0e --- /dev/null +++ b/common/web/input-processor/build.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +# +# Compile KeymanWeb's 'keyboard-processor' module, one of the components of Web's 'core' module. +# +set -eu + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")" +. "$(dirname "$THIS_SCRIPT")/../../../resources/build/build-utils.sh" +. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +# This script runs from its own folder +cd "$(dirname "$THIS_SCRIPT")" + +################################ Main script ################################ + +builder_check_color "$@" + +builder_describe "Builds the standalone, headless form of Keyman Engine for Web's input-processor module" \ + "clean" \ + "configure" \ + "build" \ + "test" \ + ":libraries Targets all in-repo libraries that this module is dependent upon" \ + ":module A headless, Node-oriented version of the module useful for unit tests" \ + ":tools Related tools useful for development and testing of this module" \ + "--ci Sets ${BUILDER_TERM_START}test${BUILDER_TERM_END} action to use CI-based test configurations & reporting" + +builder_parse "$@" + +### CONFIGURE ACTIONS + +do_configure() { + # Check if Node.JS/npm is installed. + verify_npm_setup + + "$KEYMAN_ROOT/common/web/keyman-version/build.sh" +} + +CONFIGURED= +if builder_start_action configure :libraries; then + do_configure + CONFIGURED=configure:libraries + + builder_finish_action success configure :libraries +fi + + +if builder_start_action configure :module; then + if [ -n "$CONFIGURED" ]; then + echo "Configuration already completed in ${BUILDER_TERM_START}${CONFIGURED}${BUILDER_TERM_END}; skipping." + else + do_configure + CONFIGURED=configure:module + fi + builder_finish_action success configure :module +fi + +if builder_start_action configure :tools; then + if [ -n "$CONFIGURED" ]; then + echo "Configuration already completed in ${BUILDER_TERM_START}${CONFIGURED}${BUILDER_TERM_END}; skipping." + else + do_configure + CONFIGURED=configure:tools + fi + + builder_finish_action success configure :tools +fi + +### CLEAN ACTIONS + +MODULE_OUTPUT=build + +# A nice, extensible method for -clean operations. Add to this as necessary. +do_clean() { + if [ -d $MODULE_OUTPUT ]; then + rm -rf "$MODULE_OUTPUT" + fi +} + +CLEANED= +if builder_start_action clean :libraries; then + do_clean + CLEANED=clean:libraries + + builder_finish_action success clean :libraries +fi + +if builder_start_action clean :module; then + if [ -n "$CLEANED" ]; then + echo "${BUILDER_TERM_START}clean${BUILDER_TERM_END} already completed as ${BUILDER_TERM_START}${CLEANED}${BUILDER_TERM_END}; skipping." + else + do_clean + CLEANED=clean:module + fi + builder_finish_action success clean :module +fi + +if builder_start_action clean :tools; then + if [ -n "$CLEANED" ]; then + echo "${BUILDER_TERM_START}clean${BUILDER_TERM_END} already completed as ${BUILDER_TERM_START}${CLEANED}${BUILDER_TERM_END}; skipping." + else + do_clean + CLEANED=clean:tools + fi + + builder_finish_action success clean :tools +fi + +### BUILD ACTIONS + +if builder_start_action build :libraries; then + # Ensure that the LMLayer compiles properly, readying the build product for comsumption by KMW. + "$KEYMAN_ROOT/common/predictive-text/build.sh" build:headless + # Also ensure that the keyboard-processor module builds appropriately. + "$KEYMAN_ROOT/common/web/keyboard-processor/build.sh" build + + builder_finish_action success build :libraries +fi + +if builder_start_action build :tools; then + # Used by test:module + pushd "$KEYMAN_ROOT/developer/src/kmlmc" + ./build.sh -S + popd + + builder_finish_action success build :tools +fi + +if builder_start_action build :module; then + npm run tsc -- -b src/tsconfig.json + + builder_finish_action success build :module +fi + +# TEST ACTIONS + +if builder_start_action test :tools; then + builder_finish_action success test :tools +fi + +if builder_start_action test :libraries; then + CHAINING_FLAGS= + if builder_has_option --ci; then + CHAINING_FLAGS=--ci + fi + # First, run tests on the keyboard processor. + pushd "$KEYMAN_ROOT/common/web/keyboard-processor" + ./build.sh test $CHAINING_FLAGS + popd + + # lm-layer tests are handled outside of this chain, at least by our existing CI processes. + + builder_finish_action success test :libraries +fi + +if builder_start_action test :module; then + FLAGS= + if builder_has_option --ci; then + FLAGS="--reporter mocha-teamcity-reporter" + fi + + # Build the leaf-style, bundled version of input-processor for use in testing. + npm run tsc -- -b src/tsconfig.bundled.json + + npm run mocha -- --recursive $FLAGS ./tests/cases/ + + builder_finish_action success test :module +fi \ No newline at end of file diff --git a/common/web/input-processor/src/build.sh b/common/web/input-processor/src/build.sh deleted file mode 100755 index 3fdc2e2b04..0000000000 --- a/common/web/input-processor/src/build.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash -# -# Compile KeymanWeb's 'keyboard-processor' module, one of the components of Web's 'core' module. -# -set -eu - -## START STANDARD BUILD SCRIPT INCLUDE -# adjust relative paths as necessary -THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")" -. "$(dirname "$THIS_SCRIPT")/../../../../resources/build/build-utils.sh" -. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" -## END STANDARD BUILD SCRIPT INCLUDE - -# This script runs from its own folder -cd "$(dirname "$THIS_SCRIPT")" - -display_usage ( ) { - echo "build.sh [-skip-package-install | -S] [-test]" - echo - echo " -test to compile for testing without re-fetching external dependencies" - echo " or recompiling the lm-layer module." - echo " -skip-package-install (or -S) skips the `npm install` dependency check." - echo "" - echo " If more than one target is specified, the last one will take precedence." - exit 1 -} - -# Establish default build parameters -set_default_vars ( ) { - BUILD_LMLAYER=true - BUILD_CORE=true - FETCH_DEPS=true -} - -set_default_vars - -# Parse args -while [[ $# -gt 0 ]] ; do - key="$1" - case $key in - -test) - set_default_vars - BUILD_LMLAYER=false - FETCH_DEPS=false - ;; - -skip-package-install|-S) - set_default_vars - FETCH_DEPS=false - ;; - esac - shift # past argument -done - -if [ $FETCH_DEPS = true ]; then - verify_npm_setup - # We need to build keyman-version with a script for now - "$KEYMAN_ROOT/common/web/keyman-version/build.sh" || fail "Could not build keyman-version" -fi - -if [ $BUILD_LMLAYER = true ]; then - FLAGS="-skip-package-install" - - # Ensure that the LMLayer compiles properly, readying the build product for comsumption by KMW. - cd ../../../predictive-text/ - echo "" - echo "Compiling the Language Modeling layer module..." - ./build.sh $FLAGS || fail "Failed to compile the language modeling layer module." - cd ../web/input-processor/src - # TODO: Move this back to KMW's main build script. Consider it part of the dependency update. - echo "Language Modeling layer compilation successful." - echo "" -fi - -if [ $BUILD_CORE = true ]; then - FLAGS="-skip-package-install" - - # Ensure that the KeyboardProcessor module compiles properly. - cd ../../keyboard-processor/src - echo "" - echo "Compiling Keyboard Processor module..." - ./build.sh $FLAGS || fail "Dependency build failed; aborting" - cd ../../input-processor/src - echo "Keyboard Processor module compilation successful." - echo "" -fi - -# Compile web's `keyboard-processor` module. -echo "Compiling Input Processor module..." -npm run tsc -- -b src/tsconfig.json || fail "Failed to compile the web/input-processor module." -echo "Input Processor module compilation successful." -echo "" \ No newline at end of file diff --git a/common/web/input-processor/test.sh b/common/web/input-processor/test.sh deleted file mode 100755 index 9a94dceace..0000000000 --- a/common/web/input-processor/test.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env bash - -set -eu - -# Include useful testing resource functions -## START STANDARD BUILD SCRIPT INCLUDE -# adjust relative paths as necessary -THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")" -. "$(dirname "$THIS_SCRIPT")/../../../resources/build/build-utils.sh" -## END STANDARD BUILD SCRIPT INCLUDE - -. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" - -# We should work within the script's directory, not the one we were called in. -cd "$THIS_SCRIPT_PATH" - -# A simple utility script to facilitate unit-testing for the LM Layer. -# It's rigged to be callable by NPM to facilitate testing during development when in other folders. - -display_usage ( ) { - echo "test.sh [-skip-package-install|-S] [-CI] [ -? | -h | -help]" - echo " -CI to perform continuous-integration friendly tests and reporting formatted for TeamCity" - echo " -? | -h | -help to display this help information" - echo " -skip-package-install to bypass refreshing dependencies. Useful when called by scripts that pre-fetch" - echo " (or -S)" - echo "" - exit 0 -} - -# Defaults -FLAGS= -CI_REPORTING=0 -FETCH_DEPS=true -CHAINING_FLAGS= - -# Parse args -while [[ $# -gt 0 ]] ; do - key="$1" - case $key in - -h|-help|-\?) - display_usage - exit - ;; - -CI) - CI_REPORTING=1 - CHAINING_FLAGS="$CHAINING_FLAGS --ci" - ;; - -skip-package-install|-S) - FETCH_DEPS=false - ;; - esac - shift # past argument -done - -if (( CI_REPORTING )); then - FLAGS="$FLAGS --reporter mocha-teamcity-reporter" -fi - -if [ $FETCH_DEPS = true ]; then - verify_npm_setup -fi - -# Ensures that the lexical model compiler has been built locally. -echo_heading "Preparing Lexical Model Compiler for test use" -pushd "$KEYMAN_ROOT/developer/src/kmlmc/" -./build.sh -popd - -test-headless ( ) { - npm run mocha -- --recursive $FLAGS ./tests/cases/ -} - -if [ $FETCH_DEPS = true ]; then - # Next, build the lm-worker in its proper, wrapped form - pushd "$KEYMAN_ROOT/common/web/lm-worker" - ./build.sh - popd -fi - -# First, run tests on the keyboard processor. -pushd "$KEYMAN_ROOT/common/web/keyboard-processor" -./build.sh test $CHAINING_FLAGS || fail "Tests failed by dependencies; aborting integration tests." -popd - -# Build the leaf-style, bundled version of input-processor for use in testing. -npm run tsc -- -b src/tsconfig.bundled.json || fail "Failed to compile the core/web/input-processor module." - -# Now we run our local tests. -echo_heading "Running Input Processor test suite" -test-headless || fail "Input Processor tests failed!" diff --git a/web/unit_tests/test.sh b/web/unit_tests/test.sh index a43c37e970..8d7735c5d1 100755 --- a/web/unit_tests/test.sh +++ b/web/unit_tests/test.sh @@ -132,7 +132,7 @@ cd ../tools/recorder # First: Web-core tests. pushd "$KEYMAN_ROOT/common/web/input-processor" -./test.sh $HEADLESS_FLAGS || fail "Tests failed by dependencies; aborting integration tests." +./build.sh build:tools test || fail "Tests failed by dependencies; aborting integration tests." # Once done, now we run the integrated (KeymanWeb) tests. popd From 5660fbfd04a6d9046c4fe629151326e92fb83a17 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 3 Oct 2022 15:23:08 +0700 Subject: [PATCH 06/28] fix(web): top-level -ci flag wasn't chaining correctly --- web/unit_tests/test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/unit_tests/test.sh b/web/unit_tests/test.sh index 8d7735c5d1..0dac16252c 100755 --- a/web/unit_tests/test.sh +++ b/web/unit_tests/test.sh @@ -74,7 +74,7 @@ get_browser_set_for_OS CONFIG=manual.conf.js # TODO - get/make OS-specific version DEBUG=false FLAGS= -HEADLESS_FLAGS=-skip-package-install +HEADLESS_FLAGS= # Parse args while [[ $# -gt 0 ]] ; do @@ -82,7 +82,7 @@ while [[ $# -gt 0 ]] ; do case $key in -CI) CONFIG=CI.conf.js - HEADLESS_FLAGS="$HEADLESS_FLAGS -CI" + HEADLESS_FLAGS="$HEADLESS_FLAGS --ci" ;; -log-level) shift @@ -132,7 +132,7 @@ cd ../tools/recorder # First: Web-core tests. pushd "$KEYMAN_ROOT/common/web/input-processor" -./build.sh build:tools test || fail "Tests failed by dependencies; aborting integration tests." +./build.sh build:tools test $HEADLESS_FLAGS || fail "Tests failed by dependencies; aborting integration tests." # Once done, now we run the integrated (KeymanWeb) tests. popd From d9a4d49fcfc1cada349c151e1e0a5112ee71b30b Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 4 Oct 2022 08:48:56 +0700 Subject: [PATCH 07/28] feat(common/models): pred-text library unit tests now emit CI-friendly reports --- common/predictive-text/unit_tests/test.sh | 30 +++++++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/common/predictive-text/unit_tests/test.sh b/common/predictive-text/unit_tests/test.sh index 677a399a12..d5990464ab 100755 --- a/common/predictive-text/unit_tests/test.sh +++ b/common/predictive-text/unit_tests/test.sh @@ -62,19 +62,39 @@ if builder_start_action configure :browser; then fi if builder_start_action test :libraries; then + # Note: these do not yet provide TeamCity-friendly-formatted test reports. - # Sadly, appending them to the `npm run test` doesn't work; it'll require work in these modules. + # They do not have builder-based scripts, being run directly via npm package script. + # So, for now, we add a text header to clarify what is running at each stage, in + # addition to fair bit of `pushd` and `popd`. pushd "$KEYMAN_ROOT/common/models/wordbreakers" - npm run test || fail "models/wordbreakers tests failed" + echo + echo "### Running ${BUILDER_TERM_START}common/models/wordbreaker${BUILDER_TERM_END} tests" + # NPM doesn't seem to parse the post `--` part if specified via script variable. + # So... a simple if-else will do the job for now. + if builder_has_option --ci; then + npm run test -- -reporter mocha-teamcity-reporter + else + npm run test + fi popd pushd "$KEYMAN_ROOT/common/models/templates" - npm run test || fail "models/templates tests failed" + echo + echo "### Running ${BUILDER_TERM_START}common/models/templates${BUILDER_TERM_END} tests" + if builder_has_option --ci; then + npm run test -- -reporter mocha-teamcity-reporter + else + npm run test + fi popd pushd "$KEYMAN_ROOT/common/models/types" - # Is not mocha-based. - npm run test || fail "models/types tests failed" + echo + echo "### Running ${BUILDER_TERM_START}common/models/types${BUILDER_TERM_END} tests" + # Is not mocha-based; it's TSC-based instead, as we're just ensuring that the .d.ts + # file is a proper TS declaration file. + npm run test popd builder_finish_action success test :libraries From 750a2667962f849b86048daa4bfe54f7bacc5fc2 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 4 Oct 2022 08:51:01 +0700 Subject: [PATCH 08/28] chore(common/models): adds 'flag' file for CI use --- common/predictive-text/.build-builder | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 common/predictive-text/.build-builder diff --git a/common/predictive-text/.build-builder b/common/predictive-text/.build-builder new file mode 100644 index 0000000000..2792189301 --- /dev/null +++ b/common/predictive-text/.build-builder @@ -0,0 +1,4 @@ +The presence of this file tells CI to use the new builder_ style parameters for build.sh. + +Once all branches for 16.0+ are updated to merge the chore/web/more-script-updates branch, then we +can remove this file and the corresponding bash test branches in CI. \ No newline at end of file From d97b1eba3cae770640034a0f8d5a79cc9ce06918 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 4 Oct 2022 09:20:42 +0700 Subject: [PATCH 09/28] fix(common/models): pred-text build.sh ignored --ci option --- common/predictive-text/build.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/common/predictive-text/build.sh b/common/predictive-text/build.sh index 3c7d521940..b2e48269e7 100755 --- a/common/predictive-text/build.sh +++ b/common/predictive-text/build.sh @@ -142,20 +142,25 @@ fi # Note - the actual test setup is done in a separate test script, but it's easy # enough to route the calls through. +TEST_OPTIONS= +if builder_has_option --ci; then + TEST_OPTIONS=--ci +fi + if builder_start_action test :libraries; then - ./unit_tests/test.sh test:libraries + ./unit_tests/test.sh test:libraries $TEST_OPTIONS builder_finish_action success test :libraries fi if builder_start_action test :headless; then - ./unit_tests/test.sh test:headless + ./unit_tests/test.sh test:headless $TEST_OPTIONS builder_finish_action success test :headless fi if builder_start_action test :browser; then - ./unit_tests/test.sh test:browser + ./unit_tests/test.sh test:browser $TEST_OPTIONS builder_finish_action success test :browser fi \ No newline at end of file From e347aeabfe4f69ef9e734879a75e0884f629bf4c Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 5 Oct 2022 09:04:23 +0700 Subject: [PATCH 10/28] chore(common/models): minor tweaks per review --- common/predictive-text/.build-builder | 5 +---- common/predictive-text/build.sh | 4 +--- common/predictive-text/unit_tests/test.sh | 3 +-- resources/builder.inc.sh | 4 +++- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/common/predictive-text/.build-builder b/common/predictive-text/.build-builder index 2792189301..4e15741a62 100644 --- a/common/predictive-text/.build-builder +++ b/common/predictive-text/.build-builder @@ -1,4 +1 @@ -The presence of this file tells CI to use the new builder_ style parameters for build.sh. - -Once all branches for 16.0+ are updated to merge the chore/web/more-script-updates branch, then we -can remove this file and the corresponding bash test branches in CI. \ No newline at end of file +The presence of this file tells CI to use the new builder_ style parameters for build.sh and unit_tests/test.sh. \ No newline at end of file diff --git a/common/predictive-text/build.sh b/common/predictive-text/build.sh index b2e48269e7..2433ab8481 100755 --- a/common/predictive-text/build.sh +++ b/common/predictive-text/build.sh @@ -81,9 +81,7 @@ fi # A nice, extensible method for -clean operations. Add to this as necessary. do_clean() { - if [ -d $LMLAYER_OUTPUT ]; then - rm -rf "$LMLAYER_OUTPUT" - fi + rm -rf "$LMLAYER_OUTPUT" } CLEANED= diff --git a/common/predictive-text/unit_tests/test.sh b/common/predictive-text/unit_tests/test.sh index d5990464ab..c6fc13bb4e 100755 --- a/common/predictive-text/unit_tests/test.sh +++ b/common/predictive-text/unit_tests/test.sh @@ -7,10 +7,9 @@ THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BA . "$KEYMAN_ROOT/resources/build/build-utils-ci.inc.sh" . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" -SCRIPT_ROOT="$(dirname "$THIS_SCRIPT")" # This script runs from its own folder -cd "$SCRIPT_ROOT" +cd "$THIS_SCRIPT_PATH" ################################ Main script ################################ diff --git a/resources/builder.inc.sh b/resources/builder.inc.sh index d526b60c9a..b6137ec3b0 100755 --- a/resources/builder.inc.sh +++ b/resources/builder.inc.sh @@ -189,7 +189,9 @@ _builder_failure_trap() { builder_finish_action failure $action $target - # Ensure that the exit code propagates fully. + # Make 100% sure that the exit code chains fully. + # Without this, nested scripts have failed to chain errors from npm calls past the script + # that directly executed the failed npm command. exit $trappedExitCode fi } From 18d0711ff391ceee6e8c8dfe57960a16a80bcbdf Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Wed, 17 Aug 2022 13:16:26 +0200 Subject: [PATCH 11/28] fix(linux): Fix reordering of output This change implements a commit queue which allows to control the order of the output. This ensures that any backspace we generate will be processed before the character we're adding. Requires changes in ibus (surrounding text fix (see #7072) and prefilter change) Fixes #1489, #4028, #4029, #4030, #4505, #5510, #6639 --- linux/ibus-keyman/src/engine.c | 112 +++++++++++++++++++++++---------- 1 file changed, 79 insertions(+), 33 deletions(-) diff --git a/linux/ibus-keyman/src/engine.c b/linux/ibus-keyman/src/engine.c index eda666adaa..39a56a2013 100644 --- a/linux/ibus-keyman/src/engine.c +++ b/linux/ibus-keyman/src/engine.c @@ -51,32 +51,50 @@ #define KEYMAN_LALT 56 // 0x38 #define KEYMAN_RCTRL 97 // 0x61 #define KEYMAN_RALT 100 // 0x64 +#define KEYMAN_F24_KEYCODE_OUTPUT_SENTINEL 202 +#define KEYMAN_NOCHAR_KEYSYM (0xffff | 0x1000000) // Unicode NOCHAR typedef struct _IBusKeymanEngine IBusKeymanEngine; typedef struct _IBusKeymanEngineClass IBusKeymanEngineClass; -struct _IBusKeymanEngine { - IBusEngine parent; +#define MAX_QUEUE_SIZE 100 - /* members */ - km_kbp_keyboard *keyboard; - km_kbp_state *state; - gchar *ldmlfile; - gchar *kb_name; - gchar *char_buffer; - gboolean lctrl_pressed; - gboolean rctrl_pressed; - gboolean lalt_pressed; - gboolean ralt_pressed; - gboolean emitting_keystroke; - IBusProperty *status_prop; - IBusPropList *prop_list; +struct commit_queue_item { + gchar *char_buffer; + gboolean emitting_keystroke; + guint keyval; + guint keycode; + guint state; +}; + +struct _IBusKeymanEngine { + IBusEngine parent; + + /* members */ + km_kbp_keyboard *keyboard; + km_kbp_state *state; + gchar *ldmlfile; + gchar *kb_name; + gchar *char_buffer; + gboolean lctrl_pressed; + gboolean rctrl_pressed; + gboolean lalt_pressed; + gboolean ralt_pressed; + gboolean emitting_keystroke; + guint event_keyval; + guint event_keycode; + guint event_state; + IBusProperty *status_prop; + IBusPropList *prop_list; #ifdef GDK_WINDOWING_X11 - Display *xdisplay; + Display *xdisplay; #endif #ifdef GDK_WINDOWING_WAYLAND - GdkWaylandDisplay *wldisplay; + GdkWaylandDisplay *wldisplay; #endif + + struct commit_queue_item commit_queue[MAX_QUEUE_SIZE]; + int queue_end; }; struct _IBusKeymanEngineClass { @@ -609,12 +627,8 @@ process_persist_action( return TRUE; } -static gboolean process_emit_keystroke_action(IBusKeymanEngine *keyman) { - if (keyman->char_buffer != NULL) { - ibus_keyman_engine_commit_string(keyman, keyman->char_buffer); - g_free(keyman->char_buffer); - keyman->char_buffer = NULL; - } +static gboolean +process_emit_keystroke_action(IBusKeymanEngine *keyman) { keyman->emitting_keystroke = TRUE; return TRUE; } @@ -648,17 +662,41 @@ process_capslock_action( return TRUE; } -static gboolean process_end_action(IBusKeymanEngine *keyman) { - if (keyman->char_buffer != NULL) { - ibus_keyman_engine_commit_string(keyman, keyman->char_buffer); - g_free(keyman->char_buffer); - keyman->char_buffer = NULL; - } - if (keyman->emitting_keystroke) { - keyman->emitting_keystroke = FALSE; - return FALSE; +static void +commit_text(IBusKeymanEngine *keyman) { + if (keyman->queue_end > 0) { + if (keyman->commit_queue[0].char_buffer != NULL) { + ibus_keyman_engine_commit_string(keyman, keyman->commit_queue[0].char_buffer); + g_free(keyman->commit_queue[0].char_buffer); + } + if (keyman->commit_queue[0].emitting_keystroke) { + ibus_engine_forward_key_event(keyman, keyman->commit_queue[0].keyval, keyman->commit_queue[0].keycode, keyman->commit_queue[0].state); + } + keyman->queue_end--; + memmove(keyman->commit_queue, &keyman->commit_queue[1], sizeof(struct commit_queue_item) * keyman->queue_end); } +} +static gboolean +process_end_action(IBusKeymanEngine *keyman) { + keyman->commit_queue[keyman->queue_end].char_buffer = keyman->char_buffer; + keyman->commit_queue[keyman->queue_end].emitting_keystroke = keyman->emitting_keystroke; + keyman->commit_queue[keyman->queue_end].keyval = keyman->event_keyval; + keyman->commit_queue[keyman->queue_end].keycode = keyman->event_keycode; + keyman->commit_queue[keyman->queue_end].state = keyman->event_state; + keyman->queue_end++; + if (keyman->queue_end >= MAX_QUEUE_SIZE) { + g_error("Overflow of keyman commit_queue!"); + keyman->queue_end = MAX_QUEUE_SIZE - 1; + } + keyman->char_buffer = NULL; + keyman->emitting_keystroke = FALSE; + + // Forward a fake key event to get the correct order of events so that any backspace key we + // generated will be processed before the character we're adding. We need to send a + // valid keyval/keycode combination so that it doesn't get swallowed by GTK but which + // isn't very likely used in real keyboards. F24 seems to work for that. + ibus_engine_forward_key_event(keyman, KEYMAN_NOCHAR_KEYSYM, KEYMAN_F24_KEYCODE_OUTPUT_SENTINEL, IBUS_PREFILTER_MASK); return TRUE; } @@ -724,6 +762,9 @@ ibus_keyman_engine_process_key_event( guint state ) { IBusKeymanEngine *keyman = (IBusKeymanEngine *)engine; + keyman->event_keyval = keyval; + keyman->event_keycode = keycode; + keyman->event_state = state; gboolean isKeyDown = !(state & IBUS_RELEASE_MASK); @@ -732,6 +773,11 @@ ibus_keyman_engine_process_key_event( "DAR: ibus_keyman_engine_process_key_event - keyval=0x%02x keycode=0x%02x, state=0x%02x, isKeyDown=%d", keyval, keycode, state, isKeyDown); + if (keycode == KEYMAN_F24_KEYCODE_OUTPUT_SENTINEL && (state & IBUS_PREFILTER_MASK)) { + commit_text(keyman); + return TRUE; + } + // REVIEW: why don't we handle these keys? switch (keycode) { case KEYMAN_LCTRL: @@ -805,7 +851,7 @@ ibus_keyman_engine_process_key_event( const km_kbp_action_item *action_items = km_kbp_state_action_items(keyman->state, &num_action_items); if (!process_actions(engine, action_items, num_action_items)) - return FALSE; + return TRUE; context = km_kbp_state_context(keyman->state); g_message("after processing all actions"); From e1d4169eebfcb162e8f85e3450fe5084e29f3af2 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Wed, 17 Aug 2022 16:09:53 +0200 Subject: [PATCH 12/28] feat(linux): Optimizations --- linux/ibus-keyman/src/engine.c | 105 +++++++++++++++++---------------- 1 file changed, 55 insertions(+), 50 deletions(-) diff --git a/linux/ibus-keyman/src/engine.c b/linux/ibus-keyman/src/engine.c index 39a56a2013..af473b5014 100644 --- a/linux/ibus-keyman/src/engine.c +++ b/linux/ibus-keyman/src/engine.c @@ -52,20 +52,20 @@ #define KEYMAN_RCTRL 97 // 0x61 #define KEYMAN_RALT 100 // 0x64 #define KEYMAN_F24_KEYCODE_OUTPUT_SENTINEL 202 -#define KEYMAN_NOCHAR_KEYSYM (0xffff | 0x1000000) // Unicode NOCHAR +#define KEYMAN_NOCHAR_KEYSYM (0xfdd0 | 0x1000000) // Unicode NOCHAR typedef struct _IBusKeymanEngine IBusKeymanEngine; typedef struct _IBusKeymanEngineClass IBusKeymanEngineClass; #define MAX_QUEUE_SIZE 100 -struct commit_queue_item { +typedef struct _commit_queue_item { gchar *char_buffer; gboolean emitting_keystroke; guint keyval; guint keycode; guint state; -}; +} commit_queue_item; struct _IBusKeymanEngine { IBusEngine parent; @@ -75,15 +75,11 @@ struct _IBusKeymanEngine { km_kbp_state *state; gchar *ldmlfile; gchar *kb_name; - gchar *char_buffer; gboolean lctrl_pressed; gboolean rctrl_pressed; gboolean lalt_pressed; gboolean ralt_pressed; - gboolean emitting_keystroke; - guint event_keyval; - guint event_keycode; - guint event_state; + IBusLookupTable *table; IBusProperty *status_prop; IBusPropList *prop_list; #ifdef GDK_WINDOWING_X11 @@ -93,8 +89,8 @@ struct _IBusKeymanEngine { GdkWaylandDisplay *wldisplay; #endif - struct commit_queue_item commit_queue[MAX_QUEUE_SIZE]; - int queue_end; + commit_queue_item commit_queue[MAX_QUEUE_SIZE]; + commit_queue_item *commit_item; }; struct _IBusKeymanEngineClass { @@ -268,6 +264,15 @@ static void reset_context(IBusEngine *engine) } } +static void +initialize_queue(IBusKeymanEngine *keyman, int index, int count) { + g_assert(keyman != NULL); + g_assert(index >= 0 && index < MAX_QUEUE_SIZE); + g_assert(count > 0 && count <= MAX_QUEUE_SIZE); + g_assert(index + count <= MAX_QUEUE_SIZE); + memset(&keyman->commit_queue[index], 0, sizeof(commit_queue_item) * count); +} + static void ibus_keyman_engine_init(IBusKeymanEngine *keyman) { gdk_init(NULL, NULL); @@ -333,8 +338,9 @@ ibus_keyman_engine_constructor( keyman->lctrl_pressed = FALSE; keyman->ralt_pressed = FALSE; keyman->rctrl_pressed = FALSE; - keyman->emitting_keystroke = FALSE; - gchar **split_name = g_strsplit(engine_name, ":", 2); + initialize_queue(keyman, 0, MAX_QUEUE_SIZE); + keyman->commit_item = &keyman->commit_queue[0]; + gchar **split_name = g_strsplit(engine_name, ":", 2); if (split_name[0] == NULL) { IBUS_OBJECT_CLASS (parent_class)->destroy ((IBusObject *)keyman); @@ -528,17 +534,17 @@ process_unicode_char_action( g_free(utf8); } else { g_message("unichar:U+%04x, bytes:%d, string:%s", action_item->character, numbytes, utf8); - if (keyman->char_buffer == NULL) { + if (keyman->commit_item->char_buffer == NULL) { g_message("setting buffer to converted unichar"); - keyman->char_buffer = utf8; + keyman->commit_item->char_buffer = utf8; } else { g_message("appending converted unichar to CHAR buffer"); - gchar *new_buffer = g_strjoin("", keyman->char_buffer, utf8, NULL); - g_free(keyman->char_buffer); + gchar *new_buffer = g_strjoin("", keyman->commit_item->char_buffer, utf8, NULL); + g_free(keyman->commit_item->char_buffer); g_free(utf8); - keyman->char_buffer = new_buffer; + keyman->commit_item->char_buffer = new_buffer; } - g_message("CHAR buffer is now %s", keyman->char_buffer); + g_message("CHAR buffer is now %s", keyman->commit_item->char_buffer); } return TRUE; } @@ -562,23 +568,22 @@ process_backspace_action( IBusKeymanEngine *keyman = (IBusKeymanEngine *)engine; if (action_items[i].backspace.expected_type == KM_KBP_IT_MARKER) { g_message("skipping marker type"); - } else if (keyman->char_buffer != NULL) { - // ibus_keyman_engine_commit_string(keyman, keyman->char_buffer); + } else if (keyman->commit_item->char_buffer != NULL) { g_message("removing one utf8 char from CHAR buffer"); - glong end_pos = g_utf8_strlen(keyman->char_buffer, -1); + glong end_pos = g_utf8_strlen(keyman->commit_item->char_buffer, -1); gchar *new_buffer; if (end_pos == 1) { new_buffer = NULL; g_message("resetting CHAR buffer to NULL"); } else { - new_buffer = g_utf8_substring(keyman->char_buffer, 0, end_pos - 1); + new_buffer = g_utf8_substring(keyman->commit_item->char_buffer, 0, end_pos - 1); g_message("changing CHAR buffer to :%s:", new_buffer); } - if (g_strcmp0(keyman->char_buffer, new_buffer) == 0) { + if (g_strcmp0(keyman->commit_item->char_buffer, new_buffer) == 0) { g_message("oops, CHAR buffer hasn't changed"); } - g_free(keyman->char_buffer); - keyman->char_buffer = new_buffer; + g_free(keyman->commit_item->char_buffer); + keyman->commit_item->char_buffer = new_buffer; } else { g_message( "DAR: process_backspace_action - client_capabilities=%x, %x", engine->client_capabilities, IBUS_CAP_SURROUNDING_TEXT); @@ -629,7 +634,7 @@ process_persist_action( static gboolean process_emit_keystroke_action(IBusKeymanEngine *keyman) { - keyman->emitting_keystroke = TRUE; + keyman->commit_item->emitting_keystroke = TRUE; return TRUE; } @@ -664,40 +669,38 @@ process_capslock_action( static void commit_text(IBusKeymanEngine *keyman) { - if (keyman->queue_end > 0) { - if (keyman->commit_queue[0].char_buffer != NULL) { - ibus_keyman_engine_commit_string(keyman, keyman->commit_queue[0].char_buffer); - g_free(keyman->commit_queue[0].char_buffer); + g_assert(keyman != NULL); + if (keyman->commit_item > keyman->commit_queue) { + commit_queue_item *current_item = &keyman->commit_queue[0]; + if (current_item->char_buffer != NULL) { + ibus_keyman_engine_commit_string(keyman, current_item->char_buffer); + g_free(current_item->char_buffer); } - if (keyman->commit_queue[0].emitting_keystroke) { - ibus_engine_forward_key_event(keyman, keyman->commit_queue[0].keyval, keyman->commit_queue[0].keycode, keyman->commit_queue[0].state); + if (current_item->emitting_keystroke) { + ibus_engine_forward_key_event((IBusEngine*)keyman, current_item->keyval, current_item->keycode, current_item->state); } - keyman->queue_end--; - memmove(keyman->commit_queue, &keyman->commit_queue[1], sizeof(struct commit_queue_item) * keyman->queue_end); + keyman->commit_item--; + memmove(keyman->commit_queue, &keyman->commit_queue[1], sizeof(commit_queue_item) * MAX_QUEUE_SIZE - 1); + initialize_queue(keyman, MAX_QUEUE_SIZE - 1, 1); } } static gboolean process_end_action(IBusKeymanEngine *keyman) { - keyman->commit_queue[keyman->queue_end].char_buffer = keyman->char_buffer; - keyman->commit_queue[keyman->queue_end].emitting_keystroke = keyman->emitting_keystroke; - keyman->commit_queue[keyman->queue_end].keyval = keyman->event_keyval; - keyman->commit_queue[keyman->queue_end].keycode = keyman->event_keycode; - keyman->commit_queue[keyman->queue_end].state = keyman->event_state; - keyman->queue_end++; - if (keyman->queue_end >= MAX_QUEUE_SIZE) { + g_assert(keyman != NULL); + keyman->commit_item++; + if (keyman->commit_item > &keyman->commit_queue[MAX_QUEUE_SIZE-1]) { g_error("Overflow of keyman commit_queue!"); - keyman->queue_end = MAX_QUEUE_SIZE - 1; + // TODO: log to Sentry + keyman->commit_item--; } - keyman->char_buffer = NULL; - keyman->emitting_keystroke = FALSE; // Forward a fake key event to get the correct order of events so that any backspace key we // generated will be processed before the character we're adding. We need to send a // valid keyval/keycode combination so that it doesn't get swallowed by GTK but which // isn't very likely used in real keyboards. F24 seems to work for that. - ibus_engine_forward_key_event(keyman, KEYMAN_NOCHAR_KEYSYM, KEYMAN_F24_KEYCODE_OUTPUT_SENTINEL, IBUS_PREFILTER_MASK); return TRUE; + ibus_engine_forward_key_event((IBusEngine*)keyman, KEYMAN_NOCHAR_KEYSYM, KEYMAN_F24_KEYCODE_OUTPUT_SENTINEL, IBUS_PREFILTER_MASK); } static gboolean @@ -762,9 +765,9 @@ ibus_keyman_engine_process_key_event( guint state ) { IBusKeymanEngine *keyman = (IBusKeymanEngine *)engine; - keyman->event_keyval = keyval; - keyman->event_keycode = keycode; - keyman->event_state = state; + keyman->commit_item->keyval = keyval; + keyman->commit_item->keycode = keycode; + keyman->commit_item->state = state; gboolean isKeyDown = !(state & IBUS_RELEASE_MASK); @@ -773,6 +776,8 @@ ibus_keyman_engine_process_key_event( "DAR: ibus_keyman_engine_process_key_event - keyval=0x%02x keycode=0x%02x, state=0x%02x, isKeyDown=%d", keyval, keycode, state, isKeyDown); + // This keycode is a fake keycode that we send when it's time to commit the text, ensuring the + // correct output order of backspace and text. if (keycode == KEYMAN_F24_KEYCODE_OUTPUT_SENTINEL && (state & IBUS_PREFILTER_MASK)) { commit_text(keyman); return TRUE; @@ -846,8 +851,8 @@ ibus_keyman_engine_process_key_event( // km_kbp_state_action_items to get action items size_t num_action_items; - g_free(keyman->char_buffer); - keyman->char_buffer = NULL; + g_free(keyman->commit_item->char_buffer); + keyman->commit_item->char_buffer = NULL; const km_kbp_action_item *action_items = km_kbp_state_action_items(keyman->state, &num_action_items); if (!process_actions(engine, action_items, num_action_items)) From dba20022872d91b346f67053aa45659dad19578d Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Wed, 17 Aug 2022 18:00:40 +0200 Subject: [PATCH 13/28] feat(linux): Still support older ibus versions If we're dealing with an older ibus version that doesn't have the necessary patches we fall back to the old behaviour. Otherwise we use the new output reordering. --- linux/ibus-keyman/src/engine.c | 57 ++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/linux/ibus-keyman/src/engine.c b/linux/ibus-keyman/src/engine.c index af473b5014..abc349aae1 100644 --- a/linux/ibus-keyman/src/engine.c +++ b/linux/ibus-keyman/src/engine.c @@ -634,7 +634,16 @@ process_persist_action( static gboolean process_emit_keystroke_action(IBusKeymanEngine *keyman) { - keyman->commit_item->emitting_keystroke = TRUE; + if (((IBusEngine*)keyman)->client_capabilities & IBUS_CAP_PREFILTER) { + keyman->commit_item->emitting_keystroke = TRUE; + } else { + if (keyman->commit_item->char_buffer != NULL) { + ibus_keyman_engine_commit_string(keyman, keyman->commit_item->char_buffer); + g_free(keyman->commit_item->char_buffer); + keyman->commit_item->char_buffer = NULL; + } + keyman->commit_item->emitting_keystroke = TRUE; + } return TRUE; } @@ -688,19 +697,32 @@ commit_text(IBusKeymanEngine *keyman) { static gboolean process_end_action(IBusKeymanEngine *keyman) { g_assert(keyman != NULL); - keyman->commit_item++; - if (keyman->commit_item > &keyman->commit_queue[MAX_QUEUE_SIZE-1]) { - g_error("Overflow of keyman commit_queue!"); - // TODO: log to Sentry - keyman->commit_item--; + if (((IBusEngine *)keyman)->client_capabilities & IBUS_CAP_PREFILTER) { + keyman->commit_item++; + if (keyman->commit_item > &keyman->commit_queue[MAX_QUEUE_SIZE-1]) { + g_error("Overflow of keyman commit_queue!"); + // TODO: log to Sentry + keyman->commit_item--; + } + + // Forward a fake key event to get the correct order of events so that any backspace key we + // generated will be processed before the character we're adding. We need to send a + // valid keyval/keycode combination so that it doesn't get swallowed by GTK but which + // isn't very likely used in real keyboards. F24 seems to work for that. + ibus_engine_forward_key_event((IBusEngine*)keyman, KEYMAN_NOCHAR_KEYSYM, KEYMAN_F24_KEYCODE_OUTPUT_SENTINEL, IBUS_PREFILTER_MASK); + } else { + if (keyman->commit_item->char_buffer != NULL) { + ibus_keyman_engine_commit_string(keyman, keyman->commit_item->char_buffer); + g_free(keyman->commit_item->char_buffer); + keyman->commit_item->char_buffer = NULL; + } + if (keyman->commit_item->emitting_keystroke) { + keyman->commit_item->emitting_keystroke = FALSE; + return FALSE; + } } - // Forward a fake key event to get the correct order of events so that any backspace key we - // generated will be processed before the character we're adding. We need to send a - // valid keyval/keycode combination so that it doesn't get swallowed by GTK but which - // isn't very likely used in real keyboards. F24 seems to work for that. return TRUE; - ibus_engine_forward_key_event((IBusEngine*)keyman, KEYMAN_NOCHAR_KEYSYM, KEYMAN_F24_KEYCODE_OUTPUT_SENTINEL, IBUS_PREFILTER_MASK); } static gboolean @@ -773,12 +795,13 @@ ibus_keyman_engine_process_key_event( g_message("-----------------------------------------------------------------------------------------------------------------"); g_message( - "DAR: ibus_keyman_engine_process_key_event - keyval=0x%02x keycode=0x%02x, state=0x%02x, isKeyDown=%d", keyval, keycode, - state, isKeyDown); + "DAR: ibus_keyman_engine_process_key_event - keyval=0x%02x keycode=0x%02x, state=0x%02x, isKeyDown=%d, supports_prefilter=%d", keyval, keycode, + state, isKeyDown, engine->client_capabilities & IBUS_CAP_PREFILTER); // This keycode is a fake keycode that we send when it's time to commit the text, ensuring the // correct output order of backspace and text. - if (keycode == KEYMAN_F24_KEYCODE_OUTPUT_SENTINEL && (state & IBUS_PREFILTER_MASK)) { + if ((engine->client_capabilities & IBUS_CAP_PREFILTER) && keycode == KEYMAN_F24_KEYCODE_OUTPUT_SENTINEL && + (state & IBUS_PREFILTER_MASK)) { commit_text(keyman); return TRUE; } @@ -855,8 +878,10 @@ ibus_keyman_engine_process_key_event( keyman->commit_item->char_buffer = NULL; const km_kbp_action_item *action_items = km_kbp_state_action_items(keyman->state, &num_action_items); - if (!process_actions(engine, action_items, num_action_items)) - return TRUE; + if (!process_actions(engine, action_items, num_action_items) && + !(engine->client_capabilities & IBUS_CAP_PREFILTER)) { + return FALSE; + } context = km_kbp_state_context(keyman->state); g_message("after processing all actions"); From e6601f205c0e6472bb75eae3c2dfaf8d4dcfab89 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Tue, 23 Aug 2022 18:16:46 +0200 Subject: [PATCH 14/28] feat(linux): match state of original event when creating fake event We forward a fake event for both keydown and keyup. This change will set the flag on the fake event so that we get keydown and keyup. This is probably not technically necessary, but it helps in debugging because you can then see that both fake events still belong to the same keypress. --- linux/ibus-keyman/src/engine.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/linux/ibus-keyman/src/engine.c b/linux/ibus-keyman/src/engine.c index abc349aae1..3df284f1c4 100644 --- a/linux/ibus-keyman/src/engine.c +++ b/linux/ibus-keyman/src/engine.c @@ -698,6 +698,7 @@ static gboolean process_end_action(IBusKeymanEngine *keyman) { g_assert(keyman != NULL); if (((IBusEngine *)keyman)->client_capabilities & IBUS_CAP_PREFILTER) { + guint state = keyman->commit_item->state; keyman->commit_item++; if (keyman->commit_item > &keyman->commit_queue[MAX_QUEUE_SIZE-1]) { g_error("Overflow of keyman commit_queue!"); @@ -709,7 +710,12 @@ process_end_action(IBusKeymanEngine *keyman) { // generated will be processed before the character we're adding. We need to send a // valid keyval/keycode combination so that it doesn't get swallowed by GTK but which // isn't very likely used in real keyboards. F24 seems to work for that. - ibus_engine_forward_key_event((IBusEngine*)keyman, KEYMAN_NOCHAR_KEYSYM, KEYMAN_F24_KEYCODE_OUTPUT_SENTINEL, IBUS_PREFILTER_MASK); + ibus_engine_forward_key_event((IBusEngine*)keyman, + KEYMAN_NOCHAR_KEYSYM, + KEYMAN_F24_KEYCODE_OUTPUT_SENTINEL, + (state & IBUS_RELEASE_MASK) + ? IBUS_PREFILTER_MASK | IBUS_RELEASE_MASK + : IBUS_PREFILTER_MASK); } else { if (keyman->commit_item->char_buffer != NULL) { ibus_keyman_engine_commit_string(keyman, keyman->commit_item->char_buffer); From 908dfbeefb8b7509ce10edfb337ffb3fd2b1608d Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Tue, 23 Aug 2022 18:42:24 +0200 Subject: [PATCH 15/28] feat(linux): Optimize if surrounding text is supported Only send fake event if client does not support surrounding text. --- linux/ibus-keyman/src/engine.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/linux/ibus-keyman/src/engine.c b/linux/ibus-keyman/src/engine.c index 3df284f1c4..99ecab66ea 100644 --- a/linux/ibus-keyman/src/engine.c +++ b/linux/ibus-keyman/src/engine.c @@ -634,7 +634,9 @@ process_persist_action( static gboolean process_emit_keystroke_action(IBusKeymanEngine *keyman) { - if (((IBusEngine*)keyman)->client_capabilities & IBUS_CAP_PREFILTER) { + IBusEngine *engine = (IBusEngine *)keyman; + if ((engine->client_capabilities & IBUS_CAP_PREFILTER) && + !(engine->client_capabilities & IBUS_CAP_SURROUNDING_TEXT)) { keyman->commit_item->emitting_keystroke = TRUE; } else { if (keyman->commit_item->char_buffer != NULL) { @@ -647,8 +649,8 @@ process_emit_keystroke_action(IBusKeymanEngine *keyman) { return TRUE; } -static gboolean process_invalidate_context_action(IBusEngine *engine) { - IBusKeymanEngine *keyman = (IBusKeymanEngine *)engine; +static gboolean +process_invalidate_context_action(IBusEngine *engine) { reset_context(engine); return TRUE; } @@ -697,7 +699,9 @@ commit_text(IBusKeymanEngine *keyman) { static gboolean process_end_action(IBusKeymanEngine *keyman) { g_assert(keyman != NULL); - if (((IBusEngine *)keyman)->client_capabilities & IBUS_CAP_PREFILTER) { + IBusEngine *engine = (IBusEngine *)keyman; + if ((engine->client_capabilities & IBUS_CAP_PREFILTER) && + !(engine->client_capabilities & IBUS_CAP_SURROUNDING_TEXT)) { guint state = keyman->commit_item->state; keyman->commit_item++; if (keyman->commit_item > &keyman->commit_queue[MAX_QUEUE_SIZE-1]) { @@ -806,8 +810,9 @@ ibus_keyman_engine_process_key_event( // This keycode is a fake keycode that we send when it's time to commit the text, ensuring the // correct output order of backspace and text. - if ((engine->client_capabilities & IBUS_CAP_PREFILTER) && keycode == KEYMAN_F24_KEYCODE_OUTPUT_SENTINEL && - (state & IBUS_PREFILTER_MASK)) { + if ((engine->client_capabilities & IBUS_CAP_PREFILTER) && + !(engine->client_capabilities & IBUS_CAP_SURROUNDING_TEXT) && + keycode == KEYMAN_F24_KEYCODE_OUTPUT_SENTINEL && (state & IBUS_PREFILTER_MASK)) { commit_text(keyman); return TRUE; } @@ -885,7 +890,7 @@ ibus_keyman_engine_process_key_event( const km_kbp_action_item *action_items = km_kbp_state_action_items(keyman->state, &num_action_items); if (!process_actions(engine, action_items, num_action_items) && - !(engine->client_capabilities & IBUS_CAP_PREFILTER)) { + (!(engine->client_capabilities & IBUS_CAP_PREFILTER) || (engine->client_capabilities & IBUS_CAP_SURROUNDING_TEXT))) { return FALSE; } From bcd29634bf662a5f021d18192972743f99e52919 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Fri, 9 Sep 2022 10:31:16 +0200 Subject: [PATCH 16/28] feat(linux): Address code review comments Co-authored-by: Marc Durdin --- linux/ibus-keyman/src/engine.c | 91 ++++++++++++++++++++++------------ 1 file changed, 59 insertions(+), 32 deletions(-) diff --git a/linux/ibus-keyman/src/engine.c b/linux/ibus-keyman/src/engine.c index 99ecab66ea..cc48ddffc8 100644 --- a/linux/ibus-keyman/src/engine.c +++ b/linux/ibus-keyman/src/engine.c @@ -226,9 +226,22 @@ static gchar *get_current_context_text(km_kbp_context *context) return current_context_utf8; } -static void reset_context(IBusEngine *engine) +static gboolean +client_supports_prefilter(IBusEngine *engine) { - IBusKeymanEngine *keyman = (IBusKeymanEngine *) engine; + g_assert(engine != NULL); + return (engine->client_capabilities & IBUS_CAP_PREFILTER) != 0; +} + +static gboolean +client_supports_surrounding_text(IBusEngine *engine) { + g_assert(engine != NULL); + return (engine->client_capabilities & IBUS_CAP_SURROUNDING_TEXT) != 0; +} + +static void +reset_context(IBusEngine *engine) { + IBusKeymanEngine *keyman = (IBusKeymanEngine *)engine; IBusText *text; gchar *surrounding_text, *current_context_utf8; guint cursor_pos, anchor_pos, context_start, context_pos; @@ -588,7 +601,7 @@ process_backspace_action( g_message( "DAR: process_backspace_action - client_capabilities=%x, %x", engine->client_capabilities, IBUS_CAP_SURROUNDING_TEXT); - if ((engine->client_capabilities & IBUS_CAP_SURROUNDING_TEXT) != 0) { + if (client_supports_surrounding_text(engine)) { g_message("deleting surrounding text 1 char"); ibus_engine_delete_surrounding_text(engine, -1, 1); } else { @@ -635,17 +648,13 @@ process_persist_action( static gboolean process_emit_keystroke_action(IBusKeymanEngine *keyman) { IBusEngine *engine = (IBusEngine *)keyman; - if ((engine->client_capabilities & IBUS_CAP_PREFILTER) && - !(engine->client_capabilities & IBUS_CAP_SURROUNDING_TEXT)) { - keyman->commit_item->emitting_keystroke = TRUE; - } else { - if (keyman->commit_item->char_buffer != NULL) { - ibus_keyman_engine_commit_string(keyman, keyman->commit_item->char_buffer); - g_free(keyman->commit_item->char_buffer); - keyman->commit_item->char_buffer = NULL; - } - keyman->commit_item->emitting_keystroke = TRUE; + if ((!client_supports_prefilter(engine) || client_supports_surrounding_text(engine)) && + keyman->commit_item->char_buffer != NULL) { + ibus_keyman_engine_commit_string(keyman, keyman->commit_item->char_buffer); + g_free(keyman->commit_item->char_buffer); + keyman->commit_item->char_buffer = NULL; } + keyman->commit_item->emitting_keystroke = TRUE; return TRUE; } @@ -681,27 +690,27 @@ process_capslock_action( static void commit_text(IBusKeymanEngine *keyman) { g_assert(keyman != NULL); - if (keyman->commit_item > keyman->commit_queue) { - commit_queue_item *current_item = &keyman->commit_queue[0]; - if (current_item->char_buffer != NULL) { - ibus_keyman_engine_commit_string(keyman, current_item->char_buffer); - g_free(current_item->char_buffer); - } - if (current_item->emitting_keystroke) { - ibus_engine_forward_key_event((IBusEngine*)keyman, current_item->keyval, current_item->keycode, current_item->state); - } - keyman->commit_item--; - memmove(keyman->commit_queue, &keyman->commit_queue[1], sizeof(commit_queue_item) * MAX_QUEUE_SIZE - 1); - initialize_queue(keyman, MAX_QUEUE_SIZE - 1, 1); + if (keyman->commit_item <= keyman->commit_queue) + return; + + commit_queue_item *current_item = &keyman->commit_queue[0]; + if (current_item->char_buffer != NULL) { + ibus_keyman_engine_commit_string(keyman, current_item->char_buffer); + g_free(current_item->char_buffer); } + if (current_item->emitting_keystroke) { + ibus_engine_forward_key_event((IBusEngine*)keyman, current_item->keyval, current_item->keycode, current_item->state); + } + keyman->commit_item--; + memmove(keyman->commit_queue, &keyman->commit_queue[1], sizeof(commit_queue_item) * MAX_QUEUE_SIZE - 1); + initialize_queue(keyman, MAX_QUEUE_SIZE - 1, 1); } static gboolean process_end_action(IBusKeymanEngine *keyman) { g_assert(keyman != NULL); IBusEngine *engine = (IBusEngine *)keyman; - if ((engine->client_capabilities & IBUS_CAP_PREFILTER) && - !(engine->client_capabilities & IBUS_CAP_SURROUNDING_TEXT)) { + if (client_supports_prefilter(engine) && !client_supports_surrounding_text(engine)) { guint state = keyman->commit_item->state; keyman->commit_item++; if (keyman->commit_item > &keyman->commit_queue[MAX_QUEUE_SIZE-1]) { @@ -728,10 +737,19 @@ process_end_action(IBusKeymanEngine *keyman) { } if (keyman->commit_item->emitting_keystroke) { keyman->commit_item->emitting_keystroke = FALSE; + // We have an old ibus version without prefilter support, or a client that does support + // surrounding text. In either case we return FALSE because we emitted a keystroke + // so that the processing of the event will continue. return FALSE; } } + // If we have a new ibus version that supports prefilter and a client that doesn't support + // surrounding text (e.g. Chromium as of v104) we forwarded the key event with + // IBUS_PREFILTER_MASK set and now return TRUE here to stop further processing. + // With an old ibus version without prefilter support, or with a client that does support + // surrounding text, we return TRUE if we completely processed the event and no further + // processing should happen. return TRUE; } @@ -748,6 +766,7 @@ process_actions( case KM_KBP_IT_CHAR: g_message("CHAR action %d/%d", i + 1, (int)num_action_items); continue_with_next_action = process_unicode_char_action(keyman, &action_items[i]); + g_assert(continue_with_next_action == TRUE); break; case KM_KBP_IT_MARKER: g_message("MARKER action %d/%d", i + 1, (int)num_action_items); @@ -755,26 +774,32 @@ process_actions( case KM_KBP_IT_ALERT: g_message("ALERT action %d/%d", i + 1, (int)num_action_items); continue_with_next_action = process_alert_action(); + g_assert(continue_with_next_action == TRUE); break; case KM_KBP_IT_BACK: g_message("BACK action %d/%d", i + 1, (int)num_action_items); continue_with_next_action = process_backspace_action(engine, action_items, i, num_action_items); + g_assert(continue_with_next_action == TRUE); break; case KM_KBP_IT_PERSIST_OPT: g_message("PERSIST_OPT action %d/%d", i + 1, (int)num_action_items); continue_with_next_action = process_persist_action(keyman, &action_items[i]); + g_assert(continue_with_next_action == TRUE); break; case KM_KBP_IT_EMIT_KEYSTROKE: g_message("EMIT_KEYSTROKE action %d/%d", i + 1, (int)num_action_items); continue_with_next_action = process_emit_keystroke_action(keyman); + g_assert(continue_with_next_action == TRUE); break; case KM_KBP_IT_INVALIDATE_CONTEXT: g_message("INVALIDATE_CONTEXT action %d/%d", i + 1, (int)num_action_items); continue_with_next_action = process_invalidate_context_action(engine); + g_assert(continue_with_next_action == TRUE); break; case KM_KBP_IT_CAPSLOCK: g_message("CAPSLOCK action %d/%d", i + 1, (int)num_action_items); continue_with_next_action = process_capslock_action(keyman, &action_items[i]); + g_assert(continue_with_next_action == TRUE); break; case KM_KBP_IT_END: g_message("END action %d/%d", i + 1, (int)num_action_items); @@ -806,12 +831,11 @@ ibus_keyman_engine_process_key_event( g_message("-----------------------------------------------------------------------------------------------------------------"); g_message( "DAR: ibus_keyman_engine_process_key_event - keyval=0x%02x keycode=0x%02x, state=0x%02x, isKeyDown=%d, supports_prefilter=%d", keyval, keycode, - state, isKeyDown, engine->client_capabilities & IBUS_CAP_PREFILTER); + state, isKeyDown, client_supports_prefilter(engine)); // This keycode is a fake keycode that we send when it's time to commit the text, ensuring the // correct output order of backspace and text. - if ((engine->client_capabilities & IBUS_CAP_PREFILTER) && - !(engine->client_capabilities & IBUS_CAP_SURROUNDING_TEXT) && + if (client_supports_prefilter(engine) && !client_supports_surrounding_text(engine) && keycode == KEYMAN_F24_KEYCODE_OUTPUT_SENTINEL && (state & IBUS_PREFILTER_MASK)) { commit_text(keyman); return TRUE; @@ -886,11 +910,14 @@ ibus_keyman_engine_process_key_event( // km_kbp_state_action_items to get action items size_t num_action_items; g_free(keyman->commit_item->char_buffer); - keyman->commit_item->char_buffer = NULL; + keyman->commit_item->char_buffer = NULL; const km_kbp_action_item *action_items = km_kbp_state_action_items(keyman->state, &num_action_items); if (!process_actions(engine, action_items, num_action_items) && - (!(engine->client_capabilities & IBUS_CAP_PREFILTER) || (engine->client_capabilities & IBUS_CAP_SURROUNDING_TEXT))) { + (!client_supports_prefilter(engine) || client_supports_surrounding_text(engine))) { + // If we have an old ibus version without prefilter support, or a client that supports + // surrounding text, and we forwarded a key event we want to return FALSE so that the + // processing of the event continues. return FALSE; } From 0cf307066d01bea2f88f1328fd91fd1e755f23b9 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Tue, 11 Oct 2022 16:57:11 +0200 Subject: [PATCH 17/28] chore: Ignore `.devcontainer` folder [VSCode dev containers](https://code.visualstudio.com/docs/remote/containers) allows to develop in a container and is configured through files in `.devcontainer`. However, that directory falls in the same category as `.vscode` in that it might be different on each developer's machine, depending on preference and platform he's working on. Therefore we exclude this directory similar to `.vscode`. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index cc536b78a0..b9febfb451 100644 --- a/.gitignore +++ b/.gitignore @@ -146,6 +146,7 @@ # VS Code .vscode/ .vs/ +.devcontainer/ # IDE files **/.idea/**/*.xml From 71ed48ba2e39afc98d45e7514809f2578643aa09 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Tue, 11 Oct 2022 15:56:36 -0500 Subject: [PATCH 18/28] chore(linux): Add Node.js to the docker container - used in feature-ldml Relates to #7102 --- linux/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/linux/Dockerfile b/linux/Dockerfile index 203d33aef1..8572b983f8 100644 --- a/linux/Dockerfile +++ b/linux/Dockerfile @@ -21,4 +21,6 @@ ADD debian/control /tmp/control # Answer 'yes' to install questions RUN (yes | mk-build-deps --install /tmp/control) || true # now, switch to build user +RUN curl -sL https://deb.nodesource.com/setup_16.x | bash +RUN apt-get -q -y install nodejs USER build From 1cecb87f7f28a29413c2873c4d2846db1f0ccb25 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 12 Oct 2022 12:50:26 +1100 Subject: [PATCH 19/28] feat(common): builder dependency support --- common/predictive-text/build.sh | 74 +--- common/predictive-text/unit_tests/test.sh | 28 +- common/web/input-processor/build.sh | 94 +--- common/web/keyboard-processor/build.sh | 29 +- common/web/keyman-version/build.sh | 18 +- common/web/lm-worker/build.sh | 8 +- common/web/recorder/build.sh | 27 +- common/web/utils/build.sh | 18 +- resources/build/tests/builder-deps.test.sh | 96 ++++ resources/build/tests/builder.inc.test.sh | 7 +- resources/builder.inc.sh | 491 +++++++++++++++++++-- resources/shellHelperFunctions.sh | 50 +-- 12 files changed, 673 insertions(+), 267 deletions(-) create mode 100755 resources/build/tests/builder-deps.test.sh mode change 100644 => 100755 resources/build/tests/builder.inc.test.sh diff --git a/common/predictive-text/build.sh b/common/predictive-text/build.sh index 2433ab8481..d099d6dd81 100755 --- a/common/predictive-text/build.sh +++ b/common/predictive-text/build.sh @@ -25,80 +25,51 @@ cd "$(dirname "$THIS_SCRIPT")" builder_check_color "$@" builder_describe "Builds the lm-layer module" \ + "@../web/keyman-version" \ + "@../web/lm-worker" \ "clean" \ "configure" \ "build" \ "test" \ - ":libraries Targets all in-repo libraries that this module is dependent upon" \ ":headless A headless, Node-oriented version of the module useful for unit tests" \ ":browser The standard version of the module for in-browser use" \ "--ci Sets ${BUILDER_TERM_START}test${BUILDER_TERM_END} action to use CI-based test configurations & reporting" +builder_describe_outputs \ + configure:headless /node_modules \ + configure:browser /node_modules \ + build:headless build/headless.js \ + build:browser build/index.js + builder_parse "$@" # Exit status on invalid usage. -EX_USAGE=64 LMLAYER_OUTPUT=build ### CONFIGURE ACTIONS -do_configure() { - # Check if Node.JS/npm is installed. - verify_npm_setup -} - -CONFIGURED= -if builder_start_action configure :libraries; then - do_configure - CONFIGURED=configure:libraries - - builder_finish_action success configure :libraries -fi - - if builder_start_action configure :browser; then - if [ -n "$CONFIGURED" ]; then - echo "Configuration already completed in ${BUILDER_TERM_START}${CONFIGURED}${BUILDER_TERM_END}; skipping." - else - do_configure - CONFIGURED=configure:browser - fi + verify_npm_setup builder_finish_action success configure :browser fi if builder_start_action configure :headless; then - if [ -n "$CONFIGURED" ]; then - echo "Configuration already completed in ${BUILDER_TERM_START}${CONFIGURED}${BUILDER_TERM_END}; skipping." - else - do_configure - CONFIGURED=configure:headless - fi - + verify_npm_setup builder_finish_action success configure :headless fi ### CLEAN ACTIONS # A nice, extensible method for -clean operations. Add to this as necessary. +# TODO: separate clean for different targets do_clean() { rm -rf "$LMLAYER_OUTPUT" } CLEANED= -if builder_start_action clean :libraries; then - do_clean - CLEANED=clean:libraries - - builder_finish_action success clean :libraries -fi - if builder_start_action clean :browser; then - if [ -n "$CLEANED" ]; then - echo "${BUILDER_TERM_START}clean${BUILDER_TERM_END} already completed as ${BUILDER_TERM_START}${CLEANED}${BUILDER_TERM_END}; skipping." - else - do_clean - CLEANED=clean:browser - fi + do_clean + CLEANED=clean:browser builder_finish_action success clean :browser fi @@ -115,13 +86,6 @@ fi ### BUILD ACTIONS -if builder_start_action build :libraries; then - "$KEYMAN_ROOT/common/web/keyman-version/build.sh" - "$KEYMAN_ROOT/common/web/lm-worker/build.sh" - - builder_finish_action success build :libraries -fi - # Builds the top-level JavaScript file for use in browsers if builder_start_action build :browser; then npm run tsc -- -b ./browser.tsconfig.json @@ -131,7 +95,7 @@ fi # Builds the top-level JavaScript file for use on Node if builder_start_action build :headless; then - npm run tsc -- -b ./tsconfig.json || fail + npm run tsc -- -b ./tsconfig.json builder_finish_action success build :headless fi @@ -145,14 +109,10 @@ if builder_has_option --ci; then TEST_OPTIONS=--ci fi -if builder_start_action test :libraries; then - ./unit_tests/test.sh test:libraries $TEST_OPTIONS - - builder_finish_action success test :libraries -fi - if builder_start_action test :headless; then - ./unit_tests/test.sh test:headless $TEST_OPTIONS + # We'll test the included libraries here for now, at least until we have + # converted their builds to builder scripts + ./unit_tests/test.sh test:libraries test:headless $TEST_OPTIONS builder_finish_action success test :headless fi diff --git a/common/predictive-text/unit_tests/test.sh b/common/predictive-text/unit_tests/test.sh index c6fc13bb4e..e7d23f0b55 100755 --- a/common/predictive-text/unit_tests/test.sh +++ b/common/predictive-text/unit_tests/test.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash -# We should work within the script's directory, not the one we were called in. +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")" . "$(dirname "$THIS_SCRIPT")/../../../resources/build/build-utils.sh" ## END STANDARD BUILD SCRIPT INCLUDE @@ -25,38 +26,23 @@ builder_describe "Runs all tests for the language-modeling / predictive-text lay "--ci Uses CI-based test configurations & emits CI-friendly test reports" \ "--debug,-d Activates developer-friendly debug mode for unit tests where applicable" +# TODO: consider dependencies? ideally this will be test.inc.sh? + builder_parse "$@" -do_configure() { - # Ensure all testing dependencies are in place. - verify_npm_setup -} - -CONFIGURED= - if builder_start_action configure :libraries; then # Ensure all testing dependencies are in place. - do_configure - CONFIGURED=configure:libraries + verify_npm_setup builder_finish_action success configure :libraries fi if builder_start_action configure :headless; then - if [ -n "$CONFIGURED" ]; then - echo "Configuration already completed in ${BUILDER_TERM_START}${CONFIGURED}${BUILDER_TERM_END}; skipping." - else - do_configure - CONFIGURED=configure:headless - fi + verify_npm_setup builder_finish_action success configure :headless fi if builder_start_action configure :browser; then - if [[ -n "$CONFIGURED" ]]; then - echo "Configuration already completed in ${BUILDER_TERM_START}${CONFIGURED}${BUILDER_TERM_END}; skipping." - else - do_configure - fi + verify_npm_setup builder_finish_action success configure :browser fi diff --git a/common/web/input-processor/build.sh b/common/web/input-processor/build.sh index cf280b0f0e..6fb2f8dcb8 100755 --- a/common/web/input-processor/build.sh +++ b/common/web/input-processor/build.sh @@ -18,83 +18,55 @@ cd "$(dirname "$THIS_SCRIPT")" builder_check_color "$@" +# TODO: for predictive-text, we only need :headless, perhaps we should be splitting modules? +# TODO: remove :tools once kmlmc is a dependency for test:module + builder_describe "Builds the standalone, headless form of Keyman Engine for Web's input-processor module" \ + "@../keyman-version" \ + "@../keyboard-processor" \ + "@../../predictive-text" \ "clean" \ "configure" \ "build" \ "test" \ - ":libraries Targets all in-repo libraries that this module is dependent upon" \ ":module A headless, Node-oriented version of the module useful for unit tests" \ ":tools Related tools useful for development and testing of this module" \ "--ci Sets ${BUILDER_TERM_START}test${BUILDER_TERM_END} action to use CI-based test configurations & reporting" +builder_describe_outputs \ + configure:module /node_modules \ + configure:tools /node_modules \ + build:module build/index.js \ + build:tools /developer/src/kmlmc/dist/kmlmc.js # TODO: remove this once kmlmc is a dependency + builder_parse "$@" ### CONFIGURE ACTIONS -do_configure() { - # Check if Node.JS/npm is installed. - verify_npm_setup - - "$KEYMAN_ROOT/common/web/keyman-version/build.sh" -} - -CONFIGURED= -if builder_start_action configure :libraries; then - do_configure - CONFIGURED=configure:libraries - - builder_finish_action success configure :libraries -fi - - if builder_start_action configure :module; then - if [ -n "$CONFIGURED" ]; then - echo "Configuration already completed in ${BUILDER_TERM_START}${CONFIGURED}${BUILDER_TERM_END}; skipping." - else - do_configure - CONFIGURED=configure:module - fi + verify_npm_setup builder_finish_action success configure :module fi if builder_start_action configure :tools; then - if [ -n "$CONFIGURED" ]; then - echo "Configuration already completed in ${BUILDER_TERM_START}${CONFIGURED}${BUILDER_TERM_END}; skipping." - else - do_configure - CONFIGURED=configure:tools - fi - + verify_npm_setup builder_finish_action success configure :tools fi ### CLEAN ACTIONS -MODULE_OUTPUT=build - # A nice, extensible method for -clean operations. Add to this as necessary. do_clean() { - if [ -d $MODULE_OUTPUT ]; then - rm -rf "$MODULE_OUTPUT" + if [ -d build ]; then + rm -rf build/ fi } CLEANED= -if builder_start_action clean :libraries; then - do_clean - CLEANED=clean:libraries - - builder_finish_action success clean :libraries -fi if builder_start_action clean :module; then - if [ -n "$CLEANED" ]; then - echo "${BUILDER_TERM_START}clean${BUILDER_TERM_END} already completed as ${BUILDER_TERM_START}${CLEANED}${BUILDER_TERM_END}; skipping." - else - do_clean - CLEANED=clean:module - fi + do_clean + CLEANED=clean:module builder_finish_action success clean :module fi @@ -111,17 +83,9 @@ fi ### BUILD ACTIONS -if builder_start_action build :libraries; then - # Ensure that the LMLayer compiles properly, readying the build product for comsumption by KMW. - "$KEYMAN_ROOT/common/predictive-text/build.sh" build:headless - # Also ensure that the keyboard-processor module builds appropriately. - "$KEYMAN_ROOT/common/web/keyboard-processor/build.sh" build - - builder_finish_action success build :libraries -fi - if builder_start_action build :tools; then # Used by test:module + # TODO: convert to a dependency once we have updated kmlmc to use builder script pushd "$KEYMAN_ROOT/developer/src/kmlmc" ./build.sh -S popd @@ -131,31 +95,11 @@ fi if builder_start_action build :module; then npm run tsc -- -b src/tsconfig.json - builder_finish_action success build :module fi # TEST ACTIONS -if builder_start_action test :tools; then - builder_finish_action success test :tools -fi - -if builder_start_action test :libraries; then - CHAINING_FLAGS= - if builder_has_option --ci; then - CHAINING_FLAGS=--ci - fi - # First, run tests on the keyboard processor. - pushd "$KEYMAN_ROOT/common/web/keyboard-processor" - ./build.sh test $CHAINING_FLAGS - popd - - # lm-layer tests are handled outside of this chain, at least by our existing CI processes. - - builder_finish_action success test :libraries -fi - if builder_start_action test :module; then FLAGS= if builder_has_option --ci; then diff --git a/common/web/keyboard-processor/build.sh b/common/web/keyboard-processor/build.sh index 512d40693f..86c02f9806 100755 --- a/common/web/keyboard-processor/build.sh +++ b/common/web/keyboard-processor/build.sh @@ -23,30 +23,23 @@ builder_check_color "$@" builder_describe \ "Compiles the web-oriented utility function module." \ + "@../recorder test" \ + "@../keyman-version" \ + "@../utils" \ configure \ clean \ build \ test \ "--ci For use with action ${BUILDER_TERM_START}test${BUILDER_TERM_END} - emits CI-friendly test reports" +builder_describe_outputs \ + configure /node_modules \ + build build/index.js + builder_parse "$@" -# START - Script parameter configuration -REPORT_STYLE=local # Default setting. - -if builder_has_option --ci; then - REPORT_STYLE=ci - - echo "Replacing user-friendly test reports with CI-friendly versions." -fi - -# END - Script parameter configuration - if builder_start_action configure; then verify_npm_setup - - "$KEYMAN_ROOT/common/web/keyman-version/build.sh" - builder_finish_action success configure fi @@ -61,17 +54,13 @@ if builder_start_action build; then fi if builder_start_action test; then - # Build test dependency - pushd "$KEYMAN_ROOT/common/web/recorder" - ./build.sh - popd - npm run tsc -- --build "$THIS_SCRIPT_PATH/src/tsconfig.bundled.json" echo_heading "Running Keyboard Processor test suite" FLAGS= - if [ $REPORT_STYLE == ci ]; then + if builder_has_option --ci; then + echo "Replacing user-friendly test reports with CI-friendly versions." FLAGS="$FLAGS --reporter mocha-teamcity-reporter" fi diff --git a/common/web/keyman-version/build.sh b/common/web/keyman-version/build.sh index e2e021310a..370c33ee88 100755 --- a/common/web/keyman-version/build.sh +++ b/common/web/keyman-version/build.sh @@ -15,15 +15,18 @@ THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BA . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" # This script runs from its own folder -cd "$(dirname "$THIS_SCRIPT")" +cd "$THIS_SCRIPT_PATH" ################################ Main script ################################ builder_describe "Build the include script for current Keyman version" configure clean build + +builder_describe_outputs \ + configure "/node_modules" \ + build "build/index.js" + builder_parse "$@" -# TODO: build if out-of-date if test is specified -# TODO: configure if npm has not been run, and build is specified if builder_start_action configure; then verify_npm_setup @@ -56,7 +59,12 @@ if builder_start_action build; then } " > ./version.inc.ts - # Build - npm run build -- $builder_verbose + # Note: in a dependency build, we'll expect keyman-version to be built by tsc -b + if builder_is_dep_build; then + echo "[$THIS_SCRIPT_IDENTIFIER] skipping tsc -b; will be completed by $builder_dep_parent" + else + npm run build -- $builder_verbose + fi + builder_finish_action success build fi diff --git a/common/web/lm-worker/build.sh b/common/web/lm-worker/build.sh index 487d564da6..1633e66753 100755 --- a/common/web/lm-worker/build.sh +++ b/common/web/lm-worker/build.sh @@ -76,8 +76,13 @@ wrap-worker-code ( ) { builder_describe \ "Compiles the Language Modeling Layer for common use in predictive text and autocorrective applications." \ + "@../keyman-version" \ configure clean build test +builder_describe_outputs \ + configure /node_modules \ + build build/index.js + builder_parse "$@" # TODO: build if out-of-date if test is specified @@ -105,9 +110,6 @@ if builder_start_action build; then npm run clean fi - # Ensure keyman-version is properly build (requires build script) - "$KEYMAN_ROOT/common/web/keyman-version/build.sh" || fail "Could not build keyman-version" - # Build worker with tsc first npm run build -- $builder_verbose || fail "Could not build worker." diff --git a/common/web/recorder/build.sh b/common/web/recorder/build.sh index 441272e78f..f92aa9e887 100755 --- a/common/web/recorder/build.sh +++ b/common/web/recorder/build.sh @@ -19,32 +19,23 @@ THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BA builder_describe \ "Compiles the web-oriented utility function module." \ + "@../keyman-version" \ configure \ clean \ build \ ":module Builds recorder-core module" \ ":proctor Builds headless-testing, node-oriented 'proctor' component" +builder_describe_outputs \ + configure:module "/node_modules" \ + configure:proctor "/node_modules" \ + build:module "build/index.js" \ + build:proctor "build/nodeProctor/index.js" + builder_parse "$@" -# START - Script parameter configuration -REPORT_STYLE="local" # Default setting. - -if builder_has_option --ci; then - REPORT_STYLE="ci" - - echo "Replacing user-friendly test reports with CI-friendly versions." -fi - -# END - Script parameter configuration - -function do_configure() { - verify_npm_setup - "$KEYMAN_ROOT/common/web/keyman-version/build.sh" -} - if builder_start_action configure :module; then - do_configure + verify_npm_setup builder_finish_action success configure :module fi @@ -52,7 +43,7 @@ if builder_start_action configure :proctor; then if builder_has_action configure :module; then echo "Configuration already completed in configure:module; skipping." else - do_configure + verify_npm_setup fi builder_finish_action success configure :proctor fi diff --git a/common/web/utils/build.sh b/common/web/utils/build.sh index 1ff55ecd8a..9232703a85 100755 --- a/common/web/utils/build.sh +++ b/common/web/utils/build.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # # Compiles common TS-based utility functions for use among Keyman's codebase + set -eu ## START STANDARD BUILD SCRIPT INCLUDE @@ -11,19 +12,23 @@ THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BA . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" +cd "$THIS_SCRIPT_PATH" + ################################ Main script ################################ builder_describe \ "Compiles the web-oriented utility function module." \ + "@../keyman-version" \ configure clean build +builder_describe_outputs \ + configure "/node_modules" \ + build "build/index.js" + builder_parse "$@" if builder_start_action configure; then verify_npm_setup - - "$KEYMAN_ROOT/common/web/keyman-version/build.sh" - builder_finish_action success configure fi @@ -33,6 +38,11 @@ if builder_start_action clean; then fi if builder_start_action build; then - npm run tsc -- --build "$THIS_SCRIPT_PATH/tsconfig.json" + # Note: in a dependency build, we'll expect utils to be built by tsc -b + if builder_is_dep_build; then + echo "[$THIS_SCRIPT_IDENTIFIER] skipping tsc -b; will be completed by $builder_dep_parent" + else + npm run tsc -- --build "$THIS_SCRIPT_PATH/tsconfig.json" + fi builder_finish_action success build fi \ No newline at end of file diff --git a/resources/build/tests/builder-deps.test.sh b/resources/build/tests/builder-deps.test.sh new file mode 100755 index 0000000000..ca1a59e672 --- /dev/null +++ b/resources/build/tests/builder-deps.test.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +set -eu + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")" +. "$(dirname "$THIS_SCRIPT")/../build-utils.sh" +# END STANDARD BUILD SCRIPT INCLUDE + +. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" + +builder_describe \ + "Tests dependency builds" \ + "@./dep1" \ + "@./dep2 test" \ + "@./dep3 test:*" \ + "@./dep4 *:project" \ + "@./dep5 build:bar test:*" \ + "@./dep6 build:bar test" \ + configure \ + build \ + test \ + :project \ + :bar + +builder_describe_outputs \ + configure builder.inc.test.sh \ + build non-existing-file \ + test non-existing-file + +builder_parse configure build test + +function test_dep_should_build() { + local at="$1" + local dep="$2" + + if ! _builder_should_build_dep "$at" "resources/build/tests/$dep"; then + fail "FAIL: expecting to build dependency $dep for $at" + else + echo "PASS: will build dependency $dep for $at" + fi +} + +function test_dep_should_not_build() { + local at="$1" + local dep="$2" + + if _builder_should_build_dep "$at" "resources/build/tests/$dep"; then + fail "FAIL: not expecting to build dependency $dep for $at" + else + echo "PASS: will not build dependency $dep for $at" + fi +} + +test_dep_should_build configure:project dep1 +test_dep_should_build build:project dep1 +test_dep_should_build test:project dep1 +test_dep_should_build configure:bar dep1 +test_dep_should_build build:bar dep1 +test_dep_should_build test:bar dep1 + +test_dep_should_not_build configure:project dep2 +test_dep_should_not_build build:project dep2 +test_dep_should_build test:project dep2 +test_dep_should_not_build configure:bar dep2 +test_dep_should_not_build build:bar dep2 +test_dep_should_build test:bar dep2 + +test_dep_should_not_build configure:project dep3 +test_dep_should_not_build build:project dep3 +test_dep_should_build test:project dep3 +test_dep_should_not_build configure:bar dep3 +test_dep_should_not_build build:bar dep3 +test_dep_should_build test:bar dep3 + +test_dep_should_build configure:project dep4 +test_dep_should_build build:project dep4 +test_dep_should_build test:project dep4 +test_dep_should_not_build configure:bar dep4 +test_dep_should_not_build build:bar dep4 +test_dep_should_not_build test:bar dep4 + +test_dep_should_not_build configure:project dep5 +test_dep_should_not_build build:project dep5 +test_dep_should_build test:project dep5 +test_dep_should_not_build configure:bar dep5 +test_dep_should_build build:bar dep5 +test_dep_should_build test:bar dep5 + +test_dep_should_not_build configure:project dep6 +test_dep_should_not_build build:project dep6 +test_dep_should_build test:project dep6 +test_dep_should_not_build configure:bar dep6 +test_dep_should_build build:bar dep6 +test_dep_should_build test:bar dep6 diff --git a/resources/build/tests/builder.inc.test.sh b/resources/build/tests/builder.inc.test.sh old mode 100644 new mode 100755 index 4fccb2a072..5a8003272c --- a/resources/build/tests/builder.inc.test.sh +++ b/resources/build/tests/builder.inc.test.sh @@ -163,11 +163,14 @@ fi # Due to the nature of the build-utils-traps tests, only one may be # specified at a time; each ends with an `exit`. -echo "Running separate tests" +echo "${COLOR_BLUE}## Running trap tests{$COLOR_RESET}" $THIS_SCRIPT_PATH/build-utils-traps.test.sh error $THIS_SCRIPT_PATH/build-utils-traps.test.sh error-in-function $THIS_SCRIPT_PATH/build-utils-traps.test.sh incomplete -echo "Fin" +echo "${COLOR_BLUE}## Running dependency tests{$COLOR_RESET}" +$THIS_SCRIPT_PATH/builder-deps.test.sh +echo "${COLOR_BLUE}## End external tests${COLOR_RESET}" +echo # Finally, run with --help so we can see what it looks like # Note: calls `exit`, so no further tests may be defined. diff --git a/resources/builder.inc.sh b/resources/builder.inc.sh index b6137ec3b0..acc21834e8 100755 --- a/resources/builder.inc.sh +++ b/resources/builder.inc.sh @@ -145,6 +145,31 @@ _builder_item_in_array() { return 1 } +# +# Returns `0` if first parameter is in the array passed as second parameter, +# where the array may contain globs. +# +# ### Parameters +# +# * 1: `item` item to search for in array +# * 2: `array` bash array, e.g. `array=(one two three)` +# +# ### Example +# +# ```bash +# array=(foo bar it*) +# if _builder_item_in_glob_array "item" "${array[@]}"; then ...; fi +# ``` +# +_builder_item_in_glob_array() { + local e match="$1" + shift + [[ -z "$match" ]] && return 1 + for e; do [[ "$match" == $e ]] && return 0; done + return 1 +} + + _builder_item_is_target() { local item="$1" [[ $item =~ ^: ]] && return 1 @@ -168,6 +193,8 @@ _builder_failure_trap() { local trappedExitCode=$? local action target + _builder_cleanup_deps + # Since 'exit' is also trapped, we can also handle end-of-script incomplete actions. if [[ $trappedExitCode == 0 ]]; then # While there weren't errors, were there any actions that never reported success or failure? @@ -196,6 +223,21 @@ _builder_failure_trap() { fi } +# +# Removes temporary `_builder_deps_built` file when top-level build script +# finishes. +# +_builder_cleanup_deps() { + if ! builder_is_dep_build && [[ ! -z ${_builder_deps_built+x} ]]; then + if $_builder_debug; then + echo "[DEBUG] Dependencies that were built:" + cat "$_builder_deps_built" + fi + rm -f "$_builder_deps_built" + _builder_deps_built= + fi +} + # # Builds the standardized `action:target` string for the specified action-target # pairing and also returns 0 if the user has asked to perform it on the command @@ -235,27 +277,52 @@ builder_has_action() { } # -# Returns 0 if the user has asked to perform action on target on the command line, and -# then starts the action. Should be paired with builder_finish_action +# Returns `0` if the user has asked to perform action on target on the command +# line, and then starts the action. Should be paired with +# `builder_finish_action`. # -# Usage: +# ### Usage +# +# ```bash # if builder_start_action action[:target]; then ...; fi -# Parameters: -# 1: action name of action -# 2: :target name of target, :-prefixed, as part of first param or space separated ok -# Example: -# if builder_start_action build :app; then -# if builder_start_action build:app; then +# ``` +# +# ### Parameters +# +# * 1: `action[:target]` name of action, and optionally also target, if +# target excluded starts for all defined targets +# +# ### Example +# +# ```bash +# if builder_start_action build:app; then ... +# ``` # builder_start_action() { local scope="[$THIS_SCRIPT_IDENTIFIER] " if builder_has_action $@; then + # In a dependency quick build (the default), determine whether we actually + # need to run this step. Uses data passed to builder_describe_outputs to + # verify whether a target output is present. + if builder_is_dep_build && + ! builder_is_full_dep_build && + [[ ! -z ${_builder_dep_path[$_builder_matched_action]+x} ]] && + [[ -e "$KEYMAN_ROOT/${_builder_dep_path[$_builder_matched_action]}" ]]; then + if builder_verbose; then + echo "[$THIS_SCRIPT_IDENTIFIER] skipping $_builder_matched_action, up-to-date" + fi + return 1 + fi + echo "${COLOR_BLUE}## $scope$_builder_matched_action starting...${COLOR_RESET}" if [ -n "${_builder_current_action}" ]; then _builder_warn_if_incomplete fi _builder_current_action="$_builder_matched_action" + + # Build dependencies as required + _builder_do_build_deps "$_builder_matched_action" return 0 else return 1 @@ -290,48 +357,131 @@ _builder_trim() { printf '%s' "$var" } +# +# Expands an in-repo-relative path to a repo-relative path. A path starting with +# `/` is expected to be relative to repo root, not filesystem root. Otherwise, +# it's relative to current script path, not current working directory. The +# returned path will not have a prefix `/`, and will be relative to +# `$KEYMAN_ROOT`. Assumes realpath is installed (brew coreutils on macOS). +# +_builder_expand_relative_path() { + local path="$1" + if [[ "$path" =~ ^/ ]]; then + echo "${path:1}" + else + realpath --relative-to="$KEYMAN_ROOT" "$THIS_SCRIPT_PATH/$path" + fi +} + +# +# Expands an `[action][:target]` string, replacing missing values with `*`, +# for example: +# +# * `build` --> `build:*` +# * `build:app` --> `build:app` +# * `:app` --> `*:app` +# +# Supports multiple action:targets in the string +# +_builder_expand_action_target() { + local input="$1" target= action= + if [[ "$input" =~ : ]]; then + action=$(echo "$input" | cut -d: -f 1 -) + target=$(echo "$input" | cut -d: -f 2 -) + else + action=$input + fi + + if [[ -z "$action" ]]; then + action='*' + fi + if [[ -z "$target" ]]; then + target='*' + fi + + echo "$action:$target" +} + +_builder_expand_action_targets() { + local input=($1) e output=() + for e in "${input[@]}"; do + e=`_builder_expand_action_target "$e"` + output+=($e) + done + if [[ ${#output[@]} == 0 ]]; then + echo "*:*" + else + echo "${output[@]}" + fi +} + # # Describes a build script, defines available parameters and their meanings. Use # together with `builder_parse` to process input parameters. # -# Usage: +# ### Usage +# +# ```bash # builder_describe description param_desc... -# Parameters: -# description A short description of what the script does -# param_desc Space separated name and description of parameter, e.g. -# "build Builds the target" -# May be repeated to describe all parameters +# ``` # -# There are three types of parameters that may be specified: +# ### Parameters +# +# * `description` A short description of what the script does. +# * `param_desc` Space separated name and description of parameter, e.g. +# `"build Builds the target"` +# This parameter may be repeated to describe all parameters. +# +# There are four types of parameters that may be specified: +# +# * **Option:** `"--option[,-o][=var] [One line description]"` # -# * Option, param_desc format: "--option[,-o][=var] [One line description]" # All options must have a longhand form with two prefix hyphens, -# e.g. --option. The ",-o" shorthand form is optional. When testing if -# the option is set with `builder_has_option``, always use the longhand +# e.g. `--option`. The `,-o` shorthand form is optional. When testing if +# the option is set with `builder_has_option`, always use the longhand # form. # -# if =var is specified, then the next parameter will be a variable stored -# in $var for that option. e.g. --option=opt means $opt will have the value -# 'foo' when the script is called for --option foo. +# if `=var` is specified, then the next parameter will be a variable stored in +# `$var` for that option. e.g. `--option=opt` means `$opt` will have the value +# `"foo"` when the script is called for `--option foo`. # -# * Action, param_desc format: "action [One line description]" -# Actions must be a single word, lower case. To specify an action -# as the default, append a '+' to the action name, e.g. -# "test+ Test the project". If there is no default specified, then -# it will be 'build' +# * **Action**: `"action [One line description]"` # -# * Target, param_desc format: ":target [One line description]" -# A target always starts with colon, e.g. :project. +# Actions must be a single word, lower case. To specify an action as the +# default, append a `+` to the action name, e.g. `"test+ Test the project"`. +# If there is no default specified, then it will be `build`. +# +# * **Target:** `":target [One line description]"` +# +# A target always starts with colon, e.g. `:project`. +# +# * **Dependency:** "@/path/to/dependency [action][:target] ..." +# +# A dependency always starts with `@`. The path to the dependency will be +# relative to the build script folder, or to the root of the repository, if +# the path starts with `/`, not to the root of the file system. It is an error +# to specify a dependency outside the repo root. +# +# Relative paths will be expanded to full paths, again, relative to the root +# of the repository. +# +# Dependencies may be limited to specific `action:target`. If not specified, +# dependencies will be built for all actions on all targets. Either `action` +# or `:target` may be omitted, and multiple actions and targets may be +# specified, space separated. # builder_describe() { _builder_description="$1" _builder_actions=() _builder_targets=() _builder_options=() + _builder_deps=() # array of all dependencies for this script _builder_default_action=build declare -A -g _builder_params declare -A -g _builder_options_short declare -A -g _builder_options_var + declare -A -g _builder_dep_path # array of output files for action:target pairs + declare -A -g _builder_dep_related_actions # array of action:targets associated with a given dependency shift # describe each target, action, and option possibility while [[ $# -gt 0 ]]; do @@ -339,12 +489,20 @@ builder_describe() { local value="$(echo "$key" | cut -d" " -f 1 -)" local description= if [[ $key =~ [[:space:]] ]]; then - description=$(_builder_trim "$(echo "$key" | cut -d" " -f 2- -)") + description="$(_builder_trim "$(echo "$key" | cut -d" " -f 2- -)")" fi if [[ $value =~ ^: ]]; then # Parameter is a target _builder_targets+=($value) + elif [[ $value =~ ^@ ]]; then + # Parameter is a dependency + local dependency="${value:1}" + dependency="`_builder_expand_relative_path "$dependency"`" + _builder_deps+=($dependency) + # echo "$description" + _builder_dep_related_actions[$dependency]="`_builder_expand_action_targets "$description"`" + # echo "${_builder_dep_related_actions[$dependency]}" elif [[ $value =~ ^-- ]]; then # Parameter is an option # Look for a shorthand version of the option @@ -398,6 +556,50 @@ builder_describe() { fi } +# +# Defines an output file or folder expected to be present after successful +# completion of an action for a target. Used to skip actions for dependency +# builds. If `:target` is not provided, assumes `:project`. +# +# Relative paths are relative to script folder; absolute paths are relative +# to repository root, not filesystem root. +# +# ### Usage +# +# ```bash +# builder_describe_outputs action:target filename [...] +# ``` +# +# ### Parameters +# +# * 1: `action[:target]` action and/or target associated with file +# * 2: `filename` name of file or folder to check +# * 3+: ... repeat previous arguments for additional outputs +# +# ### Example +# +# ```bash +# builder_describe_outputs \ +# configure /node_modules \ +# build build/index.js +# ``` +# +function builder_describe_outputs() { + while [[ $# -gt 0 ]]; do + local key="$1" path="$2" action target + if [[ $key =~ : ]]; then + action=$(echo "$key" | cut -d: -f 1 -) + target=$(echo "$key" | cut -d: -f 2 -) + else + action=$key + target=project + fi + path="`_builder_expand_relative_path "$path"`" + _builder_dep_path[$action:$target]="$path" + shift 2 + done +} + _builder_get_default_description() { local description= local value="$1" @@ -463,6 +665,7 @@ builder_check_color() { # Parameters # 1: $@ command-line arguments builder_parse() { + _builder_build_deps=--deps builder_verbose= builder_extra_params=() _builder_chosen_action_targets=() @@ -551,6 +754,19 @@ builder_parse() { _builder_chosen_options+=(--verbose) builder_verbose=--verbose ;; + --deps|--no-deps|--force-deps) + _builder_build_deps=$key + ;; + --builder-dep-parent) + # internal use parameter for dependency builds - identifier of parent script + shift + builder_dep_parent="$1" + ;; + --builder-deps-built) + # internal use parameter for dependency builds - path to dependency tracking file + shift + _builder_deps_built="$1" + ;; *) _builder_parameter_error "$0" parameter "$key" esac @@ -576,6 +792,19 @@ builder_parse() { done fi + if builder_is_dep_build; then + echo "[$THIS_SCRIPT_IDENTIFIER] dependency build, started by $builder_dep_parent" + if [[ -z ${_builder_deps_built+x} ]]; then + echo "FATAL ERROR: Expected --builder-deps-built parameter" + exit 1 + fi + else + # This is a top-level invocation, not a dependency build, so we want to + # track which dependencies have been built, so they don't get built multiple + # times. + _builder_deps_built=`mktemp` + fi + # Now that we've successfully parsed options adhering to the _builder spec, we may activate our # action_failure and action_hanging traps. (We don't want them active on scripts not yet using # said script.) @@ -611,11 +840,16 @@ builder_display_usage() { program="$(basename "$0")" if [[ ! -z ${_builder_description+x} ]]; then - echo "$program: $_builder_description" + echo "Summary:" + echo " $_builder_description" echo fi + echo "Script Identifier:" + echo " $THIS_SCRIPT_IDENTIFIER" + echo - echo "Usage: $program [options...] [action][:target]..." + echo "Usage:" + echo " $program [options...] [action][:target]..." echo echo "Actions: " @@ -652,8 +886,24 @@ builder_display_usage() { _builder_pad $width " --verbose, -v" "Verbose logging" _builder_pad $width " --color" "Force colorized output" _builder_pad $width " --no-color" "Never use colorized output" + if builder_has_dependencies; then + _builder_pad $width " --deps" "Build dependencies if required (default)" + _builder_pad $width " --no-deps" "Skip build of dependencies" + _builder_pad $width " --force-deps" "Reconfigure and rebuild all dependencies" + fi _builder_pad $width " --help, -h" "Show this help" + echo + echo "Dependencies: " + + if builder_has_dependencies; then + for d in "${_builder_deps[@]}"; do + echo " $d" + done + else + echo " This module has no dependencies" + fi + # Defined in `builder_use_color`; this assumes that said func has been called. local c1=$BUILDER_TERM_START local c0=$BUILDER_TERM_END @@ -696,6 +946,183 @@ builder_finish_action() { fi } +# +# Returns `0` if the dependency should be built for the given action:target +# +_builder_should_build_dep() { + local action_target="$1" + local dep="$2" + local related_actions=(${_builder_dep_related_actions[$dep]}) + # echo "bdra: ${_builder_dep_related_actions[@]}" + # echo "target: $action_target" + # echo "dep: $2" + # echo "ra: ${related_actions[@]}" + if ! _builder_item_in_glob_array "$action_target" "${related_actions[@]}"; then + return 1 + fi + return 0 +} + +# +# Configure and build all dependencies +# Later, may restrict by either action or target +# +_builder_do_build_deps() { + local action_target="$1" + + if [[ $_builder_build_deps == --no-deps ]]; then + # we've been asked to skip dependencies + return 0 + fi + + for dep in "${_builder_deps[@]}"; do + # Don't attempt to build dependencies that don't match the current + # action:target (wildcards supported for matches here) + if ! _builder_should_build_dep "$action_target" "$dep"; then + echo "[$THIS_SCRIPT_IDENTIFIER] Skipping dependency build $dep for $action_target" + continue + fi + + # Only configure and build the dependency once per invocation + if builder_has_module_been_built "$dep"; then + continue + fi + + # TODO: add --debug as a standard builder parameter + builder_set_module_has_been_built "$dep" + "$KEYMAN_ROOT/$dep/build.sh" configure build \ + $builder_verbose \ + $_builder_build_deps \ + --builder-deps-built "$_builder_deps_built" \ + --builder-dep-parent "$THIS_SCRIPT_IDENTIFIER" + done +} + +# +# returns `0` if we are in a dependency doing a build. +# +builder_is_dep_build() { + if [[ ! -z ${builder_dep_parent+x} ]]; then + return 0 + fi + return 1 +} + +# +# returns `0` if we should attempt to do quick builds in a dependency build, for +# example skipping `tsc -b` where a parent may also do it; corresponds to the +# `--deps` parameter (which is the default). +# +builder_is_quick_dep_build() { + if builder_is_dep_build && [[ $_builder_build_deps == --deps ]]; then + return 0 + fi + return 1 +} + +# +# returns `0` if we should do a full configure and build in a dependency build; +# corresponds to the `--force-deps`` parameter. +# +builder_is_full_dep_build() { + if builder_is_dep_build && [[ $_builder_build_deps == --force-deps ]]; then + return 0 + fi + return 1 +} + +# +# returns `0` if the current build script has at least one dependency. +# +builder_has_dependencies() { + if [[ ${#_builder_deps[@]} -eq 0 ]]; then + return 1 + fi + return 0 +} + +# +# Tests if a dependency module has been built already in the current script +# invocation; if not running in a builder context, always returns `1` (i.e. +# "false"). +# +# ### Usage +# +# ```bash +# builder_has_module_been_built dependency-name +# ``` +# +# ### Parameters +# +# * 1: `dependency-name` the `$SCRIPT_IDENTIFIER` of the dependency +# (repo-relative path without leading `/`); or for +# external dependencies, a path-like starting with +# `/external/`. +# +# ### Examples +# +# ```bash +# if builder_has_module_been_built common/web/keyman-version; then ... +# if builder_has_module_been_built /external/npm-ci; then ... +# ``` +# +builder_has_module_been_built() { + local module="$1" + + if [[ -z ${_builder_deps_built+x} ]]; then + # not in a builder context, so we assume a build is needed + return 1 + fi + + if [[ -f $_builder_deps_built ]] && grep -qx "$module" $_builder_deps_built; then + # dependency history file contains the dependency module + return 0 + fi + return 1 +} + +# +# Updates the dependency module build state for the current script invocation; +# if not running in a builder context, a no-op. +# +# ### Usage +# +# ```bash +# builder_set_module_has_been_built dependency-name +# ``` +# +# ### Parameters +# +# * 1: `dependency-name` the `$SCRIPT_IDENTIFIER` of the dependency +# (repo-relative path without leading `/`); or for +# external dependencies, a path-like starting with +# `/external/`. +# +# ### Examples +# +# ```bash +# builder_set_module_has_been_built common/web/keyman-version +# builder_set_module_has_been_built /external/npm-ci +# ``` +# +builder_set_module_has_been_built() { + local module="$1" + + if [[ ! -z ${_builder_deps_built+x} ]]; then + echo "$module" >> $_builder_deps_built + fi +} + +# +# returns `0` if we should be verbose in output +# +builder_verbose() { + if [[ $builder_verbose == --verbose ]]; then + return 0 + fi + return 1 +} + # # Initialize builder once all functions are declared # diff --git a/resources/shellHelperFunctions.sh b/resources/shellHelperFunctions.sh index 79e917bc32..439ee69db4 100755 --- a/resources/shellHelperFunctions.sh +++ b/resources/shellHelperFunctions.sh @@ -1,11 +1,5 @@ #!/usr/bin/env bash -# -# WARNING: this file is copied into other locations during package builds; do not -# include other files or make path assumptions when changing this file. -# See `core/build.sh` for more details. -# - _shf_base_dir=$(dirname "$BASH_SOURCE")/.. # Designed to determine which set of browsers should be available for local testing, @@ -261,32 +255,28 @@ set_npm_version () { npm version --allow-same-version --no-git-tag-version --no-commit-hooks "$VERSION_WITH_TAG" } -# Initializes use of the npm packages within the repo. -init_npm() { - if [ "${KEYMAN_ROOT}" = "" ]; then - fail "KEYMAN_ROOT not defined; cannot install repo's dependencies" +# +# Verifies that node is installed, and installs npm packages, but only once per +# build invocation +# +verify_npm_setup() { + # We'll piggy-back on the builder module dependency build state to determine + # if npm ci has been called in the current script invocation. Adding the + # prefix /external/ to module name in order to differentiate between this and + # internal modules (although it is unlikely to ever collide!); we will also + # use this pattern for other similar external dependencies in future. These + # functions are safe to call even in a non-builder context (they do nothing or + # return 1 -- not built) + if builder_has_module_been_built /external/npm-ci; then + return 0 fi + builder_set_module_has_been_built /external/npm-ci + + # Check if Node.JS/npm is installed. + type npm >/dev/null ||\ + fail "Build environment setup error detected! Please ensure Node.js is installed!" + pushd "$KEYMAN_ROOT" > /dev/null npm ci popd > /dev/null } - -# Accepts an optional parameter. -# $1 - when set to 'false', only ensures that `npm` and `node` are accessible; does not install dependencies. -# -# Designed for use with the projects/packages we have listed in the base folder package.json workspaces. -verify_npm_setup () { - if [ $# != 0 ]; then - fetch_deps=$1 - else - fetch_deps=true - fi - - # Check if Node.JS/npm is installed. - type npm >/dev/null ||\ - fail "Build environment setup error detected! Please ensure Node.js is installed!" - - if [ $fetch_deps = true ]; then - init_npm - fi -} From d803d47d1d51e6fadde39944f223e8ed0fd357fc Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 12 Oct 2022 13:28:16 +1100 Subject: [PATCH 20/28] chore(common): handle missing files in path expansion --- resources/builder.inc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/builder.inc.sh b/resources/builder.inc.sh index acc21834e8..fe2f512005 100755 --- a/resources/builder.inc.sh +++ b/resources/builder.inc.sh @@ -369,7 +369,7 @@ _builder_expand_relative_path() { if [[ "$path" =~ ^/ ]]; then echo "${path:1}" else - realpath --relative-to="$KEYMAN_ROOT" "$THIS_SCRIPT_PATH/$path" + realpath --canonicalize-missing --relative-to="$KEYMAN_ROOT" "$THIS_SCRIPT_PATH/$path" fi } From 30121d97936e9891cd22606733784acbfc439536 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 12 Oct 2022 13:02:19 +1100 Subject: [PATCH 21/28] chore(common): remove space for builder_..._action param --- common/predictive-text/build.sh | 32 +++++++++++------------ common/predictive-text/unit_tests/test.sh | 24 ++++++++--------- common/web/input-processor/build.sh | 28 ++++++++++---------- resources/build/tests/builder.inc.test.sh | 17 ++++-------- resources/builder.inc.sh | 29 +++++++++++--------- 5 files changed, 63 insertions(+), 67 deletions(-) diff --git a/common/predictive-text/build.sh b/common/predictive-text/build.sh index d099d6dd81..f182c038b2 100755 --- a/common/predictive-text/build.sh +++ b/common/predictive-text/build.sh @@ -48,14 +48,14 @@ LMLAYER_OUTPUT=build ### CONFIGURE ACTIONS -if builder_start_action configure :browser; then +if builder_start_action configure:browser; then verify_npm_setup - builder_finish_action success configure :browser + builder_finish_action success configure:browser fi -if builder_start_action configure :headless; then +if builder_start_action configure:headless; then verify_npm_setup - builder_finish_action success configure :headless + builder_finish_action success configure:headless fi ### CLEAN ACTIONS @@ -67,13 +67,13 @@ do_clean() { } CLEANED= -if builder_start_action clean :browser; then +if builder_start_action clean:browser; then do_clean CLEANED=clean:browser - builder_finish_action success clean :browser + builder_finish_action success clean:browser fi -if builder_start_action clean :headless; then +if builder_start_action clean:headless; then if [ -n "$CLEANED" ]; then echo "${BUILDER_TERM_START}clean${BUILDER_TERM_END} already completed as ${BUILDER_TERM_START}${CLEANED}${BUILDER_TERM_END}; skipping." else @@ -81,23 +81,23 @@ if builder_start_action clean :headless; then CLEANED=clean:headless fi - builder_finish_action success clean :headless + builder_finish_action success clean:headless fi ### BUILD ACTIONS # Builds the top-level JavaScript file for use in browsers -if builder_start_action build :browser; then +if builder_start_action build:browser; then npm run tsc -- -b ./browser.tsconfig.json - builder_finish_action success build :browser + builder_finish_action success build:browser fi # Builds the top-level JavaScript file for use on Node -if builder_start_action build :headless; then +if builder_start_action build:headless; then npm run tsc -- -b ./tsconfig.json - builder_finish_action success build :headless + builder_finish_action success build:headless fi ### TEST ACTIONS @@ -109,16 +109,16 @@ if builder_has_option --ci; then TEST_OPTIONS=--ci fi -if builder_start_action test :headless; then +if builder_start_action test:headless; then # We'll test the included libraries here for now, at least until we have # converted their builds to builder scripts ./unit_tests/test.sh test:libraries test:headless $TEST_OPTIONS - builder_finish_action success test :headless + builder_finish_action success test:headless fi -if builder_start_action test :browser; then +if builder_start_action test:browser; then ./unit_tests/test.sh test:browser $TEST_OPTIONS - builder_finish_action success test :browser + builder_finish_action success test:browser fi \ No newline at end of file diff --git a/common/predictive-text/unit_tests/test.sh b/common/predictive-text/unit_tests/test.sh index e7d23f0b55..8c74211dee 100755 --- a/common/predictive-text/unit_tests/test.sh +++ b/common/predictive-text/unit_tests/test.sh @@ -30,23 +30,23 @@ builder_describe "Runs all tests for the language-modeling / predictive-text lay builder_parse "$@" -if builder_start_action configure :libraries; then +if builder_start_action configure:libraries; then # Ensure all testing dependencies are in place. verify_npm_setup - builder_finish_action success configure :libraries + builder_finish_action success configure:libraries fi -if builder_start_action configure :headless; then +if builder_start_action configure:headless; then verify_npm_setup - builder_finish_action success configure :headless + builder_finish_action success configure:headless fi -if builder_start_action configure :browser; then +if builder_start_action configure:browser; then verify_npm_setup - builder_finish_action success configure :browser + builder_finish_action success configure:browser fi -if builder_start_action test :libraries; then +if builder_start_action test:libraries; then # Note: these do not yet provide TeamCity-friendly-formatted test reports. # They do not have builder-based scripts, being run directly via npm package script. @@ -82,10 +82,10 @@ if builder_start_action test :libraries; then npm run test popd - builder_finish_action success test :libraries + builder_finish_action success test:libraries fi -if builder_start_action test :headless; then +if builder_start_action test:headless; then MOCHA_FLAGS=$FLAGS if builder_has_option --ci; then @@ -94,7 +94,7 @@ if builder_start_action test :headless; then npm run mocha -- --recursive $MOCHA_FLAGS ./unit_tests/headless/*.js ./unit_tests/headless/**/*.js - builder_finish_action success test :headless + builder_finish_action success test:headless fi # If we are running a TeamCity test build, for now, only run BrowserStack @@ -124,7 +124,7 @@ get_browser_set_for_OS ( ) { fi } -if builder_start_action test :browser; then +if builder_start_action test:browser; then KARMA_FLAGS=$FLAGS KARMA_INFO_LEVEL="--log-level=warn" @@ -157,5 +157,5 @@ if builder_start_action test :browser; then fi npm run karma -- start $KARMA_INFO_LEVEL $KARMA_FLAGS $BROWSERS unit_tests/in_browser/$KARMA_CONFIG - builder_finish_action success test :browser + builder_finish_action success test:browser fi \ No newline at end of file diff --git a/common/web/input-processor/build.sh b/common/web/input-processor/build.sh index 6fb2f8dcb8..548c125ffa 100755 --- a/common/web/input-processor/build.sh +++ b/common/web/input-processor/build.sh @@ -43,14 +43,14 @@ builder_parse "$@" ### CONFIGURE ACTIONS -if builder_start_action configure :module; then +if builder_start_action configure:module; then verify_npm_setup - builder_finish_action success configure :module + builder_finish_action success configure:module fi -if builder_start_action configure :tools; then +if builder_start_action configure:tools; then verify_npm_setup - builder_finish_action success configure :tools + builder_finish_action success configure:tools fi ### CLEAN ACTIONS @@ -64,13 +64,13 @@ do_clean() { CLEANED= -if builder_start_action clean :module; then +if builder_start_action clean:module; then do_clean CLEANED=clean:module - builder_finish_action success clean :module + builder_finish_action success clean:module fi -if builder_start_action clean :tools; then +if builder_start_action clean:tools; then if [ -n "$CLEANED" ]; then echo "${BUILDER_TERM_START}clean${BUILDER_TERM_END} already completed as ${BUILDER_TERM_START}${CLEANED}${BUILDER_TERM_END}; skipping." else @@ -78,29 +78,29 @@ if builder_start_action clean :tools; then CLEANED=clean:tools fi - builder_finish_action success clean :tools + builder_finish_action success clean:tools fi ### BUILD ACTIONS -if builder_start_action build :tools; then +if builder_start_action build:tools; then # Used by test:module # TODO: convert to a dependency once we have updated kmlmc to use builder script pushd "$KEYMAN_ROOT/developer/src/kmlmc" ./build.sh -S popd - builder_finish_action success build :tools + builder_finish_action success build:tools fi -if builder_start_action build :module; then +if builder_start_action build:module; then npm run tsc -- -b src/tsconfig.json - builder_finish_action success build :module + builder_finish_action success build:module fi # TEST ACTIONS -if builder_start_action test :module; then +if builder_start_action test:module; then FLAGS= if builder_has_option --ci; then FLAGS="--reporter mocha-teamcity-reporter" @@ -111,5 +111,5 @@ if builder_start_action test :module; then npm run mocha -- --recursive $FLAGS ./tests/cases/ - builder_finish_action success test :module + builder_finish_action success test:module fi \ No newline at end of file diff --git a/resources/build/tests/builder.inc.test.sh b/resources/build/tests/builder.inc.test.sh index 5a8003272c..7723ccb40b 100755 --- a/resources/build/tests/builder.inc.test.sh +++ b/resources/build/tests/builder.inc.test.sh @@ -61,13 +61,6 @@ else fail "FAIL: should have matched action build for :project" fi -if builder_start_action clean :app; then - echo "Cleaning " - builder_finish_action success clean :app -else - fail "FAIL: should have matched action clean for :app" -fi - if builder_start_action clean:app; then echo "Cleaning " builder_finish_action success clean:app @@ -75,14 +68,14 @@ else fail "FAIL: should have matched action clean for :app" fi -if builder_start_action build :app; then +if builder_start_action build:app; then echo "Building app" - builder_finish_action success build :app + builder_finish_action success build:app else fail "FAIL: should have matched action build for :app" fi -if builder_start_action build :module; then +if builder_start_action build:module; then fail "FAIL: should not have matched action build for :module" fi @@ -163,11 +156,11 @@ fi # Due to the nature of the build-utils-traps tests, only one may be # specified at a time; each ends with an `exit`. -echo "${COLOR_BLUE}## Running trap tests{$COLOR_RESET}" +echo "${COLOR_BLUE}## Running trap tests${COLOR_RESET}" $THIS_SCRIPT_PATH/build-utils-traps.test.sh error $THIS_SCRIPT_PATH/build-utils-traps.test.sh error-in-function $THIS_SCRIPT_PATH/build-utils-traps.test.sh incomplete -echo "${COLOR_BLUE}## Running dependency tests{$COLOR_RESET}" +echo "${COLOR_BLUE}## Running dependency tests${COLOR_RESET}" $THIS_SCRIPT_PATH/builder-deps.test.sh echo "${COLOR_BLUE}## End external tests${COLOR_RESET}" echo diff --git a/resources/builder.inc.sh b/resources/builder.inc.sh index fe2f512005..5cdda99d64 100755 --- a/resources/builder.inc.sh +++ b/resources/builder.inc.sh @@ -214,7 +214,7 @@ _builder_failure_trap() { target=:project fi - builder_finish_action failure $action $target + builder_finish_action failure $action$target # Make 100% sure that the exit code chains fully. # Without this, nested scripts have failed to chain errors from npm calls past the script @@ -247,23 +247,28 @@ _builder_cleanup_deps() { # The string will be set as `_builder_matched_action`, which is for # builder.inc.sh internal use, used by `builder_start_action`. # -# Usage: +# ### Usage +# +# ```bash # if build_has_action action[:target]; then ...; fi +# ```` +# # Parameters: -# 1: action name of action -# 2: :target name of target, :-prefixed, as part of first param or space separated ok +# 1: action[:target] name of action:target # Example: -# if builder_has_action build :app; then # or build:app, that's fine too. +# +# ```bash +# if builder_has_action build:app; then ... +# ``` +# builder_has_action() { local action="$1" target if [[ $action =~ : ]]; then IFS=: read -r action target <<< $action target=:$target - elif [[ -z ${2+x} ]]; then - target=:project else - target="$2" + target=:project fi if _builder_item_in_array "$action$target" "${_builder_chosen_action_targets[@]}"; then @@ -301,7 +306,7 @@ builder_has_action() { builder_start_action() { local scope="[$THIS_SCRIPT_IDENTIFIER] " - if builder_has_action $@; then + if builder_has_action $1; then # In a dependency quick build (the default), determine whether we actually # need to run this step. Uses data passed to builder_describe_outputs to # verify whether a target output is present. @@ -310,7 +315,7 @@ builder_start_action() { [[ ! -z ${_builder_dep_path[$_builder_matched_action]+x} ]] && [[ -e "$KEYMAN_ROOT/${_builder_dep_path[$_builder_matched_action]}" ]]; then if builder_verbose; then - echo "[$THIS_SCRIPT_IDENTIFIER] skipping $_builder_matched_action, up-to-date" + echo "$scope skipping $_builder_matched_action, up-to-date" fi return 1 fi @@ -922,10 +927,8 @@ builder_finish_action() { if [[ $action =~ : ]]; then IFS=: read -r action target <<< $action target=:$target - elif [[ -z ${3+x} ]]; then - target=:project else - target="$3" + target=:project fi local scope="[$THIS_SCRIPT_IDENTIFIER] " From b7000f5b7f7c0007d7d771943f25aece84e0540e Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 12 Oct 2022 13:21:37 +1100 Subject: [PATCH 22/28] chore(common): builder_start_action wildcard support --- common/predictive-text/build.sh | 35 ++----------- common/predictive-text/unit_tests/test.sh | 15 +----- common/web/input-processor/build.sh | 36 ++----------- common/web/recorder/build.sh | 29 ++++------- resources/builder.inc.sh | 62 ++++++++++++++--------- 5 files changed, 59 insertions(+), 118 deletions(-) diff --git a/common/predictive-text/build.sh b/common/predictive-text/build.sh index f182c038b2..866efaf4e6 100755 --- a/common/predictive-text/build.sh +++ b/common/predictive-text/build.sh @@ -43,47 +43,20 @@ builder_describe_outputs \ builder_parse "$@" -# Exit status on invalid usage. -LMLAYER_OUTPUT=build - ### CONFIGURE ACTIONS -if builder_start_action configure:browser; then +if builder_start_action configure; then verify_npm_setup - builder_finish_action success configure:browser -fi - -if builder_start_action configure:headless; then - verify_npm_setup - builder_finish_action success configure:headless + builder_finish_action success configure fi ### CLEAN ACTIONS -# A nice, extensible method for -clean operations. Add to this as necessary. -# TODO: separate clean for different targets -do_clean() { - rm -rf "$LMLAYER_OUTPUT" -} - -CLEANED= -if builder_start_action clean:browser; then - do_clean - CLEANED=clean:browser +if builder_start_action clean; then + rm -rf build/ builder_finish_action success clean:browser fi -if builder_start_action clean:headless; then - if [ -n "$CLEANED" ]; then - echo "${BUILDER_TERM_START}clean${BUILDER_TERM_END} already completed as ${BUILDER_TERM_START}${CLEANED}${BUILDER_TERM_END}; skipping." - else - do_clean - CLEANED=clean:headless - fi - - builder_finish_action success clean:headless -fi - ### BUILD ACTIONS # Builds the top-level JavaScript file for use in browsers diff --git a/common/predictive-text/unit_tests/test.sh b/common/predictive-text/unit_tests/test.sh index 8c74211dee..2718483505 100755 --- a/common/predictive-text/unit_tests/test.sh +++ b/common/predictive-text/unit_tests/test.sh @@ -30,20 +30,9 @@ builder_describe "Runs all tests for the language-modeling / predictive-text lay builder_parse "$@" -if builder_start_action configure:libraries; then - # Ensure all testing dependencies are in place. +if builder_start_action configure; then verify_npm_setup - builder_finish_action success configure:libraries -fi - -if builder_start_action configure:headless; then - verify_npm_setup - builder_finish_action success configure:headless -fi - -if builder_start_action configure:browser; then - verify_npm_setup - builder_finish_action success configure:browser + builder_finish_action success configure fi if builder_start_action test:libraries; then diff --git a/common/web/input-processor/build.sh b/common/web/input-processor/build.sh index 548c125ffa..70e0cc7398 100755 --- a/common/web/input-processor/build.sh +++ b/common/web/input-processor/build.sh @@ -43,42 +43,16 @@ builder_parse "$@" ### CONFIGURE ACTIONS -if builder_start_action configure:module; then +if builder_start_action configure; then verify_npm_setup - builder_finish_action success configure:module -fi - -if builder_start_action configure:tools; then - verify_npm_setup - builder_finish_action success configure:tools + builder_finish_action success configure fi ### CLEAN ACTIONS -# A nice, extensible method for -clean operations. Add to this as necessary. -do_clean() { - if [ -d build ]; then - rm -rf build/ - fi -} - -CLEANED= - -if builder_start_action clean:module; then - do_clean - CLEANED=clean:module - builder_finish_action success clean:module -fi - -if builder_start_action clean:tools; then - if [ -n "$CLEANED" ]; then - echo "${BUILDER_TERM_START}clean${BUILDER_TERM_END} already completed as ${BUILDER_TERM_START}${CLEANED}${BUILDER_TERM_END}; skipping." - else - do_clean - CLEANED=clean:tools - fi - - builder_finish_action success clean:tools +if builder_start_action clean; then + rm -rf build/ + builder_finish_action success clean fi ### BUILD ACTIONS diff --git a/common/web/recorder/build.sh b/common/web/recorder/build.sh index f92aa9e887..969453f37c 100755 --- a/common/web/recorder/build.sh +++ b/common/web/recorder/build.sh @@ -34,36 +34,27 @@ builder_describe_outputs \ builder_parse "$@" -if builder_start_action configure :module; then +if builder_start_action configure; then verify_npm_setup - builder_finish_action success configure :module + builder_finish_action success configure fi -if builder_start_action configure :proctor; then - if builder_has_action configure :module; then - echo "Configuration already completed in configure:module; skipping." - else - verify_npm_setup - fi - builder_finish_action success configure :proctor -fi - -if builder_start_action clean :module; then +if builder_start_action clean:module; then npm run tsc -- -b --clean "$THIS_SCRIPT_PATH/src/tsconfig.json" - builder_finish_action success clean :module + builder_finish_action success clean:module fi -if builder_start_action clean :proctor; then +if builder_start_action clean:proctor; then npm run tsc -- -b --clean "$THIS_SCRIPT_PATH/src/nodeProctor.tsconfig.json" - builder_finish_action success clean :proctor + builder_finish_action success clean:proctor fi -if builder_start_action build :module; then +if builder_start_action build:module; then npm run tsc -- --build "$THIS_SCRIPT_PATH/src/tsconfig.json" - builder_finish_action success build :module + builder_finish_action success build:module fi -if builder_start_action build :proctor; then +if builder_start_action build:proctor; then npm run tsc -- --build "$THIS_SCRIPT_PATH/src/nodeProctor.tsconfig.json" - builder_finish_action success build :proctor + builder_finish_action success build:proctor fi \ No newline at end of file diff --git a/resources/builder.inc.sh b/resources/builder.inc.sh index 5cdda99d64..ebdf5fb8b9 100755 --- a/resources/builder.inc.sh +++ b/resources/builder.inc.sh @@ -36,16 +36,19 @@ function _builder_findRepoRoot() { } # Used to build script-related build variables useful for referencing the calling script -# and for prefixing builder_finish_action outputs in order to more clearly identify the calling +# and for prefixing `builder_finish_action` outputs in order to more clearly identify the calling # script. # -# Assumes that THIS_SCRIPT has been set, typically like this: +# Assumes that `THIS_SCRIPT` has been set, typically like this: +# +# ```bash +# ## START STANDARD BUILD SCRIPT INCLUDE +# # adjust relative paths as necessary +# THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")" +# . "$(dirname "$THIS_SCRIPT")/resources/builder.inc.sh" +# ## END STANDARD BUILD SCRIPT INCLUDE +# ``` # -# ## START STANDARD BUILD SCRIPT INCLUDE -# # adjust relative paths as necessary -# THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")" -# . "$(dirname "$THIS_SCRIPT")/resources/builder.inc.sh" -# ## END STANDARD BUILD SCRIPT INCLUDE function _builder_setBuildScriptIdentifiers() { if [ ! -z ${THIS_SCRIPT+x} ]; then THIS_SCRIPT_PATH="$(dirname "$THIS_SCRIPT")" @@ -141,7 +144,7 @@ _builder_item_in_array() { local e match="$1" shift [[ -z "$match" ]] && return 1 - for e; do [[ "$e" == "$match" ]] && return 0; done + for e; do [[ "$e" == $match ]] && return 0; done return 1 } @@ -268,12 +271,17 @@ builder_has_action() { IFS=: read -r action target <<< $action target=:$target else - target=:project + target=':*' fi if _builder_item_in_array "$action$target" "${_builder_chosen_action_targets[@]}"; then # To avoid WET re-processing of the $action$target string set _builder_matched_action="$action$target" + if [[ $target == ':*' ]]; then + _builder_matched_action_name="$action" + else + _builder_matched_action_name="$action$target" + fi return 0 else _builder_matched_action= @@ -315,12 +323,12 @@ builder_start_action() { [[ ! -z ${_builder_dep_path[$_builder_matched_action]+x} ]] && [[ -e "$KEYMAN_ROOT/${_builder_dep_path[$_builder_matched_action]}" ]]; then if builder_verbose; then - echo "$scope skipping $_builder_matched_action, up-to-date" + echo "$scope skipping $_builder_matched_action_name, up-to-date" fi return 1 fi - echo "${COLOR_BLUE}## $scope$_builder_matched_action starting...${COLOR_RESET}" + echo "${COLOR_BLUE}## $scope$_builder_matched_action_name starting...${COLOR_RESET}" if [ -n "${_builder_current_action}" ]; then _builder_warn_if_incomplete fi @@ -593,14 +601,14 @@ function builder_describe_outputs() { while [[ $# -gt 0 ]]; do local key="$1" path="$2" action target if [[ $key =~ : ]]; then - action=$(echo "$key" | cut -d: -f 1 -) - target=$(echo "$key" | cut -d: -f 2 -) + action="$(echo "$key" | cut -d: -f 1 -)" + target=":$(echo "$key" | cut -d: -f 2 -)" else - action=$key - target=project + action="$key" + target=':*' fi path="`_builder_expand_relative_path "$path"`" - _builder_dep_path[$action:$target]="$path" + _builder_dep_path[$action$target]="$path" shift 2 done } @@ -922,30 +930,36 @@ builder_display_usage() { builder_finish_action() { local result="$1" - local action="$2" target + local action="$2" target action_name if [[ $action =~ : ]]; then IFS=: read -r action target <<< $action - target=:$target + target=":$target" else - target=:project + target=':*' + fi + + if [[ "$target" == ":*" ]]; then + action_name="$action" + else + action_name="$action$target" fi local scope="[$THIS_SCRIPT_IDENTIFIER] " if [[ "$action$target" == "${_builder_current_action}" ]]; then if [[ $result == success ]]; then - echo "${COLOR_GREEN}## $scope$action$target completed successfully${COLOR_RESET}" + echo "${COLOR_GREEN}## $scope$action_name completed successfully${COLOR_RESET}" elif [[ $result == failure ]]; then - echo "${COLOR_RED}## $scope$action$target failed${COLOR_RESET}" + echo "${COLOR_RED}## $scope$action_name failed${COLOR_RESET}" else - echo "${COLOR_RED}## $scope$action$target failed with message: $result${COLOR_RESET}" + echo "${COLOR_RED}## $scope$action_name failed with message: $result${COLOR_RESET}" fi # Remove $action$target from the array; it is no longer a current action _builder_current_action= else - echo "${COLOR_YELLOW}## Warning: reporting result of $action$target but the action was never started!${COLOR_RESET}" + echo "${COLOR_YELLOW}## Warning: reporting result of $action_name but the action was never started!${COLOR_RESET}" fi } @@ -982,7 +996,7 @@ _builder_do_build_deps() { # Don't attempt to build dependencies that don't match the current # action:target (wildcards supported for matches here) if ! _builder_should_build_dep "$action_target" "$dep"; then - echo "[$THIS_SCRIPT_IDENTIFIER] Skipping dependency build $dep for $action_target" + echo "[$THIS_SCRIPT_IDENTIFIER] Skipping dependency build $dep for $_builder_matched_action_name" continue fi From 1e8e598cd3a236867cd6369a5ee844b2e5241a93 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Tue, 11 Oct 2022 17:00:40 +0200 Subject: [PATCH 23/28] chore(linux): Update readme This change adds separate READMEs for the legacy projects, and removes mentioning of the legacy projects in the current READMEs. The motivation is to make it later easier to remove the legacy projects, as well as reducing the efforts to testing the README with legacy projects. Since legacy projects aren't maintained anymore it doesn't make sense to still have them in the regular README, suggesting that the instructions work for the legacy projects as well. --- linux/LICENSE.md | 19 +++++----- linux/README.md | 23 ++++-------- linux/README.repo | 7 ---- linux/legacy/LICENSE.md | 13 +++++++ linux/legacy/README.md | 80 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 109 insertions(+), 33 deletions(-) delete mode 100644 linux/README.repo create mode 100644 linux/legacy/LICENSE.md create mode 100644 linux/legacy/README.md diff --git a/linux/LICENSE.md b/linux/LICENSE.md index 153a83e91b..98cdd5435c 100644 --- a/linux/LICENSE.md +++ b/linux/LICENSE.md @@ -1,17 +1,16 @@ # LICENSE -The [kmflcomp](./kmflcomp), [libkmfl](./libkmfl), and [keyman_config](./keyman_config) projects -are covered by the [MIT license](./libkmfl/COPYING). +The [keyman-config](./keyman-config) project is covered by the +[MIT license](./keyman-config/COPYING). -The [ibus-kmfl](./ibus-kmfl) and [ibus-keyman](./ibus-keyman) projects are licensed under GNU -General Public License as published by the Free Software Foundation; either -[version 2 of the License](./ibus-kmfl/COPYING), or (at your option) any later version. +The [ibus-keyman](./ibus-keyman) projects is licensed under GNU General Public License +as published by the Free Software Foundation; either +[version 2 of the License](./ibus-keyman/COPYING), or (at your option) any later version. -Two files in ibus-kmfl, [kmflutil.c](./ibus-kmfl/src/kmflutil.c) and -[kmflutil.h](./ibus-kmfl/src/kmflutil.h), as well as four files in ibus-keyman, -[keymanutil.c](./ibus-keyman/src/keymanutil.c), [keymanutil.h](./ibus-keyman/src/keymanutil.h), -[kmpdetails.c](./ibus-keyman/src/kmpdetails.c) and [kmpdetails.h](./ibus-keyman/src/kmpdetails.h), -are dual licensed by the MIT license and the GNU General Public License which is described above. +Four files in ibus-keyman, [keymanutil.c](./ibus-keyman/src/keymanutil.c), +[keymanutil.h](./ibus-keyman/src/keymanutil.h), [kmpdetails.c](./ibus-keyman/src/kmpdetails.c) +and [kmpdetails.h](./ibus-keyman/src/kmpdetails.h), are dual licensed by the MIT license and +the GNU General Public License which is described above. The MIT license alone may be chosen for them for their use in other projects. Two files in ibus-keyman, [keyman-service.c](./ibus-keyman/src/keyman-service.c) and diff --git a/linux/README.md b/linux/README.md index 870fddabb9..5aafea3dd1 100644 --- a/linux/README.md +++ b/linux/README.md @@ -16,7 +16,7 @@ See [license information](./LICENSE.md) about licensing. - It is helpful to be using the [packages.sil.org](http://packages.sil.org) repo -- Install packages required for building and developing KMFL and Keyman for Linux +- Install packages required for building and developing Keyman for Linux ```bash sudo apt install cdbs debhelper libx11-dev autotools-dev build-essential \ @@ -31,7 +31,7 @@ See [license information](./LICENSE.md) about licensing. ### Build script -#### Installing for ibus to use ibus-kmfl or ibus-keyman +#### Installing for ibus to use ibus-keyman - The process to build and install everything is: @@ -44,10 +44,8 @@ See [license information](./LICENSE.md) about licensing. To do this run `sudo make install` - This will install to `/usr/local` - - and `/usr/share/ibus/component/kmfl.xml` and `/usr/share/kmfl/icons` - and `/usr/share/ibus/component/keyman.xml` and `/usr/share/keyman/icons` - - If you already have the ibus-kmfl package installed then it will move the file `/usr/share/ibus/component/kmfl.xml` to `/usr/share/doc/ibus-kmfl/` - If you already have the ibus-keyman package installed then it will move the file `/usr/share/ibus/component/keyman.xml` to `/usr/share/doc/ibus-keyman/` - run `sudo make uninstall` to uninstall everything and put it back again @@ -56,20 +54,16 @@ See [license information](./LICENSE.md) about licensing. Used by TC for validating PRs -Run `make tmpinstall` to build and install **keyboardprocessor**, **kmflcomp**, **libkmfl**, **ibus-kmfl** and **ibus-keyman** to `/tmp/kmfl` +Run `make tmpinstall` to build and install **keyboardprocessor** and **ibus-keyman** to `/tmp/kmfl` -This is only for testing the build, not for running **ibus-kmfl** or **ibus-keyman** in ibus +This is only for testing the build, not for running **ibus-keyman** in ibus ### Manually -- **libkmfl** requires `kmfl.h` header from **kmflcomp** -- **ibus-kfml** requires `kmfl.h` and headers and lib from **libkmfl** - **ibus-keyman** requires headers and lib from **keyboardprocessor** So -- **kmflcomp** must be built and installed before **libkmfl** -- **libkmfl** must be built and installed before **ibus-kmfl** - **keyboardprocessor** must be built before **ibus-keyman** For each project run `./configure && make && make install`. @@ -77,16 +71,13 @@ For each project run `./configure && make && make install`. You may prefer to create a different directory to build in and run configure from there e.g. ```bash -mkdir ../build-kmflcomp -cd ../build-kmflcomp -../kmflcomp/configure +mkdir ../build-ibus-keyman +cd ../build-ibus-keyman +../ibus-keyman/configure make make install ``` -The install of **ibus-kmfl** doesn't install everything to the correct location for it to be -used - to be fixed - ## Continuous integration Teamcity PR builds will run `make tmpinstall` diff --git a/linux/README.repo b/linux/README.repo deleted file mode 100644 index 708cda84bd..0000000000 --- a/linux/README.repo +++ /dev/null @@ -1,7 +0,0 @@ -You need autoconf, autopoint, gettext, automake and libtool to generate the build system - -flex and bison are useful to rebuild lex.l and yacc.y in kmflcomp but not essential - -For keyboardprocessor you need rust, meson and ninja (meson package pulls in ninja) - -Run `make reconf` to run them diff --git a/linux/legacy/LICENSE.md b/linux/legacy/LICENSE.md new file mode 100644 index 0000000000..25cba465bf --- /dev/null +++ b/linux/legacy/LICENSE.md @@ -0,0 +1,13 @@ +# LICENSE + +The [kmflcomp](./kmflcomp) and [libkmfl](./libkmfl) projects are covered by the +[MIT license](./libkmfl/COPYING). + +The [ibus-kmfl](./ibus-kmfl) project is licensed under GNU General Public License +as published by the Free Software Foundation; either +[version 2 of the License](./ibus-kmfl/COPYING), or (at your option) any later version. + +Two files in ibus-kmfl, [kmflutil.c](./ibus-kmfl/src/kmflutil.c) and +[kmflutil.h](./ibus-kmfl/src/kmflutil.h) are dual licensed by the MIT license and the +GNU General Public License which is described above. +The MIT license alone may be chosen for them for their use in other projects. diff --git a/linux/legacy/README.md b/linux/legacy/README.md new file mode 100644 index 0000000000..d65723012d --- /dev/null +++ b/linux/legacy/README.md @@ -0,0 +1,80 @@ +# Keyman for Linux + +## Projects + +- [kmflcomp](./kmflcomp) - KMFL keyboard compiler +- [libkmfl](./libkmfl) - older KMFL core library +- [ibus-kmfl](./ibus-kmfl) - IBUS integration to use KMFL + +See [license information](./LICENSE.md) about licensing. + +## Linux Requirements/Setup + +- It is helpful to be using the [packages.sil.org](http://packages.sil.org) repo + +- Install packages required for building and developing KMFL + + ```bash + sudo apt install cdbs debhelper libx11-dev autotools-dev build-essential \ + dh-autoreconf flex bison libibus-1.0-dev python3-setuptools meson \ + libjson-glib-dev libgtk-3-dev libxml2-utils help2man python3-lxml \ + python3-magic python3-numpy python3-pil python3-pip python3-qrcode \ + python3-requests python3-requests-cache python3 python3-gi dconf-cli \ + dconf-editor cargo python3-dbus + ``` + +## Compiling from Command Line + +### Build script + +#### Installing for ibus to use ibus-kmfl + +- The process to build and install everything is: + + - `make reconf` to create the build system and set the version + - `make fullbuild` to configure and build + - `sudo make install` to install to `/usr/local` + +- Some of the files must be installed to `/usr/share/` so `make install` must be run as `sudo`. + + To do this run `sudo make install` + + - This will install to `/usr/local` + - and `/usr/share/ibus/component/kmfl.xml` and `/usr/share/kmfl/icons` + + - If you already have the ibus-kmfl package installed then it will move the file `/usr/share/ibus/component/kmfl.xml` to `/usr/share/doc/ibus-kmfl/` + +- run `sudo make uninstall` to uninstall everything and put it back again + +#### Tmp install + +Used by TC for validating PRs + +Run `make tmpinstall` to build and install **keyboardprocessor**, **kmflcomp**, **libkmfl**, **ibus-kmfl** and **ibus-keyman** to `/tmp/kmfl` + +This is only for testing the build, not for running **ibus-kmfl** in ibus + +### Manually + +- **libkmfl** requires `kmfl.h` header from **kmflcomp** +- **ibus-kfml** requires `kmfl.h` and headers and lib from **libkmfl** + +So + +- **kmflcomp** must be built and installed before **libkmfl** +- **libkmfl** must be built and installed before **ibus-kmfl** + +For each project run `./configure && make && make install`. + +You may prefer to create a different directory to build in and run configure from there e.g. + +```bash +mkdir ../build-kmflcomp +cd ../build-kmflcomp +../kmflcomp/configure +make +make install +``` + +The install of **ibus-kmfl** doesn't install everything to the correct location for it to be +used - to be fixed From c891f870b6949f975606506b7a5200848bfc5d49 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Tue, 11 Oct 2022 18:44:05 +0200 Subject: [PATCH 24/28] fix(linux): Fix make install Several fixes for building and installing locally. Fixes #7397. Closes #7398. Closes #7400. --- .gitignore | 11 +++++++++++ linux/README.md | 2 +- linux/keyman-config/Makefile | 11 +++++++---- linux/keyman-config/README.md | 33 ++++++++++++++++++++------------- linux/keyman-config/setup.py | 4 ++-- 5 files changed, 41 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index b9febfb451..2f5f7e6dbf 100644 --- a/.gitignore +++ b/.gitignore @@ -173,3 +173,14 @@ Thumbs.db # Temporary file for logging scripts in xcode runs, see build-utils.sh for # details /xcodebuild-scripts.log + +# Linux packaging related +/debian/ +results/ +*.deb +*.ddeb +*.dsc +*.build +*.buildinfo +*.changes +*.tar.?z diff --git a/linux/README.md b/linux/README.md index 5aafea3dd1..9a84ba4fff 100644 --- a/linux/README.md +++ b/linux/README.md @@ -54,7 +54,7 @@ See [license information](./LICENSE.md) about licensing. Used by TC for validating PRs -Run `make tmpinstall` to build and install **keyboardprocessor** and **ibus-keyman** to `/tmp/kmfl` +Run `make tmpinstall` to build and install **keyboardprocessor** and **ibus-keyman** to `/tmp/keyman` This is only for testing the build, not for running **ibus-keyman** in ibus diff --git a/linux/keyman-config/Makefile b/linux/keyman-config/Makefile index cb0fe8ab51..e539765069 100644 --- a/linux/keyman-config/Makefile +++ b/linux/keyman-config/Makefile @@ -8,17 +8,20 @@ langtags: install: # run as sudo pip3 install qrcode sentry-sdk + # eventually change this to: pip3 install . python3 setup.py install # install icons mkdir -p /usr/local/share/keyman/icons cp keyman_config/icons/* /usr/local/share/keyman/icons # install man pages mkdir -p /usr/local/share/man/man1 - cp debian/man/*.1 /usr/local/share/man/man1 + cp ../../debian/man/*.1 /usr/local/share/man/man1 install-temp: - mkdir -p /tmp/kmfl/`python3 -c 'import sys;import os;pythonver="python%d.%d" % (sys.version_info[0], sys.version_info[1]);sitedir = os.path.join("lib", pythonver, "site-packages");print(sitedir)'` - PYTHONUSERBASE=/tmp/kmfl python3 setup.py install --user + mkdir -p /tmp/keyman/$(shell python3 -c 'import sys;import os;pythonver="python%d.%d" % (sys.version_info[0], sys.version_info[1]);sitedir = os.path.join("lib", pythonver, "site-packages");print(sitedir)') + # when we no longer have to support old pip version (python > 3.6) change this to: + # pip3 install --prefix /tmp/keyman . + PYTHONUSERBASE=/tmp/keyman python3 setup.py install --user uninstall: clean # run as sudo rm -rf /usr/local/share/keyman/icons @@ -48,7 +51,7 @@ deb: dist cd make_deb && tar xf ../dist/keyman_config-$(shell cat /tmp/keyman_version).tar.gz && \ mv keyman_config-$(shell cat /tmp/keyman_version) keyman-config-$(shell cat /tmp/keyman_version) && \ tar cfz keyman-config_$(shell cat /tmp/keyman_version).orig.tar.gz keyman-config-$(shell cat /tmp/keyman_version) && \ - cd keyman-config-$(shell cat /tmp/keyman_version) && cp -a ../../debian . && dch -v$(shell cat /tmp/keyman_version)-1 "" + cd keyman-config-$(shell cat /tmp/keyman_version) && cp -a ../../../debian . && dch -v$(shell cat /tmp/keyman_version)-1 "" cd make_deb/keyman-config-$(shell cat /tmp/keyman_version) && debuild -us -uc rm /tmp/keyman_version diff --git a/linux/keyman-config/README.md b/linux/keyman-config/README.md index fdacc38f9c..59a0840e6b 100644 --- a/linux/keyman-config/README.md +++ b/linux/keyman-config/README.md @@ -8,22 +8,29 @@ then you will need to: ```bash sudo apt install python3-lxml python3-magic python3-numpy python3-qrcode python3-pil \ python3-requests python3-requests-cache python3 python3-gi gir1.2-webkit2-4.0 dconf-cli \ - python3-setuptools python3-pip python3-dbus ibus + python3-setuptools python3-pip python3-dbus ibus libglib2.0-bin liblocale-gettext-perl ``` -Either `python3-raven` or `python3-sentry-sdk` is required as well. To install it on Ubuntu 18.04 and earlier run: - -```bash -sudo apt install python3-raven -``` - -On Ubuntu 20.04 and later: +Either `python3-raven` or `python3-sentry-sdk` (>= 1.4) is required as well. On Ubuntu 22.04 and later run: ```bash sudo apt install python3-sentry-sdk ``` -(It's also possible to install it with pip: `pip3 install sentry-sdk`) +To install it on Ubuntu 18.04 and earlier run: + +```bash +sudo apt install python3-raven +``` + +For Ubuntu versions that don't provide `python3-raven` but instead provide +`python3-sentry-sdk` in a too old version (i.e. Ubuntu 20.04): +Install `python3-sentry-sdk` from `packages.sil.org`, +or install it with pip: + +```bash +pip3 install sentry-sdk +``` Run the script `./createkeymandirs.sh` to create the directories for these programs to install the packages to. @@ -42,13 +49,13 @@ Running `km-config` requires a language tag mapping file `keyman_config/standards/lang_tags_map.py`. This file gets generated during a package build, and also when running `make`. -### Installing manually from the repo +## Installing manually from the repo -`make && sudo make install` will install locally to `/usr/local` +`make && sudo make install` will install locally to `/usr/local`. -`python3 setup.py --help install` will give you more install options +`pip3 help install` will give you more install options. -You will need `sudo apt install python3-pip` to `make uninstall` +To uninstall you can run `sudo make uninstall`. ## Things to run from the command line diff --git a/linux/keyman-config/setup.py b/linux/keyman-config/setup.py index fef56b97cb..c090be9fbf 100644 --- a/linux/keyman-config/setup.py +++ b/linux/keyman-config/setup.py @@ -20,8 +20,8 @@ setup( ], # metadata to display on PyPI - author="Daniel Glassey", - author_email="wdg@debian.org", + author="Keyman team", + author_email="support@keyman.com", description="Keyman for Linux configuration", license="MIT", keywords="keyman, keyman-config, keyboard", From b7a72fe799f030e0efc6ad61ab905ff1304c78e6 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Wed, 12 Oct 2022 14:02:14 -0400 Subject: [PATCH 25/28] auto: increment master version to 16.0.79 --- HISTORY.md | 6 ++++++ VERSION.md | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index d779d0eb3f..0e82a002d3 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,11 @@ # Keyman Version History +## 16.0.78 alpha 2022-10-12 + +* fix(linux): Fix reordering of output (#7079) +* fix(linux): Fix make install (#7434) +* chore(linux): Add Node.js to the docker container (#7435) + ## 16.0.77 alpha 2022-10-10 * fix(web): possible error on change of context to a contextEditable (#7359) diff --git a/VERSION.md b/VERSION.md index fe616bd6cd..6e450c8619 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.78 \ No newline at end of file +16.0.79 \ No newline at end of file From d89c63aaf3b55512ea67768a6794762369046bc7 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 13 Oct 2022 06:04:14 +1100 Subject: [PATCH 26/28] Update common/predictive-text/build.sh Co-authored-by: Joshua Horton --- common/predictive-text/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/predictive-text/build.sh b/common/predictive-text/build.sh index 866efaf4e6..2a216b3555 100755 --- a/common/predictive-text/build.sh +++ b/common/predictive-text/build.sh @@ -54,7 +54,7 @@ fi if builder_start_action clean; then rm -rf build/ - builder_finish_action success clean:browser + builder_finish_action success clean fi ### BUILD ACTIONS From 57f305666e73ab676af41c32e8ae8e6d9283eb3f Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 13 Oct 2022 12:02:07 +1100 Subject: [PATCH 27/28] Update build.sh --- common/predictive-text/build.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/common/predictive-text/build.sh b/common/predictive-text/build.sh index d099d6dd81..d9fa5afc9b 100755 --- a/common/predictive-text/build.sh +++ b/common/predictive-text/build.sh @@ -24,6 +24,11 @@ cd "$(dirname "$THIS_SCRIPT")" builder_check_color "$@" +# TODO: once these modules are builder-based, reference here too: +# "@../models/templates" \ +# "@../models/types" \ +# "@../models/wordbreakers" + builder_describe "Builds the lm-layer module" \ "@../web/keyman-version" \ "@../web/lm-worker" \ @@ -121,4 +126,4 @@ if builder_start_action test :browser; then ./unit_tests/test.sh test:browser $TEST_OPTIONS builder_finish_action success test :browser -fi \ No newline at end of file +fi From 435aade36e66c87e1988c7fef70cbff623d35a1f Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 13 Oct 2022 12:39:30 +1100 Subject: [PATCH 28/28] chore(web): add keyboard-processor test to top-level web test script --- web/unit_tests/test.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/web/unit_tests/test.sh b/web/unit_tests/test.sh index 0dac16252c..6be07e582e 100755 --- a/web/unit_tests/test.sh +++ b/web/unit_tests/test.sh @@ -131,6 +131,11 @@ cd ../tools/recorder # Run our headless tests first. # First: Web-core tests. +pushd "$KEYMAN_ROOT/common/web/keyboard-processor" +./build.sh test $HEADLESS_FLAGS || fail "Tests failed by dependencies; aborting integration tests." +# Once done, now we run the integrated (KeymanWeb) tests. +popd + pushd "$KEYMAN_ROOT/common/web/input-processor" ./build.sh build:tools test $HEADLESS_FLAGS || fail "Tests failed by dependencies; aborting integration tests." # Once done, now we run the integrated (KeymanWeb) tests.