diff --git a/.gitignore b/.gitignore index cc536b78a0..2f5f7e6dbf 100644 --- a/.gitignore +++ b/.gitignore @@ -146,6 +146,7 @@ # VS Code .vscode/ .vs/ +.devcontainer/ # IDE files **/.idea/**/*.xml @@ -172,3 +173,14 @@ Thumbs.db # Temporary file for logging scripts in xcode runs, see build-utils.sh for # details /xcodebuild-scripts.log + +# Linux packaging related +/debian/ +results/ +*.deb +*.ddeb +*.dsc +*.build +*.buildinfo +*.changes +*.tar.?z diff --git a/HISTORY.md b/HISTORY.md index d779d0eb3f..0e82a002d3 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,11 @@ # Keyman Version History +## 16.0.78 alpha 2022-10-12 + +* fix(linux): Fix reordering of output (#7079) +* fix(linux): Fix make install (#7434) +* chore(linux): Add Node.js to the docker container (#7435) + ## 16.0.77 alpha 2022-10-10 * fix(web): possible error on change of context to a contextEditable (#7359) diff --git a/VERSION.md b/VERSION.md index fe616bd6cd..6e450c8619 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.78 \ No newline at end of file +16.0.79 \ No newline at end of file diff --git a/common/predictive-text/.build-builder b/common/predictive-text/.build-builder new file mode 100644 index 0000000000..4e15741a62 --- /dev/null +++ b/common/predictive-text/.build-builder @@ -0,0 +1 @@ +The presence of this file tells CI to use the new builder_ style parameters for build.sh and unit_tests/test.sh. \ No newline at end of file diff --git a/common/predictive-text/build.sh b/common/predictive-text/build.sh old mode 100755 new mode 100644 index cbaef012d5..7d1f3ff92b --- a/common/predictive-text/build.sh +++ b/common/predictive-text/build.sh @@ -20,92 +20,83 @@ THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BA # This script runs from its own folder cd "$(dirname "$THIS_SCRIPT")" -# Exit status on invalid usage. -EX_USAGE=64 - -LMLAYER_OUTPUT=build - -# Builds the top-level JavaScript file for use in browsers (the second stage of compilation) -build-browser () { - npm run tsc -- -b ./browser.tsconfig.json || fail "Could not build top-level browser-targeted JavaScript file." -} - -# Builds the top-level JavaScript file for use on Node (the second stage of compilation) -build-headless () { - npm run tsc -- -b ./tsconfig.json || fail "Could not build top-level node-targeted JavaScript file." -} - -# A nice, extensible method for -clean operations. Add to this as necessary. -clean ( ) { - if [ -d $LMLAYER_OUTPUT ]; then - rm -rf "$LMLAYER_OUTPUT" || fail "Failed to erase the prior build." - fi -} - -display_usage ( ) { - echo "Usage: $0 [-clean] [-skip-package-install | -S] [-test | -tdd]" - echo " $0 -help" - echo - echo " -clean to erase pre-existing build products before a re-build" - echo " -help displays this screen and exits" - echo " -skip-package-install (or -S) skips dependency updates" - echo " -tdd skips dependency updates, builds, then runs unit tests only" - echo " -test runs unit and integration tests after building" -} - ################################ Main script ################################ -run_tests=0 -fetch_deps=true -unit_tests_only=0 +builder_check_color "$@" -# Process command-line arguments -while [[ $# -gt 0 ]] ; do - key="$1" - case $key in - -clean) - clean - ;; - -help|-h) - display_usage - exit - ;; - -skip-package-install|-S) - fetch_deps=false - ;; - -test) - run_tests=1 - ;; - -tdd) - run_tests=1 - fetch_deps=false - unit_tests_only=1 - ;; - *) - echo "$0: invalid option: $key" - display_usage - exit $EX_USAGE - esac - shift # past the processed argument -done +# TODO: once these modules are builder-based, reference here too: +# "@../models/templates" \ +# "@../models/types" \ +# "@../models/wordbreakers" -# Check if Node.JS/npm is installed. -verify_npm_setup $fetch_deps +builder_describe "Builds the lm-layer module" \ + "@../web/keyman-version" \ + "@../web/lm-worker" \ + "clean" \ + "configure" \ + "build" \ + "test" \ + ":headless A headless, Node-oriented version of the module useful for unit tests" \ + ":browser The standard version of the module for in-browser use" \ + "--ci Sets ${BUILDER_TERM_START}test${BUILDER_TERM_END} action to use CI-based test configurations & reporting" -if $fetch_deps; then - # We need to build keyman-version and lm-worker with a script for now - "$KEYMAN_ROOT/common/web/keyman-version/build.sh" || fail "Could not build keyman-version" - "$KEYMAN_ROOT/common/web/lm-worker/build.sh" || fail "Could not build lm-worker" +builder_describe_outputs \ + configure:headless /node_modules \ + configure:browser /node_modules \ + build:headless build/headless.js \ + build:browser build/index.js + +builder_parse "$@" + +### CONFIGURE ACTIONS + +if builder_start_action configure; then + verify_npm_setup + builder_finish_action success configure fi -build-browser || fail "Browser-oriented compilation failed." -build-headless || fail "Headless compilation failed." -echo "Typescript compilation successful." +### CLEAN ACTIONS -if (( run_tests )); then - if (( unit_tests_only )); then - npm run test -- -headless || fail "Unit tests failed" - else - npm test || fail "Tests failed" - fi +if builder_start_action clean; then + rm -rf build/ + builder_finish_action success clean +fi + +### BUILD ACTIONS + +# Builds the top-level JavaScript file for use in browsers +if builder_start_action build:browser; then + npm run tsc -- -b ./browser.tsconfig.json + + builder_finish_action success build:browser +fi + +# Builds the top-level JavaScript file for use on Node +if builder_start_action build:headless; then + npm run tsc -- -b ./tsconfig.json + + builder_finish_action success build:headless +fi + +### TEST ACTIONS +# Note - the actual test setup is done in a separate test script, but it's easy +# enough to route the calls through. + +TEST_OPTIONS= +if builder_has_option --ci; then + TEST_OPTIONS=--ci +fi + +if builder_start_action test:headless; then + # We'll test the included libraries here for now, at least until we have + # converted their builds to builder scripts + ./unit_tests/test.sh test:libraries test:headless $TEST_OPTIONS + + builder_finish_action success test:headless +fi + +if builder_start_action test:browser; then + ./unit_tests/test.sh test:browser $TEST_OPTIONS + + builder_finish_action success test:browser fi diff --git a/common/predictive-text/unit_tests/headless/promise-store.js b/common/predictive-text/unit_tests/headless/promise-store.js index 509ca09f50..f14b9c8555 100644 --- a/common/predictive-text/unit_tests/headless/promise-store.js +++ b/common/predictive-text/unit_tests/headless/promise-store.js @@ -1,7 +1,7 @@ var assert = require('chai').assert; var sinon = require('sinon'); -let PromiseStore = require('../../build').PromiseStore; +let PromiseStore = require('../../build/headless').PromiseStore; describe('PromiseStore', function () { describe('.make()', function () { @@ -19,13 +19,13 @@ describe('PromiseStore', function () { it('should reject the promise when a token is reused', function () { var promises = new PromiseStore(); - + var reusedToken = randomToken(); // These two fakes are to ensure the original is called. var originalResolve; var overwrittenResolve; - - + + // Add a promise, and an unrelated promise. new Promise(function (resolve, reject) { originalResolve = sinon.fake(resolve); diff --git a/common/predictive-text/unit_tests/headless/top-level-lmlayer.js b/common/predictive-text/unit_tests/headless/top-level-lmlayer.js index a4645e7e27..f4b891c8e0 100644 --- a/common/predictive-text/unit_tests/headless/top-level-lmlayer.js +++ b/common/predictive-text/unit_tests/headless/top-level-lmlayer.js @@ -1,7 +1,7 @@ var assert = require('chai').assert; var sinon = require('sinon'); -let LMLayer = require('../../build'); +let LMLayer = require('../../build/headless'); // Test the top-level LMLayer interface. // Note: these tests can only be run after BOTH stages of compilation are completed. @@ -54,7 +54,7 @@ describe('LMLayer', function() { assert.notProperty(data.source, 'code'); assert.property(data.source, 'file'); assert.isString(data.source.file); - + callAsynchronously(() => fakeWorker.onmessage({ data: { message: 'ready', @@ -113,7 +113,7 @@ describe('LMLayer', function() { * Returns an object implementing *enough* of the Worker * interface to fool the LMLayer into thinking it's * communicating with a bona fide Web Worker. - * + * * @returns {Worker} an object with sinon.fake() instances. */ function createFakeWorker(postMessage) { @@ -124,10 +124,10 @@ describe('LMLayer', function() { } /** - * Call a function in the future, i.e., later in the event loop. + * Call a function in the future, i.e., later in the event loop. * The call does NOT block the current execution. - * Use this to fake asynchronous callbacks. - * + * Use this to fake asynchronous callbacks. + * * @param {Function} fn function to call */ function callAsynchronously(fn) { diff --git a/common/predictive-text/unit_tests/in_browser/base.conf.js b/common/predictive-text/unit_tests/in_browser/base.conf.js index 9671ac9e0a..2c62af1854 100644 --- a/common/predictive-text/unit_tests/in_browser/base.conf.js +++ b/common/predictive-text/unit_tests/in_browser/base.conf.js @@ -41,7 +41,6 @@ module.exports = { // We don't have anything in these locations... yet. But they'll be useful for test resources. 'json/**/*.json', // Where pre-loaded JSON resides. {pattern: 'resources/**/*.*', watched: true, served: true, included: false}, // General testing resources. - {pattern: 'fixtures/**/*.html', watched: true} // HTML structures useful for testing. ], // list of files / patterns to exclude @@ -51,7 +50,6 @@ module.exports = { // preprocess matching files before serving them to the browser // available preprocessors: https://npmjs.org/browse/keyword/karma-preprocessor preprocessors: { - 'fixtures/**/*.html' : ['html2js'], 'json/**/*.json' : ['json_fixtures'] }, diff --git a/common/predictive-text/unit_tests/in_browser/browser-test.sh b/common/predictive-text/unit_tests/in_browser/browser-test.sh deleted file mode 100755 index 28c286ecf7..0000000000 --- a/common/predictive-text/unit_tests/in_browser/browser-test.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash - -# A testing script designed for automating in-browser testing. -# It's designed to be called by this folder's ./test.sh. - -display_usage ( ) { - echo "browser_test.sh os [-CI | -debug | -? | -h | -help] [-reporter ]" - echo - echo " os Should be one of the following: win, mac, linux" - echo "" - echo " -CI to run unit tests in CI mode on BrowserStack." - echo " This script requires your credentials to be set in environment variables - see " - echo " https://stackoverflow.com/questions/32450546/hiding-browserstack-key-in-karma" - echo "" - echo " -debug to establish a Karma server that facilitates unit test debugging" - echo " Not compatible with -CI." - echo "" - echo " -reporter sets the test engine to utilize the specified ." - echo " Valid options: BrowserStack, teamcity, dots, progress, mocha" - echo "" - echo " -? | -h | -help to display this help information" - echo "" - echo " Specifying no option will perform a simple, single run of the test cases on" - echo " the dominant set of browsers for the currently-detected active OS." - echo "" - exit 0 -} - -os_id=$1 - -get_browser_set_for_OS ( ) { - if [ $os_id = "mac" ]; then - BROWSERS="--browsers Firefox,Chrome,Safari" - elif [ $os_id = "win" ]; then - BROWSERS="--browsers Chrome" - else - BROWSERS="--browsers Firefox,Chrome" - fi -} - -# Defaults -get_browser_set_for_OS - -CONFIG=manual.conf.js # TODO - get/make OS-specific version -DEBUG=false -FLAGS= - -# Parse args -while [[ $# -gt 0 ]] ; do - key="$1" - case $key in - -CI) - CONFIG=CI.conf.js - ;; - -debug) - # Disables the default 'run once, then done' configuration needed for CI. - DEBUG=true - FLAGS="--no-single-run $FLAGS" - ;; - -reporter) - shift - FLAGS="--reporters $1 $FLAGS" - ;; - -h) - display_usage - ;; - -help) - display_usage - ;; - -\?) - display_usage - ;; - esac - shift # past argument -done - -if [ $DEBUG = true ] && [ $CONFIG = CI.conf.js ]; then - echo "-CI and -debug are not compatible!" - exit 1 -fi - -if [ $CONFIG = CI.conf.js ]; then - # If doing a CI run, use the file's default browser selection. - BROWSERS= -fi - -npm --no-color run karma -- start --log-level=debug $FLAGS $BROWSERS unit_tests/in_browser/$CONFIG - -CODE=$? - -exit $CODE diff --git a/common/predictive-text/unit_tests/test.sh b/common/predictive-text/unit_tests/test.sh index 8329e8c417..2718483505 100755 --- a/common/predictive-text/unit_tests/test.sh +++ b/common/predictive-text/unit_tests/test.sh @@ -1,122 +1,150 @@ #!/usr/bin/env bash -# We should work within the script's directory, not the one we were called in. +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")" . "$(dirname "$THIS_SCRIPT")/../../../resources/build/build-utils.sh" ## END STANDARD BUILD SCRIPT INCLUDE . "$KEYMAN_ROOT/resources/build/build-utils-ci.inc.sh" . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" -SCRIPT_ROOT="$(dirname "$THIS_SCRIPT")" -# A simple utility script to facilitate unit-testing for the LM Layer. -# It's rigged to be callable by NPM to facilitate testing during development when in other folders. +# This script runs from its own folder +cd "$THIS_SCRIPT_PATH" -display_usage ( ) { - echo "test.sh [ -? | -h | -help]" - echo " -CI to perform continuous-integration friendly tests and reporting" - echo " -headless to disable the in-browser tests" - echo " -integrated to disable the 'headless' test suite" - echo " -skip-package-install (or -S) skips dependency updates" - echo " -? | -h | -help to display this help information" - echo "" - exit 0 -} +################################ Main script ################################ -init_dependencies ( ) { - # Ensure all testing dependencies are in place. - verify_npm_setup $fetch_deps -} +# Defaults +FLAGS="--require ./unit_tests/helpers" -test-headless ( ) { - _FLAGS=$FLAGS - if (( CI_REPORTING )); then - _FLAGS="$_FLAGS --reporter mocha-teamcity-reporter" - fi +builder_describe "Runs all tests for the language-modeling / predictive-text layer module" \ + "configure" \ + "test+" \ + ":libraries Runs unit tests for in-repo libraries used by this module"\ + ":headless Runs this module's headless user tests" \ + ":browser Runs this module's browser-based user tests" \ + "--ci Uses CI-based test configurations & emits CI-friendly test reports" \ + "--debug,-d Activates developer-friendly debug mode for unit tests where applicable" +# TODO: consider dependencies? ideally this will be test.inc.sh? + +builder_parse "$@" + +if builder_start_action configure; then + verify_npm_setup + builder_finish_action success configure +fi + +if builder_start_action test:libraries; then + + # Note: these do not yet provide TeamCity-friendly-formatted test reports. + # They do not have builder-based scripts, being run directly via npm package script. + # So, for now, we add a text header to clarify what is running at each stage, in + # addition to fair bit of `pushd` and `popd`. pushd "$KEYMAN_ROOT/common/models/wordbreakers" - npm run test || fail "models/wordbreakers tests failed" + echo + echo "### Running ${BUILDER_TERM_START}common/models/wordbreaker${BUILDER_TERM_END} tests" + # NPM doesn't seem to parse the post `--` part if specified via script variable. + # So... a simple if-else will do the job for now. + if builder_has_option --ci; then + npm run test -- -reporter mocha-teamcity-reporter + else + npm run test + fi popd pushd "$KEYMAN_ROOT/common/models/templates" - npm run test || fail "models/templates tests failed" + echo + echo "### Running ${BUILDER_TERM_START}common/models/templates${BUILDER_TERM_END} tests" + if builder_has_option --ci; then + npm run test -- -reporter mocha-teamcity-reporter + else + npm run test + fi popd pushd "$KEYMAN_ROOT/common/models/types" - npm run test || fail "models/types tests failed" + echo + echo "### Running ${BUILDER_TERM_START}common/models/types${BUILDER_TERM_END} tests" + # Is not mocha-based; it's TSC-based instead, as we're just ensuring that the .d.ts + # file is a proper TS declaration file. + npm run test popd - npm run mocha -- --recursive $_FLAGS ./unit_tests/headless/*.js ./unit_tests/headless/**/*.js -} - -test-browsers ( ) { - _FLAGS=$FLAGS - if (( CI_REPORTING )); then - _FLAGS="$_FLAGS -CI -reporter teamcity,BrowserStack" - fi - - $SCRIPT_ROOT/in_browser/browser-test.sh $os_id $_FLAGS -} - -# Defaults -get_builder_OS # return: os_id="linux"|"mac"|"win" - -FLAGS="--require ./unit_tests/helpers" -CI_REPORTING=0 -RUN_HEADLESS=1 -RUN_BROWSERS=1 -fetch_deps=true - -# Parse args -while [[ $# -gt 0 ]] ; do - key="$1" - case $key in - -h|-help) - display_usage - exit - ;; - -CI) - CI_REPORTING=1 - ;; - -headless) - RUN_BROWSERS=0 - ;; - -integrated) - RUN_HEADLESS=0 - ;; - -skip-package-install|-S) - fetch_deps=false - ;; - esac - shift # past argument -done - -init_dependencies - -# Run headless (browserless) tests. -if (( RUN_HEADLESS )); then - test-headless || fail "DOMless tests failed!" + builder_finish_action success test:libraries fi -if (( RUN_BROWSERS )); then - if [[ $VERSION_ENVIRONMENT == test ]]; then - # If we are running a TeamCity test build, for now, only run BrowserStack - # tests when on a PR branch with a title including "(web)" or with the label - # test-browserstack. This is because the BrowserStack tests are currently - # unreliable, and the false positive failures are masking actual failures. - # - # We do not run BrowserStack tests on master, beta, or stable-x.y test - # builds. - RUN_BROWSERS=0 - if builder_pull_get_details; then - if [[ $builder_pull_title =~ \(web\) ]] || builder_pull_has_label test-browserstack; then - RUN_BROWSERS=1 - fi +if builder_start_action test:headless; then + MOCHA_FLAGS=$FLAGS + + if builder_has_option --ci; then + MOCHA_FLAGS="$MOCHA_FLAGS --reporter mocha-teamcity-reporter" + fi + + npm run mocha -- --recursive $MOCHA_FLAGS ./unit_tests/headless/*.js ./unit_tests/headless/**/*.js + + builder_finish_action success test:headless +fi + +# If we are running a TeamCity test build, for now, only run BrowserStack +# tests when on a PR branch with a title including "(web)" or with the label +# test-browserstack. This is because the BrowserStack tests are currently +# unreliable, and the false positive failures are masking actual failures. +# +# We do not run BrowserStack tests on master, beta, or stable-x.y test +# builds. +if [[ $VERSION_ENVIRONMENT == test ]] && builder_has_action test :browser; then + if builder_pull_get_details; then + if ! ([[ $builder_pull_title =~ \(web\) ]] || builder_pull_has_label test-browserstack); then + + echo "Auto-skipping ${BUILDER_TERM_START}test:browser${BUILDER_TERM_END} for unrelated CI test build" + exit 0 fi fi fi -# Run browser-based tests. -if (( RUN_BROWSERS )); then - test-browsers || fail "Browser-based tests failed!" -fi +get_browser_set_for_OS ( ) { + if [ $os_id = "mac" ]; then + BROWSERS="--browsers Firefox,Chrome,Safari" + elif [ $os_id = "win" ]; then + BROWSERS="--browsers Chrome" + else + BROWSERS="--browsers Firefox,Chrome" + fi +} + +if builder_start_action test:browser; then + KARMA_FLAGS=$FLAGS + KARMA_INFO_LEVEL="--log-level=warn" + + if builder_has_option --ci; then + KARMA_FLAGS="$KARMA_FLAGS --reporters teamcity,BrowserStack" + KARMA_CONFIG="CI.conf.js" + KARMA_INFO_LEVEL="--log-level=debug" + + if builder_has_option --debug; then + echo "${BUILDER_TERM_START}--ci${BUILDER_TERM_END} option set; ignoring ${BUILDER_TERM_START}--debug${BUILDER_TERM_END} option" + fi + else + KARMA_CONFIG="manual.conf.js" + if builder_has_option --debug; then + KARMA_FLAGS="$KARMA_FLAGS --no-single-run" + KARMA_CONFIG="manual.conf.js" + KARMA_INFO_LEVEL="--log-level=debug" + + echo + echo "${COLOR_YELLOW}You must manually terminate this mode (CTRL-C) for the script to exit.${COLOR_RESET}" + sleep 2 + fi + fi + + if [[ KARMA_CONFIG == "manual.conf.js" ]]; then + get_builder_OS # return: os_id="linux"|"mac"|"win" + get_browser_set_for_OS + else + BROWSERS= + fi + npm run karma -- start $KARMA_INFO_LEVEL $KARMA_FLAGS $BROWSERS unit_tests/in_browser/$KARMA_CONFIG + + builder_finish_action success test:browser +fi \ No newline at end of file diff --git a/common/web/input-processor/build.sh b/common/web/input-processor/build.sh new file mode 100755 index 0000000000..70e0cc7398 --- /dev/null +++ b/common/web/input-processor/build.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# +# Compile KeymanWeb's 'keyboard-processor' module, one of the components of Web's 'core' module. +# +set -eu + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")" +. "$(dirname "$THIS_SCRIPT")/../../../resources/build/build-utils.sh" +. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +# This script runs from its own folder +cd "$(dirname "$THIS_SCRIPT")" + +################################ Main script ################################ + +builder_check_color "$@" + +# TODO: for predictive-text, we only need :headless, perhaps we should be splitting modules? +# TODO: remove :tools once kmlmc is a dependency for test:module + +builder_describe "Builds the standalone, headless form of Keyman Engine for Web's input-processor module" \ + "@../keyman-version" \ + "@../keyboard-processor" \ + "@../../predictive-text" \ + "clean" \ + "configure" \ + "build" \ + "test" \ + ":module A headless, Node-oriented version of the module useful for unit tests" \ + ":tools Related tools useful for development and testing of this module" \ + "--ci Sets ${BUILDER_TERM_START}test${BUILDER_TERM_END} action to use CI-based test configurations & reporting" + +builder_describe_outputs \ + configure:module /node_modules \ + configure:tools /node_modules \ + build:module build/index.js \ + build:tools /developer/src/kmlmc/dist/kmlmc.js # TODO: remove this once kmlmc is a dependency + +builder_parse "$@" + +### CONFIGURE ACTIONS + +if builder_start_action configure; then + verify_npm_setup + builder_finish_action success configure +fi + +### CLEAN ACTIONS + +if builder_start_action clean; then + rm -rf build/ + builder_finish_action success clean +fi + +### BUILD ACTIONS + +if builder_start_action build:tools; then + # Used by test:module + # TODO: convert to a dependency once we have updated kmlmc to use builder script + pushd "$KEYMAN_ROOT/developer/src/kmlmc" + ./build.sh -S + popd + + builder_finish_action success build:tools +fi + +if builder_start_action build:module; then + npm run tsc -- -b src/tsconfig.json + builder_finish_action success build:module +fi + +# TEST ACTIONS + +if builder_start_action test:module; then + FLAGS= + if builder_has_option --ci; then + FLAGS="--reporter mocha-teamcity-reporter" + fi + + # Build the leaf-style, bundled version of input-processor for use in testing. + npm run tsc -- -b src/tsconfig.bundled.json + + npm run mocha -- --recursive $FLAGS ./tests/cases/ + + builder_finish_action success test:module +fi \ No newline at end of file diff --git a/common/web/input-processor/src/build.sh b/common/web/input-processor/src/build.sh deleted file mode 100755 index 3fdc2e2b04..0000000000 --- a/common/web/input-processor/src/build.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash -# -# Compile KeymanWeb's 'keyboard-processor' module, one of the components of Web's 'core' module. -# -set -eu - -## START STANDARD BUILD SCRIPT INCLUDE -# adjust relative paths as necessary -THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")" -. "$(dirname "$THIS_SCRIPT")/../../../../resources/build/build-utils.sh" -. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" -## END STANDARD BUILD SCRIPT INCLUDE - -# This script runs from its own folder -cd "$(dirname "$THIS_SCRIPT")" - -display_usage ( ) { - echo "build.sh [-skip-package-install | -S] [-test]" - echo - echo " -test to compile for testing without re-fetching external dependencies" - echo " or recompiling the lm-layer module." - echo " -skip-package-install (or -S) skips the `npm install` dependency check." - echo "" - echo " If more than one target is specified, the last one will take precedence." - exit 1 -} - -# Establish default build parameters -set_default_vars ( ) { - BUILD_LMLAYER=true - BUILD_CORE=true - FETCH_DEPS=true -} - -set_default_vars - -# Parse args -while [[ $# -gt 0 ]] ; do - key="$1" - case $key in - -test) - set_default_vars - BUILD_LMLAYER=false - FETCH_DEPS=false - ;; - -skip-package-install|-S) - set_default_vars - FETCH_DEPS=false - ;; - esac - shift # past argument -done - -if [ $FETCH_DEPS = true ]; then - verify_npm_setup - # We need to build keyman-version with a script for now - "$KEYMAN_ROOT/common/web/keyman-version/build.sh" || fail "Could not build keyman-version" -fi - -if [ $BUILD_LMLAYER = true ]; then - FLAGS="-skip-package-install" - - # Ensure that the LMLayer compiles properly, readying the build product for comsumption by KMW. - cd ../../../predictive-text/ - echo "" - echo "Compiling the Language Modeling layer module..." - ./build.sh $FLAGS || fail "Failed to compile the language modeling layer module." - cd ../web/input-processor/src - # TODO: Move this back to KMW's main build script. Consider it part of the dependency update. - echo "Language Modeling layer compilation successful." - echo "" -fi - -if [ $BUILD_CORE = true ]; then - FLAGS="-skip-package-install" - - # Ensure that the KeyboardProcessor module compiles properly. - cd ../../keyboard-processor/src - echo "" - echo "Compiling Keyboard Processor module..." - ./build.sh $FLAGS || fail "Dependency build failed; aborting" - cd ../../input-processor/src - echo "Keyboard Processor module compilation successful." - echo "" -fi - -# Compile web's `keyboard-processor` module. -echo "Compiling Input Processor module..." -npm run tsc -- -b src/tsconfig.json || fail "Failed to compile the web/input-processor module." -echo "Input Processor module compilation successful." -echo "" \ No newline at end of file diff --git a/common/web/input-processor/test.sh b/common/web/input-processor/test.sh deleted file mode 100755 index 9a94dceace..0000000000 --- a/common/web/input-processor/test.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env bash - -set -eu - -# Include useful testing resource functions -## START STANDARD BUILD SCRIPT INCLUDE -# adjust relative paths as necessary -THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")" -. "$(dirname "$THIS_SCRIPT")/../../../resources/build/build-utils.sh" -## END STANDARD BUILD SCRIPT INCLUDE - -. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" - -# We should work within the script's directory, not the one we were called in. -cd "$THIS_SCRIPT_PATH" - -# A simple utility script to facilitate unit-testing for the LM Layer. -# It's rigged to be callable by NPM to facilitate testing during development when in other folders. - -display_usage ( ) { - echo "test.sh [-skip-package-install|-S] [-CI] [ -? | -h | -help]" - echo " -CI to perform continuous-integration friendly tests and reporting formatted for TeamCity" - echo " -? | -h | -help to display this help information" - echo " -skip-package-install to bypass refreshing dependencies. Useful when called by scripts that pre-fetch" - echo " (or -S)" - echo "" - exit 0 -} - -# Defaults -FLAGS= -CI_REPORTING=0 -FETCH_DEPS=true -CHAINING_FLAGS= - -# Parse args -while [[ $# -gt 0 ]] ; do - key="$1" - case $key in - -h|-help|-\?) - display_usage - exit - ;; - -CI) - CI_REPORTING=1 - CHAINING_FLAGS="$CHAINING_FLAGS --ci" - ;; - -skip-package-install|-S) - FETCH_DEPS=false - ;; - esac - shift # past argument -done - -if (( CI_REPORTING )); then - FLAGS="$FLAGS --reporter mocha-teamcity-reporter" -fi - -if [ $FETCH_DEPS = true ]; then - verify_npm_setup -fi - -# Ensures that the lexical model compiler has been built locally. -echo_heading "Preparing Lexical Model Compiler for test use" -pushd "$KEYMAN_ROOT/developer/src/kmlmc/" -./build.sh -popd - -test-headless ( ) { - npm run mocha -- --recursive $FLAGS ./tests/cases/ -} - -if [ $FETCH_DEPS = true ]; then - # Next, build the lm-worker in its proper, wrapped form - pushd "$KEYMAN_ROOT/common/web/lm-worker" - ./build.sh - popd -fi - -# First, run tests on the keyboard processor. -pushd "$KEYMAN_ROOT/common/web/keyboard-processor" -./build.sh test $CHAINING_FLAGS || fail "Tests failed by dependencies; aborting integration tests." -popd - -# Build the leaf-style, bundled version of input-processor for use in testing. -npm run tsc -- -b src/tsconfig.bundled.json || fail "Failed to compile the core/web/input-processor module." - -# Now we run our local tests. -echo_heading "Running Input Processor test suite" -test-headless || fail "Input Processor tests failed!" diff --git a/common/web/keyboard-processor/build.sh b/common/web/keyboard-processor/build.sh index 512d40693f..86c02f9806 100755 --- a/common/web/keyboard-processor/build.sh +++ b/common/web/keyboard-processor/build.sh @@ -23,30 +23,23 @@ builder_check_color "$@" builder_describe \ "Compiles the web-oriented utility function module." \ + "@../recorder test" \ + "@../keyman-version" \ + "@../utils" \ configure \ clean \ build \ test \ "--ci For use with action ${BUILDER_TERM_START}test${BUILDER_TERM_END} - emits CI-friendly test reports" +builder_describe_outputs \ + configure /node_modules \ + build build/index.js + builder_parse "$@" -# START - Script parameter configuration -REPORT_STYLE=local # Default setting. - -if builder_has_option --ci; then - REPORT_STYLE=ci - - echo "Replacing user-friendly test reports with CI-friendly versions." -fi - -# END - Script parameter configuration - if builder_start_action configure; then verify_npm_setup - - "$KEYMAN_ROOT/common/web/keyman-version/build.sh" - builder_finish_action success configure fi @@ -61,17 +54,13 @@ if builder_start_action build; then fi if builder_start_action test; then - # Build test dependency - pushd "$KEYMAN_ROOT/common/web/recorder" - ./build.sh - popd - npm run tsc -- --build "$THIS_SCRIPT_PATH/src/tsconfig.bundled.json" echo_heading "Running Keyboard Processor test suite" FLAGS= - if [ $REPORT_STYLE == ci ]; then + if builder_has_option --ci; then + echo "Replacing user-friendly test reports with CI-friendly versions." FLAGS="$FLAGS --reporter mocha-teamcity-reporter" fi diff --git a/common/web/keyman-version/build.sh b/common/web/keyman-version/build.sh index e2e021310a..370c33ee88 100755 --- a/common/web/keyman-version/build.sh +++ b/common/web/keyman-version/build.sh @@ -15,15 +15,18 @@ THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BA . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" # This script runs from its own folder -cd "$(dirname "$THIS_SCRIPT")" +cd "$THIS_SCRIPT_PATH" ################################ Main script ################################ builder_describe "Build the include script for current Keyman version" configure clean build + +builder_describe_outputs \ + configure "/node_modules" \ + build "build/index.js" + builder_parse "$@" -# TODO: build if out-of-date if test is specified -# TODO: configure if npm has not been run, and build is specified if builder_start_action configure; then verify_npm_setup @@ -56,7 +59,12 @@ if builder_start_action build; then } " > ./version.inc.ts - # Build - npm run build -- $builder_verbose + # Note: in a dependency build, we'll expect keyman-version to be built by tsc -b + if builder_is_dep_build; then + echo "[$THIS_SCRIPT_IDENTIFIER] skipping tsc -b; will be completed by $builder_dep_parent" + else + npm run build -- $builder_verbose + fi + builder_finish_action success build fi diff --git a/common/web/lm-worker/build.sh b/common/web/lm-worker/build.sh index 487d564da6..1633e66753 100755 --- a/common/web/lm-worker/build.sh +++ b/common/web/lm-worker/build.sh @@ -76,8 +76,13 @@ wrap-worker-code ( ) { builder_describe \ "Compiles the Language Modeling Layer for common use in predictive text and autocorrective applications." \ + "@../keyman-version" \ configure clean build test +builder_describe_outputs \ + configure /node_modules \ + build build/index.js + builder_parse "$@" # TODO: build if out-of-date if test is specified @@ -105,9 +110,6 @@ if builder_start_action build; then npm run clean fi - # Ensure keyman-version is properly build (requires build script) - "$KEYMAN_ROOT/common/web/keyman-version/build.sh" || fail "Could not build keyman-version" - # Build worker with tsc first npm run build -- $builder_verbose || fail "Could not build worker." diff --git a/common/web/recorder/build.sh b/common/web/recorder/build.sh index 441272e78f..969453f37c 100755 --- a/common/web/recorder/build.sh +++ b/common/web/recorder/build.sh @@ -19,60 +19,42 @@ THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BA builder_describe \ "Compiles the web-oriented utility function module." \ + "@../keyman-version" \ configure \ clean \ build \ ":module Builds recorder-core module" \ ":proctor Builds headless-testing, node-oriented 'proctor' component" +builder_describe_outputs \ + configure:module "/node_modules" \ + configure:proctor "/node_modules" \ + build:module "build/index.js" \ + build:proctor "build/nodeProctor/index.js" + builder_parse "$@" -# START - Script parameter configuration -REPORT_STYLE="local" # Default setting. - -if builder_has_option --ci; then - REPORT_STYLE="ci" - - echo "Replacing user-friendly test reports with CI-friendly versions." -fi - -# END - Script parameter configuration - -function do_configure() { +if builder_start_action configure; then verify_npm_setup - "$KEYMAN_ROOT/common/web/keyman-version/build.sh" -} - -if builder_start_action configure :module; then - do_configure - builder_finish_action success configure :module + builder_finish_action success configure fi -if builder_start_action configure :proctor; then - if builder_has_action configure :module; then - echo "Configuration already completed in configure:module; skipping." - else - do_configure - fi - builder_finish_action success configure :proctor -fi - -if builder_start_action clean :module; then +if builder_start_action clean:module; then npm run tsc -- -b --clean "$THIS_SCRIPT_PATH/src/tsconfig.json" - builder_finish_action success clean :module + builder_finish_action success clean:module fi -if builder_start_action clean :proctor; then +if builder_start_action clean:proctor; then npm run tsc -- -b --clean "$THIS_SCRIPT_PATH/src/nodeProctor.tsconfig.json" - builder_finish_action success clean :proctor + builder_finish_action success clean:proctor fi -if builder_start_action build :module; then +if builder_start_action build:module; then npm run tsc -- --build "$THIS_SCRIPT_PATH/src/tsconfig.json" - builder_finish_action success build :module + builder_finish_action success build:module fi -if builder_start_action build :proctor; then +if builder_start_action build:proctor; then npm run tsc -- --build "$THIS_SCRIPT_PATH/src/nodeProctor.tsconfig.json" - builder_finish_action success build :proctor + builder_finish_action success build:proctor fi \ No newline at end of file diff --git a/common/web/utils/build.sh b/common/web/utils/build.sh index 1ff55ecd8a..9232703a85 100755 --- a/common/web/utils/build.sh +++ b/common/web/utils/build.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # # Compiles common TS-based utility functions for use among Keyman's codebase + set -eu ## START STANDARD BUILD SCRIPT INCLUDE @@ -11,19 +12,23 @@ THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BA . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" +cd "$THIS_SCRIPT_PATH" + ################################ Main script ################################ builder_describe \ "Compiles the web-oriented utility function module." \ + "@../keyman-version" \ configure clean build +builder_describe_outputs \ + configure "/node_modules" \ + build "build/index.js" + builder_parse "$@" if builder_start_action configure; then verify_npm_setup - - "$KEYMAN_ROOT/common/web/keyman-version/build.sh" - builder_finish_action success configure fi @@ -33,6 +38,11 @@ if builder_start_action clean; then fi if builder_start_action build; then - npm run tsc -- --build "$THIS_SCRIPT_PATH/tsconfig.json" + # Note: in a dependency build, we'll expect utils to be built by tsc -b + if builder_is_dep_build; then + echo "[$THIS_SCRIPT_IDENTIFIER] skipping tsc -b; will be completed by $builder_dep_parent" + else + npm run tsc -- --build "$THIS_SCRIPT_PATH/tsconfig.json" + fi builder_finish_action success build fi \ No newline at end of file diff --git a/linux/Dockerfile b/linux/Dockerfile index 203d33aef1..8572b983f8 100644 --- a/linux/Dockerfile +++ b/linux/Dockerfile @@ -21,4 +21,6 @@ ADD debian/control /tmp/control # Answer 'yes' to install questions RUN (yes | mk-build-deps --install /tmp/control) || true # now, switch to build user +RUN curl -sL https://deb.nodesource.com/setup_16.x | bash +RUN apt-get -q -y install nodejs USER build diff --git a/linux/LICENSE.md b/linux/LICENSE.md index 153a83e91b..98cdd5435c 100644 --- a/linux/LICENSE.md +++ b/linux/LICENSE.md @@ -1,17 +1,16 @@ # LICENSE -The [kmflcomp](./kmflcomp), [libkmfl](./libkmfl), and [keyman_config](./keyman_config) projects -are covered by the [MIT license](./libkmfl/COPYING). +The [keyman-config](./keyman-config) project is covered by the +[MIT license](./keyman-config/COPYING). -The [ibus-kmfl](./ibus-kmfl) and [ibus-keyman](./ibus-keyman) projects are licensed under GNU -General Public License as published by the Free Software Foundation; either -[version 2 of the License](./ibus-kmfl/COPYING), or (at your option) any later version. +The [ibus-keyman](./ibus-keyman) projects is licensed under GNU General Public License +as published by the Free Software Foundation; either +[version 2 of the License](./ibus-keyman/COPYING), or (at your option) any later version. -Two files in ibus-kmfl, [kmflutil.c](./ibus-kmfl/src/kmflutil.c) and -[kmflutil.h](./ibus-kmfl/src/kmflutil.h), as well as four files in ibus-keyman, -[keymanutil.c](./ibus-keyman/src/keymanutil.c), [keymanutil.h](./ibus-keyman/src/keymanutil.h), -[kmpdetails.c](./ibus-keyman/src/kmpdetails.c) and [kmpdetails.h](./ibus-keyman/src/kmpdetails.h), -are dual licensed by the MIT license and the GNU General Public License which is described above. +Four files in ibus-keyman, [keymanutil.c](./ibus-keyman/src/keymanutil.c), +[keymanutil.h](./ibus-keyman/src/keymanutil.h), [kmpdetails.c](./ibus-keyman/src/kmpdetails.c) +and [kmpdetails.h](./ibus-keyman/src/kmpdetails.h), are dual licensed by the MIT license and +the GNU General Public License which is described above. The MIT license alone may be chosen for them for their use in other projects. Two files in ibus-keyman, [keyman-service.c](./ibus-keyman/src/keyman-service.c) and diff --git a/linux/README.md b/linux/README.md index 870fddabb9..9a84ba4fff 100644 --- a/linux/README.md +++ b/linux/README.md @@ -16,7 +16,7 @@ See [license information](./LICENSE.md) about licensing. - It is helpful to be using the [packages.sil.org](http://packages.sil.org) repo -- Install packages required for building and developing KMFL and Keyman for Linux +- Install packages required for building and developing Keyman for Linux ```bash sudo apt install cdbs debhelper libx11-dev autotools-dev build-essential \ @@ -31,7 +31,7 @@ See [license information](./LICENSE.md) about licensing. ### Build script -#### Installing for ibus to use ibus-kmfl or ibus-keyman +#### Installing for ibus to use ibus-keyman - The process to build and install everything is: @@ -44,10 +44,8 @@ See [license information](./LICENSE.md) about licensing. To do this run `sudo make install` - This will install to `/usr/local` - - and `/usr/share/ibus/component/kmfl.xml` and `/usr/share/kmfl/icons` - and `/usr/share/ibus/component/keyman.xml` and `/usr/share/keyman/icons` - - If you already have the ibus-kmfl package installed then it will move the file `/usr/share/ibus/component/kmfl.xml` to `/usr/share/doc/ibus-kmfl/` - If you already have the ibus-keyman package installed then it will move the file `/usr/share/ibus/component/keyman.xml` to `/usr/share/doc/ibus-keyman/` - run `sudo make uninstall` to uninstall everything and put it back again @@ -56,20 +54,16 @@ See [license information](./LICENSE.md) about licensing. Used by TC for validating PRs -Run `make tmpinstall` to build and install **keyboardprocessor**, **kmflcomp**, **libkmfl**, **ibus-kmfl** and **ibus-keyman** to `/tmp/kmfl` +Run `make tmpinstall` to build and install **keyboardprocessor** and **ibus-keyman** to `/tmp/keyman` -This is only for testing the build, not for running **ibus-kmfl** or **ibus-keyman** in ibus +This is only for testing the build, not for running **ibus-keyman** in ibus ### Manually -- **libkmfl** requires `kmfl.h` header from **kmflcomp** -- **ibus-kfml** requires `kmfl.h` and headers and lib from **libkmfl** - **ibus-keyman** requires headers and lib from **keyboardprocessor** So -- **kmflcomp** must be built and installed before **libkmfl** -- **libkmfl** must be built and installed before **ibus-kmfl** - **keyboardprocessor** must be built before **ibus-keyman** For each project run `./configure && make && make install`. @@ -77,16 +71,13 @@ For each project run `./configure && make && make install`. You may prefer to create a different directory to build in and run configure from there e.g. ```bash -mkdir ../build-kmflcomp -cd ../build-kmflcomp -../kmflcomp/configure +mkdir ../build-ibus-keyman +cd ../build-ibus-keyman +../ibus-keyman/configure make make install ``` -The install of **ibus-kmfl** doesn't install everything to the correct location for it to be -used - to be fixed - ## Continuous integration Teamcity PR builds will run `make tmpinstall` diff --git a/linux/README.repo b/linux/README.repo deleted file mode 100644 index 708cda84bd..0000000000 --- a/linux/README.repo +++ /dev/null @@ -1,7 +0,0 @@ -You need autoconf, autopoint, gettext, automake and libtool to generate the build system - -flex and bison are useful to rebuild lex.l and yacc.y in kmflcomp but not essential - -For keyboardprocessor you need rust, meson and ninja (meson package pulls in ninja) - -Run `make reconf` to run them diff --git a/linux/ibus-keyman/src/engine.c b/linux/ibus-keyman/src/engine.c index eda666adaa..cc48ddffc8 100644 --- a/linux/ibus-keyman/src/engine.c +++ b/linux/ibus-keyman/src/engine.c @@ -51,32 +51,46 @@ #define KEYMAN_LALT 56 // 0x38 #define KEYMAN_RCTRL 97 // 0x61 #define KEYMAN_RALT 100 // 0x64 +#define KEYMAN_F24_KEYCODE_OUTPUT_SENTINEL 202 +#define KEYMAN_NOCHAR_KEYSYM (0xfdd0 | 0x1000000) // Unicode NOCHAR typedef struct _IBusKeymanEngine IBusKeymanEngine; typedef struct _IBusKeymanEngineClass IBusKeymanEngineClass; -struct _IBusKeymanEngine { - IBusEngine parent; +#define MAX_QUEUE_SIZE 100 - /* members */ - km_kbp_keyboard *keyboard; - km_kbp_state *state; - gchar *ldmlfile; - gchar *kb_name; - gchar *char_buffer; - gboolean lctrl_pressed; - gboolean rctrl_pressed; - gboolean lalt_pressed; - gboolean ralt_pressed; - gboolean emitting_keystroke; - IBusProperty *status_prop; - IBusPropList *prop_list; +typedef struct _commit_queue_item { + gchar *char_buffer; + gboolean emitting_keystroke; + guint keyval; + guint keycode; + guint state; +} commit_queue_item; + +struct _IBusKeymanEngine { + IBusEngine parent; + + /* members */ + km_kbp_keyboard *keyboard; + km_kbp_state *state; + gchar *ldmlfile; + gchar *kb_name; + gboolean lctrl_pressed; + gboolean rctrl_pressed; + gboolean lalt_pressed; + gboolean ralt_pressed; + IBusLookupTable *table; + IBusProperty *status_prop; + IBusPropList *prop_list; #ifdef GDK_WINDOWING_X11 - Display *xdisplay; + Display *xdisplay; #endif #ifdef GDK_WINDOWING_WAYLAND - GdkWaylandDisplay *wldisplay; + GdkWaylandDisplay *wldisplay; #endif + + commit_queue_item commit_queue[MAX_QUEUE_SIZE]; + commit_queue_item *commit_item; }; struct _IBusKeymanEngineClass { @@ -212,9 +226,22 @@ static gchar *get_current_context_text(km_kbp_context *context) return current_context_utf8; } -static void reset_context(IBusEngine *engine) +static gboolean +client_supports_prefilter(IBusEngine *engine) { - IBusKeymanEngine *keyman = (IBusKeymanEngine *) engine; + g_assert(engine != NULL); + return (engine->client_capabilities & IBUS_CAP_PREFILTER) != 0; +} + +static gboolean +client_supports_surrounding_text(IBusEngine *engine) { + g_assert(engine != NULL); + return (engine->client_capabilities & IBUS_CAP_SURROUNDING_TEXT) != 0; +} + +static void +reset_context(IBusEngine *engine) { + IBusKeymanEngine *keyman = (IBusKeymanEngine *)engine; IBusText *text; gchar *surrounding_text, *current_context_utf8; guint cursor_pos, anchor_pos, context_start, context_pos; @@ -250,6 +277,15 @@ static void reset_context(IBusEngine *engine) } } +static void +initialize_queue(IBusKeymanEngine *keyman, int index, int count) { + g_assert(keyman != NULL); + g_assert(index >= 0 && index < MAX_QUEUE_SIZE); + g_assert(count > 0 && count <= MAX_QUEUE_SIZE); + g_assert(index + count <= MAX_QUEUE_SIZE); + memset(&keyman->commit_queue[index], 0, sizeof(commit_queue_item) * count); +} + static void ibus_keyman_engine_init(IBusKeymanEngine *keyman) { gdk_init(NULL, NULL); @@ -315,8 +351,9 @@ ibus_keyman_engine_constructor( keyman->lctrl_pressed = FALSE; keyman->ralt_pressed = FALSE; keyman->rctrl_pressed = FALSE; - keyman->emitting_keystroke = FALSE; - gchar **split_name = g_strsplit(engine_name, ":", 2); + initialize_queue(keyman, 0, MAX_QUEUE_SIZE); + keyman->commit_item = &keyman->commit_queue[0]; + gchar **split_name = g_strsplit(engine_name, ":", 2); if (split_name[0] == NULL) { IBUS_OBJECT_CLASS (parent_class)->destroy ((IBusObject *)keyman); @@ -510,17 +547,17 @@ process_unicode_char_action( g_free(utf8); } else { g_message("unichar:U+%04x, bytes:%d, string:%s", action_item->character, numbytes, utf8); - if (keyman->char_buffer == NULL) { + if (keyman->commit_item->char_buffer == NULL) { g_message("setting buffer to converted unichar"); - keyman->char_buffer = utf8; + keyman->commit_item->char_buffer = utf8; } else { g_message("appending converted unichar to CHAR buffer"); - gchar *new_buffer = g_strjoin("", keyman->char_buffer, utf8, NULL); - g_free(keyman->char_buffer); + gchar *new_buffer = g_strjoin("", keyman->commit_item->char_buffer, utf8, NULL); + g_free(keyman->commit_item->char_buffer); g_free(utf8); - keyman->char_buffer = new_buffer; + keyman->commit_item->char_buffer = new_buffer; } - g_message("CHAR buffer is now %s", keyman->char_buffer); + g_message("CHAR buffer is now %s", keyman->commit_item->char_buffer); } return TRUE; } @@ -544,28 +581,27 @@ process_backspace_action( IBusKeymanEngine *keyman = (IBusKeymanEngine *)engine; if (action_items[i].backspace.expected_type == KM_KBP_IT_MARKER) { g_message("skipping marker type"); - } else if (keyman->char_buffer != NULL) { - // ibus_keyman_engine_commit_string(keyman, keyman->char_buffer); + } else if (keyman->commit_item->char_buffer != NULL) { g_message("removing one utf8 char from CHAR buffer"); - glong end_pos = g_utf8_strlen(keyman->char_buffer, -1); + glong end_pos = g_utf8_strlen(keyman->commit_item->char_buffer, -1); gchar *new_buffer; if (end_pos == 1) { new_buffer = NULL; g_message("resetting CHAR buffer to NULL"); } else { - new_buffer = g_utf8_substring(keyman->char_buffer, 0, end_pos - 1); + new_buffer = g_utf8_substring(keyman->commit_item->char_buffer, 0, end_pos - 1); g_message("changing CHAR buffer to :%s:", new_buffer); } - if (g_strcmp0(keyman->char_buffer, new_buffer) == 0) { + if (g_strcmp0(keyman->commit_item->char_buffer, new_buffer) == 0) { g_message("oops, CHAR buffer hasn't changed"); } - g_free(keyman->char_buffer); - keyman->char_buffer = new_buffer; + g_free(keyman->commit_item->char_buffer); + keyman->commit_item->char_buffer = new_buffer; } else { g_message( "DAR: process_backspace_action - client_capabilities=%x, %x", engine->client_capabilities, IBUS_CAP_SURROUNDING_TEXT); - if ((engine->client_capabilities & IBUS_CAP_SURROUNDING_TEXT) != 0) { + if (client_supports_surrounding_text(engine)) { g_message("deleting surrounding text 1 char"); ibus_engine_delete_surrounding_text(engine, -1, 1); } else { @@ -609,18 +645,21 @@ process_persist_action( return TRUE; } -static gboolean process_emit_keystroke_action(IBusKeymanEngine *keyman) { - if (keyman->char_buffer != NULL) { - ibus_keyman_engine_commit_string(keyman, keyman->char_buffer); - g_free(keyman->char_buffer); - keyman->char_buffer = NULL; +static gboolean +process_emit_keystroke_action(IBusKeymanEngine *keyman) { + IBusEngine *engine = (IBusEngine *)keyman; + if ((!client_supports_prefilter(engine) || client_supports_surrounding_text(engine)) && + keyman->commit_item->char_buffer != NULL) { + ibus_keyman_engine_commit_string(keyman, keyman->commit_item->char_buffer); + g_free(keyman->commit_item->char_buffer); + keyman->commit_item->char_buffer = NULL; } - keyman->emitting_keystroke = TRUE; + keyman->commit_item->emitting_keystroke = TRUE; return TRUE; } -static gboolean process_invalidate_context_action(IBusEngine *engine) { - IBusKeymanEngine *keyman = (IBusKeymanEngine *)engine; +static gboolean +process_invalidate_context_action(IBusEngine *engine) { reset_context(engine); return TRUE; } @@ -648,17 +687,69 @@ process_capslock_action( return TRUE; } -static gboolean process_end_action(IBusKeymanEngine *keyman) { - if (keyman->char_buffer != NULL) { - ibus_keyman_engine_commit_string(keyman, keyman->char_buffer); - g_free(keyman->char_buffer); - keyman->char_buffer = NULL; +static void +commit_text(IBusKeymanEngine *keyman) { + g_assert(keyman != NULL); + if (keyman->commit_item <= keyman->commit_queue) + return; + + commit_queue_item *current_item = &keyman->commit_queue[0]; + if (current_item->char_buffer != NULL) { + ibus_keyman_engine_commit_string(keyman, current_item->char_buffer); + g_free(current_item->char_buffer); } - if (keyman->emitting_keystroke) { - keyman->emitting_keystroke = FALSE; - return FALSE; + if (current_item->emitting_keystroke) { + ibus_engine_forward_key_event((IBusEngine*)keyman, current_item->keyval, current_item->keycode, current_item->state); + } + keyman->commit_item--; + memmove(keyman->commit_queue, &keyman->commit_queue[1], sizeof(commit_queue_item) * MAX_QUEUE_SIZE - 1); + initialize_queue(keyman, MAX_QUEUE_SIZE - 1, 1); +} + +static gboolean +process_end_action(IBusKeymanEngine *keyman) { + g_assert(keyman != NULL); + IBusEngine *engine = (IBusEngine *)keyman; + if (client_supports_prefilter(engine) && !client_supports_surrounding_text(engine)) { + guint state = keyman->commit_item->state; + keyman->commit_item++; + if (keyman->commit_item > &keyman->commit_queue[MAX_QUEUE_SIZE-1]) { + g_error("Overflow of keyman commit_queue!"); + // TODO: log to Sentry + keyman->commit_item--; + } + + // Forward a fake key event to get the correct order of events so that any backspace key we + // generated will be processed before the character we're adding. We need to send a + // valid keyval/keycode combination so that it doesn't get swallowed by GTK but which + // isn't very likely used in real keyboards. F24 seems to work for that. + ibus_engine_forward_key_event((IBusEngine*)keyman, + KEYMAN_NOCHAR_KEYSYM, + KEYMAN_F24_KEYCODE_OUTPUT_SENTINEL, + (state & IBUS_RELEASE_MASK) + ? IBUS_PREFILTER_MASK | IBUS_RELEASE_MASK + : IBUS_PREFILTER_MASK); + } else { + if (keyman->commit_item->char_buffer != NULL) { + ibus_keyman_engine_commit_string(keyman, keyman->commit_item->char_buffer); + g_free(keyman->commit_item->char_buffer); + keyman->commit_item->char_buffer = NULL; + } + if (keyman->commit_item->emitting_keystroke) { + keyman->commit_item->emitting_keystroke = FALSE; + // We have an old ibus version without prefilter support, or a client that does support + // surrounding text. In either case we return FALSE because we emitted a keystroke + // so that the processing of the event will continue. + return FALSE; + } } + // If we have a new ibus version that supports prefilter and a client that doesn't support + // surrounding text (e.g. Chromium as of v104) we forwarded the key event with + // IBUS_PREFILTER_MASK set and now return TRUE here to stop further processing. + // With an old ibus version without prefilter support, or with a client that does support + // surrounding text, we return TRUE if we completely processed the event and no further + // processing should happen. return TRUE; } @@ -675,6 +766,7 @@ process_actions( case KM_KBP_IT_CHAR: g_message("CHAR action %d/%d", i + 1, (int)num_action_items); continue_with_next_action = process_unicode_char_action(keyman, &action_items[i]); + g_assert(continue_with_next_action == TRUE); break; case KM_KBP_IT_MARKER: g_message("MARKER action %d/%d", i + 1, (int)num_action_items); @@ -682,26 +774,32 @@ process_actions( case KM_KBP_IT_ALERT: g_message("ALERT action %d/%d", i + 1, (int)num_action_items); continue_with_next_action = process_alert_action(); + g_assert(continue_with_next_action == TRUE); break; case KM_KBP_IT_BACK: g_message("BACK action %d/%d", i + 1, (int)num_action_items); continue_with_next_action = process_backspace_action(engine, action_items, i, num_action_items); + g_assert(continue_with_next_action == TRUE); break; case KM_KBP_IT_PERSIST_OPT: g_message("PERSIST_OPT action %d/%d", i + 1, (int)num_action_items); continue_with_next_action = process_persist_action(keyman, &action_items[i]); + g_assert(continue_with_next_action == TRUE); break; case KM_KBP_IT_EMIT_KEYSTROKE: g_message("EMIT_KEYSTROKE action %d/%d", i + 1, (int)num_action_items); continue_with_next_action = process_emit_keystroke_action(keyman); + g_assert(continue_with_next_action == TRUE); break; case KM_KBP_IT_INVALIDATE_CONTEXT: g_message("INVALIDATE_CONTEXT action %d/%d", i + 1, (int)num_action_items); continue_with_next_action = process_invalidate_context_action(engine); + g_assert(continue_with_next_action == TRUE); break; case KM_KBP_IT_CAPSLOCK: g_message("CAPSLOCK action %d/%d", i + 1, (int)num_action_items); continue_with_next_action = process_capslock_action(keyman, &action_items[i]); + g_assert(continue_with_next_action == TRUE); break; case KM_KBP_IT_END: g_message("END action %d/%d", i + 1, (int)num_action_items); @@ -724,13 +822,24 @@ ibus_keyman_engine_process_key_event( guint state ) { IBusKeymanEngine *keyman = (IBusKeymanEngine *)engine; + keyman->commit_item->keyval = keyval; + keyman->commit_item->keycode = keycode; + keyman->commit_item->state = state; gboolean isKeyDown = !(state & IBUS_RELEASE_MASK); g_message("-----------------------------------------------------------------------------------------------------------------"); g_message( - "DAR: ibus_keyman_engine_process_key_event - keyval=0x%02x keycode=0x%02x, state=0x%02x, isKeyDown=%d", keyval, keycode, - state, isKeyDown); + "DAR: ibus_keyman_engine_process_key_event - keyval=0x%02x keycode=0x%02x, state=0x%02x, isKeyDown=%d, supports_prefilter=%d", keyval, keycode, + state, isKeyDown, client_supports_prefilter(engine)); + + // This keycode is a fake keycode that we send when it's time to commit the text, ensuring the + // correct output order of backspace and text. + if (client_supports_prefilter(engine) && !client_supports_surrounding_text(engine) && + keycode == KEYMAN_F24_KEYCODE_OUTPUT_SENTINEL && (state & IBUS_PREFILTER_MASK)) { + commit_text(keyman); + return TRUE; + } // REVIEW: why don't we handle these keys? switch (keycode) { @@ -800,12 +909,17 @@ ibus_keyman_engine_process_key_event( // km_kbp_state_action_items to get action items size_t num_action_items; - g_free(keyman->char_buffer); - keyman->char_buffer = NULL; + g_free(keyman->commit_item->char_buffer); + keyman->commit_item->char_buffer = NULL; const km_kbp_action_item *action_items = km_kbp_state_action_items(keyman->state, &num_action_items); - if (!process_actions(engine, action_items, num_action_items)) + if (!process_actions(engine, action_items, num_action_items) && + (!client_supports_prefilter(engine) || client_supports_surrounding_text(engine))) { + // If we have an old ibus version without prefilter support, or a client that supports + // surrounding text, and we forwarded a key event we want to return FALSE so that the + // processing of the event continues. return FALSE; + } context = km_kbp_state_context(keyman->state); g_message("after processing all actions"); diff --git a/linux/keyman-config/Makefile b/linux/keyman-config/Makefile index cb0fe8ab51..e539765069 100644 --- a/linux/keyman-config/Makefile +++ b/linux/keyman-config/Makefile @@ -8,17 +8,20 @@ langtags: install: # run as sudo pip3 install qrcode sentry-sdk + # eventually change this to: pip3 install . python3 setup.py install # install icons mkdir -p /usr/local/share/keyman/icons cp keyman_config/icons/* /usr/local/share/keyman/icons # install man pages mkdir -p /usr/local/share/man/man1 - cp debian/man/*.1 /usr/local/share/man/man1 + cp ../../debian/man/*.1 /usr/local/share/man/man1 install-temp: - mkdir -p /tmp/kmfl/`python3 -c 'import sys;import os;pythonver="python%d.%d" % (sys.version_info[0], sys.version_info[1]);sitedir = os.path.join("lib", pythonver, "site-packages");print(sitedir)'` - PYTHONUSERBASE=/tmp/kmfl python3 setup.py install --user + mkdir -p /tmp/keyman/$(shell python3 -c 'import sys;import os;pythonver="python%d.%d" % (sys.version_info[0], sys.version_info[1]);sitedir = os.path.join("lib", pythonver, "site-packages");print(sitedir)') + # when we no longer have to support old pip version (python > 3.6) change this to: + # pip3 install --prefix /tmp/keyman . + PYTHONUSERBASE=/tmp/keyman python3 setup.py install --user uninstall: clean # run as sudo rm -rf /usr/local/share/keyman/icons @@ -48,7 +51,7 @@ deb: dist cd make_deb && tar xf ../dist/keyman_config-$(shell cat /tmp/keyman_version).tar.gz && \ mv keyman_config-$(shell cat /tmp/keyman_version) keyman-config-$(shell cat /tmp/keyman_version) && \ tar cfz keyman-config_$(shell cat /tmp/keyman_version).orig.tar.gz keyman-config-$(shell cat /tmp/keyman_version) && \ - cd keyman-config-$(shell cat /tmp/keyman_version) && cp -a ../../debian . && dch -v$(shell cat /tmp/keyman_version)-1 "" + cd keyman-config-$(shell cat /tmp/keyman_version) && cp -a ../../../debian . && dch -v$(shell cat /tmp/keyman_version)-1 "" cd make_deb/keyman-config-$(shell cat /tmp/keyman_version) && debuild -us -uc rm /tmp/keyman_version diff --git a/linux/keyman-config/README.md b/linux/keyman-config/README.md index fdacc38f9c..59a0840e6b 100644 --- a/linux/keyman-config/README.md +++ b/linux/keyman-config/README.md @@ -8,22 +8,29 @@ then you will need to: ```bash sudo apt install python3-lxml python3-magic python3-numpy python3-qrcode python3-pil \ python3-requests python3-requests-cache python3 python3-gi gir1.2-webkit2-4.0 dconf-cli \ - python3-setuptools python3-pip python3-dbus ibus + python3-setuptools python3-pip python3-dbus ibus libglib2.0-bin liblocale-gettext-perl ``` -Either `python3-raven` or `python3-sentry-sdk` is required as well. To install it on Ubuntu 18.04 and earlier run: - -```bash -sudo apt install python3-raven -``` - -On Ubuntu 20.04 and later: +Either `python3-raven` or `python3-sentry-sdk` (>= 1.4) is required as well. On Ubuntu 22.04 and later run: ```bash sudo apt install python3-sentry-sdk ``` -(It's also possible to install it with pip: `pip3 install sentry-sdk`) +To install it on Ubuntu 18.04 and earlier run: + +```bash +sudo apt install python3-raven +``` + +For Ubuntu versions that don't provide `python3-raven` but instead provide +`python3-sentry-sdk` in a too old version (i.e. Ubuntu 20.04): +Install `python3-sentry-sdk` from `packages.sil.org`, +or install it with pip: + +```bash +pip3 install sentry-sdk +``` Run the script `./createkeymandirs.sh` to create the directories for these programs to install the packages to. @@ -42,13 +49,13 @@ Running `km-config` requires a language tag mapping file `keyman_config/standards/lang_tags_map.py`. This file gets generated during a package build, and also when running `make`. -### Installing manually from the repo +## Installing manually from the repo -`make && sudo make install` will install locally to `/usr/local` +`make && sudo make install` will install locally to `/usr/local`. -`python3 setup.py --help install` will give you more install options +`pip3 help install` will give you more install options. -You will need `sudo apt install python3-pip` to `make uninstall` +To uninstall you can run `sudo make uninstall`. ## Things to run from the command line diff --git a/linux/keyman-config/setup.py b/linux/keyman-config/setup.py index fef56b97cb..c090be9fbf 100644 --- a/linux/keyman-config/setup.py +++ b/linux/keyman-config/setup.py @@ -20,8 +20,8 @@ setup( ], # metadata to display on PyPI - author="Daniel Glassey", - author_email="wdg@debian.org", + author="Keyman team", + author_email="support@keyman.com", description="Keyman for Linux configuration", license="MIT", keywords="keyman, keyman-config, keyboard", diff --git a/linux/legacy/LICENSE.md b/linux/legacy/LICENSE.md new file mode 100644 index 0000000000..25cba465bf --- /dev/null +++ b/linux/legacy/LICENSE.md @@ -0,0 +1,13 @@ +# LICENSE + +The [kmflcomp](./kmflcomp) and [libkmfl](./libkmfl) projects are covered by the +[MIT license](./libkmfl/COPYING). + +The [ibus-kmfl](./ibus-kmfl) project is licensed under GNU General Public License +as published by the Free Software Foundation; either +[version 2 of the License](./ibus-kmfl/COPYING), or (at your option) any later version. + +Two files in ibus-kmfl, [kmflutil.c](./ibus-kmfl/src/kmflutil.c) and +[kmflutil.h](./ibus-kmfl/src/kmflutil.h) are dual licensed by the MIT license and the +GNU General Public License which is described above. +The MIT license alone may be chosen for them for their use in other projects. diff --git a/linux/legacy/README.md b/linux/legacy/README.md new file mode 100644 index 0000000000..d65723012d --- /dev/null +++ b/linux/legacy/README.md @@ -0,0 +1,80 @@ +# Keyman for Linux + +## Projects + +- [kmflcomp](./kmflcomp) - KMFL keyboard compiler +- [libkmfl](./libkmfl) - older KMFL core library +- [ibus-kmfl](./ibus-kmfl) - IBUS integration to use KMFL + +See [license information](./LICENSE.md) about licensing. + +## Linux Requirements/Setup + +- It is helpful to be using the [packages.sil.org](http://packages.sil.org) repo + +- Install packages required for building and developing KMFL + + ```bash + sudo apt install cdbs debhelper libx11-dev autotools-dev build-essential \ + dh-autoreconf flex bison libibus-1.0-dev python3-setuptools meson \ + libjson-glib-dev libgtk-3-dev libxml2-utils help2man python3-lxml \ + python3-magic python3-numpy python3-pil python3-pip python3-qrcode \ + python3-requests python3-requests-cache python3 python3-gi dconf-cli \ + dconf-editor cargo python3-dbus + ``` + +## Compiling from Command Line + +### Build script + +#### Installing for ibus to use ibus-kmfl + +- The process to build and install everything is: + + - `make reconf` to create the build system and set the version + - `make fullbuild` to configure and build + - `sudo make install` to install to `/usr/local` + +- Some of the files must be installed to `/usr/share/` so `make install` must be run as `sudo`. + + To do this run `sudo make install` + + - This will install to `/usr/local` + - and `/usr/share/ibus/component/kmfl.xml` and `/usr/share/kmfl/icons` + + - If you already have the ibus-kmfl package installed then it will move the file `/usr/share/ibus/component/kmfl.xml` to `/usr/share/doc/ibus-kmfl/` + +- run `sudo make uninstall` to uninstall everything and put it back again + +#### Tmp install + +Used by TC for validating PRs + +Run `make tmpinstall` to build and install **keyboardprocessor**, **kmflcomp**, **libkmfl**, **ibus-kmfl** and **ibus-keyman** to `/tmp/kmfl` + +This is only for testing the build, not for running **ibus-kmfl** in ibus + +### Manually + +- **libkmfl** requires `kmfl.h` header from **kmflcomp** +- **ibus-kfml** requires `kmfl.h` and headers and lib from **libkmfl** + +So + +- **kmflcomp** must be built and installed before **libkmfl** +- **libkmfl** must be built and installed before **ibus-kmfl** + +For each project run `./configure && make && make install`. + +You may prefer to create a different directory to build in and run configure from there e.g. + +```bash +mkdir ../build-kmflcomp +cd ../build-kmflcomp +../kmflcomp/configure +make +make install +``` + +The install of **ibus-kmfl** doesn't install everything to the correct location for it to be +used - to be fixed diff --git a/resources/build/tests/builder-deps.test.sh b/resources/build/tests/builder-deps.test.sh new file mode 100755 index 0000000000..ca1a59e672 --- /dev/null +++ b/resources/build/tests/builder-deps.test.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +set -eu + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")" +. "$(dirname "$THIS_SCRIPT")/../build-utils.sh" +# END STANDARD BUILD SCRIPT INCLUDE + +. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" + +builder_describe \ + "Tests dependency builds" \ + "@./dep1" \ + "@./dep2 test" \ + "@./dep3 test:*" \ + "@./dep4 *:project" \ + "@./dep5 build:bar test:*" \ + "@./dep6 build:bar test" \ + configure \ + build \ + test \ + :project \ + :bar + +builder_describe_outputs \ + configure builder.inc.test.sh \ + build non-existing-file \ + test non-existing-file + +builder_parse configure build test + +function test_dep_should_build() { + local at="$1" + local dep="$2" + + if ! _builder_should_build_dep "$at" "resources/build/tests/$dep"; then + fail "FAIL: expecting to build dependency $dep for $at" + else + echo "PASS: will build dependency $dep for $at" + fi +} + +function test_dep_should_not_build() { + local at="$1" + local dep="$2" + + if _builder_should_build_dep "$at" "resources/build/tests/$dep"; then + fail "FAIL: not expecting to build dependency $dep for $at" + else + echo "PASS: will not build dependency $dep for $at" + fi +} + +test_dep_should_build configure:project dep1 +test_dep_should_build build:project dep1 +test_dep_should_build test:project dep1 +test_dep_should_build configure:bar dep1 +test_dep_should_build build:bar dep1 +test_dep_should_build test:bar dep1 + +test_dep_should_not_build configure:project dep2 +test_dep_should_not_build build:project dep2 +test_dep_should_build test:project dep2 +test_dep_should_not_build configure:bar dep2 +test_dep_should_not_build build:bar dep2 +test_dep_should_build test:bar dep2 + +test_dep_should_not_build configure:project dep3 +test_dep_should_not_build build:project dep3 +test_dep_should_build test:project dep3 +test_dep_should_not_build configure:bar dep3 +test_dep_should_not_build build:bar dep3 +test_dep_should_build test:bar dep3 + +test_dep_should_build configure:project dep4 +test_dep_should_build build:project dep4 +test_dep_should_build test:project dep4 +test_dep_should_not_build configure:bar dep4 +test_dep_should_not_build build:bar dep4 +test_dep_should_not_build test:bar dep4 + +test_dep_should_not_build configure:project dep5 +test_dep_should_not_build build:project dep5 +test_dep_should_build test:project dep5 +test_dep_should_not_build configure:bar dep5 +test_dep_should_build build:bar dep5 +test_dep_should_build test:bar dep5 + +test_dep_should_not_build configure:project dep6 +test_dep_should_not_build build:project dep6 +test_dep_should_build test:project dep6 +test_dep_should_not_build configure:bar dep6 +test_dep_should_build build:bar dep6 +test_dep_should_build test:bar dep6 diff --git a/resources/build/tests/builder.inc.test.sh b/resources/build/tests/builder.inc.test.sh index 4fccb2a072..7723ccb40b 100755 --- a/resources/build/tests/builder.inc.test.sh +++ b/resources/build/tests/builder.inc.test.sh @@ -61,13 +61,6 @@ else fail "FAIL: should have matched action build for :project" fi -if builder_start_action clean :app; then - echo "Cleaning " - builder_finish_action success clean :app -else - fail "FAIL: should have matched action clean for :app" -fi - if builder_start_action clean:app; then echo "Cleaning " builder_finish_action success clean:app @@ -75,14 +68,14 @@ else fail "FAIL: should have matched action clean for :app" fi -if builder_start_action build :app; then +if builder_start_action build:app; then echo "Building app" - builder_finish_action success build :app + builder_finish_action success build:app else fail "FAIL: should have matched action build for :app" fi -if builder_start_action build :module; then +if builder_start_action build:module; then fail "FAIL: should not have matched action build for :module" fi @@ -163,11 +156,14 @@ fi # Due to the nature of the build-utils-traps tests, only one may be # specified at a time; each ends with an `exit`. -echo "Running separate tests" +echo "${COLOR_BLUE}## Running trap tests${COLOR_RESET}" $THIS_SCRIPT_PATH/build-utils-traps.test.sh error $THIS_SCRIPT_PATH/build-utils-traps.test.sh error-in-function $THIS_SCRIPT_PATH/build-utils-traps.test.sh incomplete -echo "Fin" +echo "${COLOR_BLUE}## Running dependency tests${COLOR_RESET}" +$THIS_SCRIPT_PATH/builder-deps.test.sh +echo "${COLOR_BLUE}## End external tests${COLOR_RESET}" +echo # Finally, run with --help so we can see what it looks like # Note: calls `exit`, so no further tests may be defined. diff --git a/resources/builder.inc.sh b/resources/builder.inc.sh index b6137ec3b0..ebdf5fb8b9 100755 --- a/resources/builder.inc.sh +++ b/resources/builder.inc.sh @@ -36,16 +36,19 @@ function _builder_findRepoRoot() { } # Used to build script-related build variables useful for referencing the calling script -# and for prefixing builder_finish_action outputs in order to more clearly identify the calling +# and for prefixing `builder_finish_action` outputs in order to more clearly identify the calling # script. # -# Assumes that THIS_SCRIPT has been set, typically like this: +# Assumes that `THIS_SCRIPT` has been set, typically like this: +# +# ```bash +# ## START STANDARD BUILD SCRIPT INCLUDE +# # adjust relative paths as necessary +# THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")" +# . "$(dirname "$THIS_SCRIPT")/resources/builder.inc.sh" +# ## END STANDARD BUILD SCRIPT INCLUDE +# ``` # -# ## START STANDARD BUILD SCRIPT INCLUDE -# # adjust relative paths as necessary -# THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")" -# . "$(dirname "$THIS_SCRIPT")/resources/builder.inc.sh" -# ## END STANDARD BUILD SCRIPT INCLUDE function _builder_setBuildScriptIdentifiers() { if [ ! -z ${THIS_SCRIPT+x} ]; then THIS_SCRIPT_PATH="$(dirname "$THIS_SCRIPT")" @@ -141,10 +144,35 @@ _builder_item_in_array() { local e match="$1" shift [[ -z "$match" ]] && return 1 - for e; do [[ "$e" == "$match" ]] && return 0; done + for e; do [[ "$e" == $match ]] && return 0; done return 1 } +# +# Returns `0` if first parameter is in the array passed as second parameter, +# where the array may contain globs. +# +# ### Parameters +# +# * 1: `item` item to search for in array +# * 2: `array` bash array, e.g. `array=(one two three)` +# +# ### Example +# +# ```bash +# array=(foo bar it*) +# if _builder_item_in_glob_array "item" "${array[@]}"; then ...; fi +# ``` +# +_builder_item_in_glob_array() { + local e match="$1" + shift + [[ -z "$match" ]] && return 1 + for e; do [[ "$match" == $e ]] && return 0; done + return 1 +} + + _builder_item_is_target() { local item="$1" [[ $item =~ ^: ]] && return 1 @@ -168,6 +196,8 @@ _builder_failure_trap() { local trappedExitCode=$? local action target + _builder_cleanup_deps + # Since 'exit' is also trapped, we can also handle end-of-script incomplete actions. if [[ $trappedExitCode == 0 ]]; then # While there weren't errors, were there any actions that never reported success or failure? @@ -187,7 +217,7 @@ _builder_failure_trap() { target=:project fi - builder_finish_action failure $action $target + builder_finish_action failure $action$target # Make 100% sure that the exit code chains fully. # Without this, nested scripts have failed to chain errors from npm calls past the script @@ -196,6 +226,21 @@ _builder_failure_trap() { fi } +# +# Removes temporary `_builder_deps_built` file when top-level build script +# finishes. +# +_builder_cleanup_deps() { + if ! builder_is_dep_build && [[ ! -z ${_builder_deps_built+x} ]]; then + if $_builder_debug; then + echo "[DEBUG] Dependencies that were built:" + cat "$_builder_deps_built" + fi + rm -f "$_builder_deps_built" + _builder_deps_built= + fi +} + # # Builds the standardized `action:target` string for the specified action-target # pairing and also returns 0 if the user has asked to perform it on the command @@ -205,28 +250,38 @@ _builder_failure_trap() { # The string will be set as `_builder_matched_action`, which is for # builder.inc.sh internal use, used by `builder_start_action`. # -# Usage: +# ### Usage +# +# ```bash # if build_has_action action[:target]; then ...; fi +# ```` +# # Parameters: -# 1: action name of action -# 2: :target name of target, :-prefixed, as part of first param or space separated ok +# 1: action[:target] name of action:target # Example: -# if builder_has_action build :app; then # or build:app, that's fine too. +# +# ```bash +# if builder_has_action build:app; then ... +# ``` +# builder_has_action() { local action="$1" target if [[ $action =~ : ]]; then IFS=: read -r action target <<< $action target=:$target - elif [[ -z ${2+x} ]]; then - target=:project else - target="$2" + target=':*' fi if _builder_item_in_array "$action$target" "${_builder_chosen_action_targets[@]}"; then # To avoid WET re-processing of the $action$target string set _builder_matched_action="$action$target" + if [[ $target == ':*' ]]; then + _builder_matched_action_name="$action" + else + _builder_matched_action_name="$action$target" + fi return 0 else _builder_matched_action= @@ -235,27 +290,52 @@ builder_has_action() { } # -# Returns 0 if the user has asked to perform action on target on the command line, and -# then starts the action. Should be paired with builder_finish_action +# Returns `0` if the user has asked to perform action on target on the command +# line, and then starts the action. Should be paired with +# `builder_finish_action`. # -# Usage: +# ### Usage +# +# ```bash # if builder_start_action action[:target]; then ...; fi -# Parameters: -# 1: action name of action -# 2: :target name of target, :-prefixed, as part of first param or space separated ok -# Example: -# if builder_start_action build :app; then -# if builder_start_action build:app; then +# ``` +# +# ### Parameters +# +# * 1: `action[:target]` name of action, and optionally also target, if +# target excluded starts for all defined targets +# +# ### Example +# +# ```bash +# if builder_start_action build:app; then ... +# ``` # builder_start_action() { local scope="[$THIS_SCRIPT_IDENTIFIER] " - if builder_has_action $@; then - echo "${COLOR_BLUE}## $scope$_builder_matched_action starting...${COLOR_RESET}" + if builder_has_action $1; then + # In a dependency quick build (the default), determine whether we actually + # need to run this step. Uses data passed to builder_describe_outputs to + # verify whether a target output is present. + if builder_is_dep_build && + ! builder_is_full_dep_build && + [[ ! -z ${_builder_dep_path[$_builder_matched_action]+x} ]] && + [[ -e "$KEYMAN_ROOT/${_builder_dep_path[$_builder_matched_action]}" ]]; then + if builder_verbose; then + echo "$scope skipping $_builder_matched_action_name, up-to-date" + fi + return 1 + fi + + echo "${COLOR_BLUE}## $scope$_builder_matched_action_name starting...${COLOR_RESET}" if [ -n "${_builder_current_action}" ]; then _builder_warn_if_incomplete fi _builder_current_action="$_builder_matched_action" + + # Build dependencies as required + _builder_do_build_deps "$_builder_matched_action" return 0 else return 1 @@ -290,48 +370,131 @@ _builder_trim() { printf '%s' "$var" } +# +# Expands an in-repo-relative path to a repo-relative path. A path starting with +# `/` is expected to be relative to repo root, not filesystem root. Otherwise, +# it's relative to current script path, not current working directory. The +# returned path will not have a prefix `/`, and will be relative to +# `$KEYMAN_ROOT`. Assumes realpath is installed (brew coreutils on macOS). +# +_builder_expand_relative_path() { + local path="$1" + if [[ "$path" =~ ^/ ]]; then + echo "${path:1}" + else + realpath --canonicalize-missing --relative-to="$KEYMAN_ROOT" "$THIS_SCRIPT_PATH/$path" + fi +} + +# +# Expands an `[action][:target]` string, replacing missing values with `*`, +# for example: +# +# * `build` --> `build:*` +# * `build:app` --> `build:app` +# * `:app` --> `*:app` +# +# Supports multiple action:targets in the string +# +_builder_expand_action_target() { + local input="$1" target= action= + if [[ "$input" =~ : ]]; then + action=$(echo "$input" | cut -d: -f 1 -) + target=$(echo "$input" | cut -d: -f 2 -) + else + action=$input + fi + + if [[ -z "$action" ]]; then + action='*' + fi + if [[ -z "$target" ]]; then + target='*' + fi + + echo "$action:$target" +} + +_builder_expand_action_targets() { + local input=($1) e output=() + for e in "${input[@]}"; do + e=`_builder_expand_action_target "$e"` + output+=($e) + done + if [[ ${#output[@]} == 0 ]]; then + echo "*:*" + else + echo "${output[@]}" + fi +} + # # Describes a build script, defines available parameters and their meanings. Use # together with `builder_parse` to process input parameters. # -# Usage: +# ### Usage +# +# ```bash # builder_describe description param_desc... -# Parameters: -# description A short description of what the script does -# param_desc Space separated name and description of parameter, e.g. -# "build Builds the target" -# May be repeated to describe all parameters +# ``` # -# There are three types of parameters that may be specified: +# ### Parameters +# +# * `description` A short description of what the script does. +# * `param_desc` Space separated name and description of parameter, e.g. +# `"build Builds the target"` +# This parameter may be repeated to describe all parameters. +# +# There are four types of parameters that may be specified: +# +# * **Option:** `"--option[,-o][=var] [One line description]"` # -# * Option, param_desc format: "--option[,-o][=var] [One line description]" # All options must have a longhand form with two prefix hyphens, -# e.g. --option. The ",-o" shorthand form is optional. When testing if -# the option is set with `builder_has_option``, always use the longhand +# e.g. `--option`. The `,-o` shorthand form is optional. When testing if +# the option is set with `builder_has_option`, always use the longhand # form. # -# if =var is specified, then the next parameter will be a variable stored -# in $var for that option. e.g. --option=opt means $opt will have the value -# 'foo' when the script is called for --option foo. +# if `=var` is specified, then the next parameter will be a variable stored in +# `$var` for that option. e.g. `--option=opt` means `$opt` will have the value +# `"foo"` when the script is called for `--option foo`. # -# * Action, param_desc format: "action [One line description]" -# Actions must be a single word, lower case. To specify an action -# as the default, append a '+' to the action name, e.g. -# "test+ Test the project". If there is no default specified, then -# it will be 'build' +# * **Action**: `"action [One line description]"` # -# * Target, param_desc format: ":target [One line description]" -# A target always starts with colon, e.g. :project. +# Actions must be a single word, lower case. To specify an action as the +# default, append a `+` to the action name, e.g. `"test+ Test the project"`. +# If there is no default specified, then it will be `build`. +# +# * **Target:** `":target [One line description]"` +# +# A target always starts with colon, e.g. `:project`. +# +# * **Dependency:** "@/path/to/dependency [action][:target] ..." +# +# A dependency always starts with `@`. The path to the dependency will be +# relative to the build script folder, or to the root of the repository, if +# the path starts with `/`, not to the root of the file system. It is an error +# to specify a dependency outside the repo root. +# +# Relative paths will be expanded to full paths, again, relative to the root +# of the repository. +# +# Dependencies may be limited to specific `action:target`. If not specified, +# dependencies will be built for all actions on all targets. Either `action` +# or `:target` may be omitted, and multiple actions and targets may be +# specified, space separated. # builder_describe() { _builder_description="$1" _builder_actions=() _builder_targets=() _builder_options=() + _builder_deps=() # array of all dependencies for this script _builder_default_action=build declare -A -g _builder_params declare -A -g _builder_options_short declare -A -g _builder_options_var + declare -A -g _builder_dep_path # array of output files for action:target pairs + declare -A -g _builder_dep_related_actions # array of action:targets associated with a given dependency shift # describe each target, action, and option possibility while [[ $# -gt 0 ]]; do @@ -339,12 +502,20 @@ builder_describe() { local value="$(echo "$key" | cut -d" " -f 1 -)" local description= if [[ $key =~ [[:space:]] ]]; then - description=$(_builder_trim "$(echo "$key" | cut -d" " -f 2- -)") + description="$(_builder_trim "$(echo "$key" | cut -d" " -f 2- -)")" fi if [[ $value =~ ^: ]]; then # Parameter is a target _builder_targets+=($value) + elif [[ $value =~ ^@ ]]; then + # Parameter is a dependency + local dependency="${value:1}" + dependency="`_builder_expand_relative_path "$dependency"`" + _builder_deps+=($dependency) + # echo "$description" + _builder_dep_related_actions[$dependency]="`_builder_expand_action_targets "$description"`" + # echo "${_builder_dep_related_actions[$dependency]}" elif [[ $value =~ ^-- ]]; then # Parameter is an option # Look for a shorthand version of the option @@ -398,6 +569,50 @@ builder_describe() { fi } +# +# Defines an output file or folder expected to be present after successful +# completion of an action for a target. Used to skip actions for dependency +# builds. If `:target` is not provided, assumes `:project`. +# +# Relative paths are relative to script folder; absolute paths are relative +# to repository root, not filesystem root. +# +# ### Usage +# +# ```bash +# builder_describe_outputs action:target filename [...] +# ``` +# +# ### Parameters +# +# * 1: `action[:target]` action and/or target associated with file +# * 2: `filename` name of file or folder to check +# * 3+: ... repeat previous arguments for additional outputs +# +# ### Example +# +# ```bash +# builder_describe_outputs \ +# configure /node_modules \ +# build build/index.js +# ``` +# +function builder_describe_outputs() { + while [[ $# -gt 0 ]]; do + local key="$1" path="$2" action target + if [[ $key =~ : ]]; then + action="$(echo "$key" | cut -d: -f 1 -)" + target=":$(echo "$key" | cut -d: -f 2 -)" + else + action="$key" + target=':*' + fi + path="`_builder_expand_relative_path "$path"`" + _builder_dep_path[$action$target]="$path" + shift 2 + done +} + _builder_get_default_description() { local description= local value="$1" @@ -463,6 +678,7 @@ builder_check_color() { # Parameters # 1: $@ command-line arguments builder_parse() { + _builder_build_deps=--deps builder_verbose= builder_extra_params=() _builder_chosen_action_targets=() @@ -551,6 +767,19 @@ builder_parse() { _builder_chosen_options+=(--verbose) builder_verbose=--verbose ;; + --deps|--no-deps|--force-deps) + _builder_build_deps=$key + ;; + --builder-dep-parent) + # internal use parameter for dependency builds - identifier of parent script + shift + builder_dep_parent="$1" + ;; + --builder-deps-built) + # internal use parameter for dependency builds - path to dependency tracking file + shift + _builder_deps_built="$1" + ;; *) _builder_parameter_error "$0" parameter "$key" esac @@ -576,6 +805,19 @@ builder_parse() { done fi + if builder_is_dep_build; then + echo "[$THIS_SCRIPT_IDENTIFIER] dependency build, started by $builder_dep_parent" + if [[ -z ${_builder_deps_built+x} ]]; then + echo "FATAL ERROR: Expected --builder-deps-built parameter" + exit 1 + fi + else + # This is a top-level invocation, not a dependency build, so we want to + # track which dependencies have been built, so they don't get built multiple + # times. + _builder_deps_built=`mktemp` + fi + # Now that we've successfully parsed options adhering to the _builder spec, we may activate our # action_failure and action_hanging traps. (We don't want them active on scripts not yet using # said script.) @@ -611,11 +853,16 @@ builder_display_usage() { program="$(basename "$0")" if [[ ! -z ${_builder_description+x} ]]; then - echo "$program: $_builder_description" + echo "Summary:" + echo " $_builder_description" echo fi + echo "Script Identifier:" + echo " $THIS_SCRIPT_IDENTIFIER" + echo - echo "Usage: $program [options...] [action][:target]..." + echo "Usage:" + echo " $program [options...] [action][:target]..." echo echo "Actions: " @@ -652,8 +899,24 @@ builder_display_usage() { _builder_pad $width " --verbose, -v" "Verbose logging" _builder_pad $width " --color" "Force colorized output" _builder_pad $width " --no-color" "Never use colorized output" + if builder_has_dependencies; then + _builder_pad $width " --deps" "Build dependencies if required (default)" + _builder_pad $width " --no-deps" "Skip build of dependencies" + _builder_pad $width " --force-deps" "Reconfigure and rebuild all dependencies" + fi _builder_pad $width " --help, -h" "Show this help" + echo + echo "Dependencies: " + + if builder_has_dependencies; then + for d in "${_builder_deps[@]}"; do + echo " $d" + done + else + echo " This module has no dependencies" + fi + # Defined in `builder_use_color`; this assumes that said func has been called. local c1=$BUILDER_TERM_START local c0=$BUILDER_TERM_END @@ -667,35 +930,216 @@ builder_display_usage() { builder_finish_action() { local result="$1" - local action="$2" target + local action="$2" target action_name if [[ $action =~ : ]]; then IFS=: read -r action target <<< $action - target=:$target - elif [[ -z ${3+x} ]]; then - target=:project + target=":$target" else - target="$3" + target=':*' + fi + + if [[ "$target" == ":*" ]]; then + action_name="$action" + else + action_name="$action$target" fi local scope="[$THIS_SCRIPT_IDENTIFIER] " if [[ "$action$target" == "${_builder_current_action}" ]]; then if [[ $result == success ]]; then - echo "${COLOR_GREEN}## $scope$action$target completed successfully${COLOR_RESET}" + echo "${COLOR_GREEN}## $scope$action_name completed successfully${COLOR_RESET}" elif [[ $result == failure ]]; then - echo "${COLOR_RED}## $scope$action$target failed${COLOR_RESET}" + echo "${COLOR_RED}## $scope$action_name failed${COLOR_RESET}" else - echo "${COLOR_RED}## $scope$action$target failed with message: $result${COLOR_RESET}" + echo "${COLOR_RED}## $scope$action_name failed with message: $result${COLOR_RESET}" fi # Remove $action$target from the array; it is no longer a current action _builder_current_action= else - echo "${COLOR_YELLOW}## Warning: reporting result of $action$target but the action was never started!${COLOR_RESET}" + echo "${COLOR_YELLOW}## Warning: reporting result of $action_name but the action was never started!${COLOR_RESET}" fi } +# +# Returns `0` if the dependency should be built for the given action:target +# +_builder_should_build_dep() { + local action_target="$1" + local dep="$2" + local related_actions=(${_builder_dep_related_actions[$dep]}) + # echo "bdra: ${_builder_dep_related_actions[@]}" + # echo "target: $action_target" + # echo "dep: $2" + # echo "ra: ${related_actions[@]}" + if ! _builder_item_in_glob_array "$action_target" "${related_actions[@]}"; then + return 1 + fi + return 0 +} + +# +# Configure and build all dependencies +# Later, may restrict by either action or target +# +_builder_do_build_deps() { + local action_target="$1" + + if [[ $_builder_build_deps == --no-deps ]]; then + # we've been asked to skip dependencies + return 0 + fi + + for dep in "${_builder_deps[@]}"; do + # Don't attempt to build dependencies that don't match the current + # action:target (wildcards supported for matches here) + if ! _builder_should_build_dep "$action_target" "$dep"; then + echo "[$THIS_SCRIPT_IDENTIFIER] Skipping dependency build $dep for $_builder_matched_action_name" + continue + fi + + # Only configure and build the dependency once per invocation + if builder_has_module_been_built "$dep"; then + continue + fi + + # TODO: add --debug as a standard builder parameter + builder_set_module_has_been_built "$dep" + "$KEYMAN_ROOT/$dep/build.sh" configure build \ + $builder_verbose \ + $_builder_build_deps \ + --builder-deps-built "$_builder_deps_built" \ + --builder-dep-parent "$THIS_SCRIPT_IDENTIFIER" + done +} + +# +# returns `0` if we are in a dependency doing a build. +# +builder_is_dep_build() { + if [[ ! -z ${builder_dep_parent+x} ]]; then + return 0 + fi + return 1 +} + +# +# returns `0` if we should attempt to do quick builds in a dependency build, for +# example skipping `tsc -b` where a parent may also do it; corresponds to the +# `--deps` parameter (which is the default). +# +builder_is_quick_dep_build() { + if builder_is_dep_build && [[ $_builder_build_deps == --deps ]]; then + return 0 + fi + return 1 +} + +# +# returns `0` if we should do a full configure and build in a dependency build; +# corresponds to the `--force-deps`` parameter. +# +builder_is_full_dep_build() { + if builder_is_dep_build && [[ $_builder_build_deps == --force-deps ]]; then + return 0 + fi + return 1 +} + +# +# returns `0` if the current build script has at least one dependency. +# +builder_has_dependencies() { + if [[ ${#_builder_deps[@]} -eq 0 ]]; then + return 1 + fi + return 0 +} + +# +# Tests if a dependency module has been built already in the current script +# invocation; if not running in a builder context, always returns `1` (i.e. +# "false"). +# +# ### Usage +# +# ```bash +# builder_has_module_been_built dependency-name +# ``` +# +# ### Parameters +# +# * 1: `dependency-name` the `$SCRIPT_IDENTIFIER` of the dependency +# (repo-relative path without leading `/`); or for +# external dependencies, a path-like starting with +# `/external/`. +# +# ### Examples +# +# ```bash +# if builder_has_module_been_built common/web/keyman-version; then ... +# if builder_has_module_been_built /external/npm-ci; then ... +# ``` +# +builder_has_module_been_built() { + local module="$1" + + if [[ -z ${_builder_deps_built+x} ]]; then + # not in a builder context, so we assume a build is needed + return 1 + fi + + if [[ -f $_builder_deps_built ]] && grep -qx "$module" $_builder_deps_built; then + # dependency history file contains the dependency module + return 0 + fi + return 1 +} + +# +# Updates the dependency module build state for the current script invocation; +# if not running in a builder context, a no-op. +# +# ### Usage +# +# ```bash +# builder_set_module_has_been_built dependency-name +# ``` +# +# ### Parameters +# +# * 1: `dependency-name` the `$SCRIPT_IDENTIFIER` of the dependency +# (repo-relative path without leading `/`); or for +# external dependencies, a path-like starting with +# `/external/`. +# +# ### Examples +# +# ```bash +# builder_set_module_has_been_built common/web/keyman-version +# builder_set_module_has_been_built /external/npm-ci +# ``` +# +builder_set_module_has_been_built() { + local module="$1" + + if [[ ! -z ${_builder_deps_built+x} ]]; then + echo "$module" >> $_builder_deps_built + fi +} + +# +# returns `0` if we should be verbose in output +# +builder_verbose() { + if [[ $builder_verbose == --verbose ]]; then + return 0 + fi + return 1 +} + # # Initialize builder once all functions are declared # diff --git a/resources/shellHelperFunctions.sh b/resources/shellHelperFunctions.sh index 79e917bc32..439ee69db4 100755 --- a/resources/shellHelperFunctions.sh +++ b/resources/shellHelperFunctions.sh @@ -1,11 +1,5 @@ #!/usr/bin/env bash -# -# WARNING: this file is copied into other locations during package builds; do not -# include other files or make path assumptions when changing this file. -# See `core/build.sh` for more details. -# - _shf_base_dir=$(dirname "$BASH_SOURCE")/.. # Designed to determine which set of browsers should be available for local testing, @@ -261,32 +255,28 @@ set_npm_version () { npm version --allow-same-version --no-git-tag-version --no-commit-hooks "$VERSION_WITH_TAG" } -# Initializes use of the npm packages within the repo. -init_npm() { - if [ "${KEYMAN_ROOT}" = "" ]; then - fail "KEYMAN_ROOT not defined; cannot install repo's dependencies" +# +# Verifies that node is installed, and installs npm packages, but only once per +# build invocation +# +verify_npm_setup() { + # We'll piggy-back on the builder module dependency build state to determine + # if npm ci has been called in the current script invocation. Adding the + # prefix /external/ to module name in order to differentiate between this and + # internal modules (although it is unlikely to ever collide!); we will also + # use this pattern for other similar external dependencies in future. These + # functions are safe to call even in a non-builder context (they do nothing or + # return 1 -- not built) + if builder_has_module_been_built /external/npm-ci; then + return 0 fi + builder_set_module_has_been_built /external/npm-ci + + # Check if Node.JS/npm is installed. + type npm >/dev/null ||\ + fail "Build environment setup error detected! Please ensure Node.js is installed!" + pushd "$KEYMAN_ROOT" > /dev/null npm ci popd > /dev/null } - -# Accepts an optional parameter. -# $1 - when set to 'false', only ensures that `npm` and `node` are accessible; does not install dependencies. -# -# Designed for use with the projects/packages we have listed in the base folder package.json workspaces. -verify_npm_setup () { - if [ $# != 0 ]; then - fetch_deps=$1 - else - fetch_deps=true - fi - - # Check if Node.JS/npm is installed. - type npm >/dev/null ||\ - fail "Build environment setup error detected! Please ensure Node.js is installed!" - - if [ $fetch_deps = true ]; then - init_npm - fi -} diff --git a/web/unit_tests/test.sh b/web/unit_tests/test.sh index c91a79ae01..fadab660f8 100755 --- a/web/unit_tests/test.sh +++ b/web/unit_tests/test.sh @@ -75,7 +75,7 @@ CONFIG=manual.conf.js # TODO - get/make OS-specific version SH_FLAGS= DEBUG=false FLAGS= -HEADLESS_FLAGS=-skip-package-install +HEADLESS_FLAGS= # Parse args while [[ $# -gt 0 ]] ; do @@ -83,8 +83,8 @@ while [[ $# -gt 0 ]] ; do case $key in -CI) CONFIG=CI.conf.js - HEADLESS_FLAGS="$HEADLESS_FLAGS -CI" SH_FLAGS="--ci" + HEADLESS_FLAGS="$HEADLESS_FLAGS --ci" ;; -log-level) shift @@ -133,8 +133,13 @@ cd ../tools/recorder # Run our headless tests first. # First: Web-core tests. +pushd "$KEYMAN_ROOT/common/web/keyboard-processor" +./build.sh test $HEADLESS_FLAGS || fail "Tests failed by dependencies; aborting integration tests." +popd + pushd "$KEYMAN_ROOT/common/web/input-processor" -./test.sh $HEADLESS_FLAGS || fail "Tests failed by dependencies; aborting integration tests." +./build.sh build:tools test $HEADLESS_FLAGS || fail "Tests failed by dependencies; aborting integration tests." +# Once done, now we run the integrated (KeymanWeb) tests. popd # For now, we'll also link in the gesture-recognizer unit tests here.