From 19101c6a2a10abe2a8893a2fa17ed8d57f33081a Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 21 Feb 2023 12:12:28 +0700 Subject: [PATCH] chore(core): cleanup meson build This updates our meson min version on Windows to 1.0, which means that we can avoid having Visual Studio on the path, as meson can find it anyway. With this change, we can eliminate the batch-wrapper for the build, significantly improving build performance and simplifying the scripts. This also adds a change to disable tests for dependency builds of core, along with the `--no-tests` command line option for doing builds without tests. Minor cleanup includes eliminating various warnings from meson.build files, and standardizing WASM cross-platform build options for Windows. --- core/build.bat | 118 ----------------------------- core/build.sh | 27 ++++--- core/commands.inc.sh | 49 +++--------- core/meson.build | 10 ++- core/meson_options.txt | 1 + core/tests/unit/kmnkbd/meson.build | 2 +- core/tests/unit/kmx/meson.build | 8 +- core/tests/unit/ldml/meson.build | 4 +- core/wasm.build.win.in | 4 +- docs/build/windows.md | 4 +- resources/builder.inc.sh | 25 +++++- 11 files changed, 72 insertions(+), 180 deletions(-) delete mode 100644 core/build.bat create mode 100644 core/meson_options.txt diff --git a/core/build.bat b/core/build.bat deleted file mode 100644 index e10798b931..0000000000 --- a/core/build.bat +++ /dev/null @@ -1,118 +0,0 @@ -@echo off -rem **************************************************************** -rem Run build.sh, not this file directly, to build on Windows. -rem **************************************************************** -setlocal enabledelayedexpansion - -if "%1"=="" goto help -if "%1"=="all" goto all -if "%1"=="x86" goto build -if "%1"=="x64" goto build - -echo "Invalid parameter." -goto help - -rem ---------------------------------- - -:help -echo Usage: %0 x86^|x64^|all debug^|release [configure] [build] [test] [additional params for meson/ninja] -echo or -echo Usage: %0 x86^|x64 -c -echo -c will leave your environment configured for Visual Studio for selected platform. -echo. -echo Otherwise, %0 is intended to be used by build.sh, not directly. -echo At least one of 'configure', 'build', or 'test' is required. -goto :eof - -rem ---------------------------------- - -:all - -setlocal -cd %KEYMAN_ROOT%\core -cmd /c build.bat x86 %2 %3 %4 %5 %6 %7 %8 %9 || exit !errorlevel! - -cd %KEYMAN_ROOT%\core -cmd /c build.bat x64 %2 %3 %4 %5 %6 %7 %8 %9 || exit !errorlevel! - -goto :eof - -rem ---------------------------------- - -:build - -set ARCH=%1 -shift - -if "%1"=="-c" goto :setup - -echo === Locating Visual Studio === - -rem From https://github.com/microsoft/vswhere -for /f "usebackq tokens=*" %%i in (`..\resources\build\vswhere -latest -requires Microsoft.Component.MSBuild -find **\vcvarsall.bat`) do ( - set VCVARSALL="%%i" -) - -if errorlevel 1 ( - echo vswhere failed [!errorlevel!] - exit /b !errorlevel! -) - -if not exist "!VCVARSALL!" ( - echo Could not find vcvarsall.bat [!VCVARSALL!] - exit /b 1 -) - -echo === Configuring VC++ === -call !VCVARSALL! !ARCH! || exit !errorlevel! - -cd %KEYMAN_ROOT%\core - -set BUILDTYPE=%1 -shift - -set STATIC_LIBRARY=--default-library both - -set COMMAND=%1 -shift - -if "!COMMAND!" == "configure" ( - echo === Configuring Keyman Core for Windows !ARCH! !BUILDTYPE! === - if exist build\!ARCH!\!BUILDTYPE! rd /s/q build\!ARCH!\!BUILDTYPE! - meson setup build\!ARCH!\!BUILDTYPE! !STATIC_LIBRARY! --buildtype !BUILDTYPE! --werror %1 %2 %3 %4 %5 %6 %7 %8 %9 || exit !errorlevel! - shift -) - -if "!COMMAND!" == "build" ( - echo === Building Keyman Core for Windows !ARCH! !BUILDTYPE! === - cd build\!ARCH!\!BUILDTYPE! || exit !errorlevel! - ninja %1 %2 %3 %4 %5 %6 %7 %8 %9 || exit !errorlevel! - cd ..\..\.. - shift -) - -if "!COMMAND!" == "test" ( - echo === Testing Keyman Core for Windows !ARCH! !BUILDTYPE! === - cd build\!ARCH!\!BUILDTYPE! || exit !errorlevel! - meson test --print-errorlogs %1 %2 %3 %4 %5 %6 %7 %8 %9 || exit !errorlevel! - cd ..\..\.. - shift -) - -goto :eof - -rem ---------------------------------- - -:setup - -rem Standalone build, so we'll make the environment available to the caller -rem Also setup -rem Note: Visual Studio 2022 doesn't provide vcvarsall.bat, so we'll have to find a different solution -endlocal -for /f "usebackq tokens=*" %%i in (`..\resources\build\vswhere -version [15^,17^) -latest -requires Microsoft.Component.MSBuild -find **\vcvarsall.bat`) do ( - set VCVARSALL="%%i" -) -%VCVARSALL% !ARCH! -goto :eof - -rem ---------------------------------- diff --git a/core/build.sh b/core/build.sh index 6d6ad0338b..fab558ad28 100755 --- a/core/build.sh +++ b/core/build.sh @@ -42,14 +42,6 @@ esac # ":linux Build for current Linux architecture" # ":mac Build for current macOS architecture" -archdeps=() -if type node >/dev/null 2>&1; then - # Note: Found node, can build kmc and hextobin dependencies - archdeps+=(@/common/tools/hextobin @/common/web/keyman-version @/developer/src/kmc) -else - echo "Note: could not find node, skipping hextobin and kmc dependency builds, ldml+binary tests will not be run" -fi - builder_describe \ "Build Keyman Core @@ -57,7 +49,9 @@ Libraries will be built in 'build///src'. * : 'debug' or 'release' (see --debug flag) * All parameters after '--' are passed to meson or ninja " \ - "${archdeps[@]}" \ + "@/common/tools/hextobin" \ + "@/common/web/keyman-version" \ + "@/developer/src/kmc" \ "clean" \ "configure" \ "build" \ @@ -66,11 +60,26 @@ Libraries will be built in 'build///src'. "uninstall uninstall libraries from current system" \ "${archtargets[@]}" \ "--debug,-d configuration is 'debug', not 'release'" \ + "--no-tests do not configure tests (used by other projects)" \ "--target-path=opt_target_path override for build/ target path" \ "--test=opt_tests,-t test[s] to run (space separated)" builder_parse "$@" +# +# meson forces us to configure tests, including building compilers, even +# if we don't plan to run them, for example when doing a dependency build +# in CI +# +MESON_OPTION_keyman_core_tests= + +if builder_is_dep_build || builder_has_option --no-tests; then + MESON_OPTION_keyman_core_tests="-Dkeyman_core_tests=false" + builder_remove_dep /common/tools/hextobin + builder_remove_dep /common/web/keyman-version + builder_remove_dep /developer/src/kmc +fi + if builder_has_option --debug; then CONFIGURATION=debug else diff --git a/core/commands.inc.sh b/core/commands.inc.sh index d00fac85f8..d5f48fd012 100644 --- a/core/commands.inc.sh +++ b/core/commands.inc.sh @@ -8,7 +8,6 @@ do_clean() { # clean: note build/ will be left, but build// should be gone local target=$1 builder_start_action clean:$target || return 0 - rm -rf "$MESON_PATH" builder_finish_action success clean:$target } @@ -21,7 +20,7 @@ do_configure() { local target=$1 builder_start_action configure:$target || return 0 - local STANDARD_MESON_ARGS= + local STANDARD_MESON_ARGS="$MESON_OPTION_keyman_core_tests" echo_heading "======= Configuring $target =======" @@ -29,17 +28,14 @@ do_configure() { # do_configure_wasm locate_emscripten build_meson_cross_file_for_wasm - STANDARD_MESON_ARGS="--cross-file wasm.defs.build --cross-file wasm.build --default-library static" + STANDARD_MESON_ARGS="$MESON_OPTION_keyman_core_tests --cross-file wasm.defs.build --cross-file wasm.build --default-library static" fi - if [[ $target =~ ^(x86|x64)$ ]]; then - cmd //C build.bat $target $CONFIGURATION configure "${builder_extra_params[@]}" - else - pushd "$THIS_SCRIPT_PATH" > /dev/null - # Additional arguments are used by Linux build, e.g. -Dprefix=${INSTALLDIR} - meson setup "$MESON_PATH" --werror --buildtype $CONFIGURATION $STANDARD_MESON_ARGS "${builder_extra_params[@]}" - popd > /dev/null - fi + pushd "$THIS_SCRIPT_PATH" > /dev/null + # Additional arguments are used by Linux build, e.g. -Dprefix=${INSTALLDIR} + meson setup "$MESON_PATH" --werror --buildtype $CONFIGURATION $STANDARD_MESON_ARGS "${builder_extra_params[@]}" + popd > /dev/null + builder_finish_action success configure:$target } @@ -50,21 +46,7 @@ do_configure() { do_build() { local target=$1 builder_start_action build:$target || return 0 - - echo_heading "======= Building $target =======" - - if [[ $target =~ ^(x86|x64)$ ]]; then - # Build the meson targets, both x86 and x64 also - # We need to use a batch file here so we can get - # the Visual Studio build environment with vcvarsall.bat - # TODO: if PATH is the only variable required, let's try and - # eliminate this difference in the build process - cmd //C build.bat $target $CONFIGURATION build "${builder_extra_params[@]}" - else - pushd "$MESON_PATH" > /dev/null - ninja - popd > /dev/null - fi + meson compile -C "$MESON_PATH" builder_finish_action success build:$target } @@ -75,16 +57,7 @@ do_build() { do_test() { local target=$1 builder_start_action test:$target || return 0 - - echo_heading "======= Testing $target =======" - - if [[ $target =~ ^(x86|x64)$ ]]; then - cmd //C build.bat $target $CONFIGURATION test "${builder_extra_params[@]}" - else - pushd "$MESON_PATH" > /dev/null - meson test "${builder_extra_params[@]}" - popd > /dev/null - fi + meson test -C "$MESON_PATH" "${builder_extra_params[@]}" builder_finish_action success test:$target } @@ -104,9 +77,7 @@ do_command() { local command=$1 local target=$2 builder_start_action $command:$target || return 0 - pushd "$MESON_PATH" > /dev/null - ninja $command - popd > /dev/null + meson $command -C "$MESON_PATH" builder_finish_action success $command:$target } diff --git a/core/meson.build b/core/meson.build index 26305dad22..081b29253f 100644 --- a/core/meson.build +++ b/core/meson.build @@ -6,7 +6,7 @@ # project('keyboardprocessor', 'cpp', 'c', - version: run_command(find_program('getversion.bat', 'getversion.sh')).stdout().strip(), + version: run_command(find_program('getversion.bat', 'getversion.sh'), check:true).stdout().strip(), license: 'MIT', default_options : ['buildtype=release', 'cpp_std=c++14', @@ -44,4 +44,10 @@ endif subdir('doc') subdir('include') subdir('src') -subdir('tests') + +if get_option('keyman_core_tests') + message('option "keyman_core_tests" is true, enabling tests') + subdir('tests') +else + message('option "keyman_core_tests" is false, disabling tests') +endif \ No newline at end of file diff --git a/core/meson_options.txt b/core/meson_options.txt new file mode 100644 index 0000000000..a4ef5ef3e5 --- /dev/null +++ b/core/meson_options.txt @@ -0,0 +1 @@ +option('keyman_core_tests', type: 'boolean', value: true) \ No newline at end of file diff --git a/core/tests/unit/kmnkbd/meson.build b/core/tests/unit/kmnkbd/meson.build index 4cdaa91186..16884298e5 100644 --- a/core/tests/unit/kmnkbd/meson.build +++ b/core/tests/unit/kmnkbd/meson.build @@ -39,7 +39,7 @@ foreach t : tests cpp_args: defns + warns, include_directories: [inc, libsrc], link_args: links + tests_flags, - objects: lib.extract_all_objects()) + objects: lib.extract_all_objects(recursive: false)) test(t[0], bin, args: ['--color', test_path]) endforeach diff --git a/core/tests/unit/kmx/meson.build b/core/tests/unit/kmx/meson.build index ed389d7423..d6c574c5f6 100644 --- a/core/tests/unit/kmx/meson.build +++ b/core/tests/unit/kmx/meson.build @@ -38,14 +38,14 @@ kmx = executable('kmx', cpp_args: defns + warns, include_directories: [inc, libsrc], link_args: links + tests_flags, - objects: lib.extract_all_objects()) + objects: lib.extract_all_objects(recursive: false)) test_lib = static_library('kmnkbp-tests', coretest_files, cpp_args: defns + warns + flags, include_directories: [inc, libsrc], link_args: links + tests_flags, - objects: lib.extract_all_objects(), + objects: lib.extract_all_objects(recursive: false), pic: true, install: false ) @@ -248,7 +248,7 @@ key_e = executable('key_list', 'kmx_key_list.cpp', cpp_args: defns + warns, include_directories: [inc, libsrc], link_args: links + tests_flags, - objects: lib.extract_all_objects()) + objects: lib.extract_all_objects(recursive: false)) test_kbd = 'kmx_key_list' if kmcomp.found() kbd_src = files(test_kbd + '.kmn') @@ -270,7 +270,7 @@ imx_e = executable('imx_list', 'kmx_imx.cpp', cpp_args: defns + warns, include_directories: [inc, libsrc], link_args: links + tests_flags, - objects: lib.extract_all_objects()) + objects: lib.extract_all_objects(recursive: false)) test_kbd = 'kmx_imsample' if kmcomp.found() diff --git a/core/tests/unit/ldml/meson.build b/core/tests/unit/ldml/meson.build index a29b2c1b48..7dc3d81805 100644 --- a/core/tests/unit/ldml/meson.build +++ b/core/tests/unit/ldml/meson.build @@ -54,7 +54,7 @@ ldml = executable('ldml', cpp_args: defns + warns, include_directories: [inc, libsrc, '../../../../developer/src/ext/json'], link_args: links + tests_flags, - objects: lib.extract_all_objects()) + objects: lib.extract_all_objects(recursive: false)) # Run tests on all keyboards (`tests` defined in keyboards/meson.build) @@ -79,5 +79,5 @@ e = executable('test_kmx_plus', 'test_kmx_plus.cpp', cpp_args: defns + warns, include_directories: [inc, libsrc], link_args: links + tests_flags, - objects: lib.extract_all_objects()) + objects: lib.extract_all_objects(recursive: false)) test('test_kmx_plus', e, suite: 'ldml') diff --git a/core/wasm.build.win.in b/core/wasm.build.win.in index bc61cbbcc0..e6c20c0f72 100644 --- a/core/wasm.build.win.in +++ b/core/wasm.build.win.in @@ -1,6 +1,6 @@ [binaries] -c = ['python.exe', '$EMSCRIPTEN_BASE/emcc.py'] -cpp = ['python.exe', '$EMSCRIPTEN_BASE/em++.py'] +c = ['python.exe', '$EMSCRIPTEN_BASE/emcc.py', '-s', '-O2'] +cpp = ['python.exe', '$EMSCRIPTEN_BASE/em++.py', '-s', '-O2'] ar = ['python.exe', '$EMSCRIPTEN_BASE/emar.py'] [properties] diff --git a/docs/build/windows.md b/docs/build/windows.md index 096fd764b7..25889345cf 100644 --- a/docs/build/windows.md +++ b/docs/build/windows.md @@ -134,7 +134,7 @@ PowerShell. * git for Windows * jq * Python 3 -* Meson 0.56+ +* Meson 1.0+ * Ninja * Pandoc @@ -144,7 +144,7 @@ PowerShell. $ProgressPreference = 'SilentlyContinue' choco install git jq python ninja pandoc refreshenv -# choco meson (0.55) is too old, 0.56 required: +# choco meson (0.55) is too old, 1.0 required: python -m pip install meson **Environment variables**: diff --git a/resources/builder.inc.sh b/resources/builder.inc.sh index 9c81e3b38d..2e8c570c34 100755 --- a/resources/builder.inc.sh +++ b/resources/builder.inc.sh @@ -898,6 +898,8 @@ builder_parse() { _builder_record_function_call builder_parse + local _builder_params="$@" + _builder_build_deps=--deps builder_verbose= builder_debug= @@ -1040,7 +1042,7 @@ builder_parse() { fi if builder_is_dep_build; then - echo "[$THIS_SCRIPT_IDENTIFIER] dependency build, started by $builder_dep_parent" + echo -e "${HEADING_SETMARK}${COLOR_PURPLE}[$THIS_SCRIPT_IDENTIFIER] dependency build, started by $builder_dep_parent${COLOR_RESET}" if [[ -z ${_builder_deps_built+x} ]]; then echo "FATAL ERROR: Expected --builder-deps-built parameter" exit 1 @@ -1049,6 +1051,7 @@ builder_parse() { # This is a top-level invocation, not a dependency build, so we want to # track which dependencies have been built, so they don't get built multiple # times. + echo -e "${HEADING_SETMARK}${COLOR_PURPLE}[$THIS_SCRIPT_IDENTIFIER] build.sh launched with: <${_builder_params[@]}>${COLOR_RESET}" _builder_deps_built=`mktemp` fi @@ -1215,6 +1218,26 @@ _builder_should_build_dep() { return 0 } +# +# Removes a dependency from the list of available dependencies +# +# Parameters: +# $1 path to dependency +# +builder_remove_dep() { + local dependency="$1" i + dependency="`_builder_expand_relative_path "$dependency"`" + + for i in "${!_builder_deps[@]}"; do + if [[ ${_builder_deps[i]} = $dependency ]]; then + unset '_builder_deps[i]' + fi + done + + # rebuild the array to remove the empty item + _builder_deps=( "${_builder_deps[@]}" ) +} + # # Configure and build all dependencies # Later, may restrict by either action or target