From 7942f3c23b492d5de1216aa51a4bc9741ebaeeb4 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Mon, 26 May 2025 18:24:39 +0200 Subject: [PATCH 01/81] maint(web): move TC configuration for Web to script This matches the existing TC configuration as closely as possible. It also puts the scripts in `resources/teamcity/web`, but doesn't relocate the Linux scripts until we decide where things should go. Part-of: #13399 Test-bot: skip --- resources/teamcity/includes/tc-helpers.inc.sh | 29 +++++ resources/teamcity/web/keyman-web-release.sh | 93 ++++++++++++++ resources/teamcity/web/keyman-web-test.sh | 116 ++++++++++++++++++ .../teamcity/web/zip-and-upload-artifacts.ps1 | 72 +++++++++++ 4 files changed, 310 insertions(+) create mode 100644 resources/teamcity/includes/tc-helpers.inc.sh create mode 100755 resources/teamcity/web/keyman-web-release.sh create mode 100755 resources/teamcity/web/keyman-web-test.sh create mode 100755 resources/teamcity/web/zip-and-upload-artifacts.ps1 diff --git a/resources/teamcity/includes/tc-helpers.inc.sh b/resources/teamcity/includes/tc-helpers.inc.sh new file mode 100644 index 0000000000..ad81defb29 --- /dev/null +++ b/resources/teamcity/includes/tc-helpers.inc.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# Returns 0 if we're running on Ubuntu. +is_ubuntu() { + if [[ "${OSTYPE:-}" == "linux-gnu" ]]; then + return 0 + else + return 1 + fi +} + +# Returns 0 if we're running on Windows, i.e. if the environment variable +# `OSTYPE` is set to "msys" or "cygwin". +is_windows() { + if [[ "${OSTYPE:-}" == "msys" ]] || [[ "${OSTYPE:-}" == "cygwin" ]]; then + return 0 + else + return 1 + fi +} + +# Returns 0 if we're running on macOS. +is_macos() { + if [[ "${OSTYPE:-}" == "darwin" ]]; then + return 0 + else + return 1 + fi +} diff --git a/resources/teamcity/web/keyman-web-release.sh b/resources/teamcity/web/keyman-web-release.sh new file mode 100755 index 0000000000..f83029ebeb --- /dev/null +++ b/resources/teamcity/web/keyman-web-release.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# Copyright (C) 2025 SIL International. All rights reserved. +# Distributed under the MIT License. See LICENSE.md file in the project +# root for full license information. +# +# TC build script to build release of KeymanWeb. + +# shellcheck disable=SC2164 +# shellcheck disable=SC1091 + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" +. "${THIS_SCRIPT%/*}/../../../resources/build/builder.inc.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +# shellcheck disable=SC2154 +. "${KEYMAN_ROOT}/resources/teamcity/includes/tc-helpers.inc.sh" + +################################ Main script ################################ + +builder_describe \ + "Run tests for native KeymanWeb" \ + "all run all actions" \ + "build build Web + embedded" \ + "publish publish release" + +builder_parse "$@" + +cd "${KEYMAN_ROOT}/web" + +function build_web_action() { + builder_echo start web_build "Building KeymanWeb" + + node ../resources/gosh/gosh.js ./ci.sh build + + builder_echo end web_build success "Finished building KeymanWeb" +} + +function _push_release_to_skeymancom() { + # Push release to s.keyman.com/kmw/engine (do this before updating + # downloads.keyman.com so we can ensure files are available) + builder_echo start publish "Publishing release to skeyman.com" + + cd ../../s.keyman.com + git pull https://github.com/keymanapp/s.keyman.com.git master + cd ../keyman/web + node ../resources/gosh/gosh.js ./ci.sh prepare:s.keyman.com --s.keyman.com ../../s.keyman.com + + builder_echo end publish success "Finished publishing release to skeyman.com" +} + +function _zip_and_upload_artifacts() { + if ! is_windows; then + # requires Powershell + return 0 + fi + + builder_echo start "zip and upload artifacts" "Zipping and uploading artifacts" + + powershell -NonInteractive -ExecutionPolicy Bypass -File zip-and-upload-artifacts.ps1 + + builder_echo end "zip and upload artifacts" success "Finished zipping and uploading artifacts" +} + +function _upload_help() { + builder_echo start "upload help" "Uploading new Keyman for Web help to help.keyman.com" + + node ../gosh/gosh.js ./help-keyman-com.sh web + + builder_echo end "upload help" success "Finished uploading new Keyman for Web help to help.keyman.com" +} + +function publish_web_action() { + builder_echo start publish "Publishing KeymanWeb release" + + # Push release to s.keyman.com/kmw/engine (do this before updating + # downloads.keyman.com so we can ensure files are available) + _push_release_to_skeymancom + + _zip_and_upload_artifacts + _upload_help + + builder_echo end publish success "Finished publishing KeymanWeb release" +} + +if builder_has_action all; then + build_web_action + publish_web_action +else + builder_run_action build build_web_action + builder_run_action publish publish_web_action +fi diff --git a/resources/teamcity/web/keyman-web-test.sh b/resources/teamcity/web/keyman-web-test.sh new file mode 100755 index 0000000000..bb6368415f --- /dev/null +++ b/resources/teamcity/web/keyman-web-test.sh @@ -0,0 +1,116 @@ +#!/bin/bash +# Copyright (C) 2025 SIL International. All rights reserved. +# Distributed under the MIT License. See LICENSE.md file in the project +# root for full license information. +# +# TC build script for Keyman Web/Test + +# shellcheck disable=SC2164 +# shellcheck disable=SC1091 + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" +. "${THIS_SCRIPT%/*}/../../../resources/build/builder.inc.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +# shellcheck disable=SC2154 +. "${KEYMAN_ROOT}/resources/teamcity/includes/tc-helpers.inc.sh" + +################################ Main script ################################ + +builder_describe \ + "Run tests for native KeymanWeb" \ + "all run all actions" \ + "configure install dependencies" \ + "build build Web + embedded" \ + "test run native KeymanWeb tests and check build size" + +builder_parse "$@" + +cd "${KEYMAN_ROOT}/web" + +function install_dependencies_action() { + if is_windows; then + return 0 + fi + + builder_echo start install_dependencies "Installing dependencies" + + if [[ "$(lsb_release -s -i)" == "Ubuntu" ]] && dpkg --compare-versions "$(lsb_release -r -s)" ge "24.04"; then + TOINSTALL="" + # dependencies for playwright: + for p in ibevent-2.1-7t64 libxslt1.1 libwoff1 libvpx9 libgstreamer-plugins-bad1.0-0 libwebpdemux2 libharfbuzz-icu0 libenchant-2-2 libsecret-1-0 libhyphen0 libmanette-0.2-0 libflite1 gstreamer1.0-libav + do + if ! dpkg -s "${p}" >/dev/null 2>&1; then + TOINSTALL="${TOINSTALL} ${p}" + fi + done + if [[ -n "${TOINSTALL}" ]]; then + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -qy ${TOINSTALL} + fi + fi + builder_echo end install_dependencies success "Finished installing dependencies" +} + +function build_web_action() { + builder_echo start web_build "Building KeymanWeb" + + node ../resources/gosh/gosh.js ./ci.sh build + + builder_echo end web_build success "Finished building KeymanWeb" +} + +function test_web_action() { + builder_echo start web_test "Running KeymanWeb tests" + + if is_windows; then + node ../resources/gosh/gosh.js ./ci.sh test + else + if [[ "$(lsb_release -s -i)" == "Ubuntu" ]]; then + # On Linux start Xvfb etc + PID_FILE=/tmp/keymanweb-pids + echo "Starting Xvfb..." + Xvfb -screen 0 1024x768x24 :33 &> /dev/null & + echo "kill -9 $! || true" > "$PID_FILE" + sleep 1 + echo "Starting Xephyr..." + DISPLAY=:33 Xephyr :32 -screen 1024x768 &> /dev/null & + echo "kill -9 $! || true" >> "$PID_FILE" + sleep 1 + echo "Starting metacity" + metacity --display=:32 &> /dev/null & + echo "kill -9 $! || true" >> "$PID_FILE" + export DISPLAY=:32 + else + # Don't know what's needed on MacOS... + touch /tmp/keymanweb-pids + fi + node ../resources/gosh/gosh.js ./ci.sh test + bash /tmp/keymanweb-pids + fi + + node ../resources/gosh/gosh.js ./build.sh coverage + builder_echo end web_test success "Finished running KeymanWeb tests" +} + +function check_build_size_action() { + builder_echo start check_build_size "Checking build size" + + node ../resources/gosh/gosh.js ./ci.sh validate-size + + builder_echo end check_build_size success "Finished checking build size" +} + +if builder_has_action all; then + install_dependencies_action + build_web_action + test_web_action + check_build_size_action +else + builder_run_action configure install_dependencies_action + builder_run_action build build_web_action + builder_run_action test test_web_action + builder_run_action test check_build_size_action +fi diff --git a/resources/teamcity/web/zip-and-upload-artifacts.ps1 b/resources/teamcity/web/zip-and-upload-artifacts.ps1 new file mode 100755 index 0000000000..b66234dedc --- /dev/null +++ b/resources/teamcity/web/zip-and-upload-artifacts.ps1 @@ -0,0 +1,72 @@ +# +# This script should be the identical for nightly/beta/stable for a given platform +# +$ErrorActionPreference = "Stop"; + +$tier = Get-Content ..\TIER.md +$build_number = Get-Content ..\VERSION.md +$build_counter = $build_number -replace "^\d+\.\d+\.(\d+)$", '$1' + +$upload_path = "build\upload\$build_number" +$zip = "$upload_path\keymanweb-$build_number.zip" + +$7Z_HOME = $env:7Z_HOME +$RSYNC_HOME = $env:RSYNC_HOME +$USERPROFILE = $env:USERPROFILE + +# Since shell-scripting doesn't like number-initial variables, we convert it to a friendlier name. +$env:SEVEN_Z_HOME=$7Z_HOME + +# PowerShell fun: `--` parameters will break a PowerShell command when not escaped in some form. +# We can build the command in string form first and then execute the string, fortunately. +$cmd = '"C:\Program Files\Git\bin\bash.exe" --init-file "c:\Program Files\Git\etc\profile" -l ./ci.sh prepare:downloads.keyman.com' +cmd /c $cmd + +$hash = get-filehash $zip -Algorithm MD5 + +# +# Construct .build_info +# + +$download_info = @" +{ + "name": "KeymanWeb", + "version": "$build_number", + "date": "$([DateTime]::Now.ToString("yyyy-MM-dd"))", + "platform": "web", + "stability": "$tier", + "file": "keymanweb-$build_number.zip", + "md5": "$($hash.Hash)", + "type": "zip", + "build": "$build_counter" +} +"@ + +# WriteAllLines is needed to avoid BOM +[System.IO.File]::WriteAllLines("$pwd\$upload_path\keymanweb-$build_number.zip.download_info", $download_info) +# $download_info | Out-File $upload_path\keymanweb-$build_number.zip.download_info -Encoding utf8 + +# +# Upload with rsync to downloads.keyman.com +# (rsync requires that we are in the upload folder to get folders in +# sync correctly; it is possible to resolve this but easier to just cd.) +# + +$rsync_args = @( + '-vrzltp', # verbose, recurse, zip, copy symlinks, preserve times, permissions + '--chmod=Dug=rwx,Do=rx,Fug=rw,Fo=r', # map Windows security to host security + '--stats', # show statistics for log + '--rsync-path="%downloads_rsync_path%"', # path on remote server + "--rsh=$RSYNC_HOME\ssh -i $USERPROFILE\.ssh\id_rsa -o UserKnownHostsFile=$USERPROFILE\.ssh\known_hosts", # use ssh + "$build_number", # upload the whole build folder + "%downloads_rsync_user%@%downloads_rsync_host%:%downloads_rsync_root%/web/$tier/" # target server + path +) + +# Write-Output "rsync parameters:" $rsync_args + +cd "$upload_path\.." +& $RSYNC_HOME\rsync.exe $rsync_args +if ($LASTEXITCODE -ne 0) { throw "Exit code is $LASTEXITCODE" } +cd .. + +# EOF From fd9c88fa83cf87e8e556946cb226333ca3ba0b67 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Fri, 23 May 2025 18:40:12 +0200 Subject: [PATCH 02/81] maint(web): improve TC build script for Keyman Web/Test This refactors the build script and makes some changes so that it is possible to run in a docker container on a developer's machine. Part-of: #13399 Test-bot: skip --- resources/teamcity/README.md | 3 + resources/teamcity/includes/tc-actions.inc.sh | 25 +++++ resources/teamcity/includes/tc-linux.inc.sh | 29 +++++ resources/teamcity/web/keyman-web-release.sh | 2 +- resources/teamcity/web/keyman-web-test.sh | 104 ++++++------------ 5 files changed, 92 insertions(+), 71 deletions(-) diff --git a/resources/teamcity/README.md b/resources/teamcity/README.md index 9fe012a942..ad56784b46 100644 --- a/resources/teamcity/README.md +++ b/resources/teamcity/README.md @@ -22,11 +22,14 @@ Inside of the container, run ```bash apt update && apt install sudo export DOCKER_RUNNING=1 +echo "ubuntu ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers +su ubuntu ``` and then run the build, e.g. ```bash +cd /Develop resources/teamcity/keyman-linux-test.sh configure,build,test ``` diff --git a/resources/teamcity/includes/tc-actions.inc.sh b/resources/teamcity/includes/tc-actions.inc.sh index f6973418b6..caaed1c745 100644 --- a/resources/teamcity/includes/tc-actions.inc.sh +++ b/resources/teamcity/includes/tc-actions.inc.sh @@ -58,3 +58,28 @@ linux_unit_tests_action() { builder_echo end unit_tests success "Finished running unit tests" } +web_build_action() { + builder_echo start web_build "Building web" + "${KEYMAN_ROOT}/web/ci.sh" build + builder_echo end web_build success "Finished building web" +} + +web_test_action() { + builder_echo start web_test "Running tests for native KeymanWeb" + if is_ubuntu; then + linux_start_xvfb + trap "linux_stop_xvfb" ERR + fi + + "${KEYMAN_ROOT}/web/ci.sh" test + + if is_ubuntu; then + linux_stop_xvfb + trap ERR + fi + + "${KEYMAN_ROOT}/web/build.sh" coverage + + builder_echo end web_test success "Finished running tests for native KeymanWeb" +} + diff --git a/resources/teamcity/includes/tc-linux.inc.sh b/resources/teamcity/includes/tc-linux.inc.sh index 32d311cad3..8997331dad 100644 --- a/resources/teamcity/includes/tc-linux.inc.sh +++ b/resources/teamcity/includes/tc-linux.inc.sh @@ -27,6 +27,10 @@ check_and_install_packages() { local PACKAGES=$* local TOINSTALL="" + if ! is_ubuntu; then + return 0 + fi + for p in ${PACKAGES} do if ! is_package_installed "${p}"; then @@ -83,3 +87,28 @@ linux_install_dependencies_for_tests() { builder_echo end install_dependencies_for_tests success "Finished installing dependencies for tests" } +linux_start_xvfb() { + # On Linux start Xvfb etc + local PID_FILE=/tmp/keymanweb-pids + builder_echo "Starting Xvfb..." + Xvfb -screen 0 1024x768x24 :33 &> /dev/null & + echo "kill -9 $! || true" > "${PID_FILE}" + sleep 1 + builder_echo "Starting Xephyr..." + DISPLAY=:33 Xephyr :32 -screen 1024x768 &> /dev/null & + echo "kill -9 $! || true" >> "${PID_FILE}" + sleep 1 + builder_echo "Starting metacity" + metacity --display=:32 &> /dev/null & + echo "kill -9 $! || true" >> "${PID_FILE}" + export DISPLAY=:32 +} + +linux_stop_xvfb() { + # On Linux stop Xvfb etc + local PID_FILE=/tmp/keymanweb-pids + if [[ -f "${PID_FILE}" ]]; then + bash "${PID_FILE}" + rm -f "${PID_FILE}" + fi +} diff --git a/resources/teamcity/web/keyman-web-release.sh b/resources/teamcity/web/keyman-web-release.sh index f83029ebeb..99869c632e 100755 --- a/resources/teamcity/web/keyman-web-release.sh +++ b/resources/teamcity/web/keyman-web-release.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright (C) 2025 SIL International. All rights reserved. # Distributed under the MIT License. See LICENSE.md file in the project # root for full license information. diff --git a/resources/teamcity/web/keyman-web-test.sh b/resources/teamcity/web/keyman-web-test.sh index bb6368415f..b3afe31104 100755 --- a/resources/teamcity/web/keyman-web-test.sh +++ b/resources/teamcity/web/keyman-web-test.sh @@ -1,5 +1,5 @@ -#!/bin/bash -# Copyright (C) 2025 SIL International. All rights reserved. +#!/usr/bin/env bash +# Copyright (C) 2025 SIL International # Distributed under the MIT License. See LICENSE.md file in the project # root for full license information. # @@ -15,7 +15,10 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" ## END STANDARD BUILD SCRIPT INCLUDE # shellcheck disable=SC2154 +. "${KEYMAN_ROOT}/resources/shellHelperFunctions.sh" +. "${KEYMAN_ROOT}/resources/teamcity/includes/tc-actions.inc.sh" . "${KEYMAN_ROOT}/resources/teamcity/includes/tc-helpers.inc.sh" +. "${KEYMAN_ROOT}/resources/teamcity/includes/tc-linux.inc.sh" ################################ Main script ################################ @@ -31,86 +34,47 @@ builder_parse "$@" cd "${KEYMAN_ROOT}/web" function install_dependencies_action() { - if is_windows; then + builder_echo start "install dependencies" "Install dependencies" + + # shellcheck disable=SC2086 + check_and_install_packages devscripts jq + + # TODO: we can we do something similar for Windows and macOS? + linux_install_nvm + install_playwright_dependencies + + builder_echo end "install dependencies" success "Finished installing dependencies" +} + +function install_playwright_dependencies() { + if ! is_ubuntu || ! is_os_version_or_higher 24.04; then return 0 fi - builder_echo start install_dependencies "Installing dependencies" - - if [[ "$(lsb_release -s -i)" == "Ubuntu" ]] && dpkg --compare-versions "$(lsb_release -r -s)" ge "24.04"; then - TOINSTALL="" - # dependencies for playwright: - for p in ibevent-2.1-7t64 libxslt1.1 libwoff1 libvpx9 libgstreamer-plugins-bad1.0-0 libwebpdemux2 libharfbuzz-icu0 libenchant-2-2 libsecret-1-0 libhyphen0 libmanette-0.2-0 libflite1 gstreamer1.0-libav - do - if ! dpkg -s "${p}" >/dev/null 2>&1; then - TOINSTALL="${TOINSTALL} ${p}" - fi - done - if [[ -n "${TOINSTALL}" ]]; then - sudo apt-get update - sudo DEBIAN_FRONTEND="noninteractive" apt-get install -qy ${TOINSTALL} - fi - fi - builder_echo end install_dependencies success "Finished installing dependencies" -} - -function build_web_action() { - builder_echo start web_build "Building KeymanWeb" - - node ../resources/gosh/gosh.js ./ci.sh build - - builder_echo end web_build success "Finished building KeymanWeb" -} - -function test_web_action() { - builder_echo start web_test "Running KeymanWeb tests" - - if is_windows; then - node ../resources/gosh/gosh.js ./ci.sh test - else - if [[ "$(lsb_release -s -i)" == "Ubuntu" ]]; then - # On Linux start Xvfb etc - PID_FILE=/tmp/keymanweb-pids - echo "Starting Xvfb..." - Xvfb -screen 0 1024x768x24 :33 &> /dev/null & - echo "kill -9 $! || true" > "$PID_FILE" - sleep 1 - echo "Starting Xephyr..." - DISPLAY=:33 Xephyr :32 -screen 1024x768 &> /dev/null & - echo "kill -9 $! || true" >> "$PID_FILE" - sleep 1 - echo "Starting metacity" - metacity --display=:32 &> /dev/null & - echo "kill -9 $! || true" >> "$PID_FILE" - export DISPLAY=:32 - else - # Don't know what's needed on MacOS... - touch /tmp/keymanweb-pids - fi - node ../resources/gosh/gosh.js ./ci.sh test - bash /tmp/keymanweb-pids - fi - - node ../resources/gosh/gosh.js ./build.sh coverage - builder_echo end web_test success "Finished running KeymanWeb tests" + # shellcheck disable=SC2086 + check_and_install_packages ibevent-2.1-7t64 libxslt1.1 libwoff1 libvpx9 libgstreamer-plugins-bad1.0-0 libwebpdemux2 libharfbuzz-icu0 libenchant-2-2 libsecret-1-0 libhyphen0 libmanette-0.2-0 libflite1 gstreamer1.0-libav } function check_build_size_action() { - builder_echo start check_build_size "Checking build size" - - node ../resources/gosh/gosh.js ./ci.sh validate-size - - builder_echo end check_build_size success "Finished checking build size" + builder_echo start "check build size" "Check build size" + "${KEYMAN_ROOT}/web/ci.sh" validate-size + builder_echo end "check build size" success "Finished checking build size" } if builder_has_action all; then install_dependencies_action - build_web_action - test_web_action + + set_variables_for_nvm + + web_build_action + web_test_action check_build_size_action else builder_run_action configure install_dependencies_action - builder_run_action build build_web_action - builder_run_action test test_web_action + + set_variables_for_nvm + + builder_run_action build web_build_action + builder_run_action test web_test_action builder_run_action test check_build_size_action fi From 797af43b0af48d4ec31f74ea5b61cdbfc70082b0 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Mon, 26 May 2025 19:22:48 +0200 Subject: [PATCH 03/81] maint(web): move TC configuration for Keyman Developer to scripts This matches the existing TC configuration as closely as possible. Part-of: #13399 Test-bot: skip --- .../download-symbol-server-index.ps1 | 58 +++++ .../developer/keyman-developer-release.sh | 130 +++++++++++ .../developer/keyman-developer-test-linux.sh | 40 ++++ .../developer/keyman-developer-test-macos.sh | 40 ++++ .../keyman-developer-test-windows.sh | 65 ++++++ .../developer/publish-new-symbols.ps1 | 34 +++ .../publish-to-downloads-keyman-com.ps1 | 217 ++++++++++++++++++ 7 files changed, 584 insertions(+) create mode 100644 resources/teamcity/developer/download-symbol-server-index.ps1 create mode 100755 resources/teamcity/developer/keyman-developer-release.sh create mode 100755 resources/teamcity/developer/keyman-developer-test-linux.sh create mode 100755 resources/teamcity/developer/keyman-developer-test-macos.sh create mode 100755 resources/teamcity/developer/keyman-developer-test-windows.sh create mode 100644 resources/teamcity/developer/publish-new-symbols.ps1 create mode 100644 resources/teamcity/developer/publish-to-downloads-keyman-com.ps1 diff --git a/resources/teamcity/developer/download-symbol-server-index.ps1 b/resources/teamcity/developer/download-symbol-server-index.ps1 new file mode 100644 index 0000000000..e160739b4d --- /dev/null +++ b/resources/teamcity/developer/download-symbol-server-index.ps1 @@ -0,0 +1,58 @@ +# +# This script should be the identical for nightly/beta/stable for a given platform +# + +$ErrorActionPreference = "Stop" + +$RSYNC_HOME = $env:RSYNC_HOME +$USERPROFILE = $env:USERPROFILE + +New-Item -Force -ItemType Directory symbols\000admin +cd symbols\000admin + +# +# Download the symbol server index files with rsync from downloads.keyman.com +# (rsync requires that we are in the symbols folder to get folders in +# sync correctly; it is possible to resolve this but easier to just cd.) +# + +$rsync_args = @( + '-vrzltp', # verbose, recurse, zip, copy symlinks, preserve times, permissions + '--chmod=Dug=rwx,Do=rx,Fug=rw,Fo=r', # map Windows security to host security + '--stats', # show statistics for log + '--rsync-path="sudo -u vu2009 rsync"', # path on remote server + "--rsh=$RSYNC_HOME\ssh -i $USERPROFILE\.ssh\id_rsa -o UserKnownHostsFile=$USERPROFILE\.ssh\known_hosts", # use ssh + "root@sysops.downloads.keyman.com:/var/www/virtual/downloads.keyman.com/htdocs/windows/symbols/000admin/lastid.txt", # target server + path + "." # download the whole symbols 000Admin folder +) + +& $RSYNC_HOME\rsync.exe $rsync_args +if ($LASTEXITCODE -ne 0) { throw "Exit code is $LASTEXITCODE" } + +$rsync_args = @( + '-vrzltp', # verbose, recurse, zip, copy symlinks, preserve times, permissions + '--chmod=Dug=rwx,Do=rx,Fug=rw,Fo=r', # map Windows security to host security + '--stats', # show statistics for log + '--rsync-path="sudo -u vu2009 rsync"', # path on remote server + "--rsh=$RSYNC_HOME\ssh -i $USERPROFILE\.ssh\id_rsa -o UserKnownHostsFile=$USERPROFILE\.ssh\known_hosts", # use ssh + "root@sysops.downloads.keyman.com:/var/www/virtual/downloads.keyman.com/htdocs/windows/symbols/000admin/history.txt", # target server + path + "." # download the whole symbols 000Admin folder +) + +& $RSYNC_HOME\rsync.exe $rsync_args +if ($LASTEXITCODE -ne 0) { throw "Exit code is $LASTEXITCODE" } + +$rsync_args = @( + '-vrzltp', # verbose, recurse, zip, copy symlinks, preserve times, permissions + '--chmod=Dug=rwx,Do=rx,Fug=rw,Fo=r', # map Windows security to host security + '--stats', # show statistics for log + '--rsync-path="sudo -u vu2009 rsync"', # path on remote server + "--rsh=$RSYNC_HOME\ssh -i $USERPROFILE\.ssh\id_rsa -o UserKnownHostsFile=$USERPROFILE\.ssh\known_hosts", # use ssh + "root@sysops.downloads.keyman.com:/var/www/virtual/downloads.keyman.com/htdocs/windows/symbols/000admin/server.txt", # target server + path + "." # download the whole symbols 000Admin folder +) + +& $RSYNC_HOME\rsync.exe $rsync_args +if ($LASTEXITCODE -ne 0) { throw "Exit code is $LASTEXITCODE" } + +# EOF diff --git a/resources/teamcity/developer/keyman-developer-release.sh b/resources/teamcity/developer/keyman-developer-release.sh new file mode 100755 index 0000000000..698ffbc3cb --- /dev/null +++ b/resources/teamcity/developer/keyman-developer-release.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +# Copyright (C) 2025 SIL International +# Distributed under the MIT License. See LICENSE.md file in the project +# root for full license information. +# +# TC build script to build release of Keyman Developer + +# shellcheck disable=SC2164 +# shellcheck disable=SC1091 + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" +. "${THIS_SCRIPT%/*}/../../../resources/build/builder.inc.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +# shellcheck disable=SC2154 +. "${KEYMAN_ROOT}/resources/teamcity/includes/tc-helpers.inc.sh" + +################################ Main script ################################ + +builder_describe \ + "Build Keyman Developer on Windows" \ + "all run all actions" \ + "build build Keyman Developer and test keyboards" \ + "publish publish release of Keyman Developer" + +builder_parse "$@" + +# shellcheck disable=SC2154 +cd "${KEYMAN_ROOT}/developer/src" + +function _build_developer() { + builder_echo start "build developer" "Building Keyman Developer" + + ./build.sh configure build test api publish --npm-publish + + builder_echo end "build developer" success "Finished building Keyman Developer" +} + +function _build_testkeyboards() { + builder_echo start "build testkeyboards" "Building test keyboards" + + "${KEYMAN_ROOT}/common/test/keyboards/build.sh" + + builder_echo end "build testkeyboards" success "Finished building test keyboards" +} + +function _publish_sentry() { + builder_echo start "publish sentry" "Publishing debug information files to Sentry" + + "${KEYMAN_ROOT}/developer/src/tools/sentry-upload-difs.sh" + + builder_echo end "publish sentry" success "Finished publishing debug information files to Sentry" +} + +function _download_symbol_server_index() { + # Download symbol server index from symbol server + if ! is_windows; then + # requires Powershell + return 0 + fi + + builder_echo start "download symbol server index" "Downloading symbol server index" + + # shellcheck disable=SC2154 + powershell -NonInteractive -ExecutionPolicy Bypass -File "${THIS_SCRIPT_PATH}/download-symbol-server-index.ps1" + + builder_echo end "download symbol server index" success "Finished downloading symbol server index" +} + +function _publish_new_symbols() { + # Publish new symbols to symbol server + if ! is_windows; then + # requires Powershell + return 0 + fi + + builder_echo start "publish new symbols" "Publishing new symbols to symbol server" + + # shellcheck disable=SC2154 + powershell -NonInteractive -ExecutionPolicy Bypass -File "${THIS_SCRIPT_PATH}/publish-new-symbols.ps1" + + builder_echo end "publish new symbols" success "Finished publishing new symbols to symbol server" +} + +function _publish_to_downloads_keyman_com() { + # Publish to downloads.keyman.com + if ! is_windows; then + # requires Powershell + return 0 + fi + + builder_echo start "publish to downloads.keyman.com" "Publishing release to downloads.keyman.com" + + # shellcheck disable=SC2154 + powershell -NonInteractive -ExecutionPolicy Bypass -File "${THIS_SCRIPT_PATH}/publish-to-downloads-keyman-com.ps1" + + builder_echo end "publish to downloads.keyman.com" success "Finished publishing release to downloads.keyman.com" +} + +function _publish_api_documentation() { + # Upload new Keyman Developer API documentation to help.keyman.com + builder_echo start "publish api documentation" "Uploading new Keyman Developer API documentation to help.keyman.com" + + "${KEYMAN_ROOT}/resouyrces/build/help-keyman-com.sh" developer + + builder_echo end "publish api documentation" success "Finished uploading new Keyman Developer API documentation to help.keyman.com" +} + +function build_developer_action() { + _build_developer + _build_testkeyboards +} + +function publish_action() { + _publish_sentry + _download_symbol_server_index + _publish_new_symbols + _publish_to_downloads_keyman_com + _publish_api_documentation +} + +if builder_has_action all; then + build_developer_action + publish_action +else + builder_run_action build build_developer_action + builder_run_action publish publish_action +fi diff --git a/resources/teamcity/developer/keyman-developer-test-linux.sh b/resources/teamcity/developer/keyman-developer-test-linux.sh new file mode 100755 index 0000000000..a195abe2c1 --- /dev/null +++ b/resources/teamcity/developer/keyman-developer-test-linux.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Copyright (C) 2025 SIL International +# Distributed under the MIT License. See LICENSE.md file in the project +# root for full license information. +# +# TC build script for Keyman Developer on Linux + +# shellcheck disable=SC2164 +# shellcheck disable=SC1091 + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" +. "${THIS_SCRIPT%/*}/../../../resources/build/builder.inc.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +################################ Main script ################################ + +builder_describe \ + "Build Keyman Developer on Linux" \ + "all run all actions" \ + "build build" + +builder_parse "$@" + +cd "${KEYMAN_ROOT}/developer/src" + +function build_developer_action() { + builder_echo start "build developer" "Building Keyman Developer" + + ./build.sh configure build test + + builder_echo end "build developer" success "Finished building Keyman Developer" +} + +if builder_has_action all; then + build_developer_action +else + builder_run_action build build_developer_action +fi diff --git a/resources/teamcity/developer/keyman-developer-test-macos.sh b/resources/teamcity/developer/keyman-developer-test-macos.sh new file mode 100755 index 0000000000..75b2fba7b2 --- /dev/null +++ b/resources/teamcity/developer/keyman-developer-test-macos.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Copyright (C) 2025 SIL International +# Distributed under the MIT License. See LICENSE.md file in the project +# root for full license information. +# +# TC build script for Keyman Developer on macOS + +# shellcheck disable=SC2164 +# shellcheck disable=SC1091 + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" +. "${THIS_SCRIPT%/*}/../../../resources/build/builder.inc.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +################################ Main script ################################ + +builder_describe \ + "Build Keyman Developer on macOS" \ + "all run all actions" \ + "build build" + +builder_parse "$@" + +cd "${KEYMAN_ROOT}/developer/src" + +function build_developer_action() { + builder_echo start "build developer" "Building Keyman Developer" + + ./build.sh configure build test + + builder_echo end "build developer" success "Finished building Keyman Developer" +} + +if builder_has_action all; then + build_developer_action +else + builder_run_action build build_developer_action +fi diff --git a/resources/teamcity/developer/keyman-developer-test-windows.sh b/resources/teamcity/developer/keyman-developer-test-windows.sh new file mode 100755 index 0000000000..1bfe45b264 --- /dev/null +++ b/resources/teamcity/developer/keyman-developer-test-windows.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Copyright (C) 2025 SIL International +# Distributed under the MIT License. See LICENSE.md file in the project +# root for full license information. +# +# TC build script for Keyman Developer on Windows + +# shellcheck disable=SC2164 +# shellcheck disable=SC1091 + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" +. "${THIS_SCRIPT%/*}/../../../resources/build/builder.inc.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +################################ Main script ################################ + +builder_describe \ + "Build Keyman Developer on Windows" \ + "all run all actions" \ + "build build Keyman Developer and test keyboards" \ + "publish publish debug information files to sentry" + +builder_parse "$@" + +# shellcheck disable=SC2154 +cd "${KEYMAN_ROOT}/developer/src" + +function build_developer_action() { + _build_developer + _build_testkeyboards +} + +function _build_developer() { + builder_echo start "build developer" "Building Keyman Developer" + + ./build.sh configure build test api publish --dry-run + + builder_echo end "build developer" success "Finished building Keyman Developer" +} + +function _build_testkeyboards() { + builder_echo start "build testkeyboards" "Building test keyboards" + + "${KEYMAN_ROOT}/common/test/keyboards/build.sh" --zip-source --index + + builder_echo end "build testkeyboards" success "Finished building test keyboards" +} + +function publish_sentry_action() { + builder_echo start "publish sentry" "Publishing debug information files to Sentry" + + "${KEYMAN_ROOT}/developer/src/tools/sentry-upload-difs.sh" + + builder_echo end "publish sentry" success "Finished publishing debug information files to Sentry" +} + +if builder_has_action all; then + build_developer_action + publish_sentry_action +else + builder_run_action build build_developer_action + builder_run_action publish publish_sentry_action +fi diff --git a/resources/teamcity/developer/publish-new-symbols.ps1 b/resources/teamcity/developer/publish-new-symbols.ps1 new file mode 100644 index 0000000000..82432579be --- /dev/null +++ b/resources/teamcity/developer/publish-new-symbols.ps1 @@ -0,0 +1,34 @@ +# +# This script should be the identical for nightly/beta/stable for a given platform +# + +$ErrorActionPreference = "Stop" + +$RSYNC_HOME = $env:RSYNC_HOME +$USERPROFILE = $env:USERPROFILE + +# Rename 000Admin to 000admin + +ren 000Admin 000admin_ +ren 000admin_ 000admin + +# +# Upload with rsync to downloads.keyman.com +# (rsync requires that we are in the symbols folder to get folders in +# sync correctly; it is possible to resolve this but easier to just cd.) +# + +$rsync_args = @( + '-vrzltp', # verbose, recurse, zip, copy symlinks, preserve times, permissions + '--chmod=Dug=rwx,Do=rx,Fug=rw,Fo=r', # map Windows security to host security + '--stats', # show statistics for log + '--rsync-path="sudo -u vu2009 rsync"', # path on remote server + "--rsh=$RSYNC_HOME\ssh -i $USERPROFILE\.ssh\id_rsa -o UserKnownHostsFile=$USERPROFILE\.ssh\known_hosts", # use ssh + ".", # upload the whole symbols folder + "%downloads_rsync_user%@%downloads_rsync_host%:%downloads_rsync_root%/windows/symbols/" # target server + path +) + +& $RSYNC_HOME\rsync.exe $rsync_args +if ($LASTEXITCODE -ne 0) { throw "Exit code is $LASTEXITCODE" } + +# EOF diff --git a/resources/teamcity/developer/publish-to-downloads-keyman-com.ps1 b/resources/teamcity/developer/publish-to-downloads-keyman-com.ps1 new file mode 100644 index 0000000000..29a1d8372b --- /dev/null +++ b/resources/teamcity/developer/publish-to-downloads-keyman-com.ps1 @@ -0,0 +1,217 @@ +$ErrorActionPreference = "Stop" + +$tier = Get-Content ..\TIER.md +$build_number = Get-Content ..\VERSION.md +$build_counter = $build_number -replace "^\d+\.\d+\.(\d+)$", '$1' +$msi_major_version = $build_number -replace "^(\d+)\.(\d+)\.(\d+)$", '$1$2' + +if ( $tier -eq "alpha" -or $tier -eq "beta") { + $version_tag = $tier + $version_with_tag = $build_number # we may go with $build_number + "-" + $version_tag in the future one day +} else { + $version_tag = "" + $version_with_tag = $build_number +} + +$upload_path = "upload\$build_number" +$keyboards_path = "$upload_path\keyboards" + +# Keyman Developer installers +$kmcomp_zip = "kmcomp-$version_with_tag.zip" +$keyman_developer_exe = "keymandeveloper-$version_with_tag.exe" + +$setup_exe = "setup.exe" + +# Debug files +$debug_zip = "debug-$build_number.zip" + +$7Z_HOME = $env:7Z_HOME +$RSYNC_HOME = $env:RSYNC_HOME +$USERPROFILE = $env:USERPROFILE + +########################################################################################### +## Developer upload +########################################################################################### + +# +# Preparation +# + +if((Test-Path $upload_path) -ne 0) { + Remove-Item -Path $upload_path -Recurse +} + +mkdir $upload_path + +# +# Build Keyman Compiler WINE archive +# + +if((Test-Path ..\release\$kmcomp_zip) -ne 0) { + copy ..\release\$kmcomp_zip ..\$upload_path\$kmcomp_zip +} else { + cd bin + if((Test-Path ..\..\common\schemas\keyboard_info\keyboard_info.source.json) -ne 0) { + # Keyman versions through -16.0 + copy ..\..\common\schemas\keyboard_info\keyboard_info.source.json . + copy ..\..\common\schemas\keyboard_info\keyboard_info.distribution.json . + & "$7Z_HOME\7z.exe" a -bd -bb0 ..\$upload_path\$kmcomp_zip kmcomp.exe kmcmpdll.dll kmcomp.x64.exe kmcmpdll.x64.dll kmconvert.exe keyboard_info.source.json keyboard_info.distribution.json xml\layoutbuilder\*.keyman-touch-layout projects\ + + # Add Keyman Developer Server to the archive (15.0 late alpha - after 171) + if((Test-Path ..\src\server\build) -ne 0) { + copy ..\src\server\build\ server\ -Recurse + & "$7Z_HOME\7z.exe" a -bd -bb0 ..\$upload_path\$kmcomp_zip server\ + } + } else { + # Keyman versions 17.0+; note, use npm install for most modules + copy ..\..\common\schemas\keyboard_info\keyboard_info.schema.json . + & "$7Z_HOME\7z.exe" a -bd -bb0 ..\$upload_path\$kmcomp_zip kmconvert.exe keyboard_info.schema.json xml\layoutbuilder\*.keyman-touch-layout projects\ server\ + } + cd .. +} + +# +# Copy source files +# + +copy release\$build_number\$keyman_developer_exe $upload_path\$keyman_developer_exe + +# +# Construct keyman_developer.download_info +# + +$hash = get-filehash $upload_path\$keyman_developer_exe -Algorithm MD5 + +$keyman_developer_download_info = @" +{ + "name": "Keyman Developer", + "version": "$version_with_tag", + "date": "$([DateTime]::Now.ToString("yyyy-MM-dd"))", + "platform": "win", + "stability": "$tier", + "file": "keymandeveloper-$version_with_tag.exe", + "md5": "$($hash.Hash)", + "type": "exe", + "build": "$build_counter" +} +"@ + +[System.IO.File]::WriteAllLines("$pwd\$upload_path\$keyman_developer_exe.download_info", $keyman_developer_download_info) + +# +# Construct kmcomp.download_info +# + +$hash = get-filehash $upload_path\$kmcomp_zip -Algorithm MD5 + +$kmcomp_download_info = @" +{ + "name": "Keyman Developer Command-Line Compiler", + "version": "$version_with_tag", + "date": "$([DateTime]::Now.ToString("yyyy-MM-dd"))", + "platform": "win", + "stability": "$tier", + "file": "kmcomp-$version_with_tag.zip", + "md5": "$($hash.Hash)", + "type": "zip", + "build": "$build_counter" +} +"@ + +[System.IO.File]::WriteAllLines("$pwd\$upload_path\$kmcomp_zip.download_info", $kmcomp_download_info) + +# +# Copy common/test/keyboards/*/build/*.kmp to keyboards/ +# +if((Test-Path ..\common\test\keyboards) -ne 0) { + mkdir $keyboards_path + Copy-Item ..\common\test\keyboards\*\build\*.kmp $keyboards_path\ +} + +# +# Upload with rsync to downloads.keyman.com +# (rsync requires that we are in the upload folder to get folders in +# sync correctly; it is possible to resolve this but easier to just cd.) +# + +$rsync_args = @( + '-vrzltp', # verbose, recurse, zip, copy symlinks, preserve times, permissions + '--chmod=Dug=rwx,Do=rx,Fug=rw,Fo=r', # map Windows security to host security + '--stats', # show statistics for log + '--rsync-path="sudo -u vu2009 rsync"', # path on remote server + "--rsh=$RSYNC_HOME\ssh -i $USERPROFILE\.ssh\id_rsa -o UserKnownHostsFile=$USERPROFILE\.ssh\known_hosts", # use ssh + "$build_number", # upload the whole build folder + "%downloads_rsync_user%@%downloads_rsync_host%:%downloads_rsync_root%/developer/$tier/" # target server + path +) + +# Write-Output "rsync parameters:" $rsync_args + +cd upload +& $RSYNC_HOME\rsync.exe $rsync_args +if( $LASTEXITCODE -ne 0 ) { + Write-Output "rsync failed." + exit 99 +} +cd .. + +################# +# DEBUG.ZIP +################# + +if((Test-Path release\$build_number\$debug_zip) -ne 0) { + # + # Copy source files + # + + copy release\$build_number\$debug_zip $upload_path\$debug_zip + + # + # Construct debug-.download_info + # + + $hash = get-filehash $upload_path\$debug_zip -Algorithm MD5 + + $debug_zip_download_info = @" +{ + "name": "Keyman Developer debug files", + "version": "$version_with_tag", + "date": "$([DateTime]::Now.ToString("yyyy-MM-dd"))", + "platform": "win", + "stability": "$tier", + "file": "debug-$build_number.zip", + "md5": "$($hash.Hash)", + "type": "zip", + "build": "$build_counter" +} +"@ + + [System.IO.File]::WriteAllLines("$pwd\$upload_path\$debug_zip.download_info", $debug_zip_download_info) +} + +# +# Upload with rsync to downloads.keyman.com +# (rsync requires that we are in the upload folder to get folders in +# sync correctly; it is possible to resolve this but easier to just cd.) +# + +$rsync_args = @( + '-vrzltp', # verbose, recurse, zip, copy symlinks, preserve times, permissions + '--chmod=Dug=rwx,Do=rx,Fug=rw,Fo=r', # map Windows security to host security + '--stats', # show statistics for log + '--rsync-path="%downloads_rsync_path%"', # path on remote server + "--rsh=$RSYNC_HOME\ssh -i $USERPROFILE\.ssh\id_rsa -o UserKnownHostsFile=$USERPROFILE\.ssh\known_hosts", # use ssh + "$build_number", # upload the whole build folder + "%downloads_rsync_user%@%downloads_rsync_host%:%downloads_rsync_root%/developer/$tier/" # target server + path +) + +# Write-Output "rsync parameters:" $rsync_args + +cd upload +& $RSYNC_HOME\rsync.exe $rsync_args +if( $LASTEXITCODE -ne 0 ) { + Write-Output "rsync failed." + exit 99 +} +cd .. + +# EOF From ea8eea1cfacad5378e64c56995e11b421a573af5 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Tue, 27 May 2025 17:56:32 +0200 Subject: [PATCH 04/81] maint(developer): add docker image to build developer node modules Also update build documentation. Fixes: #14045 Test-bot: skip --- docs/build/linux-ubuntu.md | 14 ++++- resources/docker-images/build.sh | 11 ++-- resources/docker-images/developer/Dockerfile | 54 ++++++++++++++++++++ resources/docker-images/run.sh | 17 ++++-- 4 files changed, 86 insertions(+), 10 deletions(-) create mode 100644 resources/docker-images/developer/Dockerfile diff --git a/docs/build/linux-ubuntu.md b/docs/build/linux-ubuntu.md index 98e15185bf..899a5d7504 100644 --- a/docs/build/linux-ubuntu.md +++ b/docs/build/linux-ubuntu.md @@ -6,6 +6,7 @@ On Linux, you can build the following projects: - [Keyman Core](#keyman-core) (aka core) - [Keyman for Linux](#keyman-for-linux) +- [Keyman Developer (node modules only)](#keyman-developer-node-modules-only) - [Keyman Web](#keyman-web) - [Keyman for Android](#keyman-for-android)