diff --git a/.github/workflows/deb-packaging.README.md b/.github/workflows/deb-packaging.README.md index 9dcfce20db..662526a875 100644 --- a/.github/workflows/deb-packaging.README.md +++ b/.github/workflows/deb-packaging.README.md @@ -18,7 +18,8 @@ You can manually trigger a deb-packaging action on GitHub, e.g. to test changes: \"baseBranch\": \"master\", \"baseRef\": \"$(git rev-parse refs/heads/master^)\", \"user\": \"${USER}\", - \"isTestBuild\": \"true\"}" \ + \"isTestBuild\": \"true\", + \"force\": \"true\"}" \ https://api.github.com/repos//keyman/dispatches ``` @@ -36,6 +37,7 @@ You can manually trigger a deb-packaging action on GitHub, e.g. to test changes: \"baseBranch\": \"master\", \"baseRef\": \"$(git rev-parse refs/heads/master)\", \"user\": \"${USER}\", - \"isTestBuild\": \"true\"}" \ + \"isTestBuild\": \"true\", + \"force\": \"true\"}" \ https://api.github.com/repos//keyman/dispatches ``` diff --git a/.github/workflows/deb-packaging.yml b/.github/workflows/deb-packaging.yml index 472adf91b0..28440c4211 100644 --- a/.github/workflows/deb-packaging.yml +++ b/.github/workflows/deb-packaging.yml @@ -35,7 +35,7 @@ env: jobs: sourcepackage: name: Build source package - if: github.repository == 'keymanapp/keyman' + if: github.repository == 'keymanapp/keyman' || github.event.client_payload.force runs-on: ubuntu-24.04 outputs: KEYMAN_VERSION: ${{ steps.version_step.outputs.KEYMAN_VERSION }} @@ -185,28 +185,36 @@ jobs: runs-on: ubuntu-latest steps: - # - name: Download Artifacts - # uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - # with: - # path: artifacts - # merge-multiple: true + - name: Download Source Artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: keyman-srcpkg + path: artifacts - # - name: Install dependencies - # run: | - # sudo DEBIAN_FRONTEND=noninteractive apt-get -q -y install autopkgtest qemu-system qemu-utils autodep8 genisoimage python3-distro-info + - name: Download Binary Artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + path: artifacts + pattern: keyman-binarypkgs-* + merge-multiple: true - # - name: Build test image - # run: | - # cd "${GITHUB_WORKSPACE}/artifacts" - # autopkgtest-buildvm-ubuntu-cloud -v --release=jammy - - # - name: Run tests - # run: | - # cd "${GITHUB_WORKSPACE}/artifacts" - # autopkgtest -B *.deb keyman_*.dsc -- qemu autopkgtest-jammy-amd64.img - - name: Ignore + - name: Install dependencies run: | - echo "Ignored for now - until working solution is in place (#13777)" + sudo apt update + sudo DEBIAN_FRONTEND=noninteractive apt-get -q -y install autopkgtest qemu-system qemu-utils autodep8 genisoimage python3-distro-info + + - name: Build test image + run: | + cd "${GITHUB_WORKSPACE}/artifacts" + sudo chown ${USER} /dev/kvm + autopkgtest-buildvm-ubuntu-cloud -v --release=$(lsb_release -c -s) + + - name: Run tests + run: | + # Tests are defined in linux/debian/tests/test-build + cd "${GITHUB_WORKSPACE}/artifacts" + DIST="$(lsb_release -c -s)" + autopkgtest -B *${DIST}*.deb keyman_*.dsc -- qemu autopkgtest-${DIST}-amd64.img deb_signing: name: Sign source and binary packages diff --git a/HISTORY.md b/HISTORY.md index 6f299aa2c9..894337e6da 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,27 @@ # Keyman Version History +## 19.0.123 alpha 2025-09-19 + +* chore(linux): Update debian changelog (#14783) + +## 19.0.122 alpha 2025-09-18 + +* fix(android): Refresh the banner theme when system keyboard is changed (#14768) +* maint(common): update HISTORY.md for 18.0.241 (#14780) + +## 19.0.121 alpha 2025-09-17 + +* docs(developer): document `fileVersion` (#14766) +* maint(linux): allow to force package build on non-Keyman repo (#14749) +* feat(linux): add breadcrumbs and system information to Sentry error reports (#14765) +* maint(resources): fix broken test build trigger for `master` branch (#14770) +* chore(linux): remove automatic installation of `onboard-keyman` (#14772) + +## 19.0.120 alpha 2025-09-16 + +* refactor(linux): remove warning (#14761) +* fix(linux): don't crash on invalid languages in keyboard (#14760) + ## 19.0.119 alpha 2025-09-15 * fix(developer): transform slashes to backslashes in paths in Package Editor (#14721) @@ -757,6 +779,22 @@ * refactor(windows): rename `TKeymanMutex.MutexOwned` to `TakeOwnership` and add `ReleaseOwnership` (#13168) * chore: increment to alpha 19.0 (#13187) +## 18.0.241 stable 2025-09-18 + +* maint(resources): use correct version for PR title for history cherry-pick (#14614) +* chore(linux): Update debian changelog (#14589) +* fix(android): Update: Android Target API to 35, Java to 21, and Gradle dependencies (#14584) +* fix(android): Convert Keyman app layouts for edge to edge (#14597) +* fix(android): Fix insets for in-app vs system keyboard (#14642) +* maint(linux): create temporary worktree for packaging (#14599) +* fix(android): Cleanup layouts for Engine, Sample apps, and FirstVoices apps for edge to edge (#14663) +* fix(web): fix error trying to calculate style scaling effects (#14696) +* fix(linux): don't crash if `kmp.json` is missing `keyboards` section (#14714) +* fix(developer): transform slashes to backslashes in paths in Package Editor (#14722) +* chore(linux): remove automatic installation of `onboard-keyman` (#14773) +* docs(linux): update release notes for stable release (#14774) +* fix(linux): don't crash on invalid languages in keyboard (#14771) + ## 18.0.240 stable 2025-08-27 * maint(resources): automatically cherry-pick history updates to master (#14596) diff --git a/VERSION.md b/VERSION.md index ce3bffa569..55bb75ca00 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -19.0.120 \ No newline at end of file +19.0.124 \ No newline at end of file diff --git a/android/KMAPro/kMAPro/src/main/java/com/keyman/android/SystemKeyboard.java b/android/KMAPro/kMAPro/src/main/java/com/keyman/android/SystemKeyboard.java index f6acd350da..2c19e1dde7 100644 --- a/android/KMAPro/kMAPro/src/main/java/com/keyman/android/SystemKeyboard.java +++ b/android/KMAPro/kMAPro/src/main/java/com/keyman/android/SystemKeyboard.java @@ -263,6 +263,8 @@ public class SystemKeyboard extends InputMethodService implements OnKeyboardEven @Override public void onKeyboardChanged(String newKeyboard) { + // Refresh banner theme + BannerController.setHTMLBanner(this, KeyboardType.KEYBOARD_TYPE_SYSTEM); KMManager.showSystemKeyboard(); } diff --git a/developer/docs/help/reference/file-types/metadata.md b/developer/docs/help/reference/file-types/metadata.md index 59d30375dc..8630872cf8 100644 --- a/developer/docs/help/reference/file-types/metadata.md +++ b/developer/docs/help/reference/file-types/metadata.md @@ -88,10 +88,15 @@ The `System` object is used by Keyman Desktop to install keyboards The version of Keyman Developer used to create the package file. If undefined, use `'0.0.0.0'` -`fileVersion,` +`fileVersion` : `string` + The minimum version of Keyman required to load the package metadata. Note that + the files within the package may have other version requirements. The lowest + version here will be `MIN_KBD_FILEVERSION_KMP_JSON` (7.0) for keyboard packages, + and `MIN_LM_FILEVERSION_KMP_JSON` (12.0) for lexical model packages. + ### The Options object The `Options` object is used by Keyman Desktop to install keyboards diff --git a/linux/debian/changelog b/linux/debian/changelog index b87a0a9dc9..92691ada9a 100644 --- a/linux/debian/changelog +++ b/linux/debian/changelog @@ -1,3 +1,10 @@ +keyman (18.0.241-1) unstable; urgency=medium + + * remove onboard-keyman recommends (closes: 1115423) + * New upstream release. + + -- Eberhard Beilharz Thu, 18 Sep 2025 11:35:01 +0200 + keyman (18.0.240-1) unstable; urgency=medium * New upstream release. diff --git a/linux/debian/control b/linux/debian/control index e4a0c0437a..87367f9f31 100644 --- a/linux/debian/control +++ b/linux/debian/control @@ -58,8 +58,6 @@ Depends: ibus-keyman (>= ${binary:Version}), ibus-keyman (<< ${binary:Version}.1~), ${misc:Depends}, -Recommends: - onboard-keyman, Description: Type in your language with Keyman for Linux Keyman makes it possible for you to type in over 2,000 languages on Windows, macOS, Linux, iPhone, iPad, Android tablets and phones, and even instantly diff --git a/linux/ibus-keyman/src/test/meson.build b/linux/ibus-keyman/src/test/meson.build index f1f34a63c5..d377756e38 100644 --- a/linux/ibus-keyman/src/test/meson.build +++ b/linux/ibus-keyman/src/test/meson.build @@ -9,7 +9,7 @@ test_env = [ 'G_TEST_SRCDIR=' + meson.current_source_dir(), 'G_TEST_BUILDDIR=' + meson.current_build_dir(), 'TOP_SRCDIR=' + meson.global_source_root(), - 'TOP_BINDIR=' + meson.build_root(), + 'TOP_BINDIR=' + meson.global_build_root(), ] test_include_dirs = [ diff --git a/linux/ibus-keyman/tests/scripts/run-single-test.sh b/linux/ibus-keyman/tests/scripts/run-single-test.sh index 39075421fc..b2ee42cc40 100755 --- a/linux/ibus-keyman/tests/scripts/run-single-test.sh +++ b/linux/ibus-keyman/tests/scripts/run-single-test.sh @@ -1,20 +1,21 @@ #!/usr/bin/env bash set -eu -TESTDIR=${XDG_DATA_HOME:-$HOME/.local/share}/keyman/test_kmx +TESTDIR=${XDG_DATA_HOME:-${HOME}/.local/share}/keyman/test_kmx +# shellcheck source=linux/ibus-keyman/tests/scripts/test-helper.inc.sh . "$(dirname "$0")"/test-helper.inc.sh -if [ -v KEYMAN_PKG_BUILD ]; then +if [[ -v KEYMAN_PKG_BUILD ]]; then # During package builds we skip these tests that require to start ibus because # ibus requires to find /var/lib/dbus/machine-id or /etc/machine-id, otherwise it fails with: # "Bail out! IBUS-FATAL-WARNING: Unable to load /var/lib/dbus/machine-id: Failed to open file - # “/var/lib/dbus/machine-id”: No such file or directory" + # "/var/lib/dbus/machine-id": No such file or directory" echo "TAP version 14" echo "1..0 # SKIP on package build" exit 0 fi -if ! which Xvfb > /dev/null || ! which Xephyr > /dev/null || ! which metacity > /dev/null || ! which mutter > /dev/null; then +if ! command -v Xvfb > /dev/null || ! command -v Xephyr > /dev/null || ! command -v metacity > /dev/null || ! command -v mutter > /dev/null; then echo "Please install Xvfb, Xephyr, metacity and mutter before running these tests!" echo "sudo apt install xvfb xserver-xephyr metacity mutter" exit 1 @@ -53,7 +54,7 @@ function run_tests() { # shellcheck disable=SC2086 "${G_TEST_BUILDDIR:-.}"/ibus-keyman-tests ${ARG_K-} ${ARG_TAP-} \ ${ARG_VERBOSE-} ${ARG_DEBUG-} ${ARG_SURROUNDING_TEXT-} ${ARG_NO_SURROUNDING_TEXT-} \ - --directory "$TESTDIR" ${ARG_DISPLAY_SERVER} "$TESTFILE" + --directory "${TESTDIR}" ${ARG_DISPLAY_SERVER} "${TESTFILE}" echo "# Finished tests." } @@ -79,14 +80,14 @@ while (( $# )); do done # shellcheck disable=SC2236 -if [ -n "${ARG_PIDS:-}" ] && [ ! -n "${ARG_CLEANUP:-}" ]; then +if [[ -n "${ARG_PIDS:-}" ]] && [[ ! -n "${ARG_CLEANUP:-}" ]]; then echo "Error: '--check' also requires '--cleanup'. Exiting." exit 6 fi -check_processes_running "$ARG_DISPLAY_SERVER" "$ARG_ENV" "$ARG_CLEANUP" "$ARG_PIDS" "$ARG_TESTNAME" >&2 +check_processes_running "${ARG_DISPLAY_SERVER}" "${ARG_ENV}" "${ARG_CLEANUP}" "${ARG_PIDS}" "${ARG_TESTNAME}" >&2 # shellcheck source=/dev/null -. "$ARG_ENV" +. "${ARG_ENV}" run_tests diff --git a/linux/keyman-config/keyman_config/__init__.py b/linux/keyman-config/keyman_config/__init__.py index 26fbeddf7d..88371a8be9 100644 --- a/linux/keyman-config/keyman_config/__init__.py +++ b/linux/keyman-config/keyman_config/__init__.py @@ -110,27 +110,30 @@ def _set_dbus_started_for_session(value): def verify_dbus_running(): - if not 'DBUS_SESSION_BUS_ADDRESS' in os.environ: - try: - # Seems dbus isn't running for the current user. Try to start it - # and set these environment variables - logging.info('Starting dbus with dbus-launch') - stdout = subprocess.run( - ('dbus-launch', '--exit-with-session'), - stdout=subprocess.PIPE, check=False).stdout - _set_dbus_started_for_session(True) - lines = stdout.decode('utf-8').splitlines() - for line in lines: - equal_sign = line.find('=') - if equal_sign <= 0: - logging.warning('Got unexpected line from dbus-launch: %s', line) - continue - name = line[:equal_sign] - value = line[equal_sign+1:] - logging.debug('Setting environment %s=%s', name, value) - os.environ[name] = value - except Exception as e: - logging.error('Starting dbus-launch failed with %s', e) + if 'DBUS_SESSION_BUS_ADDRESS' in os.environ: + # already running - nothing to do + return + + try: + # Seems dbus isn't running for the current user. Try to start it + # and set these environment variables + logging.info('Starting dbus with dbus-launch') + stdout = subprocess.run( + ('dbus-launch', '--exit-with-session'), + stdout=subprocess.PIPE, check=False).stdout + _set_dbus_started_for_session(True) + lines = stdout.decode('utf-8').splitlines() + for line in lines: + equal_sign = line.find('=') + if equal_sign <= 0: + logging.warning('Got unexpected line from dbus-launch: %s', line) + continue + name = line[:equal_sign] + value = line[equal_sign+1:] + logging.debug('Setting environment %s=%s', name, value) + os.environ[name] = value + except Exception as e: + logging.error('Starting dbus-launch failed with %s', e) def add_standard_arguments(parser): diff --git a/linux/keyman-config/keyman_config/downloadkeyboard.py b/linux/keyman-config/keyman_config/downloadkeyboard.py index ca3b091f92..7eac44b5ec 100755 --- a/linux/keyman-config/keyman_config/downloadkeyboard.py +++ b/linux/keyman-config/keyman_config/downloadkeyboard.py @@ -94,8 +94,7 @@ class DownloadKmpWindow(Gtk.Dialog): qs = urllib.parse.parse_qs(parsed.query) package_id = parsed.path.split('/')[-1] downloadfile = os.path.join(get_download_folder(), package_id) - download_url = KeymanComUrl + '/go/package/download/' + package_id + \ - '?platform=linux&tier=' + __tier__ + download_url = f'{KeymanComUrl}/go/package/download/{package_id}?platform=linux&tier={__tier__}' if 'bcp47' in qs: self.language = qs['bcp47'][0] download_url += '&bcp47=' + qs['bcp47'][0] diff --git a/linux/keyman-config/keyman_config/get_kmp.py b/linux/keyman-config/keyman_config/get_kmp.py index 3addda759b..104036c7fb 100755 --- a/linux/keyman-config/keyman_config/get_kmp.py +++ b/linux/keyman-config/keyman_config/get_kmp.py @@ -13,6 +13,7 @@ from gi.repository import GObject from keyman_config import _ from keyman_config import KeymanApiUrl, KeymanDownloadsUrl from keyman_config.deprecated_decorator import deprecated +from keyman_config.sentry_handling import SentryErrorHandling class InstallLocation(GObject.GEnum): @@ -105,7 +106,7 @@ def get_keyboard_data(keyboardID, weekCache=False): dict: Keyboard data """ logging.info('Getting data for keyboard %s', keyboardID) - api_url = KeymanApiUrl + '/keyboard/' + keyboardID + api_url = f'{KeymanApiUrl}/keyboard/{keyboardID}' logging.debug('At URL %s', api_url) cache_dir = keyman_cache_dir() current_dir = os.getcwd() @@ -123,10 +124,7 @@ def get_keyboard_data(keyboardID, weekCache=False): logging.debug('Time: {0} / Used Cache: {1}'.format(now, _did_use_cache(response))) os.chdir(current_dir) _uninstall_cache() - if response.status_code == 200: - return response.json() - else: - return None + return response.json() if response.status_code == 200 else None def get_download_folder(): @@ -234,6 +232,8 @@ def download_kmp_file(url, kmpfile, cache=False): """ logging.info('Download URL: %s', url) downloadfile = None + sentry = SentryErrorHandling() + sentry.add_breadcrumb(category='download', message=f'Downloading keyboard from {url}') if cache: cache_dir = keyman_cache_dir() diff --git a/linux/keyman-config/keyman_config/gnome_keyboards_util.py b/linux/keyman-config/keyman_config/gnome_keyboards_util.py index 8d9a6fae2f..030f4d9234 100644 --- a/linux/keyman-config/keyman_config/gnome_keyboards_util.py +++ b/linux/keyman-config/keyman_config/gnome_keyboards_util.py @@ -3,6 +3,7 @@ import logging import os from keyman_config.gsettings import GSettings +from keyman_config.sentry_handling import SentryErrorHandling # pylint: disable=global-statement @@ -52,13 +53,20 @@ def _reset_gnome_shell(): def get_ibus_keyboard_id(keyboard, packageDir, language=None, ignore_language=False): if not keyboard: return None + sentry = SentryErrorHandling() kmx_file = os.path.join(packageDir, keyboard['id'] + ".kmx") if ignore_language: + sentry.add_breadcrumb(category='keyboard', message=f'ibus_keyboard_id(1): {kmx_file}') return kmx_file if language is not None and language != '': logging.debug(language) + sentry.add_breadcrumb(category='keyboard', message=f'ibus_keyboard_id(2): {language}:{kmx_file}') return f"{language}:{kmx_file}" if "languages" in keyboard and len(keyboard["languages"]) > 0: - logging.debug(keyboard["languages"][0]) - return f"{keyboard['languages'][0]['id']}:{kmx_file}" + firstLanguage = keyboard["languages"][0] + logging.debug(firstLanguage) + if 'id' in firstLanguage: + sentry.add_breadcrumb(category='keyboard', message=f'ibus_keyboard_id(3): {firstLanguage["id"]}:{kmx_file}') + return f"{firstLanguage['id']}:{kmx_file}" + sentry.add_breadcrumb(category='keyboard', message=f'ibus_keyboard_id(4): {kmx_file}') return kmx_file diff --git a/linux/keyman-config/keyman_config/handle_install.py b/linux/keyman-config/keyman_config/handle_install.py index f551a63dd6..9933978a57 100644 --- a/linux/keyman-config/keyman_config/handle_install.py +++ b/linux/keyman-config/keyman_config/handle_install.py @@ -6,6 +6,7 @@ from zipfile import is_zipfile from keyman_config import KeymanComUrl, __tier__ from keyman_config.get_kmp import download_kmp_file, get_download_folder from keyman_config.install_window import InstallKmpWindow +from keyman_config.sentry_handling import SentryErrorHandling def download_and_install_package(url): @@ -18,6 +19,7 @@ def download_and_install_package(url): url: a .kmp file, a keyman:// URL, or a file:// URL pointing to a .kmp file, possibly with a bcp47= specified """ + sentry = SentryErrorHandling() parsedUrl = urlparse(url) bcp47 = _extract_bcp47(parsedUrl.query) @@ -34,11 +36,13 @@ def download_and_install_package(url): downloadFile = os.path.join(get_download_folder(), packageId) downloadUrl = f'{KeymanComUrl}/go/package/download/{packageId}?platform=linux&tier={__tier__}' + sentry.add_breadcrumb(category='download', message=f'About to download keyboard {packageId} from {downloadUrl}') packageFile = download_kmp_file(downloadUrl, downloadFile) if packageFile is None: return elif parsedUrl.scheme in ['', 'file']: packageFile = parsedUrl.path + sentry.add_breadcrumb(category='keyboard', message=f'adding keyboard from file {packageFile}') else: logging.error(f"Invalid URL: {url}") return diff --git a/linux/keyman-config/keyman_config/install_kmp.py b/linux/keyman-config/keyman_config/install_kmp.py index 5320a00d3a..4848416a5b 100755 --- a/linux/keyman-config/keyman_config/install_kmp.py +++ b/linux/keyman-config/keyman_config/install_kmp.py @@ -23,6 +23,7 @@ from keyman_config.gnome_keyboards_util import (GnomeKeyboardsUtil, from keyman_config.ibus_util import get_ibus_bus, install_to_ibus, restart_ibus from keyman_config.kmpmetadata import KMFileTypes, get_metadata from keyman_config.kvk2ldml import convert_kvk_to_ldml, output_ldml +from keyman_config.sentry_handling import SentryErrorHandling # TODO userdir install # special processing for kmn if needed @@ -105,6 +106,9 @@ class InstallKmp(): self.kmpdocdir = get_keyman_doc_dir(area, self.packageID) self.kmpfontdir = get_keyman_font_dir(area, self.packageID) + sentry = SentryErrorHandling() + sentry.add_breadcrumb(category='install', message=f'Installing kmp: {self.packageID} from "{inputfile}" for lang: {language}; area: {"OS" if area == 1 else "Shared" if area == 2 else "User" if area == 3 else "Unknown"}') + if not os.path.isfile(inputfile): message = _("File {kmpfile} doesn't exist").format(kmpfile=inputfile) logging.error("install_kmp.py: %s", message) @@ -258,6 +262,9 @@ class InstallKmp(): if not language: language = self._add_custom_keyboard(firstKeyboard, packageDir, requested_language) + sentry = SentryErrorHandling() + sentry.add_breadcrumb(category='install', message=f'Installing keyboards for lang: {language}') + if is_fcitx_running(): return self._install_keyboards_to_fcitx() diff --git a/linux/keyman-config/keyman_config/kmpmetadata.py b/linux/keyman-config/keyman_config/kmpmetadata.py index 82e90b138d..42fa9774a6 100755 --- a/linux/keyman-config/keyman_config/kmpmetadata.py +++ b/linux/keyman-config/keyman_config/kmpmetadata.py @@ -11,6 +11,7 @@ from json.decoder import JSONDecodeError import magic from keyman_config import secure_lookup +from keyman_config.sentry_handling import SentryErrorHandling class KMFileTypes(Enum): @@ -455,6 +456,7 @@ def parsemetadata(jsonfile, verbose=False): """ info = system = keyboards = files = options = nonexistent = None extracted_dir = os.path.dirname(jsonfile) + sentry = SentryErrorHandling() logging.debug("parsemetadata: reading file:%s dir:%s", jsonfile, extracted_dir) @@ -496,6 +498,11 @@ def parsemetadata(jsonfile, verbose=False): print_options(options) print_keyboards(keyboards) print_files(files, extracted_dir) + + if system: + sentry.add_breadcrumb( + category='keyboard', message=f'kmp.json: pkg: "{secure_lookup(info, "name", "description")}", KeymanDeveloperVersion: {secure_lookup(system, "keymanDeveloperVersion")}, dir: {os.path.basename(extracted_dir)}') + return info, system, options, keyboards, files @@ -532,19 +539,22 @@ def get_and_convert_infdata(tmpdirname): see kmpmetadata.parseinfdata for details """ kmpinf = os.path.join(tmpdirname, "kmp.inf") - if os.path.isfile(kmpinf): - info, system, options, keyboards, files = parseinfdata(kmpinf, False) - j = infmetadata_to_json(info, system, options, keyboards, files) - kmpjson = os.path.join(tmpdirname, "kmp.json") - try: - with open(kmpjson, "w") as write_file: - print(j, file=write_file) - except Exception as e: - logging.warning('Exception %s writing metadata %s %s', type(e), kmpjson, e.args) - return info, system, options, keyboards, files - else: + if not os.path.isfile(kmpinf): return None, None, None, None, None + sentry = SentryErrorHandling() + info, system, options, keyboards, files = parseinfdata(kmpinf, False) + j = infmetadata_to_json(info, system, options, keyboards, files) + kmpjson = os.path.join(tmpdirname, "kmp.json") + try: + with open(kmpjson, "w") as write_file: + print(j, file=write_file) + except Exception as e: + logging.warning('Exception %s writing metadata %s %s', type(e), kmpjson, e.args) + sentry.add_breadcrumb( + category='keyboard', message=f'kmp.inf: pkg: "{secure_lookup(info, "name", "description")}", KeymanDeveloperVersion: {secure_lookup(system, "keymanDeveloperVersion")}, dir: {os.path.basename(tmpdirname)}') + return info, system, options, keyboards, files + def infmetadata_to_json(info, system, options, keyboards, files): jsonfiles = [] diff --git a/linux/keyman-config/keyman_config/sentry_handling.py b/linux/keyman-config/keyman_config/sentry_handling.py index e7b3d40da3..b153066a1a 100644 --- a/linux/keyman-config/keyman_config/sentry_handling.py +++ b/linux/keyman-config/keyman_config/sentry_handling.py @@ -30,6 +30,9 @@ gi.require_version('Gtk', '3.0') from gi.repository import Gio, Gtk +is_sentry_sdk = False + + class SentryErrorHandling: def __init__(self) -> None: self.error_reporting_setting = KeymanOption('error-reporting') @@ -78,6 +81,15 @@ class SentryErrorHandling: if enabled != was_enabled: self._handle_enabled(enabled) + def add_breadcrumb(self, category, message, level='info'): + global is_sentry_sdk + if not self.is_sentry_enabled() or not is_sentry_sdk: + return + + # sentry_sdk = importlib.import_module('sentry_sdk') + import sentry_sdk + sentry_sdk.add_breadcrumb(category=category, message=message, level=level) + def _get_environ_nosentry(self): keyman_nosentry = os.environ.get('KEYMAN_NOSENTRY') return keyman_nosentry and (int(keyman_nosentry) == 1) @@ -114,47 +126,61 @@ class SentryErrorHandling: def _sentry_sdk_initialize(self): # Try new sentry-sdk first + global is_sentry_sdk sentry_sdk = importlib.import_module('sentry_sdk') from sentry_sdk import configure_scope, set_user from sentry_sdk.integrations.logging import LoggingIntegration + is_sentry_sdk = True + sentry_logging = LoggingIntegration( level=logging.INFO, # Capture info and above as breadcrumbs event_level=logging.CRITICAL # Send critical errors as events ) SentryUrl = "https://1d0edbf2d0dc411b87119b6e92e2c357@o1005580.ingest.sentry.io/5983525" + + os_release = None + dist = None + try: + os_release = platform.freedesktop_os_release() + dist = os_release['VERSION_CODENAME'] + except OSError as e: + logging.debug(f'System does not have os_release file: {e.strerror}') + except AttributeError: + logging.debug('System does not have platform.freedesktop_os_release() method') + except Exception: + logging.debug( + 'Got exception trying to access platform.freedesktop_os_release() method or os_release information') + sentry_sdk.init( dsn=SentryUrl, environment=__environment__, release=__versiongittag__, integrations=[sentry_logging], + dist=dist, + in_app_include=['keyman_config'], before_send=self._before_send ) - hash = hashlib.md5() - hash.update(getpass.getuser().encode()) - set_user({'id': hash.hexdigest()}) + md5Hash = hashlib.md5() + md5Hash.update(getpass.getuser().encode()) + set_user({'id': md5Hash.hexdigest()}) with configure_scope() as scope: - scope.set_tag("app", os.path.basename(sys.argv[0])) - scope.set_tag("pkgversion", __pkgversion__) - scope.set_tag("platform", platform.platform()) - scope.set_tag("system", platform.system()) - scope.set_tag("tier", __tier__) - scope.set_tag("device", platform.node()) - try: - os_release = platform.freedesktop_os_release() - scope.set_tag('os', os_release['PRETTY_NAME']) - scope.set_tag('os.name', os_release['NAME']) - if 'VERSION' in os_release: - scope.set_tag('os.version', os_release['VERSION']) - except OSError as e: - logging.debug(f'System does not have os_release file: {e.strerror}') - except AttributeError: - logging.debug('System does not have platform.freedesktop_os_release() method') - except: - logging.debug( - 'Got exception trying to access platform.freedesktop_os_release() method or os_release information') + self._set_system_info(scope, os_release) logging.info("Initialized Sentry error reporting") + def _set_system_info(self, scope, os_release): + scope.set_tag("app", os.path.basename(sys.argv[0])) + scope.set_tag("pkgversion", __pkgversion__) + scope.set_tag("platform", platform.platform()) + scope.set_tag("system", platform.system()) + scope.set_tag("tier", __tier__) + scope.set_tag("device", platform.node()) + if os_release: + scope.set_tag('os', os_release['PRETTY_NAME']) + scope.set_tag('os.name', os_release['NAME']) + if 'VERSION' in os_release: + scope.set_tag('os.version', os_release['VERSION']) + def _raven_initialize(self): # sentry-sdk is not available, so use older raven raven = importlib.import_module('raven') diff --git a/linux/keyman-config/km-config b/linux/keyman-config/km-config index 21072329a8..bfb2063c90 100755 --- a/linux/keyman-config/km-config +++ b/linux/keyman-config/km-config @@ -71,7 +71,7 @@ if __name__ == '__main__': download_and_install_package(args.url) else: # Workaround for bug in webkit2gtk (#12587) - if not 'WEBKIT_DISABLE_DMABUF_RENDERER' in os.environ: + if 'WEBKIT_DISABLE_DMABUF_RENDERER' not in os.environ: os.environ['WEBKIT_DISABLE_DMABUF_RENDERER'] = '1' w = ViewInstalledWindow() diff --git a/linux/keyman-config/tests/gnome_keyboards_util_tests.py b/linux/keyman-config/tests/gnome_keyboards_util_tests.py index f86443567c..fac51253a0 100644 --- a/linux/keyman-config/tests/gnome_keyboards_util_tests.py +++ b/linux/keyman-config/tests/gnome_keyboards_util_tests.py @@ -4,7 +4,7 @@ import unittest from unittest import mock from unittest.mock import patch -from keyman_config.gnome_keyboards_util import is_gnome_desktop, _reset_gnome_shell +from keyman_config.gnome_keyboards_util import get_ibus_keyboard_id, is_gnome_desktop, _reset_gnome_shell class GnomeKeyboardsUtilTests(unittest.TestCase): @@ -35,6 +35,37 @@ class GnomeKeyboardsUtilTests(unittest.TestCase): # Execute/Verify self.assertEqual(is_gnome_desktop(), True) + def test_GetIbusKeyboardId_NoKeyboard(self): + self.assertIsNone(get_ibus_keyboard_id(None, '/tmp')) + + def test_GetIbusKeyboardId_IgnoreLanguage(self): + keyboard = {'id': 'foo'} + self.assertEqual(get_ibus_keyboard_id(keyboard, '/tmp', ignore_language=True), '/tmp/foo.kmx') + + def test_GetIbusKeyboardId_NoLanguage(self): + keyboard = {'id': 'foo'} + self.assertEqual(get_ibus_keyboard_id(keyboard, '/tmp'), '/tmp/foo.kmx') + + def test_GetIbusKeyboardId_EmptyLanguage(self): + keyboard = {'id': 'foo'} + self.assertEqual(get_ibus_keyboard_id(keyboard, '/tmp', ''), '/tmp/foo.kmx') + + def test_GetIbusKeyboardId_Language(self): + keyboard = {'id': 'foo'} + self.assertEqual(get_ibus_keyboard_id(keyboard, '/tmp', 'en'), 'en:/tmp/foo.kmx') + + def test_GetIbusKeyboardId_LanguagesInKeyboard(self): + keyboard = {'id': 'foo', 'languages': [ {'id': 'fr'}]} + self.assertEqual(get_ibus_keyboard_id(keyboard, '/tmp'), 'fr:/tmp/foo.kmx') + + def test_GetIbusKeyboardId_MultipleLanguagesInKeyboard(self): + keyboard = {'id': 'foo', 'languages': [{'id': 'km'}, {'id': 'fr'}]} + self.assertEqual(get_ibus_keyboard_id(keyboard, '/tmp'), 'km:/tmp/foo.kmx') + + def test_GetIbusKeyboardId_InvalidLanguagesInKeyboard(self): # 14748 + keyboard = {'id': 'foo', 'languages': ['es']} + self.assertEqual(get_ibus_keyboard_id(keyboard, '/tmp'), '/tmp/foo.kmx') + if __name__ == '__main__': unittest.main() diff --git a/linux/scripts/upload-to-debian.sh b/linux/scripts/upload-to-debian.sh index be6e6e6394..fb91d66053 100755 --- a/linux/scripts/upload-to-debian.sh +++ b/linux/scripts/upload-to-debian.sh @@ -108,9 +108,9 @@ function push_to_github_and_create_pr() { if [[ -n ${PR_NUMBER} ]]; then builder_echo "PR #${PR_NUMBER} already exists" else - ${NOOP} gh pr create --draft --base "${BASE}" --title "${PR_TITLE}" --body "${PR_BODY}" - sleep 2s - PR_NUMBER=$(gh pr list --draft --search "${PR_TITLE}" --base "${BASE}" --json number --jq '.[].number') + local PR_URL + PR_URL=$(gh pr create --draft --base "${BASE}" --title "${PR_TITLE}" --body "${PR_BODY}") + PR_NUMBER="${PR_URL##*/}" fi else PR_NUMBER="" @@ -191,13 +191,15 @@ cp debianpackage/keyman-*/debian/changelog debian/ git add debian/changelog COMMIT_MESSAGE="chore(linux): Update debian changelog" git commit -m "${COMMIT_MESSAGE}" -push_to_github_and_create_pr chore/linux/changelog "${DEPLOY_BRANCH}" "${COMMIT_MESSAGE} 🏠" "Test-bot: skip" +push_to_github_and_create_pr chore/linux/changelog "${DEPLOY_BRANCH}" "${COMMIT_MESSAGE} 🏠" "Build-bot: skip +Test-bot: skip" # Create cherry-pick on master branch git checkout -B chore/linux/cherry-pick/changelog origin/master git cherry-pick -x chore/linux/changelog push_to_github_and_create_pr chore/linux/cherry-pick/changelog master "${COMMIT_MESSAGE} 🍒" \ "Cherry-pick-of: #${PR_NUMBER} +Build-bot: skip Test-bot: skip" builder_heading "Finishing" diff --git a/resources/docker-images/docker-build.inc.sh b/resources/docker-images/docker-build.inc.sh index b020559b45..6366e7b05d 100644 --- a/resources/docker-images/docker-build.inc.sh +++ b/resources/docker-images/docker-build.inc.sh @@ -83,10 +83,12 @@ docker_wrapper() { } setup_docker() { + DOCKER_RUN_ARGS=() if [[ "${MSYSTEM:-}" == "MINGW64" ]]; then - DOCKER_RUN_ARGS="--env DOCKER_RUN_AS_ROOT=1" - else - DOCKER_RUN_ARGS= + DOCKER_RUN_ARGS+=(--env DOCKER_RUN_AS_ROOT=1) + fi + if [[ -z ${DOCKER_RUNNING:-} ]] ; then + DOCKER_RUN_ARGS+=(-t) fi } diff --git a/resources/docker-images/run.sh b/resources/docker-images/run.sh index b06c0b2d5b..8c5b3bcff5 100755 --- a/resources/docker-images/run.sh +++ b/resources/docker-images/run.sh @@ -3,12 +3,14 @@ ## START STANDARD BUILD SCRIPT INCLUDE # adjust relative paths as necessary THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" +# shellcheck source=resources/build/builder-full.inc.sh . "${THIS_SCRIPT%/*}/../../resources/build/builder-full.inc.sh" ## END STANDARD BUILD SCRIPT INCLUDE +# shellcheck disable=SC2154 . "${KEYMAN_ROOT}/resources/build/minimum-versions.inc.sh" . "${KEYMAN_ROOT}/resources/docker-images/docker-build.inc.sh" -. "$KEYMAN_ROOT/resources/build/utils.inc.sh" +. "${KEYMAN_ROOT}/resources/build/utils.inc.sh" ################################ Main script ################################ @@ -42,21 +44,21 @@ else fi run_android() { - docker_wrapper run ${DOCKER_RUN_ARGS} -i --rm -v "${KEYMAN_ROOT}":/home/build/build \ + docker_wrapper run "${DOCKER_RUN_ARGS[@]}" -i --rm -v "${KEYMAN_ROOT}":/home/build/build \ -v "${KEYMAN_ROOT}/core/build/docker-core/${build_dir}":/home/build/build/core/build \ "${registry_slash}keymanapp/keyman-android-ci:${image_version}" \ "${builder_extra_params[@]}" } run_core() { - docker_wrapper run ${DOCKER_RUN_ARGS} -i --rm -v "${KEYMAN_ROOT}":/home/build/build \ + docker_wrapper run "${DOCKER_RUN_ARGS[@]}" -i --rm -v "${KEYMAN_ROOT}":/home/build/build \ -v "${KEYMAN_ROOT}/core/build/docker-core/${build_dir}":/home/build/build/core/build \ "${registry_slash}keymanapp/keyman-core-ci:${image_version}" \ "${builder_extra_params[@]}" } run_developer() { - docker_wrapper run ${DOCKER_RUN_ARGS} -i --rm -v "${KEYMAN_ROOT}":/home/build/build \ + docker_wrapper run "${DOCKER_RUN_ARGS[@]}" -i --rm -v "${KEYMAN_ROOT}":/home/build/build \ -v "${KEYMAN_ROOT}/core/build/docker-core/${build_dir}":/home/build/build/core/build \ "${registry_slash}keymanapp/keyman-developer-ci:${image_version}" \ "${builder_extra_params[@]}" @@ -65,8 +67,7 @@ run_developer() { run_linux() { mkdir -p "${KEYMAN_ROOT}/linux/build/docker-linux/${build_dir}" mkdir -p "${KEYMAN_ROOT}/linux/keyman-system-service/build/docker-linux/${build_dir}" - docker_wrapper run ${DOCKER_RUN_ARGS} -i --privileged --network host \ - -v "${KEYMAN_ROOT}":/home/build/build \ + docker_wrapper run "${DOCKER_RUN_ARGS[@]}" -i --privileged --rm -v "${KEYMAN_ROOT}":/home/build/build \ -v "${KEYMAN_ROOT}/core/build/docker-core/${build_dir}":/home/build/build/core/build \ -v "${KEYMAN_ROOT}/linux/build/docker-linux/${build_dir}":/home/build/build/linux/build \ -v "${KEYMAN_ROOT}/linux/keyman-system-service/build/docker-linux/${build_dir}":/home/build/build/linux/keyman-system-service/build \ @@ -76,7 +77,7 @@ run_linux() { } run_web() { - docker_wrapper run ${DOCKER_RUN_ARGS} -i --privileged --rm -v "${KEYMAN_ROOT}":/home/build/build \ + docker_wrapper run "${DOCKER_RUN_ARGS[@]}" -i --privileged --rm -v "${KEYMAN_ROOT}":/home/build/build \ -v "${KEYMAN_ROOT}/core/build/docker-core/${build_dir}":/home/build/build/core/build \ "${registry_slash}keymanapp/keyman-web-ci:${image_version}" \ "${builder_extra_params[@]}" diff --git a/resources/teamcity/triggers/trigger-test-builds.sh b/resources/teamcity/triggers/trigger-test-builds.sh index cbd317ec8a..f23439c93b 100755 --- a/resources/teamcity/triggers/trigger-test-builds.sh +++ b/resources/teamcity/triggers/trigger-test-builds.sh @@ -119,6 +119,39 @@ function postSkippedBuildsStatusResult() { fi } +# +# Find the platforms that have changes based on the watch_ variables in trigger-definitions.inc.sh +# +function find_platform_changes() { + builder_echo grey "# Find platforms that have changes" + declare -gA build_platforms + local platform watch + + # Scan the files found + while IFS= read -r line; do + # for each platform + for platform in "${available_platforms[@]}"; do + if [[ ! " ${!build_platforms[*]} " =~ " ${platform} " ]]; then + # Which platform are we watching? + eval watch="'$'watch_${platform}" + + # Add common patterns to the watch list + watch="^(${platform}|(oem/[^/]+/${platform})|resources/((?!teamcity|docker-images)|teamcity/(${platform}|includes)|docker-images/(${platform}|base))|${watch})" + # Since bash doesn't support negative look-aheads we use perl to test + # (grep has a --perl-regexp option, but not the version on macOS) + if perl -e 'exit($ARGV[0] =~ /$ARGV[1]/ ? 0 : 1)' "${line}" "${watch}"; then + # By default, we'll build a 'release' test build for touched platforms + build_platforms[${platform}]=${build_level_release} + fi + fi + done + done <<< "${prfiles}" + + builder_echo blue "Default build platforms: ${!build_platforms[*]}" +} + +KEYMAN_API_CHECK_SKIP="false" + # # Following is not an error; this script can run against master/beta/stable # branches as well as pull requests @@ -201,38 +234,6 @@ debug_echo "Files found: ${prfiles[*]}" popd > /dev/null -# -# Find the platforms that have changes based on the watch_ variables in trigger-definitions.inc.sh -# - -function find_platform_changes() { - builder_echo grey "# Find platforms that have changes" - declare -gA build_platforms - local platform watch - - # Scan the files found - while IFS= read -r line; do - # for each platform - for platform in "${available_platforms[@]}"; do - if [[ ! " ${!build_platforms[*]} " =~ " ${platform} " ]]; then - # Which platform are we watching? - eval watch="'$'watch_${platform}" - - # Add common patterns to the watch list - watch="^(${platform}|(oem/[^/]+/${platform})|resources/((?!teamcity|docker-images)|teamcity/(${platform}|includes)|docker-images/(${platform}|base))|${watch})" - # Since bash doesn't support negative look-aheads we use perl to test - # (grep has a --perl-regexp option, but not the version on macOS) - if perl -e 'exit($ARGV[0] =~ /$ARGV[1]/ ? 0 : 1)' "${line}" "${watch}"; then - # By default, we'll build a 'release' test build for touched platforms - build_platforms[${platform}]=${build_level_release} - fi - fi - done - done <<< "${prfiles}" - - builder_echo blue "Default build platforms: ${!build_platforms[*]}" -} - find_platform_changes # @@ -240,8 +241,6 @@ find_platform_changes # This will modify the build_platforms array # -KEYMAN_API_CHECK_SKIP="false" - if [[ "${prremote}" == "origin" ]]; then # We only accept Build-bot commands on trusted local origin PRs cd "${KEYMAN_ROOT}/resources/build/ci/github"