Merge branch 'feat/developer/kmc-convert' into feat/developer/kmc-convert-createSeperateUtil

HISTORY.md

@@ -1,5 +1,30 @@
 # Keyman Version History
 
+## 19.0.229 alpha 2026-05-08
+
+* maint(linux): fix source tarball used for packaging (#15926)
+
+## 19.0.228 alpha 2026-05-06
+
+* fix(windows): caps lock stores to work in compliant applications (#15771)
+* chore: move localization documentation to /docs (#15909)
+
+## 19.0.227 alpha 2026-05-05
+
+* maint(linux): make emscripten setup on build agents idempotent (#15908)
+
+## 19.0.226 alpha 2026-05-04
+
+* chore(common): cleanup optionals in ldml keyboard reader (#15865)
+
+## 19.0.225 alpha 2026-05-01
+
+* fix(windows): contextual help chm file has blank pages (#15724)
+
+## 19.0.224 alpha 2026-04-30
+
+* fix(linux): fix context after typing Bksp with Wayland (#15820)
+
 ## 19.0.223 alpha 2026-04-23
 
 * fix(android): Reorder display language list (#15863)

VERSION.md

@@ -1 +1 @@
-19.0.224
+19.0.230

developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts

@@ -100,31 +100,31 @@ export class LDMLKeyboardXMLSourceFileReader implements EventResolver {
     boxXmlArray(source?.keyboard3?.locales, 'locale');
     boxXmlArray(source?.keyboard3, 'transforms');
     if(source?.keyboard3?.layers) {
-      for(const layers of source?.keyboard3?.layers) {
+      for(const layers of source.keyboard3.layers) {
         boxXmlArray(layers, 'layer');
-        if(layers?.layer) {
-          for(const layer of layers?.layer) {
+        if(layers.layer) {
+          for(const layer of layers.layer) {
             boxXmlArray(layer, 'row');
           }
         }
       }
     }
     if(source?.keyboard3?.forms?.form) {
-      boxXmlArray(source?.keyboard3?.forms, 'form');
-      for(const form of source?.keyboard3?.forms?.form) {
+      boxXmlArray(source.keyboard3.forms, 'form');
+      for(const form of source.keyboard3.forms.form) {
         boxXmlArray(form, 'scanCodes');
       }
     }
     if(source?.keyboard3?.flicks) {
-      boxXmlArray(source?.keyboard3?.flicks, 'flick');
-      for(const flick of source?.keyboard3?.flicks?.flick) {
+      boxXmlArray(source.keyboard3.flicks, 'flick');
+      for(const flick of source.keyboard3.flicks.flick) {
         boxXmlArray(flick, 'flickSegment');
       }
     }
     if(source?.keyboard3?.variables) {
-      boxXmlArray(source?.keyboard3?.variables, 'set');
-      boxXmlArray(source?.keyboard3?.variables, 'string');
-      boxXmlArray(source?.keyboard3?.variables, 'uset');
+      boxXmlArray(source.keyboard3.variables, 'set');
+      boxXmlArray(source.keyboard3.variables, 'string');
+      boxXmlArray(source.keyboard3.variables, 'uset');
     }
     if(source?.keyboard3?.transforms) {
       for(const transforms of source.keyboard3.transforms)  {

docs/localization/README.md

@@ -2,9 +2,9 @@
 
 Localization for Keyman is maintained at https://translate.keyman.com
 
-Downloading and updating files between Keyman and Crowdin happens 
+Downloading and updating files between Keyman and Crowdin happens
 on GitHub by way of the Crowdin git integration. The configuration file for all platforms
-is a YAML file named [crowdin.yml](../../../crowdin.yml). Currently, the git integration tracks the `master` branch.
+is a YAML file named [crowdin.yml](../../crowdin.yml). Currently, the git integration tracks the `master` branch.
 
 A daily GitHub action to update the source files in Crowdin, which are organized in a top-level folder "master".
 
@@ -60,7 +60,7 @@ project.
 https://crowdin.com/project/keyman/settings#languages
 
 Only the project owner (sil_ltops) can edit custom language codes (at the bottom of the "Target Languages" list).
-For some of these, the language mapping needs to be maintained in the "Language Mapping" button on the project settings menu (instead of the [crowdin.yml](../../../crowdin.yml)) configuration file.
+For some of these, the language mapping needs to be maintained in the "Language Mapping" button on the project settings menu (instead of the [crowdin.yml](../../crowdin.yml)) configuration file.
 
 Language mappings have been set up for "Greek (Polytonic)" and "Shuwa (Latin)" so downloading with the Crowdin CLI would be:
 ```

linux/docs/internal/gtk-input-methods.md

@@ -18,25 +18,41 @@ support.
 
 ```mermaid
 flowchart LR
-  subgraph GTK
-    app(application)
-    im-module["im-ibus
-    (IBusIMContext)"]
-    app --- im-module
+  subgraph App["Application"]
+    style App fill:#9999ff
+    subgraph ClientApp["Client App"]
+      N1("GTK 3/4 Application")
+    end
+    subgraph GtkIMContext["Gtk.IMContext"]
+      N2["GTK 3:<br/>in /usr/lib/x86_64-linux-gnu/<br/>gtk-3.0/3.0.0/immodules:<br>im-ibus.so<br/>im-wayland.so<br/>im-xim.so"]
+      N3["GTK 4:<br/>in /usr/lib/x86_64-linux-gnu/<br/>gtk-4.0/4.0.0/immodules:<br>libim-ibus.so<br>(wayland built-in)"]
+      subgraph S2["&nbsp;"]
+        IBusIMContext
+        N4["Source for IBus clients:<br>https://github.com/ibus/<br>ibus/tree/main/client"]
+      end
+    end
   end
 
-  subgraph ibus[ibus-daemon]
-    direction LR
-    IBusInputContext <--> IBusEngine
+  subgraph IBus["IBus Daemon"]
+    style IBus fill:#ff4dff
+    IBusInputContext["IBusInputContext"]
+    IBusEngine["IBusEngine"]
+    N5["Source for IBus Daemon in https://github.com/ibus/ibus/tree/main/src"]
   end
 
-  subgraph keyman
-    ibus-engine-keyman
-  end
+  KeymanEngine["Keyman Engine"]
 
-  GTK <--> ibus <--> keyman
+  ClientApp ~~~ GtkIMContext
+  IBusIMContext <-- IBus--> IBusInputContext
+  IBusInputContext <--> IBusEngine
+  IBusEngine <-- IBus --> KeymanEngine
 
-  style app color:#FFFFFF, stroke:#00C853, fill:#00C853
+style N1 color:#FFFFFF, stroke:#00C853, fill:#00C853
+style N2 fill:none
+style N3 fill:none
+style N4 fill:none
+style N5 fill:none
+style S2 fill:none, stroke-width:0
 ```
 
 The im-module implements a [GIOExtensionPoint](https://docs.gtk.org/gio/struct.IOExtensionPoint.html)
@@ -129,4 +145,4 @@ After installing a new module the database needs to be updated with
 
 ## Links
 
-- [GTK4 Input Handling](https://docs.gtk.org/gtk4/input-handling.html)
+- [GTK4 Input Handling](https://docs.gtk.org/gtk4/input-handling.html)

linux/docs/internal/ibus-sequence-diagram.md

@@ -0,0 +1,48 @@
+# Communication between Application, IBus, and Keyman
+
+The sequence diagram below shows the sequence of events that happen in
+response to IBus sending the "Enable" signal to the Keyman Engine.
+This illustrates why `client_supports_surrounding_text()` might return
+a wrong value initially.
+
+```mermaid
+sequenceDiagram
+    autonumber
+    box rgb(153, 153, 255) Application
+    participant A as Client
+    participant IMC as IBusIMContext
+    end
+    box rgb(255, 77, 255) IBus
+    participant InC as IBusInputContext
+    participant E as IBusEngine
+    end
+    participant K as Keyman Engine
+
+    E->>+K: enable signal
+    K->>+E: IBusEngine.get_surrounding_text(NULL, NULL, NULL)
+    Note left of K: Keyman wants to use surrounding text
+    E->>+InC: RequireSurroundingText
+    InC->>InC: set needs_surrounding_text flag = TRUE
+    InC->>+IMC: require-surrounding-text signal
+    IMC->>+A: retrieve_surrounding signal
+    A->>+IMC: gtk_im_context_set_surrounding()
+    IMC->>+InC: ibus_input_context_set_surrounding_text()
+    InC->>+E: SetSurroundingText
+    E->>+K: set_surrounding_text signal
+    Note right of K: client_supports_surrounding_text() might<br>report a wrong value<br>because the processing of<br>get_surrounding_text() hasn't finished yet
+    K->>-E: return from<br>set_surrounding_text signal
+    E->>-InC: return from<br>SetSurroundingText
+    InC->>-IMC: return from<br>ibus_input_context_set_surrounding_text()
+    IMC->>-A: return from<br>gtk_im_context_set_surrounding()
+    A->>-IMC: return from<br>retrieve_surrounding signal
+    IMC->>-InC: return from<br>require-surrounding-text signal
+    InC->>-E: return from<br>RequireSurroundingText
+    E->>-K: return from<br>get_surrounding_text()
+    K->>-E: return from<br>enable signal
+```
+
+## Links
+
+- <https://docs.gtk.org/gtk3/class.IMContext.html>
+- <https://ibus.github.io/docs/ibus-1.5/IBusInputContext.html>
+- <https://ibus.github.io/docs/ibus-1.5/IBusEngine.html>

linux/ibus-keyman/src/engine.c

@@ -83,10 +83,13 @@ struct _IBusKeymanEngine {
   gboolean         rctrl_pressed;
   gboolean         lalt_pressed;
   gboolean         ralt_pressed;
-  IBusLookupTable *table;
   IBusProperty    *status_prop;
   IBusPropList    *prop_list;
   void            *settings;
+  // Flag indicating if there are pending changes (forwarded keystrokes,
+  // deleted surrounding text) that need to be applied by committing an
+  //empty text to the input engine
+  gboolean         is_dirty;
 
   commit_queue_item commit_queue[MAX_QUEUE_SIZE];
   commit_queue_item *commit_item;
@@ -246,45 +249,66 @@ debug_utf8_with_codepoints(const gchar *utf8) {
 
 static gboolean
 client_supports_surrounding_text(IBusEngine *engine) {
+  // This is not always reliable: IBus detects whether the client supports
+  // surrounding text by emitting the retrieve-surrounding signal. As part
+  // of that signal handler, the client is expected to call
+  // gtk_im_context_set_surrounding_with_selection which ends calling
+  // ibus_keyman_engine_set_surrounding_text at a time when IBus is still
+  // in the middle of determining whether the client supports surrounding text.
   g_assert(engine != NULL);
+  if (!engine->enabled) {
+    g_warning("%s: engine is not enabled, so return value is likley incorrect.", __FUNCTION__);
+  }
   return (engine->client_capabilities & IBUS_CAP_SURROUNDING_TEXT) != 0;
 }
 
 static void
-set_context_if_needed(IBusEngine *engine) {
-  IBusKeymanEngine *keyman = (IBusKeymanEngine *)engine;
+set_context_impl(
+  IBusEngine *engine,
+  IBusText    *text,
+  guint       cursor_pos,
+  guint       anchor_pos
+) {
+  IBusKeymanEngine* keyman                   = (IBusKeymanEngine*)engine;
+  g_autofree gchar* application_context_utf8 = NULL;
+  guint context_start, context_end;
+  context_end              = anchor_pos < cursor_pos ? anchor_pos : cursor_pos;
+  context_start            = context_end > MAXCONTEXT_ITEMS ? context_end - MAXCONTEXT_ITEMS : 0;
+  application_context_utf8 = g_utf8_substring(ibus_text_get_text(text), context_start, context_end);
+  g_message(
+      "%s: new application context: |%s| (len:%u) cursor:%d anchor:%d", __FUNCTION__, application_context_utf8,
+      context_end - context_start, cursor_pos, anchor_pos);
 
+  km_core_cu* application_context_utf16 = g_utf8_to_utf16(application_context_utf8, -1, NULL, NULL, NULL);
+  km_core_context_status result;
+  result = km_core_state_context_set_if_needed(keyman->state, application_context_utf16);
+  g_free(application_context_utf16);
+
+  g_message(
+      "%s: context %s", __FUNCTION__,
+      result == KM_CORE_CONTEXT_STATUS_UNCHANGED ? "unchanged"
+      : result == KM_CORE_CONTEXT_STATUS_UPDATED ? "updated"
+      : result == KM_CORE_CONTEXT_STATUS_CLEARED ? "cleared"
+      : result == KM_CORE_CONTEXT_STATUS_ERROR   ? "error"
+                                                 : "invalid argument");
+}
+
+static void
+set_context_if_needed(IBusEngine *engine) {
   if (!client_supports_surrounding_text(engine)) {
     g_message("%s: not a compliant client app", __FUNCTION__);
     return;
   }
 
   IBusText *text;
-  g_autofree gchar *application_context_utf8 = NULL;
-  guint cursor_pos, anchor_pos, context_start, context_end;
+  guint cursor_pos, anchor_pos;
 
   g_autofree gchar *debug_context = NULL;
   g_message("%s: current core context   : %s", __FUNCTION__, debug_context = get_context_debug(engine));
 
   ibus_engine_get_surrounding_text(engine, &text, &cursor_pos, &anchor_pos);
 
-  context_end         = anchor_pos < cursor_pos ? anchor_pos : cursor_pos;
-  context_start       = context_end > MAXCONTEXT_ITEMS ? context_end - MAXCONTEXT_ITEMS : 0;
-  application_context_utf8 = g_utf8_substring(ibus_text_get_text(text), context_start, context_end);
-  g_message("%s: new application context: |%s| (len:%u) cursor:%d anchor:%d", __FUNCTION__,
-    application_context_utf8, context_end - context_start, cursor_pos, anchor_pos);
-
-  km_core_cu *application_context_utf16 = g_utf8_to_utf16(application_context_utf8, -1, NULL, NULL, NULL);
-  km_core_context_status result;
-  result = km_core_state_context_set_if_needed(keyman->state, application_context_utf16);
-  g_free(application_context_utf16);
-
-  g_message("%s: context %s", __FUNCTION__,
-    result == KM_CORE_CONTEXT_STATUS_UNCHANGED ? "unchanged"
-    : result == KM_CORE_CONTEXT_STATUS_UPDATED ? "updated"
-    : result == KM_CORE_CONTEXT_STATUS_CLEARED ? "cleared"
-    : result == KM_CORE_CONTEXT_STATUS_ERROR   ? "error"
-                                               : "invalid argument");
+  set_context_impl(engine, text, cursor_pos, anchor_pos);
 }
 
 static void
@@ -461,6 +485,7 @@ ibus_keyman_engine_constructor(
     keyman->lctrl_pressed = FALSE;
     keyman->ralt_pressed = FALSE;
     keyman->rctrl_pressed = FALSE;
+    keyman->is_dirty = FALSE;
     initialize_queue_items(keyman, 0, MAX_QUEUE_SIZE);
     keyman->commit_item    = &keyman->commit_queue[0];
     gchar **split_name     = g_strsplit(engine_name, ":", 2);
@@ -557,8 +582,6 @@ ibus_keyman_engine_constructor(
 
     ping_keyman_system_service();
 
-    set_context_if_needed(engine);
-
     return (GObject *) keyman;
 }
 
@@ -629,10 +652,29 @@ static void commit_string(IBusKeymanEngine *keyman, const gchar *string)
     IBusText *text;
     g_autofree gchar *debug = NULL;
     g_message("DAR: %s - %s", __FUNCTION__, debug = debug_utf8_with_codepoints(string));
-    text = ibus_text_new_from_static_string (string);
+    text = ibus_text_new_from_string(string);
     g_object_ref_sink(text);
     ibus_engine_commit_text ((IBusEngine *)keyman, text);
     g_object_unref (text);
+    keyman->is_dirty = FALSE;
+}
+
+/**
+ * Commit an empty string to flush the surrounding text buffer.
+ *
+ * Wayland uses double-buffering for surrounding text and some other
+ * functionality, so we have to commit to apply the changes.
+ * See https://wayland.app/protocols/input-method-unstable-v2
+ */
+static void
+apply_changes(IBusKeymanEngine* keyman) {
+  IBusText* text;
+  g_message("%s - committing", __FUNCTION__);
+  text = ibus_text_new_from_static_string("");
+  g_object_ref_sink(text);
+  ibus_engine_commit_text((IBusEngine*)keyman, text);
+  g_object_unref(text);
+  keyman->is_dirty = FALSE;
 }
 
 //
@@ -643,7 +685,7 @@ is_core_options_end(km_core_option_item *option) {
 }
 
 static void
-process_output_action(IBusEngine *engine, const km_core_usv* output_utf32) {
+process_output_action(IBusEngine* engine, const km_core_usv* output_utf32) {
   if (output_utf32 == NULL || output_utf32[0] == '\0') {
     return;
   }
@@ -683,12 +725,13 @@ process_backspace_action(IBusEngine *engine, unsigned int code_points_to_delete)
     return;
   }
 
+  IBusKeymanEngine* keyman = (IBusKeymanEngine*)engine;
   if (client_supports_surrounding_text(engine)) {
     g_message("%s: compliant app: deleting surrounding text %d codepoints", __FUNCTION__, code_points_to_delete);
     ibus_engine_delete_surrounding_text(engine, -code_points_to_delete, code_points_to_delete);
+    keyman->is_dirty = TRUE;
   } else {
     g_message("%s: non-compliant app: queueing %d backspaces", __FUNCTION__, code_points_to_delete);
-    IBusKeymanEngine *keyman = (IBusKeymanEngine *)engine;
     keyman->commit_item->code_points_to_delete = code_points_to_delete;
   }
 }
@@ -761,6 +804,7 @@ commit_current_queue_item(IBusKeymanEngine *keyman) {
       ibus_engine_forward_key_event(engine, KEYMAN_BACKSPACE_KEYSYM, KEYMAN_BACKSPACE, 0);
       current_item->code_points_to_delete--;
     }
+    keyman->is_dirty = TRUE;
     // don't remove the item from the queue yet - we need to process it
     // again for the output and keystrokes. Instead emit the sentinel key
     // again.
@@ -780,6 +824,7 @@ commit_current_queue_item(IBusKeymanEngine *keyman) {
     g_message("%s: Forwarding key from commit queue: keyval=0x%02x, keycode=0x%02x, state=0x%02x",
       __FUNCTION__, current_item->keyval, current_item->keycode, current_item->state);
     ibus_engine_forward_key_event(engine, current_item->keyval, current_item->keycode, current_item->state);
+    keyman->is_dirty = TRUE;
   }
   keyman->commit_item--;
   memmove(keyman->commit_queue, &keyman->commit_queue[1], sizeof(commit_queue_item) * (MAX_QUEUE_SIZE - 1));
@@ -790,6 +835,10 @@ static void
 finish_process_actions(IBusEngine *engine) {
   g_assert(engine != NULL);
   IBusKeymanEngine *keyman = (IBusKeymanEngine *)engine;
+  if (keyman->is_dirty) {
+    apply_changes(keyman);
+  }
+
   if (client_supports_surrounding_text(engine)) {
     // compliant app
     return;
@@ -840,6 +889,8 @@ process_actions(
   IBusEngine *engine,
   km_core_actions const *actions
 ) {
+  IBusKeymanEngine* keyman = (IBusKeymanEngine*)engine;
+  keyman->is_dirty = FALSE;
   process_backspace_action(engine, actions->code_points_to_delete);
   process_output_action(engine, actions->output);
   process_persist_action(engine, actions->persist_options);
@@ -1032,8 +1083,12 @@ ibus_keyman_engine_set_surrounding_text(
   guint       cursor_pos,
   guint       anchor_pos
 ){
-    parent_class->set_surrounding_text(engine, text, cursor_pos, anchor_pos);
-    set_context_if_needed(engine);
+  g_message(
+      "%s: text=%s (len: %u), cursor_pos=%d, anchor_pos=%d", __FUNCTION__,
+      ibus_text_get_text(text), ibus_text_get_length(text),cursor_pos, anchor_pos);
+
+  parent_class->set_surrounding_text(engine, text, cursor_pos, anchor_pos);
+  set_context_impl(engine, text, cursor_pos, anchor_pos);
 }
 
 // static void ibus_keyman_engine_set_cursor_location (IBusEngine             *engine,
@@ -1062,7 +1117,11 @@ ibus_keyman_engine_focus_in (IBusEngine *engine)
     g_message("%s", __FUNCTION__);
     ibus_engine_register_properties (engine, keyman->prop_list);
 
-    set_context_if_needed(engine);
+    if (engine->enabled) {
+      // While the engine is not enabled ibus might not yet know if
+      // the client app supports surrounding text.
+      set_context_if_needed(engine);
+    }
     parent_class->focus_in (engine);
 }
 
@@ -1123,7 +1182,8 @@ ibus_keyman_engine_enable (IBusEngine *engine)
         km_service_set_ldmlfile (service, keyman->ldmlfile);
         km_service_set_name (service, keyman->kb_name);
     }
-    parent_class->enable (engine);
+    engine->enabled = TRUE;
+    parent_class->enable(engine);
 }
 
 /**
@@ -1148,6 +1208,8 @@ ibus_keyman_engine_disable (IBusEngine *engine)
     km_service_set_name (service, "None");
     // g_clear_object(&service);
 
+    engine->enabled = FALSE;
+
     parent_class->disable (engine);
 }
 

linux/scripts/dist.sh

@@ -70,6 +70,7 @@ to_exclude=(
   linux/docs/help \
   linux/keyman-config/keyman_config/version.py \
   linux/keyman-config/buildtools/build-langtags.py \
+  linux/upload \
 )
 
 if [[ -z "${create_origdist+x}" ]]; then

linux/scripts/package-build.inc.sh

@@ -43,7 +43,7 @@ function downloadSource() {
   cd ..
   mv "${proj}-${version}" "${BASEDIR}/${packageDir}"
   mv "${proj}_${version}.orig.tar.xz" "${BASEDIR}/${packageDir}"
-  mv "${proj}-${version}.tar.xz" "${BASEDIR}/${packageDir}"
+  mv "${proj}_${version}.pkg.tar.xz" "${BASEDIR}/${packageDir}"
   mv "${proj}"*.asc "${BASEDIR}/${packageDir}"
   rm "${proj}"*.debian.tar.xz
   cd "${BASEDIR}/${packageDir}" || exit

linux/scripts/watch.in

@@ -1,3 +1,3 @@
 version=4
 # Tier replaced by package-build.inc.sh script
-opts=pgpsigurlmangle=s/$/.asc/ https://downloads.keyman.com/linux/$tier/@ANY_VERSION@/@PACKAGE@@ANY_VERSION@@ARCHIVE_EXT@ debian uupdate
+opts=pgpsigurlmangle=s/$/.asc/ https://downloads.keyman.com/linux/$tier/@ANY_VERSION@/@PACKAGE@@ANY_VERSION@.pkg@ARCHIVE_EXT@ debian uupdate

resources/build/htm-link.lua

@@ -21,10 +21,8 @@ end
 
 -- Support for redirect meta key, adds a meta refresh to the resulting document
 function Meta(meta)
-  for k, v in pairs(meta) do
-    if k == 'redirect' and type(v) == 'table' and v.t == 'MetaInlines' then
-      redirect = v[1].c
-    end
+  if meta.redirect then
+    redirect = pandoc.utils.stringify(meta.redirect)
   end
   return meta
 end

resources/teamcity/includes/tc-linux.inc.sh

@@ -67,13 +67,25 @@ ba_linux_install_emscripten() {
     # shellcheck disable=SC2154
     . "${KEYMAN_ROOT}/resources/build/minimum-versions.inc.sh"
 
+    local EMSDK_DIR
+    if [[ ! -z "${EMSCRIPTEN_BASE:-}" ]] && [[ -d "${EMSCRIPTEN_BASE}" ]]; then
+      # EMSCRIPTEN_BASE points to emsdk/upstream/emscripten
+      EMSDK_DIR="$(realpath "${EMSCRIPTEN_BASE}/../..")"
+    else
+      EMSDK_DIR="${HOME}/emsdk"
+    fi
+
     builder_echo "Installing emscripten version ${KEYMAN_MIN_VERSION_EMSCRIPTEN}"
     export EMSDK_KEEP_DOWNLOADS=1
+    if [[ -d "${EMSDK_DIR}" ]]; then
+      builder_echo "emsdk directory already exists at ${EMSDK_DIR}, skipping clone"
+    else
+      # shellcheck disable=SC2164
+      cd "$(dirname "${EMSDK_DIR}")"
+      git clone https://github.com/emscripten-core/emsdk.git
+    fi
     # shellcheck disable=SC2164
-    cd "${HOME}"
-    git clone https://github.com/emscripten-core/emsdk.git
-    # shellcheck disable=SC2164
-    cd emsdk
+    cd "${EMSDK_DIR}"
     ./emsdk install "${KEYMAN_MIN_VERSION_EMSCRIPTEN}"
     ./emsdk activate "${KEYMAN_MIN_VERSION_EMSCRIPTEN}"
   fi

resources/teamcity/linux/keyman-linux-release.sh

@@ -57,10 +57,14 @@ function _cleanup_before_creating_source_package() {
 function _make_release_source_tarball() {
   builder_echo start "make source tarball" "Make source tarball"
   rm -rf dist
+  mkdir -p "upload/${KEYMAN_VERSION}"
   ./scripts/reconf.sh
   PKG_CONFIG_PATH="${KEYMAN_ROOT}/core/build/arch/release/meson-private" ./scripts/dist.sh
-  mkdir -p "upload/${KEYMAN_VERSION}"
-  cp -a dist/*.tar.xz "upload/${KEYMAN_VERSION}"
+  mv dist/*.tar.xz "upload/${KEYMAN_VERSION}/"
+  builder_echo heading "Make source for packaging"
+  PKG_CONFIG_PATH="${KEYMAN_ROOT}/core/build/arch/release/meson-private" ./scripts/dist.sh origdist
+  mv "dist/keyman_${KEYMAN_VERSION}.orig.tar.xz" "dist/keyman_${KEYMAN_VERSION}.pkg.tar.xz"
+  mv dist/*.tar.xz "upload/${KEYMAN_VERSION}/"
   (
     cd "upload/${KEYMAN_VERSION}"
     sha256sum ./*.tar.xz > SHA256SUMS
@@ -84,10 +88,9 @@ function _sign_source_tarball() {
 function _publish_to_downloads() {
   builder_echo start "publish to downloads" "Publish to downloads.keyman.com"
 
-  local UPLOAD_DIR KEYMAN_TXZ
+  local UPLOAD_DIR
 
   UPLOAD_DIR="upload/${KEYMAN_VERSION}"
-  KEYMAN_TXZ="keyman-${KEYMAN_VERSION}.tar.xz"
 
   # Set permissions as required on download site
   builder_echo "Setting upload file permissions for downloads.keyman.com"
@@ -96,7 +99,8 @@ function _publish_to_downloads() {
   chmod g+w  "${UPLOAD_DIR}"/*
   chmod a+r  "${UPLOAD_DIR}"/*
 
-  write_download_info "${UPLOAD_DIR}" "${KEYMAN_TXZ}" "Keyman for Linux" tar.xz linux
+  write_download_info "${UPLOAD_DIR}" "keyman-${KEYMAN_VERSION}.tar.xz" "Keyman for Linux source tarball" tar.xz linux
+  write_download_info "${UPLOAD_DIR}" "keyman_${KEYMAN_VERSION}.pkg.tar.xz" "Keyman for Linux source for packaging" tar.xz linux
   tc_rsync_upload "${UPLOAD_DIR}" "linux/${KEYMAN_TIER}"
 
   builder_echo end "publish to downloads" success "Publish to downloads.keyman.com"

windows/src/engine/keyman32/appint/aiTIP.cpp

@@ -114,6 +114,12 @@ void ProcessToggleChange(UINT key) {   // I4793
   }
 }
 
+void UpdateLastKeyCache(PKEYMAN64THREADDATA _td, WPARAM wParam, BYTE scan, BYTE keyTransition) {
+  _td->LastKey = wParam;
+  _td->LastScanCode = scan;
+  _td->LastTransition = keyTransition;
+}
+
 BOOL TIPProcessKeyInternal(
   PKEYMAN64THREADDATA _td,
   WPARAM wParam,
@@ -128,23 +134,34 @@ BOOL TIPProcessKeyInternal(
   BOOL isUp = keyFlags & KF_UP ? TRUE : FALSE;
   BOOL extended = keyFlags & KF_EXTENDED ? TRUE : FALSE;
   BYTE scan = keyFlags & 0xFF;
+  BYTE keyTransition =  KEYMSG_FLAG_TRANSITION(lParam);
+  WPARAM prevKey = _td->LastKey;
+  BYTE prevScanCode = _td->LastScanCode;
+  BYTE prevKeyTransition = _td->LastTransition;
 
+  UpdateLastKeyCache(_td, wParam, scan, keyTransition);
   SendDebugEntry();
   SendDebugMessageFormat("VirtualKey=%s lParam=%x   IsUp=%d Extended=%d Updateable=%d Preserved=%d",
     Debug_VirtualKey((WORD) wParam), lParam, isUp, extended, Updateable, Preserved);
 
-  if(_td->LastKey == wParam && scan == 0) {   // I4642
+  if (_td->LastKey == wParam && scan == 0) {  // I4642 - handle issue with Logos application.
     scan = _td->LastScanCode;
     SendDebugMessageFormat("Scan code was zero so using cached scan code %x", scan);
   }
 
-  if(scan == SCAN_FLAG_KEYMAN_KEY_EVENT) {   // I4370
+  // If this key event was generated by Keyman, then we should return FALSE without
+  // processing it in the core processor. If it is a CapsLock key we update our
+  // Globals::ShiftState. Since this could run before kmhook_getmessage processes the event,
+  // we also check for the scan code flag that we set when generating a key event from Keyman.
+  if ((prevKey == wParam && prevScanCode == SCAN_FLAG_KEYMAN_KEY_EVENT && prevKeyTransition == keyTransition)
+    || scan == SCAN_FLAG_KEYMAN_KEY_EVENT) {
     if (wParam == VK_CAPITAL && !isUp) {
       // Must also record toggle state change when Keyman has generated
       // a Caps Lock event
       ProcessToggleChange((UINT)wParam);   // I4793
     }
-    SendDebugMessageFormat("Virtual Key was generated by Keyman [Scan=0xFF]");
+    SendDebugMessageFormat("Virtual Key was generated by Keyman [Scan=%x wParam=%x] [prevKey=%x prevScanCode=%x prevKeyTransition=%x]",
+      scan, wParam, prevKey, prevScanCode, prevKeyTransition);
     return_SendDebugExit(FALSE);
   }
 

windows/src/engine/keyman32/appint/aiTIP.h

@@ -80,10 +80,27 @@ public:
 
 /**
  * ProcessToggleChange
- * Toggles the state of FLAGS in the Globals::ShiftState bit mask
+ * Sets or clears the state of FLAGS in the Globals::ShiftState bit mask
+ * Using the status of the key as determined by GetKeyState,
+ * ensuring it is consistent with the actual state of the key.
  * Supports VK_CAPITAL and VK_NUMLOCK
  * It DOES NOT generate a system event change for these flags
  * @param  key
  */
 void ProcessToggleChange(UINT key);
+
+// Forward declaration - final in globals.h
+struct tagKEYMAN64THREADDATA;
+typedef struct tagKEYMAN64THREADDATA *PKEYMAN64THREADDATA;
+
+/**
+ * Update the cache of the last key event received, this is set here - TIP processing, and by
+ * the GetMessage hook.
+ * @param  _td      Thread data to update
+ * @param  wParam   WPARAM of the key event
+ * @param  scan     Scan code of the key event
+ * @param  keyTransition   Transition state of the key event (00b = key down, 01b = repeat, 11b = keyup)
+ */
+void UpdateLastKeyCache(PKEYMAN64THREADDATA _td, WPARAM wParam, BYTE scan, BYTE keyTransition);
+
 #endif

windows/src/engine/keyman32/globals.h

@@ -239,6 +239,7 @@ typedef struct tagKEYMAN64THREADDATA
 
   WPARAM LastKey;   // I4642
   BYTE LastScanCode;   // I4642
+  BYTE LastTransition;
 
   /* Serialized key events */
 

windows/src/engine/keyman32/keyman64.h

@@ -118,8 +118,15 @@
 #define KEYMSG_FLAG_DLGMODE(lParam) (HIWORD(lParam) & KF_DLGMODE ? 1 : 0)
 #define KEYMSG_FLAG_MENUMODE(lParam) (HIWORD(lParam) & KF_MENUMODE ? 1 : 0)
 #define KEYMSG_FLAG_ALTDOWN(lParam) (HIWORD(lParam) & KF_ALTDOWN ? 1 : 0)
+// Repeat is actually previous KF_UP value of the key.
+// It is always set to 1 for WM_KEYUP and WM_SYSKEYUP messages.
+// It is set to 1 for WM_KEYDOWN and WM_SYSKEYDOWN keystroke messages generated by the automatic repeat feature.
+// see https://learn.microsoft.com/en-us/windows/win32/inputdev/about-keyboard-input#previous-key-state-flag
 #define KEYMSG_FLAG_REPEAT(lParam) (HIWORD(lParam) & KF_REPEAT ? 1 : 0)
 #define KEYMSG_FLAG_UP(lParam) (HIWORD(lParam) & KF_UP ? 1 : 0)
+// Combine the transition flags into a single value for higher chance of identification
+// 00b = key down, 01b = repeat, 11b = keyup
+#define KEYMSG_FLAG_TRANSITION(lParam) ((BYTE)((HIWORD(lParam) & (KF_UP | KF_REPEAT)) >> 14))
 
 // TODO: Deprecate overloading of scancodes and use dwExtraInfo instead
 #define SCAN_FLAG_KEYMAN_KEY_EVENT          0xFF

windows/src/engine/keyman32/kmhook_getmessage.cpp

@@ -156,9 +156,9 @@ LRESULT _kmnGetMessageProc(int nCode, WPARAM wParam, LPARAM lParam)
     }
 
     BYTE scan = KEYMSG_LPARAM_SCAN(mp->lParam);
+    BYTE keyTransitionEvent = KEYMSG_FLAG_TRANSITION(mp->lParam);
     CheckScheduledRefresh();
-    _td->LastScanCode = scan;
-    _td->LastKey = mp->wParam;
+    UpdateLastKeyCache(_td, mp->wParam, scan, keyTransitionEvent);
 
     switch (mp->wParam) {
     case VK_MENU:
@@ -174,6 +174,11 @@ LRESULT _kmnGetMessageProc(int nCode, WPARAM wParam, LPARAM lParam)
     if (mp->wParam != VK_BACK) {
       if(scan == SCAN_FLAG_KEYMAN_KEY_EVENT) {
         mp->lParam = (mp->lParam & 0xFF00FFFFL) | (MapVirtualKey((UINT)mp->wParam, 0) << 16);
+        if (mp->wParam == VK_CAPITAL) {
+          ProcessToggleChange(VK_CAPITAL);
+        }
+        SendDebugMessageFormat("WMKEY=%x Clear `SCAN_FLAG_KEYMAN_KEY_EVENT`  wParam=%x lParam=%x Set LastKey=%x LastScanCode=%x LastTransition=%x",
+            mp->message, mp->wParam, mp->lParam, _td->LastKey, _td->LastScanCode, _td->LastTransition);
       }
     }
   }

windows/src/engine/keyman32/kmprocess.cpp

@@ -131,7 +131,7 @@ BOOL ProcessHook()
         _td->lpActiveKeyboard->lpCoreKeyboardState,
         KM_CORE_DEBUG_CONTEXT_CACHED
       );
-      SendDebugMessageFormatW(L"Key %s: %hs Context '%s'",
+      SendDebugMessageFormatW(L"Key %s: %hs Core Cached Context '%s'",
         _td->state.isDown ? L"pressed" : L"released",
         Debug_VirtualKey(_td->state.vkey), debug_context);
       km_core_cu_dispose(debug_context);

windows/src/engine/keyman32/kmprocessactions.cpp

@@ -77,38 +77,29 @@ processPersistOpt(km_core_actions const* actions, LPINTKEYBOARDINFO activeKeyboa
   }
 }
 
-static void processCapsLock(const km_core_caps_state caps_lock_state, BOOL isUp, BOOL Updateable, BOOL externalEvent) {
+static void processCapsLock(const km_core_caps_state caps_state_change, BOOL isUp, BOOL Updateable, BOOL externalEvent) {
+  BOOL isCapsOn            = IsCapsLockOn();
 
-  // We only want to process the Caps Lock key event once --
-  // in the first pass (!Updateable).
-  if (Updateable){
+  // This debug message is useful for understanding the sequence of events around caps lock changes
+  //SendDebugMessageFormat("ACTION CAPS STATE:%d FIsUp=%d Updateable=%d ExternalEvent=%d CapsState=%d", caps_state_change, isUp, Updateable,
+  //    externalEvent, isCapsOn);
+
+  // We only want to process the Caps Lock key event once;
+  // it has to be when updateble=1 as TSF does not consistently
+  // have updateable=0 events.
+  if (!Updateable || caps_state_change == KM_CORE_CAPS_UNCHANGED) {
     return;
   }
+  // Turn three state value into a boolean for whether capslock should be on or off,
+  // we only want to process the key event if the state is changing.
+  BOOL required_caps_state = (caps_state_change == KM_CORE_CAPS_ON);
 
-  if (caps_lock_state == KM_CORE_CAPS_ON) {
-    // This case would occur for the keyboard system store setting `store(&CapsOnOnly) '1'`
-    if (isUp && !IsCapsLockOn()) {  // I267 - 24/11/2006 invert GetKeyState test
-      SendDebugMessageFormat("TURN CAPS ON: FIsUp=%d CapsState=%d", isUp, IsCapsLockOn());
-      keybd_event(VK_CAPITAL, SCAN_FLAG_KEYMAN_KEY_EVENT, 0, 0);
-      keybd_event(VK_CAPITAL, SCAN_FLAG_KEYMAN_KEY_EVENT, KEYEVENTF_KEYUP, 0);
-    }
-
-    // This case would occur for the keyboard system store setting `store(&CapsAlwaysOff) '1'`
-    // A trick is being played here of synthesising a release the CAPSLOCK key event
-    // then a depress CAPSLOCK key event
-    else if (!isUp && IsCapsLockOn()) {  // I267 - 24/11/2006 invert GetKeyState test
-      SendDebugMessageFormat("TURN CAPS OFF: FIsUp=%d CapsState=%d", isUp, IsCapsLockOn());
-      keybd_event(VK_CAPITAL, SCAN_FLAG_KEYMAN_KEY_EVENT, KEYEVENTF_KEYUP, 0);
-      keybd_event(VK_CAPITAL, SCAN_FLAG_KEYMAN_KEY_EVENT, 0, 0);
-    }
-  } else if (caps_lock_state == KM_CORE_CAPS_OFF) {
-    // This case would occur for the keyboard system store setting `store(&ShiftFreesCaps) '1'`
-    // OR selecting a keyboard with CAPs always off rule
-    if ((!isUp && IsCapsLockOn()) || (externalEvent && IsCapsLockOn())) {
-      SendDebugMessageFormat("TURN CAPS OFF: FIsUp=%d CapsState=%d", isUp, IsCapsLockOn());
-      keybd_event(VK_CAPITAL, SCAN_FLAG_KEYMAN_KEY_EVENT, 0, 0);
-      keybd_event(VK_CAPITAL, SCAN_FLAG_KEYMAN_KEY_EVENT, KEYEVENTF_KEYUP, 0);
-    }
+  if (isCapsOn != required_caps_state) {
+    SendDebugMessageFormat(
+      "Simulate CapsLock %s: FIsUp=%d CurrentCapsState=%d ExternalEvent=%d",
+      required_caps_state ? "ON" : "OFF", isUp, isCapsOn, externalEvent);
+    keybd_event(VK_CAPITAL, SCAN_FLAG_KEYMAN_KEY_EVENT, 0, 0);
+    keybd_event(VK_CAPITAL, SCAN_FLAG_KEYMAN_KEY_EVENT, KEYEVENTF_KEYUP, 0);
   }
 }
 
@@ -151,7 +142,6 @@ ProcessActionsNonUpdatableParse(BOOL* emitKeystroke) {
 
   km_core_actions const* core_actions = km_core_state_get_actions(_td->lpActiveKeyboard->lpCoreKeyboardState);
 
-  processCapsLock(core_actions->new_caps_lock_state, !_td->state.isDown, _td->TIPFUpdateable, FALSE);
   if (core_actions->emit_keystroke) {
     *emitKeystroke = TRUE;
     SendDebugMessageFormat("EMIT_KEYSTROKE");
@@ -167,6 +157,6 @@ ProcessActionsExternalEvent() {
     return FALSE;
   }
   km_core_actions const* core_actions = km_core_state_get_actions(_td->lpActiveKeyboard->lpCoreKeyboardState);
-  processCapsLock(core_actions->new_caps_lock_state, !_td->state.isDown, FALSE, TRUE);
+  processCapsLock(core_actions->new_caps_lock_state, !_td->state.isDown, TRUE, TRUE);
   return TRUE;
 }

windows/src/engine/kmtip/keys.cpp

@@ -75,7 +75,8 @@ BOOL CKMTipTextService::_InitKeystrokeSink()
 
   pKeystrokeMgr->Release();
 
-	memset(fEatenBuf, 0, sizeof(fEatenBuf));
+	memset(fEatenBuf, 0, sizeof(fEatenBuf));          // OnKeyDown/Up
+  memset(fOnTestEatenBuf, 0, sizeof(fOnTestEatenBuf));  // OnTestKeyDown/Up
 
   return_SendDebugExit(_keystrokeSinkInitialized = (hr == S_OK));
 }
@@ -178,17 +179,8 @@ STDAPI CKMTipTextService::OnTestKeyDown(ITfContext *pContext, WPARAM wParam, LPA
 {
   SendDebugEntry();
   LogKey(0, wParam, lParam);
-  // If the keystroke is a Keyman-generated key, ignore it
-  // But we need to pass Caps Lock through, even if we generated it, so we can track Caps Lock state.
-  // TODO: Fix magic constants
-  if ((lParam & 0x00FF0000L) == 0xFF0000L &&
-    wParam != VK_CAPITAL) {
-    *pfEaten = FALSE;
-  }
-  else {
-    *pfEaten = _KeymanProcessKeystroke(pContext, wParam, lParam, FALSE, FALSE);   // I3588
-//  SendDebugMessageFormat("pfEaten=%s", *pfEaten ? "TRUE" : "FALSE");
-  }
+  fOnTestEatenBuf[wParam] = *pfEaten = _KeymanProcessKeystroke(pContext, wParam, lParam, FALSE, FALSE);  // I3588
+  SendDebugMessageFormat(L"pfEaten=%s, wParam=%x, lParam=%x", *pfEaten ? L"TRUE" : L"FALSE", wParam, lParam);
   SendDebugExit();
   return S_OK;
 }
@@ -205,8 +197,8 @@ STDAPI CKMTipTextService::OnKeyDown(ITfContext *pContext, WPARAM wParam, LPARAM
 {
   SendDebugEntry();
   LogKey(1, wParam, lParam);
-  fEatenBuf[wParam] = *pfEaten = _KeymanProcessKeystroke(pContext, wParam, lParam, TRUE, FALSE);   // I3588
-//  SendDebugMessageFormat("pfEaten=%s", *pfEaten ? "TRUE" : "FALSE");
+  fEatenBuf[wParam] = *pfEaten = _KeymanProcessKeystroke(pContext, wParam, lParam, TRUE, FALSE);  // I3588
+  SendDebugMessageFormat(L"pfEaten=%s wParam=%x lParam=%x", *pfEaten ? L"TRUE" : L"FALSE", wParam, lParam);
   SendDebugExit();
 	return S_OK;
 }
@@ -222,17 +214,9 @@ STDAPI CKMTipTextService::OnTestKeyUp(ITfContext *pContext, WPARAM wParam, LPARA
 {
   SendDebugEntry();
   LogKey(2, wParam, lParam);
-  // If the keystroke is a Keyman-generated key, ignore it
-  // But we need to pass Caps Lock through, even if we generated it, so we can track Caps Lock state.
-  if ((lParam & 0x00FF0000L) == 0xFF0000L &&
-    wParam != VK_CAPITAL) {  // I3566
-    *pfEaten = FALSE;
-  }
-  else {
-    _KeymanProcessKeystroke(pContext, wParam, lParam, FALSE, FALSE);   // I3588
-    *pfEaten = fEatenBuf[wParam];
-  }
-//  SendDebugMessageFormat("pfEaten=%s", *pfEaten ? "TRUE" : "FALSE");
+  _KeymanProcessKeystroke(pContext, wParam, lParam, FALSE, FALSE);   // I3588
+  *pfEaten = fOnTestEatenBuf[wParam];
+  SendDebugMessageFormat(L"pfEaten=%s wParam=%x lParam=%x", *pfEaten ? L"TRUE" : L"FALSE", wParam, lParam);
   SendDebugExit();
   return S_OK;
 }
@@ -251,16 +235,11 @@ STDAPI CKMTipTextService::OnKeyUp(ITfContext *pContext, WPARAM wParam, LPARAM lP
   LogKey(3, wParam, lParam);
   // If the keystroke is a Keyman-generated key, ignore it
   // But we need to pass Caps Lock through, even if we generated it, so we can track Caps Lock state.
-  if ((lParam & 0x00FF0000L) == 0xFF0000L &&
-    wParam != VK_CAPITAL) {   // I3566   // I3605
-    *pfEaten = FALSE;
-  }
-  else
-  {
-  	_KeymanProcessKeystroke(pContext, wParam, lParam, TRUE, FALSE);   // I3588   // I3605
-    *pfEaten = fEatenBuf[wParam];
-  }
-//  SendDebugMessageFormat("pfEaten=%s", *pfEaten ? "TRUE" : "FALSE");
+
+  _KeymanProcessKeystroke(pContext, wParam, lParam, TRUE, FALSE);   // I3588   // I3605
+  *pfEaten = fEatenBuf[wParam];
+
+  SendDebugMessageFormat(L"pfEaten=%s wParam=%x lParam=%x", *pfEaten ? L"TRUE" : L"FALSE", wParam, lParam);
   SendDebugExit();
   return S_OK;
 }

windows/src/engine/kmtip/kmkey.cpp

@@ -102,15 +102,16 @@ BOOL CKMTipTextService::_KeymanProcessKeystroke(ITfContext *pContext, WPARAM wPa
 
   // Don't process Unicode characters injected or ProcessKey events which are generated by Windows
   if (wParam == VK_PACKET || wParam == VK_PROCESSKEY) {
+    SendDebugMessageFormat(L"wParam=%x early return FALSE", wParam);
     return FALSE;   // I3608   // I4201
   }
 
   SendDebugEntry();
   SendDebugMessageFormat(L"%x %x %s %s ex=%x", wParam, lParam, fUpdate ? L"update" : L"", fPreserved ? L"preserved" : L"", GetMessageExtraInfo());   // I4378
 
-  // Don't process keystrokes generated by Keyman (scan code = 0xFF)
-  // But we need to pass Caps Lock through, even if we generated it, so we can track Caps Lock state.
-  // TODO: This is done in multiple places, but we probably only need to do it once
+  // Don't process keystrokes generated by Keyman (scan code = 0xFF = SCAN_FLAG_KEYMAN_KEY_EVENT)
+  // But we need to pass Caps Lock through to the keyman engine (but not the core),
+  // even if we generated it, so we can track Caps Lock state.
   if ((lParam & 0xFF0000) == 0xFF0000 &&
     wParam != VK_CAPITAL) {
     return_SendDebugExit(FALSE);

windows/src/engine/kmtip/kmtip.h

@@ -117,6 +117,7 @@ private:
 
     BOOL _keystrokeSinkInitialized;
     BOOL fEatenBuf[256];
+    BOOL fOnTestEatenBuf[256];
 
     ITfThreadMgr *_pThreadMgr;
     TfClientId _tfClientId;