From b5bddc025f8a817856de55acb8644ddd25d23913 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 26 Jul 2022 13:06:16 +1000 Subject: [PATCH 001/132] chore(core): rename json.hpp to jsonpp.hpp The name json.hpp conflicts with the JSON for Modern C++ library that is also used by Developer. Renames json.hpp and json.cpp to jsonpp.hpp and jsonpp.cpp aka "JSON Pretty Printer". --- core/doc/meson.build | 2 +- core/src/{json.cpp => jsonpp.cpp} | 2 +- core/src/{json.hpp => jsonpp.hpp} | 0 core/src/keyboard.cpp | 2 +- core/src/km_kbp_context_api.cpp | 2 +- core/src/km_kbp_options_api.cpp | 2 +- core/src/km_kbp_state_api.cpp | 2 +- core/src/meson.build | 2 +- core/src/path.hpp | 2 +- core/tests/unit/json/jsontest.cpp | 2 +- core/tests/unit/json/meson.build | 2 +- linux/ibus-keyman/tests/Makefile.am | 2 +- 12 files changed, 11 insertions(+), 11 deletions(-) rename core/src/{json.cpp => jsonpp.cpp} (99%) rename core/src/{json.hpp => jsonpp.hpp} (100%) diff --git a/core/doc/meson.build b/core/doc/meson.build index cae6b4be60..2772819ce7 100644 --- a/core/doc/meson.build +++ b/core/doc/meson.build @@ -17,7 +17,7 @@ if hotdoc.found() configuration: cfg) deps = files( '../include/keyman/keyboardprocessor.h.in', - '../src/json.hpp', + '../src/jsonpp.hpp', '../src/utfcodec.hpp' ) diff --git a/core/src/json.cpp b/core/src/jsonpp.cpp similarity index 99% rename from core/src/json.cpp rename to core/src/jsonpp.cpp index 40cb86130c..69c791c2cc 100644 --- a/core/src/json.cpp +++ b/core/src/jsonpp.cpp @@ -12,7 +12,7 @@ #include #include -#include "json.hpp" +#include "jsonpp.hpp" #if defined(_MSC_VER) diff --git a/core/src/json.hpp b/core/src/jsonpp.hpp similarity index 100% rename from core/src/json.hpp rename to core/src/jsonpp.hpp diff --git a/core/src/keyboard.cpp b/core/src/keyboard.cpp index ac5e978a5d..977da597d3 100644 --- a/core/src/keyboard.cpp +++ b/core/src/keyboard.cpp @@ -6,7 +6,7 @@ History: 7 Oct 2018 - TSE - Refactored out of km_kbp_keyboard_api.cpp */ #include "keyboard.hpp" -#include "json.hpp" +#include "jsonpp.hpp" using namespace km::kbp; diff --git a/core/src/km_kbp_context_api.cpp b/core/src/km_kbp_context_api.cpp index 97b439f9e3..a9111f66c4 100644 --- a/core/src/km_kbp_context_api.cpp +++ b/core/src/km_kbp_context_api.cpp @@ -15,7 +15,7 @@ #include #include "context.hpp" -#include "json.hpp" +#include "jsonpp.hpp" #include "utfcodec.hpp" namespace { diff --git a/core/src/km_kbp_options_api.cpp b/core/src/km_kbp_options_api.cpp index f95159a8ab..7dc4879098 100644 --- a/core/src/km_kbp_options_api.cpp +++ b/core/src/km_kbp_options_api.cpp @@ -14,7 +14,7 @@ #include #include "processor.hpp" -#include "json.hpp" +#include "jsonpp.hpp" #include "state.hpp" diff --git a/core/src/km_kbp_state_api.cpp b/core/src/km_kbp_state_api.cpp index 1d750536d7..b966688950 100644 --- a/core/src/km_kbp_state_api.cpp +++ b/core/src/km_kbp_state_api.cpp @@ -13,7 +13,7 @@ #include #include -#include "json.hpp" +#include "jsonpp.hpp" #include "processor.hpp" #include "state.hpp" diff --git a/core/src/meson.build b/core/src/meson.build index 1e3b6e82fc..d97a5c9ca6 100644 --- a/core/src/meson.build +++ b/core/src/meson.build @@ -68,7 +68,7 @@ lib = library('kmnkbp0', 'km_kbp_state_api.cpp', 'km_kbp_debug_api.cpp', 'km_kbp_processevent_api.cpp', - 'json.cpp', + 'jsonpp.cpp', 'mock/mock_processor.cpp', 'kmx/kmx_consts.cpp', 'kmx/kmx_processevent.cpp', diff --git a/core/src/path.hpp b/core/src/path.hpp index f7600f3841..e13360248e 100644 --- a/core/src/path.hpp +++ b/core/src/path.hpp @@ -13,7 +13,7 @@ #include #include -#include "json.hpp" +#include "jsonpp.hpp" #include "utfcodec.hpp" // Forward declarations diff --git a/core/tests/unit/json/jsontest.cpp b/core/tests/unit/json/jsontest.cpp index fabd24eca6..3ff64d4cd5 100644 --- a/core/tests/unit/json/jsontest.cpp +++ b/core/tests/unit/json/jsontest.cpp @@ -14,7 +14,7 @@ #include #include -#include +#include #ifdef __EMSCRIPTEN__ #include diff --git a/core/tests/unit/json/meson.build b/core/tests/unit/json/meson.build index 5ad7b1ffd8..42106976cd 100644 --- a/core/tests/unit/json/meson.build +++ b/core/tests/unit/json/meson.build @@ -13,7 +13,7 @@ endif e = executable('jsontest', 'jsontest.cpp', include_directories: [libsrc], link_args: links + tests_flags, - objects: lib.extract_objects('json.cpp')) + objects: lib.extract_objects('jsonpp.cpp')) test('jsontest', e, args: 'jsontest.json') test('jsontestOutput', python, is_parallel: false, args: cmpfiles + ['jsontest.json', join_paths(stnds, 'jsontest.json')]) diff --git a/linux/ibus-keyman/tests/Makefile.am b/linux/ibus-keyman/tests/Makefile.am index 2a70552048..af175e392f 100644 --- a/linux/ibus-keyman/tests/Makefile.am +++ b/linux/ibus-keyman/tests/Makefile.am @@ -54,7 +54,7 @@ ibus_keyman_tests_SOURCES = \ testfixture.cpp \ testmodule.c \ testmodule.h \ - ../../../core/src/json.cpp \ + ../../../core/src/jsonpp.cpp \ ../../../core/src/keyboard.cpp \ ../../../core/src/kmx/kmx_actions.cpp \ ../../../core/src/kmx/kmx_actions.h \ From 97817b8724f87a6dafb2616910ae8875530e70b0 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 27 Jul 2022 14:58:13 +1000 Subject: [PATCH 002/132] chore(developer): remove unused dependencies from KeymanWeb compiler --- developer/src/kmcomp/kmcomp.dpr | 1 - developer/src/kmcomp/kmcomp.dproj | 1 - developer/src/kmconvert/kmconvert.dpr | 1 - developer/src/kmconvert/kmconvert.dproj | 1 - .../KeyboardPackageVersionsTestSuite.dpr | 6 ------ .../KeyboardPackageVersionsTestSuite.dproj | 6 ------ developer/src/tike/compile/CompileKeymanWeb.pas | 3 --- 7 files changed, 19 deletions(-) diff --git a/developer/src/kmcomp/kmcomp.dpr b/developer/src/kmcomp/kmcomp.dpr index 1fcf977155..8ac6e124cf 100644 --- a/developer/src/kmcomp/kmcomp.dpr +++ b/developer/src/kmcomp/kmcomp.dpr @@ -53,7 +53,6 @@ uses CompileErrorCodes in '..\common\delphi\compiler\CompileErrorCodes.pas', TouchLayout in '..\tike\oskbuilder\TouchLayout.pas', TouchLayoutDefinitions in '..\tike\oskbuilder\TouchLayoutDefinitions.pas', - TouchLayoutUtils in '..\tike\oskbuilder\TouchLayoutUtils.pas', KeyboardFonts in '..\common\delphi\general\KeyboardFonts.pas', KeyboardParser in '..\tike\main\KeyboardParser.pas', WindowsLanguages in '..\common\delphi\general\WindowsLanguages.pas', diff --git a/developer/src/kmcomp/kmcomp.dproj b/developer/src/kmcomp/kmcomp.dproj index 8453e92639..f6f183330f 100644 --- a/developer/src/kmcomp/kmcomp.dproj +++ b/developer/src/kmcomp/kmcomp.dproj @@ -202,7 +202,6 @@ - diff --git a/developer/src/kmconvert/kmconvert.dpr b/developer/src/kmconvert/kmconvert.dpr index 1b0b9edda2..233c07d43c 100644 --- a/developer/src/kmconvert/kmconvert.dpr +++ b/developer/src/kmconvert/kmconvert.dpr @@ -73,7 +73,6 @@ uses OnScreenKeyboardData in '..\..\..\common\windows\delphi\visualkeyboard\OnScreenKeyboardData.pas', TouchLayout in '..\TIKE\oskbuilder\TouchLayout.pas', TouchLayoutDefinitions in '..\TIKE\oskbuilder\TouchLayoutDefinitions.pas', - TouchLayoutUtils in '..\TIKE\oskbuilder\TouchLayoutUtils.pas', KeyboardFonts in '..\common\delphi\general\KeyboardFonts.pas', Keyman.System.Util.RenderLanguageIcon in '..\..\..\common\windows\delphi\ui\Keyman.System.Util.RenderLanguageIcon.pas', utilicon in '..\..\..\common\windows\delphi\general\utilicon.pas', diff --git a/developer/src/kmconvert/kmconvert.dproj b/developer/src/kmconvert/kmconvert.dproj index 148585112e..48626bc67b 100644 --- a/developer/src/kmconvert/kmconvert.dproj +++ b/developer/src/kmconvert/kmconvert.dproj @@ -179,7 +179,6 @@ - diff --git a/developer/src/test/auto/keyboard-package-versions/KeyboardPackageVersionsTestSuite.dpr b/developer/src/test/auto/keyboard-package-versions/KeyboardPackageVersionsTestSuite.dpr index 44640edc55..bfe61aa975 100644 --- a/developer/src/test/auto/keyboard-package-versions/KeyboardPackageVersionsTestSuite.dpr +++ b/developer/src/test/auto/keyboard-package-versions/KeyboardPackageVersionsTestSuite.dpr @@ -72,14 +72,8 @@ uses WindowsLanguages in '..\..\..\common\delphi\general\WindowsLanguages.pas', KeymanWebKeyCodes in '..\..\..\tike\compile\KeymanWebKeyCodes.pas', kmxfileutils in '..\..\..\..\..\common\windows\delphi\keyboards\kmxfileutils.pas', - TikeUnicodeData in '..\..\..\tike\main\TikeUnicodeData.pas', - UnicodeData in '..\..\..\..\..\common\windows\delphi\charmap\UnicodeData.pas', - ADODB_TLB in '..\..\..\..\..\common\windows\delphi\tlb\ADODB_TLB.pas', - ADOX_TLB in '..\..\..\..\..\common\windows\delphi\tlb\ADOX_TLB.pas', - ttinfo in '..\..\..\..\..\common\windows\delphi\general\ttinfo.pas', TouchLayoutDefinitions in '..\..\..\tike\oskbuilder\TouchLayoutDefinitions.pas', TouchLayout in '..\..\..\tike\oskbuilder\TouchLayout.pas', - TouchLayoutUtils in '..\..\..\tike\oskbuilder\TouchLayoutUtils.pas', KeyboardFonts in '..\..\..\common\delphi\general\KeyboardFonts.pas', Keyman.Developer.System.Project.kpsProjectFile in '..\..\..\tike\project\Keyman.Developer.System.Project.kpsProjectFile.pas', Keyman.Developer.System.Project.kpsProjectFileAction in '..\..\..\tike\project\Keyman.Developer.System.Project.kpsProjectFileAction.pas', diff --git a/developer/src/test/auto/keyboard-package-versions/KeyboardPackageVersionsTestSuite.dproj b/developer/src/test/auto/keyboard-package-versions/KeyboardPackageVersionsTestSuite.dproj index 0cb4eb7ad9..c1d5645e40 100644 --- a/developer/src/test/auto/keyboard-package-versions/KeyboardPackageVersionsTestSuite.dproj +++ b/developer/src/test/auto/keyboard-package-versions/KeyboardPackageVersionsTestSuite.dproj @@ -153,14 +153,8 @@ - - - - - - diff --git a/developer/src/tike/compile/CompileKeymanWeb.pas b/developer/src/tike/compile/CompileKeymanWeb.pas index 76fcd8398f..0ffa8d797f 100644 --- a/developer/src/tike/compile/CompileKeymanWeb.pas +++ b/developer/src/tike/compile/CompileKeymanWeb.pas @@ -273,14 +273,11 @@ uses CompileErrorCodes, JsonUtil, KeymanDeveloperOptions, - KeyboardParser, Keyman.System.KeyboardUtils, KeymanWebKeyCodes, kmxfileutils, TouchLayout, - TouchLayoutUtils, Unicode, - UnicodeData, utilstr, VisualKeyboard, VKeys; From bdb7b5e45a6b383a0da1285f8aec22a80d5d8bb3 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 27 Jul 2022 15:16:02 +1000 Subject: [PATCH 003/132] fix(android): verify browser before starting activity Fixes #6899. --- .../java/com/tavultesoft/kmapro/KMPBrowserActivity.java | 4 +++- .../com/tavultesoft/kmapro/KeyboardSettingsActivity.java | 4 +++- .../main/java/com/tavultesoft/kmapro/MainActivity.java | 9 +++++++-- .../java/com/tavultesoft/kmea/KeyboardInfoActivity.java | 4 +++- .../main/java/com/firstvoices/keyboards/FVShared.java | 4 +++- .../java/com/firstvoices/keyboards/MainActivity.java | 4 +++- 6 files changed, 22 insertions(+), 7 deletions(-) diff --git a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/KMPBrowserActivity.java b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/KMPBrowserActivity.java index 7cff5b5e80..ff74db201f 100644 --- a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/KMPBrowserActivity.java +++ b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/KMPBrowserActivity.java @@ -113,7 +113,9 @@ public class KMPBrowserActivity extends BaseActivity { // All links that aren't internal Keyman keyboard links open in user's browser Intent intent = new Intent(Intent.ACTION_VIEW, uri); - startActivity(intent); + if (intent.resolveActivity(getPackageManager()) != null) { + startActivity(intent); + } return true; } if (lowerURL.startsWith("keyman:")) { diff --git a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/KeyboardSettingsActivity.java b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/KeyboardSettingsActivity.java index 8aee2a6acb..b471fc1cee 100644 --- a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/KeyboardSettingsActivity.java +++ b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/KeyboardSettingsActivity.java @@ -160,7 +160,9 @@ public final class KeyboardSettingsActivity extends AppCompatActivity { Bundle args = kbd.buildDownloadBundle(); Intent i = new Intent(getApplicationContext(), KMKeyboardDownloaderActivity.class); i.putExtras(args); - startActivity(i); + if (i.resolveActivity(getPackageManager()) != null) { + startActivity(i); + } finish(); // "Help" link clicked diff --git a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/MainActivity.java b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/MainActivity.java index 6f3affe4fb..af9b16fbc1 100644 --- a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/MainActivity.java +++ b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/MainActivity.java @@ -762,10 +762,15 @@ public class MainActivity extends BaseActivity implements OnKeyboardEventListene // Launch PlayStore to update Chrome try { Intent intent = new Intent(Intent.ACTION_VIEW, Uri.parse("market://details?id=com.android.chrome")); - startActivity(intent); + if (intent.resolveActivity(getPackageManager()) != null) { + startActivity(intent); + } } catch (android.content.ActivityNotFoundException e) { // Link to Chrome if user is not signed in to Play Store - startActivity(new Intent(Intent.ACTION_VIEW, Uri.parse("https://play.google.com/store/apps/details?id=com.android.chrome"))); + Intent intent = new Intent(Intent.ACTION_VIEW, Uri.parse("https://play.google.com/store/apps/details?id=com.android.chrome")); + if (intent.resolveActivity(getPackageManager()) != null) { + startActivity(intent); + } } } }); diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardInfoActivity.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardInfoActivity.java index d9b15f0ea8..89bab42c66 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardInfoActivity.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardInfoActivity.java @@ -136,7 +136,9 @@ public final class KeyboardInfoActivity extends BaseActivity { } else { Intent i = new Intent(Intent.ACTION_VIEW); i.setData(Uri.parse(customHelpLink)); - startActivity(i); + if (i.resolveActivity(getPackageManager()) != null) { + startActivity(i); + } } } } diff --git a/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/FVShared.java b/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/FVShared.java index 854175e4cc..71ace943b5 100644 --- a/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/FVShared.java +++ b/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/FVShared.java @@ -252,7 +252,9 @@ final class FVShared { String helpUrl = String.format("%s%s", FVKeyboardHelpLink, id); Intent i = new Intent(Intent.ACTION_VIEW); i.setData(Uri.parse(helpUrl)); - localContext.startActivity(i); + if (i.resolveActivity(getPackageManager()) != null) { + localContext.startActivity(i); + } } private void updateActiveKeyboardsList() { diff --git a/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/MainActivity.java b/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/MainActivity.java index 5d3e5c5939..897145ec5f 100644 --- a/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/MainActivity.java +++ b/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/MainActivity.java @@ -105,7 +105,9 @@ public class MainActivity extends AppCompatActivity implements OnKeyboardDownloa } else { Intent i = new Intent(Intent.ACTION_VIEW, Uri.parse(url)); - startActivity(i); + if (i.resolveActivity(getPackageManager()) != null) { + startActivity(i); + } } return true; From 127fb3b4e0a877a7b6ae526b88163ca96c923a90 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 27 Jul 2022 15:59:50 +1000 Subject: [PATCH 004/132] chore(android): fixup getPackageManager ref --- .../app/src/main/java/com/firstvoices/keyboards/FVShared.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/FVShared.java b/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/FVShared.java index 71ace943b5..9e8790acda 100644 --- a/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/FVShared.java +++ b/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/FVShared.java @@ -252,7 +252,7 @@ final class FVShared { String helpUrl = String.format("%s%s", FVKeyboardHelpLink, id); Intent i = new Intent(Intent.ACTION_VIEW); i.setData(Uri.parse(helpUrl)); - if (i.resolveActivity(getPackageManager()) != null) { + if (i.resolveActivity(localContext.getPackageManager()) != null) { localContext.startActivity(i); } } From 9b7d44398de704afa038588e596fea6b497631c3 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 28 Jul 2022 09:24:18 +1000 Subject: [PATCH 005/132] chore(developer): Coinit required for test suite Removing unused dependencies also removed the coinit call. Re-adding coinit. --- .../Keyman.Test.System.CompilePackageVersioningTest.pas | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/developer/src/test/auto/keyboard-package-versions/Keyman.Test.System.CompilePackageVersioningTest.pas b/developer/src/test/auto/keyboard-package-versions/Keyman.Test.System.CompilePackageVersioningTest.pas index 9d4222195b..fae07d5eed 100644 --- a/developer/src/test/auto/keyboard-package-versions/Keyman.Test.System.CompilePackageVersioningTest.pas +++ b/developer/src/test/auto/keyboard-package-versions/Keyman.Test.System.CompilePackageVersioningTest.pas @@ -36,6 +36,8 @@ implementation uses System.SysUtils, + Winapi.ActiveX, + compile, CompilePackage, Keyman.Developer.System.Project.kmnProjectFileAction, @@ -154,5 +156,8 @@ begin end; initialization + CoInitializeEx(nil, COINIT_APARTMENTTHREADED); TDUnitX.RegisterTestFixture(TCompilePackageVersioningTest); +finalization + CoUninitialize; end. From af219def62342355e71642547af5e60237d8ea38 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 4 Aug 2022 14:38:48 +0700 Subject: [PATCH 006/132] fix(web): enhanced timer for prediction alg --- .../src/correction/distance-modeler.ts | 78 +++++++++++++++++-- 1 file changed, 73 insertions(+), 5 deletions(-) diff --git a/common/web/lm-worker/src/correction/distance-modeler.ts b/common/web/lm-worker/src/correction/distance-modeler.ts index 7b4456e4d8..d4c13ff78f 100644 --- a/common/web/lm-worker/src/correction/distance-modeler.ts +++ b/common/web/lm-worker/src/correction/distance-modeler.ts @@ -526,8 +526,6 @@ namespace correction { let searchSpace = this; let currentReturns: {[mapKey: string]: SearchNode} = {}; - // JS measures time by the number of milliseconds since Jan 1, 1970. - let timeStart = Date.now(); let maxTime: number; if(waitMillis == 0) { maxTime = Infinity; @@ -537,6 +535,70 @@ namespace correction { maxTime = waitMillis; } + class ExecutionTimer { + private start: number; + private loopStart: number; + private maxExecutionTime: number; + private maxTrueTime: number; + + private executionTime: number; + + /** + * Used to track intervals in which potential context swaps by the OS may + * have occurred. + */ + private largestIntervals: number[] = [0]; + private iterationCount: number = 0; + + constructor(maxExecutionTime: number, maxTrueTime: number) { + // JS measures time by the number of milliseconds since Jan 1, 1970. + this.loopStart = this.start = Date.now(); + this.maxExecutionTime = maxExecutionTime; + this.maxTrueTime = maxTrueTime; + } + + startLoop() { + this.loopStart = Date.now(); + } + + markIteration() { + const now = Date.now(); + const delta = now - this.loopStart; + this.executionTime += delta; + this.iterationCount++; + + if(delta) { + if(this.largestIntervals.length > 2 && delta > this.largestIntervals[0]) { + this.largestIntervals[0] = delta; + } else { + this.largestIntervals.push(delta); + } + this.largestIntervals.sort(); + } + } + + shouldTimeout(): boolean { + const now = Date.now(); + if(this.start - now > this.maxTrueTime) { + return true; + } + + // Look at the 'intervals' for a possible over-large outliers. + if(this.largestIntervals.length > 2) { + if(this.largestIntervals[2] > 2 * (this.largestIntervals[0] + this.largestIntervals[1])) { + this.executionTime -= this.largestIntervals[2]; + this.largestIntervals.pop(); + } + } + + return this.executionTime > this.maxExecutionTime; + } + + resetOutlierCheck() { + this.largestIntervals = []; + } + } + class BatchingAssistant { currentCost = Number.MIN_SAFE_INTEGER; entries: SearchResult[] = []; @@ -582,17 +644,22 @@ namespace correction { let batcher = new BatchingAssistant(); + const timer = new ExecutionTimer(maxTime*3, maxTime); + // Stage 1 - if we already have extracted results, build a queue just for them and iterate over it first. let returnedValues = Object.values(this.returnedValues); if(returnedValues.length > 0) { let preprocessedQueue = new models.PriorityQueue(QUEUE_NODE_COMPARATOR, returnedValues); // Build batches of same-cost entries. + timer.startLoop(); while(preprocessedQueue.count > 0) { let entry = preprocessedQueue.dequeue(); let batch = batcher.checkAndAdd(entry); + timer.markIteration(); if(batch) { + // Do not track yielded time. yield batch; } } @@ -601,11 +668,13 @@ namespace correction { // finalize the last preprocessed group without issue. let batch = batcher.tryFinalize(); if(batch) { + // Do not track yielded time. yield batch; } } // Stage 2: the fun part; actually searching! + timer.startLoop(); let timedOut = false; do { let newResult: PathResult; @@ -613,10 +682,9 @@ namespace correction { // Search for a 'complete' path, skipping all partial paths as long as time remains. do { newResult = this.handleNextNode(); + timer.markIteration(); - // (Naive) timeout check! - let now = Date.now(); - if(now - timeStart > maxTime) { + if(timer.shouldTimeout()) { timedOut = true; } } while(!timedOut && newResult.type == 'intermediate') From 814a4922770802b7e2acb7173fb82391b7786987 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 9 Aug 2022 09:00:29 +0700 Subject: [PATCH 007/132] chore(web): tweaks, docs for execution-timer inner-class --- .../src/correction/distance-modeler.ts | 89 ++++++++++++++++--- 1 file changed, 78 insertions(+), 11 deletions(-) diff --git a/common/web/lm-worker/src/correction/distance-modeler.ts b/common/web/lm-worker/src/correction/distance-modeler.ts index d4c13ff78f..b0cd2179a9 100644 --- a/common/web/lm-worker/src/correction/distance-modeler.ts +++ b/common/web/lm-worker/src/correction/distance-modeler.ts @@ -535,9 +535,25 @@ namespace correction { maxTime = waitMillis; } + /** + * This inner class is designed to help the algorithm detect its active execution time. + * While there's no official JS way to do this, we can approximate it by polling the + * current system time (in ms) after each iteration of a short-duration loop. Unusual + * spikes in system time for a single iteration is likely to indicate that an OS + * context switch occurred at some point during the iteration's execution. + */ class ExecutionTimer { + /** + * The system time when this instance was created. + */ private start: number; + + /** + * Marks the system time at the start of the currently-running loop, as noted + * by a call to the `startLoop` function. + */ private loopStart: number; + private maxExecutionTime: number; private maxTrueTime: number; @@ -545,10 +561,14 @@ namespace correction { /** * Used to track intervals in which potential context swaps by the OS may - * have occurred. + * have occurred. Context switches generally seem to pause threads for + * at least 16 ms, while we expect each loop iteration to complete + * within just 1 ms. So, any possible context switch should have the + * longest observed change in system time. + * + * See `updateOutliers` for more details. */ private largestIntervals: number[] = [0]; - private iterationCount: number = 0; constructor(maxExecutionTime: number, maxTrueTime: number) { // JS measures time by the number of milliseconds since Jan 1, 1970. @@ -565,15 +585,69 @@ namespace correction { const now = Date.now(); const delta = now - this.loopStart; this.executionTime += delta; - this.iterationCount++; + /** + * Update the list of the three longest system-time intervals observed + * for execution of a single loop iteration. + * + * Ignore any zero-ms length intervals; they'd make the logic much + * messier than necessary otherwise. + */ if(delta) { + // If the currently-observed interval is longer than the shortest of the 3 + // previously-observed longest intervals, replace it. if(this.largestIntervals.length > 2 && delta > this.largestIntervals[0]) { this.largestIntervals[0] = delta; } else { this.largestIntervals.push(delta); } + // Puts the list in ascending order. Shortest of the list becomes the head, + // longest one the tail. this.largestIntervals.sort(); + + // Then, determine if we need to update our outlier-based tweaks. + this.updateOutliers(); + } + } + + updateOutliers() { + /* Base assumption: since each loop of the search should evaluate within ~1ms, + * notably longer execution times are probably context switches. + * + * Base assumption: OS context switches generally last at least 16ms. (Based on + * a window.setTimeout() usually not evaluating for at least + * that long, even if set to 1ms.) + * + * To mitigate these assumptions: we'll track the execution time of every loop + * iteration. If the longest observation somehow matches or exceeds the length of + * the next two almost-longest observations twice over... we have a very strong + * 'context switch' candidate. + * + * Or, in near-formal math/stats: we expect a very low variance in execution + * time among the iterations of the search's loops. With a very low variance, + * ANY significant proportional spikes in execution time are outliers - outliers + * likely caused by an OS context switch. + * + * Rather than do intensive math, we use a somewhat lazy approach below that + * achieves the same net results given our assumptions, even when relaxed somewhat. + * + * The logic below relaxes the base assumptions a bit to be safe: + * - [2ms, 2ms, 8ms] will cause 8ms to be seen as an outlier. + * - [2ms, 3ms, 10ms] will cause 10ms to be seen as an outlier. + * + * Ideally: + * - [1ms, 1ms, 4ms] will view 4ms as an outlier. + * + * So we can safely handle slightly longer average intervals and slightly shorter + * OS context-switch time intervals. + */ + if(this.largestIntervals.length > 2) { + // Precondition: the `largestIntervals` array is sorted in ascending order. + // Shortest entry is at the head, longest at the tail. + if(this.largestIntervals[2] >= 2 * (this.largestIntervals[0] + this.largestIntervals[1])) { + this.executionTime -= this.largestIntervals[2]; + this.largestIntervals.pop(); + } } } @@ -583,14 +657,6 @@ namespace correction { return true; } - // Look at the 'intervals' for a possible over-large outliers. - if(this.largestIntervals.length > 2) { - if(this.largestIntervals[2] > 2 * (this.largestIntervals[0] + this.largestIntervals[1])) { - this.executionTime -= this.largestIntervals[2]; - this.largestIntervals.pop(); - } - } - return this.executionTime > this.maxExecutionTime; } @@ -674,6 +740,7 @@ namespace correction { } // Stage 2: the fun part; actually searching! + timer.resetOutlierCheck(); timer.startLoop(); let timedOut = false; do { From 3fe24996b31ce811f7d2d4c05c75761cbc2fb313 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Tue, 16 Aug 2022 14:02:02 -0400 Subject: [PATCH 008/132] auto: increment master version to 16.0.49 --- HISTORY.md | 4 ++++ VERSION.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 775866a8cf..1878d3ad4a 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,9 @@ # Keyman Version History +## 16.0.48 alpha 2022-08-16 + +* fix(web): button, float init timer cleanup (#7036) + ## 16.0.47 alpha 2022-08-15 * chore(core): refactor kmx_file.h to common (#7062) diff --git a/VERSION.md b/VERSION.md index 84f71677bf..214271b7a3 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.48 \ No newline at end of file +16.0.49 \ No newline at end of file From f2fe3b16938ffd03fc553b5927ac6dba0652fce7 Mon Sep 17 00:00:00 2001 From: Ross Date: Wed, 17 Aug 2022 11:39:16 +1000 Subject: [PATCH 009/132] fix: remove saving and restoring context kbd options Remove the saving of context and keyboard options on the non updateable call to process hook. The windows Text Input Processor calls twice one call is not updateable. With the core integeration we run the processor only once, preserving the actions and context to be applied on the updatable call. --- windows/src/engine/keyman32/appint/aiTIP.cpp | 26 ++++++-------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/windows/src/engine/keyman32/appint/aiTIP.cpp b/windows/src/engine/keyman32/appint/aiTIP.cpp index 9043c9ece1..23240b2a06 100644 --- a/windows/src/engine/keyman32/appint/aiTIP.cpp +++ b/windows/src/engine/keyman32/appint/aiTIP.cpp @@ -258,15 +258,12 @@ extern "C" __declspec(dllexport) BOOL WINAPI TIPProcessKey(WPARAM wParam, LPARAM _td->TIPGetContext = ctfunc; AppContextWithStores *savedContext = NULL; // I4370 // I4978 - AppContext *savedContextUsingCore = NULL; // used for common core - km_kbp_option_item *SavedKBDOptions = NULL; // used for common core + if (!Updateable) { - if (isUsingCoreProcessor) { - savedContextUsingCore = new AppContext(); - _td->app->CopyContext(savedContextUsingCore); - SavedKBDOptions = SaveKeyboardOptionsCore(_td->lpActiveKeyboard); - } else { // I4370 - savedContext = new AppContextWithStores(_td->lpActiveKeyboard->Keyboard->cxStoreArray); // I4978 + // The core processor km_kbp_process_event is only called once per key stroke + // therefore there is no need to preserve context and keyboard actions + if (!isUsingCoreProcessor) { + savedContext = new AppContextWithStores(_td->lpActiveKeyboard->Keyboard->cxStoreArray); // I4370 // I4978 _td->app->SaveContext(savedContext); } } @@ -274,17 +271,8 @@ extern "C" __declspec(dllexport) BOOL WINAPI TIPProcessKey(WPARAM wParam, LPARAM BOOL res = ProcessHook(); if (!Updateable) { - if (isUsingCoreProcessor) { - if (res) { - // Reset the context if match found - _td->app->RestoreContextOnly(savedContextUsingCore); - RestoreKeyboardOptionsCore(_td->lpActiveKeyboard->lpCoreKeyboardState, SavedKBDOptions); - DisposeKeyboardOptionsCore(&SavedKBDOptions); - delete savedContextUsingCore; - savedContextUsingCore = NULL; - } - } else { // I4370 - if (res) { // I4585 + if (!isUsingCoreProcessor) { + if (res) { // I4585 // I4370 // Reset the context if match found _td->app->RestoreContext(savedContext); delete savedContext; From ea50559fc155f40e840e14d564a6fcfb67898731 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Aug 2022 19:58:19 +0000 Subject: [PATCH 010/132] chore(deps): bump @actions/core from 1.8.2 to 1.9.1 Bumps [@actions/core](https://github.com/actions/toolkit/tree/HEAD/packages/core) from 1.8.2 to 1.9.1. - [Release notes](https://github.com/actions/toolkit/releases) - [Changelog](https://github.com/actions/toolkit/blob/main/packages/core/RELEASES.md) - [Commits](https://github.com/actions/toolkit/commits/HEAD/packages/core) --- updated-dependencies: - dependency-name: "@actions/core" dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- package-lock.json | 30 +++++++++++++++------------- resources/build/version/package.json | 2 +- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/package-lock.json b/package-lock.json index ad25cc7212..03dd8ee2b0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -577,8 +577,8 @@ "typescript": "^4.5.4" }, "optionalDependencies": { - "hetrodo-node-hide-console-window-napi": "keymanapp/hetrodo-node-hide-console-window-napi#keyman-15.0", - "node-windows-trayicon": "keymanapp/node-windows-trayicon#keyman-15.0" + "hetrodo-node-hide-console-window-napi": "git+ssh://git@github.com/keymanapp/hetrodo-node-hide-console-window-napi.git#858b23036a9963b40ad6ff3c5bacd421e5839b92", + "node-windows-trayicon": "git+ssh://git@github.com/keymanapp/node-windows-trayicon.git#1e46786082213f3edcddd5953e33f5abdc7ea05f" } }, "developer/src/server/node_modules/@cspotcode/source-map-support": { @@ -865,11 +865,12 @@ "dev": true }, "node_modules/@actions/core": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.8.2.tgz", - "integrity": "sha512-FXcBL7nyik8K5ODeCKlxi+vts7torOkoDAKfeh61EAkAy1HAvwn9uVzZBY0f15YcQTcZZ2/iSGBFHEuioZWfDA==", + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.9.1.tgz", + "integrity": "sha512-5ad+U2YGrmmiw6du20AQW5XuWo7UKN2052FjSV7MX+Wfjf8sCqcsZe62NfgHys4QI4/Y+vQvLKYL8jWtA1ZBTA==", "dependencies": { - "@actions/http-client": "^2.0.1" + "@actions/http-client": "^2.0.1", + "uuid": "^8.3.2" } }, "node_modules/@actions/github": { @@ -6867,7 +6868,7 @@ "name": "@keymanapp/auto-history-action", "license": "MIT", "dependencies": { - "@actions/core": "^1.2.6", + "@actions/core": "^1.9.1", "@actions/github": "^2.1.0", "typescript": "^4.5.4", "yargs": "^15.1.0" @@ -7122,11 +7123,12 @@ }, "dependencies": { "@actions/core": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.8.2.tgz", - "integrity": "sha512-FXcBL7nyik8K5ODeCKlxi+vts7torOkoDAKfeh61EAkAy1HAvwn9uVzZBY0f15YcQTcZZ2/iSGBFHEuioZWfDA==", + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.9.1.tgz", + "integrity": "sha512-5ad+U2YGrmmiw6du20AQW5XuWo7UKN2052FjSV7MX+Wfjf8sCqcsZe62NfgHys4QI4/Y+vQvLKYL8jWtA1ZBTA==", "requires": { - "@actions/http-client": "^2.0.1" + "@actions/http-client": "^2.0.1", + "uuid": "^8.3.2" } }, "@actions/github": { @@ -7188,7 +7190,7 @@ "@keymanapp/auto-history-action": { "version": "file:resources/build/version", "requires": { - "@actions/core": "^1.2.6", + "@actions/core": "^1.9.1", "@actions/github": "^2.1.0", "@types/node": "^13.7.0", "@types/semver": "^7.1.0", @@ -7355,11 +7357,11 @@ "chalk": "^4.1.2", "copyfiles": "^2.4.1", "express": "^4.17.2", - "hetrodo-node-hide-console-window-napi": "keymanapp/hetrodo-node-hide-console-window-napi#keyman-15.0", + "hetrodo-node-hide-console-window-napi": "git+ssh://git@github.com/keymanapp/hetrodo-node-hide-console-window-napi.git#858b23036a9963b40ad6ff3c5bacd421e5839b92", "mocha": "^9.1.4", "multer": "^1.4.4", "ngrok": "^4.2.2", - "node-windows-trayicon": "keymanapp/node-windows-trayicon#keyman-15.0", + "node-windows-trayicon": "git+ssh://git@github.com/keymanapp/node-windows-trayicon.git#1e46786082213f3edcddd5953e33f5abdc7ea05f", "open": "^8.4.0", "ts-node": "^10.4.0", "tsc-watch": "^4.5.0", diff --git a/resources/build/version/package.json b/resources/build/version/package.json index 1a4e8355c8..33698434fb 100644 --- a/resources/build/version/package.json +++ b/resources/build/version/package.json @@ -1,6 +1,6 @@ { "dependencies": { - "@actions/core": "^1.2.6", + "@actions/core": "^1.9.1", "@actions/github": "^2.1.0", "typescript": "^4.5.4", "yargs": "^15.1.0" From ecad0209cb0a33d7d2be952b182ce7610a029912 Mon Sep 17 00:00:00 2001 From: Ross Date: Mon, 22 Aug 2022 11:51:01 +1000 Subject: [PATCH 011/132] fix: Add invalidate context action to non-updatable parse Rename ProcessActionsTestParse to NonUpdatable parse. It does more then just test. Add handling for the invalidate context action to this parse, so that if we emit a keystroke we invalidate the context --- windows/src/engine/keyman32/kmprocess.cpp | 2 +- windows/src/engine/keyman32/kmprocessactions.cpp | 7 +++++-- windows/src/engine/keyman32/kmprocessactions.h | 6 +++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/windows/src/engine/keyman32/kmprocess.cpp b/windows/src/engine/keyman32/kmprocess.cpp index 8b7286afbc..35e30c6857 100644 --- a/windows/src/engine/keyman32/kmprocess.cpp +++ b/windows/src/engine/keyman32/kmprocess.cpp @@ -190,7 +190,7 @@ BOOL ProcessHook() } if (!_td->TIPFUpdateable) { - ProcessActionsTestParse(&fOutputKeystroke); + ProcessActionsNonUpdatableParse(&fOutputKeystroke); } else { ProcessActions(&fOutputKeystroke); } diff --git a/windows/src/engine/keyman32/kmprocessactions.cpp b/windows/src/engine/keyman32/kmprocessactions.cpp index f0aa36256d..6a5e03c123 100644 --- a/windows/src/engine/keyman32/kmprocessactions.cpp +++ b/windows/src/engine/keyman32/kmprocessactions.cpp @@ -167,7 +167,7 @@ BOOL ProcessActions(BOOL* emitKeyStroke) } BOOL -ProcessActionsTestParse(BOOL* emitKeyStroke) { +ProcessActionsNonUpdatableParse(BOOL* emitKeyStroke) { PKEYMAN64THREADDATA _td = ThreadGlobals(); if (!_td) { return FALSE; @@ -184,13 +184,16 @@ ProcessActionsTestParse(BOOL* emitKeyStroke) { switch (act->type) { case KM_KBP_IT_EMIT_KEYSTROKE: *emitKeyStroke = TRUE; - SendDebugMessageFormat(0, sdmGlobal, 0, "ProcessActionsTestParse EMIT_KEYSTROKE: act->type=%d", act->type); + SendDebugMessageFormat(0, sdmGlobal, 0, "ProcessActionsNonUpdatableParse EMIT_KEYSTROKE: act->type=[%d]", act->type); continueProcessingActions = TRUE; _td->CoreProcessEventRun = FALSE; // If we emit the key stroke on this parse we don't need the second parse break; case KM_KBP_IT_CAPSLOCK: continueProcessingActions = processCapsLock(act, !_td->state.isDown, _td->TIPFUpdateable); break; + case KM_KBP_IT_INVALIDATE_CONTEXT: + continueProcessingActions = processInvalidateContext(_td->app, _td->lpActiveKeyboard->lpCoreKeyboardState); + break; } if (!continueProcessingActions) { return FALSE; diff --git a/windows/src/engine/keyman32/kmprocessactions.h b/windows/src/engine/keyman32/kmprocessactions.h index e96f2226d1..6d269493b2 100644 --- a/windows/src/engine/keyman32/kmprocessactions.h +++ b/windows/src/engine/keyman32/kmprocessactions.h @@ -18,12 +18,12 @@ BOOL ProcessActions(BOOL* emitKeyStroke); /** * This function process the actions queued in the core processor in - * the non updateable parse of a keystroke. - * Emit keystroke and capslock are required to be processed in this phase. + * the non-updateable parse of a keystroke. + * Emit keystroke , capslock, and possibly invalidate key stroke are required to be processed in this phase. * * @param [in, out] emitKeyStroke is set to true if requested by the core action queue * @return BOOL True if actions were successfully processed */ -BOOL ProcessActionsTestParse(BOOL* emitKeyStroke); +BOOL ProcessActionsNonUpdatableParse(BOOL* emitKeyStroke); #endif From 04a0f5a32fc461d17a4961ba3f1aeee056ca97ef Mon Sep 17 00:00:00 2001 From: Ross Date: Mon, 22 Aug 2022 13:42:31 +1000 Subject: [PATCH 012/132] fix: grammar and typos --- windows/src/engine/keyman32/kmprocessactions.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/windows/src/engine/keyman32/kmprocessactions.h b/windows/src/engine/keyman32/kmprocessactions.h index 6d269493b2..80827ea623 100644 --- a/windows/src/engine/keyman32/kmprocessactions.h +++ b/windows/src/engine/keyman32/kmprocessactions.h @@ -19,7 +19,7 @@ BOOL ProcessActions(BOOL* emitKeyStroke); /** * This function process the actions queued in the core processor in * the non-updateable parse of a keystroke. - * Emit keystroke , capslock, and possibly invalidate key stroke are required to be processed in this phase. + * Emit keystroke, capslock and possibly invalidate key stroke are required to be processed in this parse. * * @param [in, out] emitKeyStroke is set to true if requested by the core action queue * @return BOOL True if actions were successfully processed From b03e7ca32f7eadf368714b4d14f11f5f9316977a Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Mon, 22 Aug 2022 17:46:27 +0200 Subject: [PATCH 013/132] chore(core): Remove obsolete python keyboardprocessor This seems to be not used anywhere and has diverged from the API. --- core/python/keyman/keyboardprocessor/_api.py | 573 ------------------- core/python/keyman/keyboardprocessor/api.py | 118 ---- 2 files changed, 691 deletions(-) delete mode 100644 core/python/keyman/keyboardprocessor/_api.py delete mode 100644 core/python/keyman/keyboardprocessor/api.py diff --git a/core/python/keyman/keyboardprocessor/_api.py b/core/python/keyman/keyboardprocessor/_api.py deleted file mode 100644 index 0f2f59bd2c..0000000000 --- a/core/python/keyman/keyboardprocessor/_api.py +++ /dev/null @@ -1,573 +0,0 @@ -# Copyright: © 2018 SIL International. -# Description: Lowlevel C style python API. Intended to be wrapped by a more -# Pythonic higher level API. -# Create Date: 18 Oct 2018 -# Authors: Tim Eves (TSE) -# - -import ctypes -import ctypes.util -import operator -import os -from enum import auto, IntEnum, IntFlag -from ctypes import (c_uint8, - c_uint16, c_uint32, - c_size_t, - c_void_p, c_char_p, - Structure, Union, POINTER, CFUNCTYPE) -from typing import Any, Tuple - -CP = c_uint16 -USV = c_uint32 -VirtualKey = c_uint16 - -libpath = os.environ.get('PYKMNKBD_LIBRARY_PATH', - ctypes.util.find_library("kmnkbp0")) - -libkbp = ctypes.cdll.LoadLibrary(libpath) - - -# Error handling -# ============== -class StatusCode(IntEnum): - OK = 0 - NO_MEM = auto() - IO_ERROR = auto() - INVALID_ARGUMENT = auto() - KEY_ERROR = auto() - OS_ERROR = 0x80000000 - - -Status = c_uint32 - - -def __map_oserror(code: Status) -> str: - code = StatusCode.OS_ERROR ^ code - msg = '{0!s}: OS error code (' + code + '): ' + os.strerror(code) - return msg, lambda m: OSError(code, m) - - -__exceptions_map = [ - (None, '{0!s}: Success'), - (MemoryError, '{0!s}: memory allocation failed.'), - (RuntimeError, '{0!s}: IO Error: {1!s}'), - (ValueError, '{0!s}: Invalid argument passed.'), - (LookupError, '{0!s}: Item does not exist in: {1!s}'), - (OSError, __map_oserror)] - - -def status_code(code: Status, func, args): - if code == StatusCode.OK: return args - exc, msg = __exceptions_map[code] - if callable(msg): msg = msg(code) - raise exc(msg.format(libkbp._name, *args)) - - -def null_check(code, func, args): - if code is not None: return args - raise KeyError(args[1] + ': Not found in collection') - - -class Dir(IntFlag): - IN = auto() - OUT = auto() - OPT = auto() - - -def __method(iface: str, method: str, result, - *args: Tuple[Any, Dir, str], **kwds): - proto = CFUNCTYPE(result, *map(operator.itemgetter(0), args)) - params = tuple(a[1:] for a in args) - c_name = iface+'_'+method if iface else method - f = proto(('km_kbp_' + c_name, libkbp), params) - if 'errcheck' in kwds: f.errcheck = kwds.get('errcheck') - globals()[c_name] = f - - -# Context processing -# ================== -Context_p = c_void_p - - -class ContextType(IntEnum): - END = 0 - CHAR = auto() - MARKER = auto() - - -class ContextItem(Structure): - class __ContextValue(Union): - _fields_ = (('character', USV), - ('marker', c_uint32)) - _anonymous_ = ('value',) - _fields_ = (('type', c_uint8), - ('value', __ContextValue)) - - -__method('context_items', 'from_utf16', Status, - (c_void_p, Dir.IN, 'text'), - (POINTER(POINTER(ContextItem)), Dir.OUT, 'context_items'), - errcheck=status_code) - -__method('context_items', 'to_utf16', c_size_t, - (POINTER(ContextItem), Dir.IN, 'context_items'), - (c_void_p, Dir.IN | Dir.OPT, 'buffer'), - (c_size_t, Dir.IN | Dir.OPT, 'buffer_size')) - -__method('context_items', 'dispose', None, - (POINTER(ContextItem), Dir.IN, 'context_items')) - -__method('context', 'set', Status, - (Context_p, Dir.IN, 'context'), - (POINTER(ContextItem), Dir.IN, 'context_items'), - errcheck=status_code) - -__method('context', 'get', POINTER(ContextItem), - (Context_p, Dir.IN, 'context')) - -__method('context', 'clear', None, (Context_p, Dir.IN, 'context')) - -__method('context', 'length', c_size_t, (Context_p, Dir.IN, 'context')) - -__method('context', 'append', Status, - (Context_p, Dir.IN, 'context'), - (POINTER(ContextItem), Dir.IN, 'context_items'), - errcheck=status_code) - -__method('context', 'shrink', Status, - (Context_p, Dir.IN, 'context'), - (c_size_t, Dir.IN, 'num'), - (POINTER(ContextItem), Dir.IN, 'prefix'), - errcheck=status_code) - - -class ActionItem(Structure): - class __ActionItem(Union): - _fields_ = (('marker', c_size_t), - ('option', c_char_p), - ('character', USV), - ('vkey', VirtualKey)) - _anonymous_ = ('data',) - _fields_ = (('type', c_uint8), - ('reserved', c_uint8*3), - ('data', __ActionItem)) - - -class ActionType(IntEnum): - END = 0 # Marks end of action items list. - CHAR = 1 - MARKER = 2 # correlates to kmn's "deadkey" markers - ALERT = 3 - BACK = 4 - PERSIST_OPT = 5 - RESET_OPT = 6 - VKEYDOWN = 7 - VKEYUP = 8 - VSHIFTDOWN = 9 - VSHIFTUP = 10 - MAX_TYPE_ID = auto() - - -# Option processing -# ================= -OptionSet_p = c_void_p - - -class OptionScope(IntEnum): - UNKNOWN = auto() - KEYBOARD = auto() - ENVIRONMENT = auto() - - -class Option(Structure): - _fields_ = (('key', c_char_p), - ('value', c_char_p), - ('scope', c_uint8)) - - -Option.END = Option(None, None) - -__method('options_set', 'size', c_size_t, (OptionSet_p, Dir.IN, 'opts')) - -__method('options_set', 'lookup', POINTER(Option), - (OptionSet_p, Dir.IN, 'opts'), - (c_char_p, Dir.IN, 'key'), - errcheck=null_check) - -__method('options_set', 'update', Status, - (OptionSet_p, Dir.IN, 'opts'), - (POINTER(Option), Dir.IN, 'new_opts'), - errcheck=status_code) - -__method('options_set', 'to_json', Status, - (OptionSet_p, Dir.IN, 'opts'), - (c_char_p, Dir.IN | Dir.OPT, 'buffer'), - (c_size_t, Dir.IN | Dir.OUT, 'space'), - errcheck=status_code) - - -# Keyboards -# ========= -Keyboard_p = c_void_p - - -class KeyboardAttrs(Structure): - _fields_ = (('version_string', c_char_p), - ('id', c_char_p), - ('folder_path', c_char_p), - ('default_options', OptionSet_p)) - - -__method('keyboard', 'load', Status, - (c_char_p, Dir.IN, 'path'), - (POINTER(Keyboard_p), Dir.OUT, 'kb'), - errcheck=status_code) - -__method('keyboard', 'dispose', None, (Keyboard_p, Dir.IN, 'kb')) - -__method('keyboard', 'get_attrs', POINTER(KeyboardAttrs), - (Keyboard_p, Dir.IN, 'keyboard')) - - -# State processing -# ================ -State_p = c_void_p - -__method('state', 'create', Status, - (Keyboard_p, Dir.IN, 'keyboard'), - (POINTER(Option), Dir.IN, 'env',), - (POINTER(State_p), Dir.OUT, 'out'), - errcheck=status_code) - -__method('state', 'clone', Status, - (State_p, Dir.IN, 'state'), - (POINTER(State_p), Dir.OUT, 'out'), - errcheck=status_code) - -__method('state', 'dispose', None, (State_p, Dir.IN, 'state')) - -__method('state', 'context', Context_p, (State_p, Dir.IN, 'state')) - -__method('state', 'options', OptionSet_p, (State_p, Dir.IN, 'state')) - -__method('state', 'action_items', POINTER(ActionItem), - (State_p, Dir.IN, 'state'), - (POINTER(c_size_t), Dir.OUT, 'num_items')) - -__method('state', 'to_json', Status, - (State_p, Dir.IN, 'state'), - (c_char_p, Dir.IN | Dir.OPT, 'buffer'), - (c_size_t, Dir.IN | Dir.OUT, 'space'), - errcheck=status_code) - - -# Processor -# ========= -class Attributes(Structure): - _fields_ = (('max_context', c_size_t), - ('current', c_uint16), - ('revision', c_uint16), - ('age', c_uint16), - ('technology', c_uint16), - ('vendor', c_char_p)) - - -class Tech(IntFlag): - UNSPECIFIED = 0 - KMN = 1 - LDML = 2 - - -__method(None, 'get_engine_attrs', POINTER(Attributes)) - -__method(None, 'process_event', Status, - (State_p, Dir.IN, 'state'), - (VirtualKey, Dir.IN, 'vkey'), - (c_uint16, Dir.IN, 'modifier_state')) - - -class Modifier(IntFlag): - LCTRL = 1 << 0 - RCTRL = 1 << 1 - LALT = 1 << 2 - RALT = 1 << 3 - SHIFT = 1 << 4 - CTRL = 1 << 5 - ALT = 1 << 6 - CAPS = 1 << 7 - NOCAPS = 1 << 8 - NUMLOCK = 1 << 9 - NONUMLOCK = 1 << 10 - SCROLLOCK = 1 << 11 - NOSCROLLOCK = 1 << 12 - VIRTUALKEY = 1 << 13 - - -class ModifierMask(IntFlag): - ALL = 0x7f - ALT_GR_SIM = Modifier.LCTRL | Modifier.LALT - CHIRAL = 0x1f - IS_CHIRAL = 0x0f - NON_CHIRAL = 0x7f - CAPS = 0x0300 - NUMLOCK = 0x0C00 - SCROLLLOCK = 0x3000 - - -class VKey(IntEnum): - _00 = auto() - LBUTTON = auto() - RBUTTON = auto() - CANCEL = auto() - MBUTTON = auto() - _05 = auto() - _06 = auto() - _07 = auto() - BKSP = auto() - KTAB = auto() - _0A = auto() - _0B = auto() - KP5 = auto() - ENTER = auto() - _0E = auto() - _0F = auto() - SHIFT = auto() - CONTROL = auto() - ALT = auto() - PAUSE = auto() - CAPS = auto() - _15 = auto() - _16 = auto() - _17 = auto() - _18 = auto() - _19 = auto() - _1A = auto() - ESC = auto() - _1C = auto() - _1D = auto() - _1E = auto() - _1F = auto() - SPACE = auto() - PGUP = auto() - PGDN = auto() - END = auto() - HOME = auto() - LEFT = auto() - UP = auto() - RIGHT = auto() - DOWN = auto() - SEL = auto() - PRINT = auto() - EXEC = auto() - PRTSCN = auto() - INS = auto() - DEL = auto() - HELP = auto() - K0 = auto() - K1 = auto() - K2 = auto() - K3 = auto() - K4 = auto() - K5 = auto() - K6 = auto() - K7 = auto() - K8 = auto() - K9 = auto() - _3A = auto() - _3B = auto() - _3C = auto() - _3D = auto() - _3E = auto() - _3F = auto() - _40 = auto() - KA = auto() - KB = auto() - KC = auto() - KD = auto() - KE = auto() - KF = auto() - KG = auto() - KH = auto() - KI = auto() - KJ = auto() - KK = auto() - KL = auto() - KM = auto() - KN = auto() - KO = auto() - KP = auto() - KQ = auto() - KR = auto() - KS = auto() - KT = auto() - KU = auto() - KV = auto() - KW = auto() - KX = auto() - KY = auto() - KZ = auto() - _5B = auto() - _5C = auto() - _5D = auto() - _5E = auto() - _5F = auto() - NP0 = auto() - NP1 = auto() - NP2 = auto() - NP3 = auto() - NP4 = auto() - NP5 = auto() - NP6 = auto() - NP7 = auto() - NP8 = auto() - NP9 = auto() - NPSTAR = auto() - NPPLUS = auto() - SEPARATOR = auto() - NPMINUS = auto() - NPDOT = auto() - NPSLASH = auto() - F1 = auto() - F2 = auto() - F3 = auto() - F4 = auto() - F5 = auto() - F6 = auto() - F7 = auto() - F8 = auto() - F9 = auto() - F10 = auto() - F11 = auto() - F12 = auto() - F13 = auto() - F14 = auto() - F15 = auto() - F16 = auto() - F17 = auto() - F18 = auto() - F19 = auto() - F20 = auto() - F21 = auto() - F22 = auto() - F23 = auto() - F24 = auto() - _88 = auto() - _89 = auto() - _8A = auto() - _8B = auto() - _8C = auto() - _8D = auto() - _8E = auto() - _8F = auto() - NUMLOCK = auto() - SCROLL = auto() - _92 = auto() - _93 = auto() - _94 = auto() - _95 = auto() - _96 = auto() - _97 = auto() - _98 = auto() - _99 = auto() - _9A = auto() - _9B = auto() - _9C = auto() - _9D = auto() - _9E = auto() - _9F = auto() - _A0 = auto() - _A1 = auto() - _A2 = auto() - _A3 = auto() - _A4 = auto() - _A5 = auto() - _A6 = auto() - _A7 = auto() - _A8 = auto() - _A9 = auto() - _AA = auto() - _AB = auto() - _AC = auto() - _AD = auto() - _AE = auto() - _AF = auto() - _B0 = auto() - _B1 = auto() - _B2 = auto() - _B3 = auto() - _B4 = auto() - _B5 = auto() - _B6 = auto() - _B7 = auto() - _B8 = auto() - _B9 = auto() - COLON = auto() - EQUAL = auto() - COMMA = auto() - HYPHEN = auto() - PERIOD = auto() - SLASH = auto() - BKQUOTE = auto() - _C1 = auto() - _C2 = auto() - _C3 = auto() - _C4 = auto() - _C5 = auto() - _C6 = auto() - _C7 = auto() - _C8 = auto() - _C9 = auto() - _CA = auto() - _CB = auto() - _CC = auto() - _CD = auto() - _CE = auto() - _CF = auto() - _D0 = auto() - _D1 = auto() - _D2 = auto() - _D3 = auto() - _D4 = auto() - _D5 = auto() - _D6 = auto() - _D7 = auto() - _D8 = auto() - _D9 = auto() - _DA = auto() - LBRKT = auto() - BKSLASH = auto() - RBRKT = auto() - QUOTE = auto() - oDF = auto() - oE0 = auto() - oE1 = auto() - oE2 = auto() - oE3 = auto() - oE4 = auto() - _E5 = auto() - oE6 = auto() - _E7 = auto() - _E8 = auto() - oE9 = auto() - oEA = auto() - oEB = auto() - oEC = auto() - oED = auto() - oEE = auto() - oEF = auto() - oF0 = auto() - oF1 = auto() - oF2 = auto() - oF3 = auto() - oF4 = auto() - oF5 = auto() - _F6 = auto() - _F7 = auto() - _F8 = auto() - _F9 = auto() - _FA = auto() - _FB = auto() - _FC = auto() - _FD = auto() - _FE = auto() - _FF = auto() diff --git a/core/python/keyman/keyboardprocessor/api.py b/core/python/keyman/keyboardprocessor/api.py deleted file mode 100644 index 6fcedf18e5..0000000000 --- a/core/python/keyman/keyboardprocessor/api.py +++ /dev/null @@ -1,118 +0,0 @@ -import pathlib -from typing import NamedTuple, NewType, Tuple, List, Union -from enum import Enum -from collections.abc import Sequence, Mapping - -USV = int -Marker = int -VirtualKey = int - - -class Context(Sequence): - Item = Union[USV, Marker] - - def __init__(initial: str): - pass - - def __del__(self): - pass - - def __str__(self): - pass - - def set(self, ctxt: List[Item]): - self.clear() - self.apped(ctxt) - - def clear(self): - pass - - def __getitem__(self, key: int) -> Item: - pass - - def __len__(self): - pass - - def append(self, ctxt: List[Item]): - pass - - def delete(self, remove_n: int, prefix: List[Item]): - pass - - -Option = Tuple[str, str] - - -class OptionSet(Mapping): - def __getitem__(self, key: str) -> Option: - pass - - def __iter__(self): - pass - - def __len__(self): - pass - - def __str__(self): - pass - - -class Keyboard(NamedTuple('__kb_attrs', version=str, id=str, folder_path=pathlib.Path, default_options=OptionSet)): - def __new__(cls, _): - return super(Keyboard, cls).__new__(cls, *[None]*4) - - def __init__(self, kb_path: pathlib.Path): - pass - - def __del__(self): - pass - - -class Action: - VKeyDown = NewType('Action.VirtualKey', VirtualKey) - # VKeyUp = VirtualKey - # VShiftDown = VirtualKey - # VShiftUp = VirtualKey - # Char = int - # Marker = int - # Bell = NewType('Bell', None) - # Back = NewType('Back', None) - # PersistOpt = str - # ResetOpt = str - - -ActionList = List[Action] - - -class State: - def __init__(self, kb: Keyboard, env: OptionSet): - pass - - def __del__(self): - pass - - @property - def flags(self) -> int: - pass - - @property - def context(self) -> Context: - pass - - @property - def environment(self) -> OptionSet: - pass - - @property - def options(self) -> OptionSet: - pass - - def indentify_option_src(opt: Option): - pass - - -def process_event(vk: VirtualKey, - modifier_state, - state: State, - acts: ActionList): - pass From d92ac6e4d1889f257d55247aa7143a8660588bec Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Mon, 22 Aug 2022 14:02:00 -0400 Subject: [PATCH 014/132] auto: increment master version to 16.0.50 --- HISTORY.md | 5 +++++ VERSION.md | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 1878d3ad4a..0c1646e28e 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,10 @@ # Keyman Version History +## 16.0.49 alpha 2022-08-22 + +* fix: remove saving and restoring context kbd options (#7077) +* chore(deps): bump @actions/core from 1.8.2 to 1.9.1 (#7087) + ## 16.0.48 alpha 2022-08-16 * fix(web): button, float init timer cleanup (#7036) diff --git a/VERSION.md b/VERSION.md index 214271b7a3..245e8fdca3 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.49 \ No newline at end of file +16.0.50 \ No newline at end of file From 9bff83fa00df2be69e28e1833a1c29cebdcc90e3 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 23 Aug 2022 06:43:46 +1000 Subject: [PATCH 015/132] chore: Change platform advocates per discussion As discussed, we are changing some of the platform advocates for (a) geographical, and (b) platform familiarity reasons. This will be reflected in the automatic assignment of reviewers on PRs. We are pretty much all affected in this shuffle! @keymanapp-test-bot skip --- docs/CODEOWNERS | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/docs/CODEOWNERS b/docs/CODEOWNERS index eff7811f5e..021a6fc8bc 100644 --- a/docs/CODEOWNERS +++ b/docs/CODEOWNERS @@ -3,32 +3,31 @@ # # @darcywong00 @mcdurdin @ermshiperete @rc-swag @SabineSIL @sgschantz -/android/ @darcywong00 @rc-swag +/android/ @darcywong00 @mcdurdin -/common/ @mcdurdin @jahorton -/core/ @mcdurdin @jahorton +/common/ @mcdurdin @rc-swag +/core/ @mcdurdin @rc-swag /common/lexical-model-types/ @jahorton @mcdurdin /common/models/ @jahorton @mcdurdin /common/predictive-text/ @jahorton @mcdurdin /common/schemas/ @mcdurdin @jahorton /common/test/ @mcdurdin @ermshiperete -/common/web/ @jahorton @ermshiperete @mcdurdin +/common/web/ @jahorton @sgschantz @mcdurdin /developer/ @mcdurdin @darcywong00 /docs/ @mcdurdin @jahorton -/ios/ @sgschantz @mcdurdin +/ios/ @sgschantz @jahorton /linux/ @ermshiperete @darcywong00 /mac/ @sgschantz @SabineSIL -/oem/firstvoices/android/ @darcywong00 @rc-swag -/oem/firstvoices/common/ @mcdurdin @jahorton -/oem/firstvoices/ios/ @sgschantz @mcdurdin -/oem/firstvoices/windows/ @rc-swag @sgschantz +/oem/firstvoices/android/ @darcywong00 @mcdurdin +/oem/firstvoices/common/ @mcdurdin @rc-swag +/oem/firstvoices/ios/ @sgschantz @jahorton +/oem/firstvoices/windows/ @rc-swag @ermshiperete /resources/ @mcdurdin @jahorton # Web is currently shared between Marc and Joshua: -/web/ @jahorton @ermshiperete @mcdurdin +/web/ @jahorton @sgschantz @mcdurdin + +/windows/ @rc-swag @ermshiperete -/windows/ @rc-swag @sgschantz -# Override for windows/src: -/windows/src/developer/ @mcdurdin @darcywong00 From f90eaee6ddbbbdeb372b3a21351f56de4e1aace6 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 23 Aug 2022 07:41:56 +1000 Subject: [PATCH 016/132] fix(android): add toast when browser fails to open --- .../java/com/tavultesoft/kmapro/KMPBrowserActivity.java | 3 +++ .../com/tavultesoft/kmapro/KeyboardSettingsActivity.java | 3 +++ .../main/java/com/tavultesoft/kmapro/MainActivity.java | 9 +++++++++ android/KMAPro/kMAPro/src/main/res/values/strings.xml | 3 +++ .../java/com/tavultesoft/kmea/KeyboardInfoActivity.java | 2 ++ android/KMEA/app/src/main/res/values/strings.xml | 6 ++++-- .../main/java/com/firstvoices/keyboards/FVShared.java | 3 +++ .../java/com/firstvoices/keyboards/MainActivity.java | 2 ++ .../android/app/src/main/res/values/strings.xml | 2 ++ 9 files changed, 31 insertions(+), 2 deletions(-) diff --git a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/KMPBrowserActivity.java b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/KMPBrowserActivity.java index ff74db201f..fe217c57e4 100644 --- a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/KMPBrowserActivity.java +++ b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/KMPBrowserActivity.java @@ -18,6 +18,7 @@ import android.webkit.WebChromeClient; import android.webkit.WebSettings; import android.webkit.WebView; import android.webkit.WebViewClient; +import android.widget.Toast; import androidx.appcompat.app.AppCompatActivity; import com.tavultesoft.kmea.BaseActivity; @@ -115,6 +116,8 @@ public class KMPBrowserActivity extends BaseActivity { Intent intent = new Intent(Intent.ACTION_VIEW, uri); if (intent.resolveActivity(getPackageManager()) != null) { startActivity(intent); + } else { + Toast.makeText(context, getString(R.string.unable_to_open_browser), Toast.LENGTH_SHORT).show(); } return true; } diff --git a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/KeyboardSettingsActivity.java b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/KeyboardSettingsActivity.java index b471fc1cee..dc5519095a 100644 --- a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/KeyboardSettingsActivity.java +++ b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/KeyboardSettingsActivity.java @@ -26,6 +26,7 @@ import android.widget.ListAdapter; import android.widget.ListView; import android.widget.SimpleAdapter; import android.widget.TextView; +import android.widget.Toast; import com.tavultesoft.kmea.ConfirmDialogFragment; import com.tavultesoft.kmea.KMHelpFileActivity; @@ -162,6 +163,8 @@ public final class KeyboardSettingsActivity extends AppCompatActivity { i.putExtras(args); if (i.resolveActivity(getPackageManager()) != null) { startActivity(i); + } else { + Toast.makeText(getApplicationContext(), getString(R.string.unable_to_open_browser), Toast.LENGTH_SHORT).show(); } finish(); diff --git a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/MainActivity.java b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/MainActivity.java index af9b16fbc1..2a9626984a 100644 --- a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/MainActivity.java +++ b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/MainActivity.java @@ -764,12 +764,21 @@ public class MainActivity extends BaseActivity implements OnKeyboardEventListene Intent intent = new Intent(Intent.ACTION_VIEW, Uri.parse("market://details?id=com.android.chrome")); if (intent.resolveActivity(getPackageManager()) != null) { startActivity(intent); + } else { + intent = new Intent(Intent.ACTION_VIEW, Uri.parse("https://play.google.com/store/apps/details?id=com.android.chrome")); + if (intent.resolveActivity(getPackageManager()) != null) { + startActivity(intent); + } else { + Toast.makeText(getApplicationContext(), getString(R.string.unable_to_open_browser), Toast.LENGTH_SHORT).show(); + } } } catch (android.content.ActivityNotFoundException e) { // Link to Chrome if user is not signed in to Play Store Intent intent = new Intent(Intent.ACTION_VIEW, Uri.parse("https://play.google.com/store/apps/details?id=com.android.chrome")); if (intent.resolveActivity(getPackageManager()) != null) { startActivity(intent); + } else { + Toast.makeText(getApplicationContext(), getString(R.string.unable_to_open_browser), Toast.LENGTH_SHORT).show(); } } } diff --git a/android/KMAPro/kMAPro/src/main/res/values/strings.xml b/android/KMAPro/kMAPro/src/main/res/values/strings.xml index 09a91b5f60..ed4e5b85a1 100644 --- a/android/KMAPro/kMAPro/src/main/res/values/strings.xml +++ b/android/KMAPro/kMAPro/src/main/res/values/strings.xml @@ -247,4 +247,7 @@ Keyboard requires a newer version of Keyman + + + Unable to launch web browser diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardInfoActivity.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardInfoActivity.java index 89bab42c66..4b818acf7c 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardInfoActivity.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardInfoActivity.java @@ -138,6 +138,8 @@ public final class KeyboardInfoActivity extends BaseActivity { i.setData(Uri.parse(customHelpLink)); if (i.resolveActivity(getPackageManager()) != null) { startActivity(i); + } else { + Toast.makeText(context, getString(R.string.unable_to_open_browser), Toast.LENGTH_SHORT).show(); } } } diff --git a/android/KMEA/app/src/main/res/values/strings.xml b/android/KMEA/app/src/main/res/values/strings.xml index 8f7d20150d..50f52c5ef3 100644 --- a/android/KMEA/app/src/main/res/values/strings.xml +++ b/android/KMEA/app/src/main/res/values/strings.xml @@ -3,10 +3,10 @@ AndroidMobile - + KMEA - + Keyboard @@ -267,4 +267,6 @@ Tap here to change keyboard + + Unable to launch web browser diff --git a/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/FVShared.java b/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/FVShared.java index 9e8790acda..d308049417 100644 --- a/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/FVShared.java +++ b/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/FVShared.java @@ -6,6 +6,7 @@ import android.content.SharedPreferences; import android.content.res.AssetManager; import android.net.Uri; import android.util.Log; +import android.widget.Toast; import com.tavultesoft.kmea.KMManager; import com.tavultesoft.kmea.data.Keyboard; import com.tavultesoft.kmea.packages.PackageProcessor; @@ -254,6 +255,8 @@ final class FVShared { i.setData(Uri.parse(helpUrl)); if (i.resolveActivity(localContext.getPackageManager()) != null) { localContext.startActivity(i); + } else { + Toast.makeText(localContext, localContext.getString(R.string.unable_to_open_browser), Toast.LENGTH_SHORT).show(); } } diff --git a/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/MainActivity.java b/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/MainActivity.java index 897145ec5f..95309efe48 100644 --- a/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/MainActivity.java +++ b/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/MainActivity.java @@ -107,6 +107,8 @@ public class MainActivity extends AppCompatActivity implements OnKeyboardDownloa Intent i = new Intent(Intent.ACTION_VIEW, Uri.parse(url)); if (i.resolveActivity(getPackageManager()) != null) { startActivity(i); + } else { + Toast.makeText(context, getString(R.string.unable_to_open_browser), Toast.LENGTH_SHORT).show(); } } diff --git a/oem/firstvoices/android/app/src/main/res/values/strings.xml b/oem/firstvoices/android/app/src/main/res/values/strings.xml index 83edf8f86b..e443e0f717 100644 --- a/oem/firstvoices/android/app/src/main/res/values/strings.xml +++ b/oem/firstvoices/android/app/src/main/res/values/strings.xml @@ -42,4 +42,6 @@ Invalid/Missing metadata in package + + Unable to launch web browser From 8ab42029d089d389807db3bea1bbd7f432f461e4 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 23 Aug 2022 11:20:44 +1000 Subject: [PATCH 017/132] chore(common): add variable support for --options in builder script --- resources/build/build-utils.sh | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/resources/build/build-utils.sh b/resources/build/build-utils.sh index ab9c2a469c..968204c64f 100755 --- a/resources/build/build-utils.sh +++ b/resources/build/build-utils.sh @@ -410,12 +410,16 @@ _builder_trim() { # # There are three types of parameters that may be specified: # -# * Option, param_desc format: "--option[,-o] [One line description]" +# * Option, param_desc format: "--option[,-o][=var] [One line description]" # All options must have a longhand form with two prefix hyphens, # e.g. --option. The ",-o" shorthand form is optional. When testing if # the option is set with `builder_has_option``, always use the longhand # form. # +# if =var is specified, then the next parameter will be a variable stored +# in $var for that option. e.g. --option=opt means $opt will have the value +# 'foo' when the script is called for --option foo. +# # * Action, param_desc format: "action [One line description]" # Actions must be a single word, lower case. To specify an action # as the default, append a '+' to the action name, e.g. @@ -433,6 +437,7 @@ builder_describe() { _builder_default_action=build declare -A -g _builder_params declare -A -g _builder_options_short + declare -A -g _builder_options_var shift # describe each target, action, and option possibility while [[ $# -gt 0 ]]; do @@ -449,14 +454,26 @@ builder_describe() { elif [[ $value =~ ^-- ]]; then # Parameter is an option # Look for a shorthand version of the option + local option_var= + if [[ $value =~ = ]]; then + option_var="$(echo "$value" | cut -d= -f 2 -)" + value="$(echo "$value" | cut -d= -f 1 -)" + fi + if [[ $value =~ , ]]; then local option_long="$(echo "$value" | cut -d, -f 1 -)" local option_short="$(echo "$value" | cut -d, -f 2 -)" _builder_options+=($option_long) _builder_options_short[$option_short]="$option_long" + _builder_options_var[$option_long]="$option_var" value="$option_long, $option_short" else _builder_options+=($value) + _builder_options_var[$value]="$option_var" + fi + + if [[ ! -z $option_var ]]; then + value="$value $option_var" fi else # Parameter is an action @@ -574,6 +591,15 @@ builder_parse() { _builder_chosen_action_targets+=("$_builder_default_action$target") elif (( has_option )); then _builder_chosen_options+=("$key") + if [[ ! -z ${_builder_options_var[$key]+x} ]]; then + shift + if [[ $# -eq 0 ]]; then + _builder_parameter_error "$0" parameter "$key" + fi + # Set the variable associated with this option to the next parameter value + eval ${_builder_options_var[$key]}="$1" + fi + else case "$key" in --help|-h) @@ -632,7 +658,10 @@ builder_display_usage() { local width=12 for e in "${!_builder_params[@]}"; do - if (( ${#e} > $width )); then width = ${#e}; fi + if (( ${#e} > $width )); then + echo ${#e} + width=${#e} + fi done width=$((width + 6)) From e27bf34c7dc3e6df654ca9d1d4baacbd78068056 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 24 Aug 2022 03:40:57 +1000 Subject: [PATCH 018/132] chore(common): tweak variable parameter support --- resources/build/build-utils.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/resources/build/build-utils.sh b/resources/build/build-utils.sh index 968204c64f..7cff578625 100755 --- a/resources/build/build-utils.sh +++ b/resources/build/build-utils.sh @@ -27,6 +27,11 @@ # Note: keep changes to version, tier and tag determination in sync with mkver (windows/src/buildutils/mkver) # +# +# Prevents 'clear' on exit of mingw64 bash shell +# +SHLVL=0 + # Setup variable for calling script's path and name if [ ! -z ${THIS_SCRIPT+x} ]; then THIS_SCRIPT_PATH="$(dirname "$THIS_SCRIPT")" @@ -597,7 +602,10 @@ builder_parse() { _builder_parameter_error "$0" parameter "$key" fi # Set the variable associated with this option to the next parameter value - eval ${_builder_options_var[$key]}="$1" + # A little bit of hoop jumping here to avoid issues with cygwin paths being + # corrupted too early in the game + local varname=${_builder_options_var[$key]} + declare -g $varname="$1" fi else From 026809f9adf0707d8dcb91673fd6de7a9674ba90 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 24 Aug 2022 03:54:39 +1000 Subject: [PATCH 019/132] chore(common): remove debug code --- resources/build/build-utils.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/resources/build/build-utils.sh b/resources/build/build-utils.sh index 7cff578625..3268cbd55e 100755 --- a/resources/build/build-utils.sh +++ b/resources/build/build-utils.sh @@ -667,7 +667,6 @@ builder_display_usage() { for e in "${!_builder_params[@]}"; do if (( ${#e} > $width )); then - echo ${#e} width=${#e} fi done From e0366f1eb44f5943995bea0d17fcf9643fc04d04 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Tue, 23 Aug 2022 15:04:57 -0400 Subject: [PATCH 020/132] auto: increment master version to 16.0.51 --- HISTORY.md | 4 ++++ VERSION.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 0c1646e28e..f3ba952cd4 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,9 @@ # Keyman Version History +## 16.0.50 alpha 2022-08-23 + +* chore(core): Remove obsolete python keyboardprocessor (#7094) + ## 16.0.49 alpha 2022-08-22 * fix: remove saving and restoring context kbd options (#7077) diff --git a/VERSION.md b/VERSION.md index 245e8fdca3..251a25b345 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.50 \ No newline at end of file +16.0.51 \ No newline at end of file From 748a4ca63f263e3ed2a14de96a4e07f0740297de Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 24 Aug 2022 09:55:26 +1000 Subject: [PATCH 021/132] chore(common): fixup support for options which have no variable param --- resources/build/build-utils.sh | 8 ++++++-- resources/build/build-utils.test.sh | 20 ++++++++++++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/resources/build/build-utils.sh b/resources/build/build-utils.sh index 3268cbd55e..9a26531bf7 100755 --- a/resources/build/build-utils.sh +++ b/resources/build/build-utils.sh @@ -470,11 +470,15 @@ builder_describe() { local option_short="$(echo "$value" | cut -d, -f 2 -)" _builder_options+=($option_long) _builder_options_short[$option_short]="$option_long" - _builder_options_var[$option_long]="$option_var" + if [[ ! -z "$option_var" ]]; then + _builder_options_var[$option_long]="$option_var" + fi value="$option_long, $option_short" else _builder_options+=($value) - _builder_options_var[$value]="$option_var" + if [[ ! -z "$option_var" ]]; then + _builder_options_var[$value]="$option_var" + fi fi if [[ ! -z $option_var ]]; then diff --git a/resources/build/build-utils.test.sh b/resources/build/build-utils.test.sh index 3fa7902940..efc06ae483 100755 --- a/resources/build/build-utils.test.sh +++ b/resources/build/build-utils.test.sh @@ -94,7 +94,7 @@ builder_parse_test() { shift shift local parameters="$@" - echo "Testing: builder_parse $parameters" + echo "${COLOR_BLUE}## Testing: builder_parse $parameters${COLOR_RESET}" builder_parse $parameters || fail "builder_parse died under curious circumstances" if [[ "$expected" != "${_builder_chosen_action_targets[@]}" ]]; then fail " Test: builder_parse $parameters action:target != \"$expected\"" @@ -112,7 +112,8 @@ builder_describe \ ":app" \ ":engine Thomas, y'know" \ "--power,-p Use powerful mode" \ - "--zoom,-z Use zoom mode" + "--zoom,-z Use zoom mode" \ + "--feature=FOO Enable feature foo" # Test --options @@ -132,6 +133,21 @@ else fail "FAIL: --zoom option not found" fi +# Test --feature + +echo "${COLOR_BLUE}## Testing: builder_parse --feature xyzzy${COLOR_RESET}" +builder_parse --feature xyzzy + +if builder_has_option --feature; then + if [[ $FOO == xyzzy ]]; then + echo "PASS: --feature option variable \$FOO has expected value 'xyzzy'" + else + echo "FAIL: --feature option variable \$FOO had value '$FOO' but should have had 'xyzzy'" + fi +else + echo "FAIL: --feature option not found" +fi + # Finally, run with --help so we can see what it looks like echo "${COLOR_BLUE}## Testing --help${COLOR_RESET}" From 987e2d37d297a6c908256daf1d83c2292a0c8c4c Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Wed, 24 Aug 2022 15:20:35 +0700 Subject: [PATCH 022/132] fix(android/engine): Lower the max height for landscape orientation --- android/KMEA/app/src/main/res/values-land/dimens.xml | 2 +- android/KMEA/app/src/main/res/values-sw600dp-land/dimens.xml | 2 +- android/KMEA/app/src/main/res/values-sw720dp-land/dimens.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/android/KMEA/app/src/main/res/values-land/dimens.xml b/android/KMEA/app/src/main/res/values-land/dimens.xml index 1e0bfcf55f..00cb4ae358 100644 --- a/android/KMEA/app/src/main/res/values-land/dimens.xml +++ b/android/KMEA/app/src/main/res/values-land/dimens.xml @@ -2,7 +2,7 @@ 40dp - 150dp + 100dp 49.25dp 49.25dp 21dp diff --git a/android/KMEA/app/src/main/res/values-sw600dp-land/dimens.xml b/android/KMEA/app/src/main/res/values-sw600dp-land/dimens.xml index 020aab6a3b..613a8b8526 100644 --- a/android/KMEA/app/src/main/res/values-sw600dp-land/dimens.xml +++ b/android/KMEA/app/src/main/res/values-sw600dp-land/dimens.xml @@ -2,7 +2,7 @@ 50dp - 200dp + 140dp 73.5dp 58.5dp 32dp diff --git a/android/KMEA/app/src/main/res/values-sw720dp-land/dimens.xml b/android/KMEA/app/src/main/res/values-sw720dp-land/dimens.xml index 88a93389bb..3944826815 100644 --- a/android/KMEA/app/src/main/res/values-sw720dp-land/dimens.xml +++ b/android/KMEA/app/src/main/res/values-sw720dp-land/dimens.xml @@ -2,7 +2,7 @@ 60dp - 300dp + 200dp 98.5dp 98.5dp 42dp From c486486275ee6ec361bab0f695334c9601822bf0 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Wed, 24 Aug 2022 14:01:55 -0400 Subject: [PATCH 023/132] auto: increment master version to 16.0.52 --- HISTORY.md | 7 +++++++ VERSION.md | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index f3ba952cd4..12579740fa 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,12 @@ # Keyman Version History +## 16.0.51 alpha 2022-08-24 + +* fix(android): verify browser before starting activity (#7001) +* chore: Change platform advocates per discussion (#7096) +* fix(windows): Add invalidate context action to non-updatable parse (#7089) +* feat(common): add parameter variable support to `builder_` functions (#7103) + ## 16.0.50 alpha 2022-08-23 * chore(core): Remove obsolete python keyboardprocessor (#7094) diff --git a/VERSION.md b/VERSION.md index 251a25b345..6e328ac836 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.51 \ No newline at end of file +16.0.52 \ No newline at end of file From 7169723d543d46b284fde837a66ebb1c88788714 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Thu, 25 Aug 2022 18:35:42 +0200 Subject: [PATCH 024/132] fix(linux): Remove wrong `ok_for_single_backspace` method The implementation of `ok_for_single_backspace` checked the current action_item multiple times. However, we already know that that item is `KM_KBP_IT_BACK` because the only place this method gets called is from the `process_backspace_action` method. So basically this method always returns `FALSE`. "Fixing" this method to check the next action_items makes things fail. Removing this method since it doesn't do anything except burn processing cycles... Fixes #5614. --- linux/ibus-keyman/src/engine.c | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/linux/ibus-keyman/src/engine.c b/linux/ibus-keyman/src/engine.c index 81625cc672..bfc267486c 100644 --- a/linux/ibus-keyman/src/engine.c +++ b/linux/ibus-keyman/src/engine.c @@ -508,16 +508,6 @@ static void forward_backspace(IBusKeymanEngine *keyman, unsigned int state) ibus_engine_forward_key_event((IBusEngine *)keyman, KEYMAN_BACKSPACE_KEYSYM, KEYMAN_BACKSPACE, state); } -static gboolean ok_for_single_backspace(const km_kbp_action_item *action_items, int i, size_t num_actions) -{ - for (int j=i+1; j < num_actions; j++) { - if (action_items[i].type == KM_KBP_IT_BACK || action_items[i].type == KM_KBP_IT_CHAR || action_items[i].type == KM_KBP_IT_EMIT_KEYSTROKE) { - return FALSE; - } - } - return TRUE; -} - static gboolean process_unicode_char_action( IBusKeymanEngine *keyman, @@ -601,10 +591,6 @@ process_backspace_action( } g_free(keyman->char_buffer); keyman->char_buffer = new_buffer; - } else if (ok_for_single_backspace(action_items, i, num_action_items)) { - // single backspace can be handled by ibus as normal - g_message("no char actions, just single back"); - return FALSE; } else { g_message( "DAR: process_backspace_action - client_capabilities=%x, %x", engine->client_capabilities, IBUS_CAP_SURROUNDING_TEXT); From e304f054db0893bc7a766f1f2bfe2f1582beed49 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Fri, 26 Aug 2022 14:01:39 -0400 Subject: [PATCH 025/132] auto: increment master version to 16.0.53 --- HISTORY.md | 5 +++++ VERSION.md | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 12579740fa..da3efb69d9 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,10 @@ # Keyman Version History +## 16.0.52 alpha 2022-08-26 + +* fix(android/engine): Lower the max height for landscape orientation (#7119) +* fix(linux): Remove wrong `ok_for_single_backspace` method (#7123) + ## 16.0.51 alpha 2022-08-24 * fix(android): verify browser before starting activity (#7001) diff --git a/VERSION.md b/VERSION.md index 6e328ac836..8e52754086 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.52 \ No newline at end of file +16.0.53 \ No newline at end of file From 4816e590cf2af883d5cc38c835d99b29da733497 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 26 Aug 2022 17:08:56 -0500 Subject: [PATCH 026/132] feat(linux): Dockerfile for linux builder - add Dockerfile in linux/Dockerfile, referencing linux/debian/control - add docs in docs/build/linux-ubuntu.md Fixes: #7102 --- docs/build/linux-ubuntu.md | 20 ++++++++++++++++++++ linux/Dockerfile | 24 ++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 linux/Dockerfile diff --git a/docs/build/linux-ubuntu.md b/docs/build/linux-ubuntu.md index 1a4be89885..570f233298 100644 --- a/docs/build/linux-ubuntu.md +++ b/docs/build/linux-ubuntu.md @@ -64,3 +64,23 @@ All dependencies are already installed if you followed the instructions under [P Building: * [Building Keyman Core](../../core/doc/BUILDING.md) + +## Docker Builder + +The Docker builder allows you to perform a linux build from anywhere Docker is supported. +To build the docker image: + +```shell +cd ../../linux +docker pull ubuntu:latest +docker build . -t keymanapp/keyman-linux-builder:latest +``` + +Once the image is built, it may be used to build parts of Keyman. + +```shell +cd ../core +# keep linux build artifacts separate +mkdir -p build/linux +docker run -it --rm -v $(pwd)/..:/home/build -v $(pwd)/build/linux:/home/build/core/build keyman-linux-builder:latest bash -c 'cd core; bash build.sh -d' +``` diff --git a/linux/Dockerfile b/linux/Dockerfile new file mode 100644 index 0000000000..203d33aef1 --- /dev/null +++ b/linux/Dockerfile @@ -0,0 +1,24 @@ +# Copyright (c) 2022 SIL International. All rights reserved. +# +# builder image for a linux build +# see ../docs/build/linux-ubuntu.md + +FROM --platform=amd64 ubuntu:latest +LABEL org.opencontainers.image.authors="SIL International." +LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git" +LABEL org.opencontainers.image.title="Keyman Linux Build Image" +# We will switch to a build user after some installation +USER root +RUN useradd -c "Build user" -d $HOME -m build +ENV HOME /home/build +VOLUME /home/build +WORKDIR /home/build +ENV DEBIAN_FRONTEND noninteractive +# Update to the latest +RUN apt-get -q -y update && apt-get -q -y upgrade +RUN apt-get -q -y install devscripts equivs meson python3 python3-setuptools +ADD debian/control /tmp/control +# Answer 'yes' to install questions +RUN (yes | mk-build-deps --install /tmp/control) || true +# now, switch to build user +USER build From 18b6dee963a0eb611239ba56c65ed16aaf20ea5e Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 29 Aug 2022 09:59:07 +1000 Subject: [PATCH 027/132] chore: update core/ label --- .github/labeler.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index 70068cd1d5..e0abc29f2f 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -18,7 +18,6 @@ android/samples/: android/Samples/** common/: - common/** - - core/** - resources/** common/models/: common/models/** @@ -29,7 +28,7 @@ common/models/wordbreakers/: common/models/wordbreakers/** common/resources/: resources/** common/web/: common/web/** -common/core/: +core/: - core/** developer/: From 72b9fa3d21d750aeff87bed8570a87a57419eb9a Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Mon, 29 Aug 2022 15:14:29 +0200 Subject: [PATCH 028/132] chore(linux): Update debian changelog (cherry picked from commit f2b1277166f849ea917ba4aa73671368daf423b7) --- linux/debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/linux/debian/changelog b/linux/debian/changelog index 6990b7dbec..a6a93469c5 100644 --- a/linux/debian/changelog +++ b/linux/debian/changelog @@ -1,3 +1,10 @@ +keyman (15.0.269-1) unstable; urgency=medium + + * New upstream release. + * Re-release to Debian + + -- Eberhard Beilharz Mon, 29 Aug 2022 15:14:16 +0200 + keyman (15.0.268-1) unstable; urgency=medium * New upstream release. From fc3c991293973875a829b138c3f80c90ad651a10 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Mon, 29 Aug 2022 19:06:33 +0200 Subject: [PATCH 029/132] fix(linux): Allow downgrades for installing build deps When install the keyman-build-deps_*.deb file it can happen that we end up with a lower version number than what we previously built. This change allows downgrades so that the build dependencies can be installed. --- linux/scripts/jenkins.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linux/scripts/jenkins.sh b/linux/scripts/jenkins.sh index c7406d8831..bc40800fa2 100755 --- a/linux/scripts/jenkins.sh +++ b/linux/scripts/jenkins.sh @@ -55,7 +55,7 @@ checkAndInstallRequirements() fi sudo mk-build-deps debian/control - sudo apt-get -qy install ./keyman-build-deps_*.deb + sudo apt-get -qy --allow-downgrades install ./keyman-build-deps_*.deb sudo rm -f keyman-buid-deps_* } From f2b672cf9061afcfbf621a4667020dd00599a38b Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Mon, 29 Aug 2022 14:01:55 -0400 Subject: [PATCH 030/132] auto: increment master version to 16.0.54 --- HISTORY.md | 4 ++++ VERSION.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index da3efb69d9..04b545986d 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,9 @@ # Keyman Version History +## 16.0.53 alpha 2022-08-29 + +* feat(linux): Dockerfile for linux builder (#7133) + ## 16.0.52 alpha 2022-08-26 * fix(android/engine): Lower the max height for landscape orientation (#7119) diff --git a/VERSION.md b/VERSION.md index 8e52754086..13dbad0099 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.53 \ No newline at end of file +16.0.54 \ No newline at end of file From 9edabec2972fbf57ce723c72652542ad36c315c2 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Sat, 27 Aug 2022 14:53:06 +1000 Subject: [PATCH 031/132] chore(core): emcc off path for linux --- core/wasm.build.linux.in | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/wasm.build.linux.in b/core/wasm.build.linux.in index 126f856341..95dd08f12b 100644 --- a/core/wasm.build.linux.in +++ b/core/wasm.build.linux.in @@ -1,2 +1,7 @@ +[binaries] +c = ['$EMSCRIPTEN_BASE/emcc.py', '-s', 'WASM=1', '-O2'] +cpp = ['$EMSCRIPTEN_BASE/em++.py', '-s', 'WASM=1','-O2'] +ar = ['$EMSCRIPTEN_BASE/emar.py'] + [properties] root = '$EMSCRIPTEN_BASE/system' From d7430c3efc83a4422a1277503ed52c3288b8dc1f Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Tue, 30 Aug 2022 14:27:07 +0700 Subject: [PATCH 032/132] fix(android/engine): Cleanup list of subkeys when dismissing window --- .../app/src/main/java/com/tavultesoft/kmea/KMKeyboard.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KMKeyboard.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KMKeyboard.java index 1c1dfb5a6a..893686c368 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KMKeyboard.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KMKeyboard.java @@ -372,8 +372,12 @@ final class KMKeyboard extends WebView { public void dismissSubKeysWindow() { try { - if (subKeysWindow != null && subKeysWindow.isShowing()) + if (subKeysWindow != null && subKeysWindow.isShowing()) { subKeysWindow.dismiss(); + } + if (subKeysList != null) { + subKeysList = null; + } } catch (Exception e) { KMLog.LogException(TAG, "", e); } From 957c2b04c3c3b5de3d42009b2a86ff0651a21e62 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Tue, 30 Aug 2022 14:02:40 -0400 Subject: [PATCH 033/132] auto: increment master version to 16.0.55 --- HISTORY.md | 9 +++++++++ VERSION.md | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 04b545986d..de2b5958e0 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,14 @@ # Keyman Version History +## 16.0.54 alpha 2022-08-30 + +* chore(core): rename json.hpp to jsonpp.hpp (#6993) +* chore(developer): remove unused dependencies from KeymanWeb compiler (#7000) +* chore: update core/ label (#7138) +* chore(linux): Update debian changelog (#7145) +* fix(linux): Allow downgrades for installing build deps (#7147) +* chore(core): emcc off path for linux (#7149) + ## 16.0.53 alpha 2022-08-29 * feat(linux): Dockerfile for linux builder (#7133) diff --git a/VERSION.md b/VERSION.md index 13dbad0099..0e2a4103f1 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.54 \ No newline at end of file +16.0.55 \ No newline at end of file From 9198ffbff0baa2e5a075c19cbb50c296b36401db Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Wed, 31 Aug 2022 07:39:11 +0700 Subject: [PATCH 034/132] Update android/KMEA/app/src/main/java/com/tavultesoft/kmea/KMKeyboard.java Co-authored-by: Marc Durdin --- .../app/src/main/java/com/tavultesoft/kmea/KMKeyboard.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KMKeyboard.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KMKeyboard.java index 893686c368..b67e28eef1 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KMKeyboard.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KMKeyboard.java @@ -375,9 +375,7 @@ final class KMKeyboard extends WebView { if (subKeysWindow != null && subKeysWindow.isShowing()) { subKeysWindow.dismiss(); } - if (subKeysList != null) { - subKeysList = null; - } + subKeysList = null; } catch (Exception e) { KMLog.LogException(TAG, "", e); } From ede572a46413a80cad5ee37c44962c26faf66f93 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Wed, 31 Aug 2022 11:30:16 +0700 Subject: [PATCH 035/132] chore(android): Reduce Toast notification noise --- .../src/main/java/com/tavultesoft/kmapro/MainActivity.java | 4 ---- .../cloud/impl/CloudLexicalModelMetaDataDownloadCallback.java | 2 +- .../main/java/com/tavultesoft/kmea/data/CloudRepository.java | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/MainActivity.java b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/MainActivity.java index 2a9626984a..2d2710142a 100644 --- a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/MainActivity.java +++ b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/MainActivity.java @@ -993,10 +993,6 @@ public class MainActivity extends BaseActivity implements OnKeyboardEventListene String _downloadid = CloudLexicalModelMetaDataDownloadCallback.createDownloadId(languageID); CloudLexicalModelMetaDataDownloadCallback _callback = new CloudLexicalModelMetaDataDownloadCallback(); - Toast.makeText(context, - context.getString(R.string.query_associated_model), - Toast.LENGTH_SHORT).show(); - ArrayList aPreparedCloudApiParams = new ArrayList<>(); String url = CloudRepository.prepareLexicalModelQuery(languageID); aPreparedCloudApiParams.add(new CloudApiTypes.CloudApiParam( diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/cloud/impl/CloudLexicalModelMetaDataDownloadCallback.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/cloud/impl/CloudLexicalModelMetaDataDownloadCallback.java index fba9a8e1ad..f98c2fc40f 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/cloud/impl/CloudLexicalModelMetaDataDownloadCallback.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/cloud/impl/CloudLexicalModelMetaDataDownloadCallback.java @@ -148,7 +148,7 @@ public class CloudLexicalModelMetaDataDownloadCallback implements ICloudDownload KMLog.LogException(TAG, "Error parsing lexical model from api.keyman.com. ", e); } } else { - BaseActivity.makeToast(aContext, R.string.no_associated_model, Toast.LENGTH_SHORT); + // Don't make a Toast notification about nothing to download } } diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/data/CloudRepository.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/data/CloudRepository.java index 5bafb4a2df..1090d264cd 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/data/CloudRepository.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/data/CloudRepository.java @@ -400,7 +400,7 @@ public class CloudRepository { BaseActivity.makeToast(context, R.string.catalog_download_is_running_in_background, Toast.LENGTH_SHORT); } else { updateIsRunning = true; - BaseActivity.makeToast(context, R.string.catalog_download_start_in_background, Toast.LENGTH_SHORT); + // Don't make a Toast notification about cloud update CloudDownloadMgr.getInstance().executeAsDownload( context, DOWNLOAD_IDENTIFIER_CATALOGUE, memCachedDataset, _download_callback, params); } From ece48562535582fb18796f9148f497f3297f7ce7 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Wed, 31 Aug 2022 14:26:54 +0700 Subject: [PATCH 036/132] chore(android/engine): Remove Toast when resources up to date --- .../java/com/tavultesoft/kmea/logic/ResourcesUpdateTool.java | 1 - 1 file changed, 1 deletion(-) diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/logic/ResourcesUpdateTool.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/logic/ResourcesUpdateTool.java index 9495ce30bd..0b057cfdb8 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/logic/ResourcesUpdateTool.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/logic/ResourcesUpdateTool.java @@ -106,7 +106,6 @@ public class ResourcesUpdateTool implements KeyboardEventHandler.OnKeyboardDownl return; } - BaseActivity.makeToast(currentContext, R.string.update_check_current, Toast.LENGTH_SHORT); lastUpdateCheck = Calendar.getInstance(); SharedPreferences prefs = currentContext.getSharedPreferences(currentContext.getString(R.string.kma_prefs_name), Context.MODE_PRIVATE); SharedPreferences.Editor editor = prefs.edit(); From d57d77670c72b23ddaf0568d1cedfb7d727d6d0c Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Fri, 2 Sep 2022 09:30:01 +0700 Subject: [PATCH 037/132] Update android/KMEA/app/src/main/java/com/tavultesoft/kmea/data/CloudRepository.java Co-authored-by: Marc Durdin --- .../src/main/java/com/tavultesoft/kmea/data/CloudRepository.java | 1 - 1 file changed, 1 deletion(-) diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/data/CloudRepository.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/data/CloudRepository.java index 1090d264cd..1baba069ed 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/data/CloudRepository.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/data/CloudRepository.java @@ -400,7 +400,6 @@ public class CloudRepository { BaseActivity.makeToast(context, R.string.catalog_download_is_running_in_background, Toast.LENGTH_SHORT); } else { updateIsRunning = true; - // Don't make a Toast notification about cloud update CloudDownloadMgr.getInstance().executeAsDownload( context, DOWNLOAD_IDENTIFIER_CATALOGUE, memCachedDataset, _download_callback, params); } From b81f3795d085f4fed1e1811b6f52a56f9100a1bd Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Fri, 2 Sep 2022 11:00:03 +0700 Subject: [PATCH 038/132] chore(android/engine): Apply suggestion for refactor --- ...dLexicalModelMetaDataDownloadCallback.java | 53 +++++++++++-------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/cloud/impl/CloudLexicalModelMetaDataDownloadCallback.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/cloud/impl/CloudLexicalModelMetaDataDownloadCallback.java index f98c2fc40f..6791a3097d 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/cloud/impl/CloudLexicalModelMetaDataDownloadCallback.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/cloud/impl/CloudLexicalModelMetaDataDownloadCallback.java @@ -125,31 +125,12 @@ public class CloudLexicalModelMetaDataDownloadCallback implements ICloudDownload * @param aMetaDataResult the meta data results */ private void processCloudResults(Context aContext, List aMetaDataResult) { - for(MetaDataResult _r:aMetaDataResult) { - if (_r.returnjson.target== CloudApiTypes.ApiTarget.Keyboard) { + for (MetaDataResult _r : aMetaDataResult) { + if (_r.returnjson.target == CloudApiTypes.ApiTarget.Keyboard) { //handleKeyboardMetaData(_r); } - if(_r.returnjson.target== CloudApiTypes.ApiTarget.KeyboardLexicalModels) { - JSONArray lmData = _r.returnjson.jsonArray; - if (lmData != null && lmData.length() > 0) { - try { - JSONObject modelInfo = lmData.getJSONObject(0); - - if (modelInfo.has("packageFilename") && modelInfo.has("id")) { - String _modelID = modelInfo.getString("id"); - ArrayList urls = new ArrayList<>(); - urls.add(new CloudApiTypes.CloudApiParam( - CloudApiTypes.ApiTarget.LexicalModelPackage, - modelInfo.getString("packageFilename"))); - _r.additionalDownloadid = CloudLexicalPackageDownloadCallback.createDownloadId(_modelID); - _r.additionalDownloads= urls; - } - } catch (JSONException e) { - KMLog.LogException(TAG, "Error parsing lexical model from api.keyman.com. ", e); - } - } else { - // Don't make a Toast notification about nothing to download - } + if (_r.returnjson.target == CloudApiTypes.ApiTarget.KeyboardLexicalModels) { + processCloudResultForModel(aContext, _r); } if (_r.returnjson.target == CloudApiTypes.ApiTarget.PackageVersion) { @@ -164,6 +145,32 @@ public class CloudLexicalModelMetaDataDownloadCallback implements ICloudDownload } } + private void processCloudResultForModel(Context aContext, MetaDataResult _r) { + JSONArray lmData = _r.returnjson.jsonArray; + if (lmData == null || lmData.length() == 0) { + KMLog.LogError(TAG, "Error in lexical model metadata from api.keyman.com - zero or null"); + return; + } + + try { + JSONObject modelInfo = lmData.getJSONObject(0); + if (!modelInfo.has("packageFilename") || !modelInfo.has("id")) { + KMLog.LogError(TAG, "Error in lexical model metadata from api.keyman.com - missing metadata"); + return; + } + + String _modelID = modelInfo.getString("id"); + ArrayList urls = new ArrayList<>(); + urls.add(new CloudApiTypes.CloudApiParam( + CloudApiTypes.ApiTarget.LexicalModelPackage, + modelInfo.getString("packageFilename"))); + _r.additionalDownloadid = CloudLexicalPackageDownloadCallback.createDownloadId(_modelID); + _r.additionalDownloads= urls; + } catch (JSONException e) { + KMLog.LogException(TAG, "Error in lexical model metadata from api.keyman.com. ", e); + } + } + /** * create a download id for the lexical model metadata. * @param aLanguageId the language id From 75139ed9199823c2c0b0dfc3cc7d59e066035a11 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Fri, 2 Sep 2022 17:15:27 +1000 Subject: [PATCH 039/132] fix(developer): compiler mismatch on currentLine Fixes #7122. The compiler was updating currentLine when checking for unreachable rules, but not restoring it afterwards. This led to mismatches in the debugger and in compiler warnings. --- developer/src/kmcmpdll/DeprecationChecks.cpp | 4 ++-- developer/src/kmcmpdll/UnreachableRules.cpp | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/developer/src/kmcmpdll/DeprecationChecks.cpp b/developer/src/kmcmpdll/DeprecationChecks.cpp index 0e13eaa0ec..a1e19cb27b 100644 --- a/developer/src/kmcmpdll/DeprecationChecks.cpp +++ b/developer/src/kmcmpdll/DeprecationChecks.cpp @@ -25,7 +25,7 @@ BOOL CheckForDeprecatedFeatures(PFILE_KEYBOARD fk) { // Keyman 7 #define TSS_WINDOWSLANGUAGES 29 */ - int currentLineBackup = currentLine; + int oldCurrentLine = currentLine; DWORD i; PFILE_STORE sp; @@ -46,7 +46,7 @@ BOOL CheckForDeprecatedFeatures(PFILE_KEYBOARD fk) { } } - currentLine = currentLineBackup; + currentLine = oldCurrentLine; return TRUE; } diff --git a/developer/src/kmcmpdll/UnreachableRules.cpp b/developer/src/kmcmpdll/UnreachableRules.cpp index 70f4331966..a5c6519ef3 100644 --- a/developer/src/kmcmpdll/UnreachableRules.cpp +++ b/developer/src/kmcmpdll/UnreachableRules.cpp @@ -24,6 +24,8 @@ DWORD VerifyUnreachableRules(PFILE_GROUP gp) { PFILE_KEY kp = gp->dpKeyArray; DWORD i; + int oldCurrentLine = currentLine; + std::unordered_map map; std::unordered_set reportedLines; @@ -43,5 +45,7 @@ DWORD VerifyUnreachableRules(PFILE_GROUP gp) { } } + currentLine = oldCurrentLine; + return CERR_None; } From 9d1004b9ec14dd48f4d51aeeaa607e7c6259e277 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Fri, 2 Sep 2022 18:05:52 +0200 Subject: [PATCH 040/132] chore(linux): Fix ibus-keyman.postinst script When installing ibus-keyman it could happen that we got an (ignored) error "user name does not exist". This change tries to work around this problem. --- linux/debian/ibus-keyman.postinst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/linux/debian/ibus-keyman.postinst b/linux/debian/ibus-keyman.postinst index 5b66b64c2a..57c22b8d6d 100644 --- a/linux/debian/ibus-keyman.postinst +++ b/linux/debian/ibus-keyman.postinst @@ -37,8 +37,8 @@ case "$1" in fi # Verify that it's running now - if [ ! -z $SUDO_USER ]; then - ! ibusdaemon=`ps --user $SUDO_USER -o s= -o cmd | grep --regexp="^[^ZT] ibus-daemon .*--xim.*"` + if [ ! -z $SUDO_USER ] && id $SUDO_USER > /dev/null 2>/dev/null; then + ! ibusdaemon=$(ps --user $SUDO_USER -o s= -o cmd | grep --regexp="^[^ZT] ibus-daemon .*--xim.*") if [ "x$ibusdaemon" = "x" ]; then # otherwise try to start it for the user installing the package if [ "x$is_gnome_shell" = "x1" ]; then From 830ccf243862aacf49bad9da3e56aca72f272555 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Sat, 3 Sep 2022 14:01:55 -0400 Subject: [PATCH 041/132] auto: increment master version to 16.0.56 --- HISTORY.md | 4 ++++ VERSION.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index de2b5958e0..421263d29c 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,9 @@ # Keyman Version History +## 16.0.55 alpha 2022-09-03 + +* chore(android): Reduce Toast notification noise (#7178) + ## 16.0.54 alpha 2022-08-30 * chore(core): rename json.hpp to jsonpp.hpp (#6993) diff --git a/VERSION.md b/VERSION.md index 0e2a4103f1..b54e091f14 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.55 \ No newline at end of file +16.0.56 \ No newline at end of file From ad8c30c6c07b7f5272c1b1e55976a9b146fbbf06 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 5 Sep 2022 10:26:31 +0700 Subject: [PATCH 042/132] fix(web): pred-text context tracking when wordbreak not caused by whitespace --- .../src/correction/context-tracker.ts | 78 +++++++++++-------- common/web/lm-worker/src/model-compositor.ts | 36 ++------- common/web/lm-worker/src/transformUtils.ts | 25 ++++++ 3 files changed, 78 insertions(+), 61 deletions(-) create mode 100644 common/web/lm-worker/src/transformUtils.ts diff --git a/common/web/lm-worker/src/correction/context-tracker.ts b/common/web/lm-worker/src/correction/context-tracker.ts index 52e89bedc8..1a94723546 100644 --- a/common/web/lm-worker/src/correction/context-tracker.ts +++ b/common/web/lm-worker/src/correction/context-tracker.ts @@ -89,7 +89,7 @@ namespace correction { if(token.replacementText) { copy.replacementText = token.replacementText; } - + return copy; }); this.searchSpace = obj.searchSpace; @@ -139,8 +139,10 @@ namespace correction { // Track the Transform that resulted in the whitespace 'token'. // Will be needed for phrase-level correction/prediction. - whitespaceToken.transformDistributions = [transformDistribution]; - + if(transformDistribution) { + whitespaceToken.transformDistributions = [transformDistribution]; + } + whitespaceToken.raw = null; this.tokens.push(whitespaceToken); } @@ -149,19 +151,19 @@ namespace correction { * Used for 14.0's backspace workaround, which flattens all previous Distribution * entries because of limitations with direct use of backspace transforms. * @param tokenText - * @param transformId + * @param transformId */ replaceTailForBackspace(tokenText: USVString, transformId: number) { this.tokens.pop(); // It's a backspace transform; time for special handling! // - // For now, with 14.0, we simply compress all remaining Transforms for the token into - // multiple single-char transforms. Probabalistically modeling BKSP is quite complex, + // For now, with 14.0, we simply compress all remaining Transforms for the token into + // multiple single-char transforms. Probabalistically modeling BKSP is quite complex, // so we simplify by assuming everything remaining after a BKSP is 'true' and 'intended' text. // // Note that we cannot just use a single, monolithic transform at this point b/c - // of our current edit-distance optimization strategy; diagonalization is currently... + // of our current edit-distance optimization strategy; diagonalization is currently... // not very compatible with that. let backspacedTokenContext: Distribution[] = textToCharTransforms(tokenText, transformId).map(function(transform) { return [{sample: transform, p: 1.0}]; @@ -175,7 +177,7 @@ namespace correction { updateTail(transformDistribution: Distribution, tokenText?: USVString) { let editedToken = this.tail; - + // Preserve existing text if new text isn't specified. tokenText = tokenText || (tokenText === '' ? '' : editedToken.raw); @@ -191,7 +193,7 @@ namespace correction { toRawTokenization() { let sequence: USVString[] = []; - + for(let token of this.tokens) { // Hide any tokens representing wordbreaks. (Thinking ahead to phrase-level possibilities) if(token.currentText !== null) { @@ -281,7 +283,7 @@ namespace correction { /** * Returns items contained within the circular array, ordered from 'oldest' to 'newest' - * the same order in which the items will be dequeued. - * @param index + * @param index */ item(index: number) { if(index >= this.count) { @@ -294,7 +296,7 @@ namespace correction { } export class ContextTracker extends CircularArray { - static attemptMatchContext(tokenizedContext: USVString[], + static attemptMatchContext(tokenizedContext: USVString[], matchState: TrackedContextState, transformDistribution?: Distribution,): TrackedContextState { // Map the previous tokenized state to an edit-distance friendly version. @@ -335,7 +337,7 @@ namespace correction { } // Can happen for the first text input after backspace deletes a wordbreaking character, - // thus the new input continues a previous word while dropping the empty word after + // thus the new input continues a previous word while dropping the empty word after // that prior wordbreaking character. // // We can't handle it reliably from this match state, but a previous entry (without the empty token) @@ -353,7 +355,7 @@ namespace correction { // If we've made it here... success! We have a context match! let state: TrackedContextState; - + if(pushedTail) { // On suggestion acceptance, we should update the previous final token. // We do it first so that the acceptance is replicated in the new TrackedContextState @@ -376,7 +378,9 @@ namespace correction { if(primaryInput && primaryInput.insert == "" && primaryInput.deleteLeft == 0 && !primaryInput.deleteRight) { primaryInput = null; } - const isBackspace = primaryInput && primaryInput.insert == "" && primaryInput.deleteLeft > 0 && !primaryInput.deleteRight; + + const isWhitespace = primaryInput && TransformUtils.isWhitespace(primaryInput); + const isBackspace = primaryInput && TransformUtils.isBackspace(primaryInput); const finalToken = tokenizedContext[tokenizedContext.length-1]; /* Assumption: This is an adequate check for its two sub-branches. @@ -388,7 +392,7 @@ namespace correction { * - Assumption: one keystroke may only cause a single token to be appended to the context * - That is, no "reasonable" keystroke would emit a Transform adding two separate word tokens * - For languages using whitespace to word-break, said keystroke would have to include said whitespace to break the assumption. - */ + */ // If there is/was more than one context token available... if(editPath.length > 1) { @@ -399,17 +403,29 @@ namespace correction { // We're adding an additional context token. if(pushedTail) { - // ASSUMPTION: any transform that triggers this case is a pure-whitespace Transform, as we - // need a word-break before beginning a new word's context. - // Worth note: when invalid, the lm-layer already has problems in other aspects too. - state.pushWhitespaceToTail(transformDistribution); + const tokenizedTail = tokenizedContext[tokenizedContext.length - 1]; + /* + * Common-case: most transforms that trigger this case are from pure-whitespace Transforms. MOST. + * + * Less-common, but noteworthy: some wordbreaks may occur without whitespace. Example: + * `"o` => ['"', 'o']. Make sure to double-check against `tokenizedContext`! + */ + let pushedToken = new TrackedContextToken(); + pushedToken.raw = tokenizedTail; - let emptyToken = new TrackedContextToken(); - emptyToken.raw = ''; - // Continuing the earlier assumption, that 'pure-whitespace Transform' does not emit any initial characters - // for the new word (token), so the input keystrokes do not correspond to the new text token. - emptyToken.transformDistributions = []; - state.pushTail(emptyToken); + if(isWhitespace) { + state.pushWhitespaceToTail(transformDistribution); + // Continuing the earlier assumption, that 'pure-whitespace Transform' does not emit any initial characters + // for the new word (token), so the input keystrokes do not correspond to the new text token. + pushedToken.transformDistributions = []; + } else { + state.pushWhitespaceToTail(); + // Assumption: Since we only allow one-transform-at-a-time changes between states, we shouldn't be missing + // any metadata used to construct the new context state token. + pushedToken.transformDistributions = [transformDistribution]; + } + + state.pushTail(pushedToken); } else { // We're editing the final context token. // TODO: Assumption: we didn't 'miss' any inputs somehow. // As is, may be prone to fragility should the lm-layer's tracked context 'desync' from its host's. @@ -483,13 +499,13 @@ namespace correction { * Compares the current, post-input context against the most recently-seen contexts from previous prediction calls, returning * the most information-rich `TrackedContextState` possible. If a match is found, the state will be annotated with the * input information provided to previous prediction calls and persisted correction-search calculations for re-use. - * - * @param model - * @param context - * @param mainTransform - * @param transformDistribution + * + * @param model + * @param context + * @param mainTransform + * @param transformDistribution */ - analyzeState(model: LexicalModel, + analyzeState(model: LexicalModel, context: Context, transformDistribution?: Distribution): TrackedContextState { if(!model.traverseFromRoot) { diff --git a/common/web/lm-worker/src/model-compositor.ts b/common/web/lm-worker/src/model-compositor.ts index 329460776d..84687c867d 100644 --- a/common/web/lm-worker/src/model-compositor.ts +++ b/common/web/lm-worker/src/model-compositor.ts @@ -16,30 +16,6 @@ class ModelCompositor { this.punctuation = ModelCompositor.determinePunctuationFromModel(lexicalModel); } - protected isWhitespace(transform: Transform): boolean { - // Matches prefixed text + any instance of a character with Unicode general property Z* or the following: CR, LF, and Tab. - let whitespaceRemover = /.*[\u0009\u000A\u000D\u0020\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b\u2028\u2029\u202f\u205f\u3000]/i; - - // Filter out null-inserts; their high probability can cause issues. - if(transform.insert == '') { // Can actually register as 'whitespace'. - return false; - } - - let insert = transform.insert; - - insert = insert.replace(whitespaceRemover, ''); - - return insert == ''; - } - - protected isBackspace(transform: Transform): boolean { - return transform.insert == "" && transform.deleteLeft > 0; - } - - protected isEmpty(transform: Transform): boolean { - return transform.insert == '' && transform.deleteLeft == 0; - } - private predictFromCorrections(corrections: ProbabilityMass[], context: Context): Distribution { let returnedPredictions: Distribution = []; @@ -98,8 +74,8 @@ class ModelCompositor { })[0].sample; // Only allow new-word suggestions if space was the most likely keypress. - let allowSpace = this.isWhitespace(inputTransform); - let allowBksp = this.isBackspace(inputTransform); + let allowSpace = TransformUtils.isWhitespace(inputTransform); + let allowBksp = TransformUtils.isBackspace(inputTransform); let postContext = models.applyTransform(inputTransform, context); let keepOptionText = this.wordbreak(postContext); @@ -146,9 +122,9 @@ class ModelCompositor { } else { contextState = this.contextTracker.analyzeState(this.lexicalModel, postContext, - !this.isEmpty(inputTransform) ? - transformDistribution: - null + !TransformUtils.isEmpty(inputTransform) ? + transformDistribution: + null ); // TODO: Should we filter backspaces & whitespaces out of the transform distribution? @@ -164,7 +140,7 @@ class ModelCompositor { // Detect if we're starting a new context state. let contextTokens = contextState.tokens; if(contextTokens.length == 0 || contextTokens[contextTokens.length - 1].isNew) { - if(this.isEmpty(inputTransform) || this.isWhitespace(inputTransform)) { + if(TransformUtils.isEmpty(inputTransform) || TransformUtils.isWhitespace(inputTransform)) { newEmptyToken = true; prefixTransform = inputTransform; context = postContext; // Ensure the whitespace token is preapplied! diff --git a/common/web/lm-worker/src/transformUtils.ts b/common/web/lm-worker/src/transformUtils.ts new file mode 100644 index 0000000000..8512d4e9a2 --- /dev/null +++ b/common/web/lm-worker/src/transformUtils.ts @@ -0,0 +1,25 @@ +class TransformUtils { + static isWhitespace(transform: Transform): boolean { + // Matches prefixed text + any instance of a character with Unicode general property Z* or the following: CR, LF, and Tab. + let whitespaceRemover = /.*[\u0009\u000A\u000D\u0020\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b\u2028\u2029\u202f\u205f\u3000]/i; + + // Filter out null-inserts; their high probability can cause issues. + if(transform.insert == '') { // Can actually register as 'whitespace'. + return false; + } + + let insert = transform.insert; + + insert = insert.replace(whitespaceRemover, ''); + + return insert == ''; + } + + static isBackspace(transform: Transform): boolean { + return transform.insert == "" && transform.deleteLeft > 0 && !transform.deleteRight; + } + + static isEmpty(transform: Transform): boolean { + return transform.insert == '' && transform.deleteLeft == 0; + } +} \ No newline at end of file From be562b30225410afa77d65d1fbed58f6257f4784 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 5 Sep 2022 10:37:50 +0700 Subject: [PATCH 043/132] fix(web): missed method references --- common/web/lm-worker/src/model-compositor.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/web/lm-worker/src/model-compositor.ts b/common/web/lm-worker/src/model-compositor.ts index 84687c867d..657a7b1ec1 100644 --- a/common/web/lm-worker/src/model-compositor.ts +++ b/common/web/lm-worker/src/model-compositor.ts @@ -100,18 +100,18 @@ class ModelCompositor { predictionRoots = [{sample: inputTransform, p: 1.0}]; prefixTransform = inputTransform; } else { - predictionRoots = transformDistribution.map(function(alt) { + predictionRoots = transformDistribution.map((alt) => { let transform = alt.sample; // Filter out special keys unless they're expected. - if(this.isWhitespace(transform) && !allowSpace) { + if(TransformUtils.isWhitespace(transform) && !allowSpace) { return null; - } else if(this.isBackspace(transform) && !allowBksp) { + } else if(TransformUtils.isBackspace(transform) && !allowBksp) { return null; } return alt; - }, this); + }); } // Remove `null` entries. From b0184ec98bcf1106119afab36b6b7d09b9db70e1 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 5 Sep 2022 11:08:26 +0700 Subject: [PATCH 044/132] fix(web): adds unit test targeting issue, handler for edge case --- .../headless/edit-distance/context-tracker.js | 26 ++++++++++++++++--- .../src/correction/context-tracker.ts | 2 +- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js b/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js index 6a4148b669..5960a47658 100644 --- a/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js +++ b/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js @@ -45,7 +45,7 @@ describe('ContextTracker', function() { assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); }); - it("properly matches and aligns when a 'wordbreak' is added'", function() { + it("properly matches and aligns when a 'wordbreak' is added", function() { let existingContext = ["an", "apple", "a", "day", "keeps", "the", "doctor"]; let transform = { insert: ' ', @@ -56,7 +56,7 @@ describe('ContextTracker', function() { let rawTokens = ["an", null, "apple", null, "a", null, "day", null, "keeps", null, "the", null, "doctor", null, ""]; let existingState = ContextTracker.modelContextState(existingContext); - let state = ContextTracker.attemptMatchContext(newContext, existingState, null, toWrapperDistribution(transform)); + let state = ContextTracker.attemptMatchContext(newContext, existingState, toWrapperDistribution(transform)); assert.isNotNull(state); assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); @@ -65,6 +65,26 @@ describe('ContextTracker', function() { assert.isEmpty(state.tokens[state.tokens.length - 1].transformDistributions); }); + it("properly matches and aligns when an implied 'wordbreak' occurs \"'\"", function() { + let existingContext = ["'"]; + let transform = { + insert: 'a', + deleteLeft: 0 + } + let newContext = Array.from(existingContext); + newContext.push('a'); // The incoming transform should produce a new token WITH TEXT. + let rawTokens = ["'", null, "a"]; + + let existingState = ContextTracker.modelContextState(existingContext); + let state = ContextTracker.attemptMatchContext(newContext, existingState, toWrapperDistribution(transform)); + assert.isNotNull(state); + assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); + + // The 'wordbreak' transform + assert.isEmpty(state.tokens[state.tokens.length - 2].transformDistributions); + assert.isNotEmpty(state.tokens[state.tokens.length - 1].transformDistributions); + }); + it("properly matches and aligns when lead token is removed AND a 'wordbreak' is added'", function() { let existingContext = ["an", "apple", "a", "day", "keeps", "the", "doctor"]; let transform = { @@ -77,7 +97,7 @@ describe('ContextTracker', function() { let rawTokens = ["apple", null, "a", null, "day", null, "keeps", null, "the", null, "doctor", null, ""]; let existingState = ContextTracker.modelContextState(existingContext); - let state = ContextTracker.attemptMatchContext(newContext, existingState, null, toWrapperDistribution(transform)); + let state = ContextTracker.attemptMatchContext(newContext, existingState, toWrapperDistribution(transform)); assert.isNotNull(state); assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); diff --git a/common/web/lm-worker/src/correction/context-tracker.ts b/common/web/lm-worker/src/correction/context-tracker.ts index 1a94723546..0f50657f0e 100644 --- a/common/web/lm-worker/src/correction/context-tracker.ts +++ b/common/web/lm-worker/src/correction/context-tracker.ts @@ -413,7 +413,7 @@ namespace correction { let pushedToken = new TrackedContextToken(); pushedToken.raw = tokenizedTail; - if(isWhitespace) { + if(isWhitespace || !primaryInput) { state.pushWhitespaceToTail(transformDistribution); // Continuing the earlier assumption, that 'pure-whitespace Transform' does not emit any initial characters // for the new word (token), so the input keystrokes do not correspond to the new text token. From 0a237f800fbbca7aa78483ae0f3371a71534b52f Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 5 Sep 2022 11:31:57 +0700 Subject: [PATCH 045/132] chore(web): test name tweak --- .../unit_tests/headless/edit-distance/context-tracker.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js b/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js index 5960a47658..fa09ca20de 100644 --- a/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js +++ b/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js @@ -65,7 +65,7 @@ describe('ContextTracker', function() { assert.isEmpty(state.tokens[state.tokens.length - 1].transformDistributions); }); - it("properly matches and aligns when an implied 'wordbreak' occurs \"'\"", function() { + it("properly matches and aligns when an implied 'wordbreak' occurs (as when following \"'\")", function() { let existingContext = ["'"]; let transform = { insert: 'a', From d33a9f05bf999cbc476bb600f50eada8ee934395 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 5 Sep 2022 11:41:42 +0700 Subject: [PATCH 046/132] docs(web): fixes missed doc update --- common/web/lm-worker/src/correction/context-tracker.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/common/web/lm-worker/src/correction/context-tracker.ts b/common/web/lm-worker/src/correction/context-tracker.ts index 0f50657f0e..bc187fa03f 100644 --- a/common/web/lm-worker/src/correction/context-tracker.ts +++ b/common/web/lm-worker/src/correction/context-tracker.ts @@ -502,7 +502,6 @@ namespace correction { * * @param model * @param context - * @param mainTransform * @param transformDistribution */ analyzeState(model: LexicalModel, From 57ce30e0685fb6e6c8c3d06207045d4960e2131d Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 5 Sep 2022 12:17:17 +0700 Subject: [PATCH 047/132] fix(web): post-suggestion-apply error --- common/web/lm-worker/src/correction/context-tracker.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/web/lm-worker/src/correction/context-tracker.ts b/common/web/lm-worker/src/correction/context-tracker.ts index bc187fa03f..99842b8299 100644 --- a/common/web/lm-worker/src/correction/context-tracker.ts +++ b/common/web/lm-worker/src/correction/context-tracker.ts @@ -414,7 +414,7 @@ namespace correction { pushedToken.raw = tokenizedTail; if(isWhitespace || !primaryInput) { - state.pushWhitespaceToTail(transformDistribution); + state.pushWhitespaceToTail(transformDistribution ?? []); // Continuing the earlier assumption, that 'pure-whitespace Transform' does not emit any initial characters // for the new word (token), so the input keystrokes do not correspond to the new text token. pushedToken.transformDistributions = []; @@ -422,7 +422,7 @@ namespace correction { state.pushWhitespaceToTail(); // Assumption: Since we only allow one-transform-at-a-time changes between states, we shouldn't be missing // any metadata used to construct the new context state token. - pushedToken.transformDistributions = [transformDistribution]; + pushedToken.transformDistributions = transformDistribution ? [transformDistribution] : []; } state.pushTail(pushedToken); From b47596a803cc9c65c329d85a33c8c1d3736f74e0 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 5 Sep 2022 12:35:11 +0700 Subject: [PATCH 048/132] fix(common/models): context token .isNew maintenance --- common/web/lm-worker/src/correction/context-tracker.ts | 5 ++++- common/web/lm-worker/src/model-compositor.ts | 9 ++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/common/web/lm-worker/src/correction/context-tracker.ts b/common/web/lm-worker/src/correction/context-tracker.ts index 99842b8299..08ca11b136 100644 --- a/common/web/lm-worker/src/correction/context-tracker.ts +++ b/common/web/lm-worker/src/correction/context-tracker.ts @@ -27,13 +27,14 @@ namespace correction { export class TrackedContextToken { raw: string; replacementText: string; + newFlag: boolean = false; transformDistributions: Distribution[] = []; replacements: TrackedContextSuggestion[]; activeReplacementId: number = -1; get isNew(): boolean { - return this.transformDistributions.length == 0; + return this.newFlag; } get currentText(): string { @@ -126,6 +127,7 @@ namespace correction { } else { this.searchSpace = []; } + token.newFlag = true; this.tokens.push(token); let state = this; @@ -189,6 +191,7 @@ namespace correction { } // Replace old token's raw-text with new token's raw-text. editedToken.raw = tokenText; + editedToken.newFlag = false; } toRawTokenization() { diff --git a/common/web/lm-worker/src/model-compositor.ts b/common/web/lm-worker/src/model-compositor.ts index 657a7b1ec1..6c40d36e98 100644 --- a/common/web/lm-worker/src/model-compositor.ts +++ b/common/web/lm-worker/src/model-compositor.ts @@ -136,12 +136,15 @@ class ModelCompositor { // The 'eventual' logic will be significantly more complex, though still manageable. let searchSpace = contextState.searchSpace[0]; - let newEmptyToken = false; + let newToken = false; // Detect if we're starting a new context state. let contextTokens = contextState.tokens; if(contextTokens.length == 0 || contextTokens[contextTokens.length - 1].isNew) { + // Always note if we have a new token (so that we don't try to delete existing context) + newToken = true; + // If the new token is due to whitespace, or if we had a context-reset trigger this (thus, no input...) + // (Lingering question: do we need the .isEmpty check here? Track `prefixTransform` and find out.) if(TransformUtils.isEmpty(inputTransform) || TransformUtils.isWhitespace(inputTransform)) { - newEmptyToken = true; prefixTransform = inputTransform; context = postContext; // Ensure the whitespace token is preapplied! } @@ -170,7 +173,7 @@ class ModelCompositor { let deleteLeft = 0; // remove actual token string. If new token, there should be nothing to delete. - if(!newEmptyToken) { + if(!newToken) { // If this is triggered from a backspace, make sure to use its results // and also include its left-deletions! It's the one post-input context case. if(allowBksp) { From 26c5150977cf60de364b6d2ef8aa893668d00aeb Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 5 Sep 2022 14:02:55 +0700 Subject: [PATCH 049/132] fix(web): context-tracker newFlag management for new contexts --- .../lm-worker/src/correction/context-tracker.ts | 17 +++++++++++++++-- common/web/lm-worker/src/model-compositor.ts | 2 +- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/common/web/lm-worker/src/correction/context-tracker.ts b/common/web/lm-worker/src/correction/context-tracker.ts index 08ca11b136..fba8826168 100644 --- a/common/web/lm-worker/src/correction/context-tracker.ts +++ b/common/web/lm-worker/src/correction/context-tracker.ts @@ -461,7 +461,9 @@ namespace correction { return state; } - static modelContextState(tokenizedContext: USVString[], lexicalModel: LexicalModel): TrackedContextState { + static modelContextState(tokenizedContext: USVString[], + transformDistribution: Distribution, + lexicalModel: LexicalModel): TrackedContextState { let baseTokens = tokenizedContext.map(function(entry) { let token = new TrackedContextToken(); token.raw = entry; @@ -495,6 +497,17 @@ namespace correction { state.pushTail(token); } + for(let i = 0; i < state.tokens.length - 1; i++) { + state.tokens[i].newFlag = false; + } + + const finalToken = state.tokens[state.tokens.length - 1]; + const baseTransform = (transformDistribution && transformDistribution.length > 0) ? transformDistribution[0] : null; + + if(!baseTransform || baseTransform.sample.insert != finalToken.raw) { + finalToken.newFlag = false; + } + return state; } @@ -537,7 +550,7 @@ namespace correction { // // Assumption: as a caret needs to move to context before any actual transform distributions occur, // this state is only reached on caret moves; thus, transformDistribution is actually just a single null transform. - let state = ContextTracker.modelContextState(tokenizedContext.left, model); + let state = ContextTracker.modelContextState(tokenizedContext.left, transformDistribution, model); state.taggedContext = context; this.enqueue(state); return state; diff --git a/common/web/lm-worker/src/model-compositor.ts b/common/web/lm-worker/src/model-compositor.ts index 6c40d36e98..8709ab0c6b 100644 --- a/common/web/lm-worker/src/model-compositor.ts +++ b/common/web/lm-worker/src/model-compositor.ts @@ -638,7 +638,7 @@ class ModelCompositor { // than before. if(this.contextTracker) { let tokenizedContext = models.tokenize(this.lexicalModel.wordbreaker || wordBreakers.default, context); - let contextState = correction.ContextTracker.modelContextState(tokenizedContext.left, this.lexicalModel); + let contextState = correction.ContextTracker.modelContextState(tokenizedContext.left, null, this.lexicalModel); this.contextTracker.enqueue(contextState); } } From eddec014ee83c397839d8797699762c8765e65de Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Mon, 5 Sep 2022 14:01:31 -0400 Subject: [PATCH 050/132] auto: increment master version to 16.0.57 --- HISTORY.md | 4 ++++ VERSION.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 421263d29c..76eb2baed3 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,9 @@ # Keyman Version History +## 16.0.56 alpha 2022-09-05 + +* chore(linux): Fix ibus-keyman.postinst script (#7192) + ## 16.0.55 alpha 2022-09-03 * chore(android): Reduce Toast notification noise (#7178) diff --git a/VERSION.md b/VERSION.md index b54e091f14..2eda1c465b 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.56 \ No newline at end of file +16.0.57 \ No newline at end of file From a6e65333c55e9ba781893da78acc939ed00fae18 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 6 Sep 2022 08:54:56 +0700 Subject: [PATCH 051/132] fix(common/models): wordbreaker test require paths --- .../models/wordbreakers/src/default/index.ts | 10 +++- .../test/test-ascii-word-breaker.js | 2 +- .../test/test-default-word-breaker.js | 46 ++++++++++++------- .../test/test-placeholder-word-breaker.js | 2 +- 4 files changed, 39 insertions(+), 21 deletions(-) diff --git a/common/models/wordbreakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts index c587ca08de..357372b228 100644 --- a/common/models/wordbreakers/src/default/index.ts +++ b/common/models/wordbreakers/src/default/index.ts @@ -3,8 +3,8 @@ namespace wordBreakers { /** * Word breaker based on Unicode Standard Annex #29, Section 4.1: - * Default Word Boundary Specification. - * + * Default Word Boundary Specification. + * * @see http://unicode.org/reports/tr29/#Word_Boundaries * @see https://github.com/eddieantonio/unicode-default-word-boundary/tree/v12.0.0 */ @@ -349,6 +349,12 @@ namespace wordBreakers { /** * Binary search for the word break property of a given CODE POINT. + * + * The auto-generated data.ts master array defines a **character range** + * lookup table. If a character's codepoint is equal to or greater than + * the I.Start value for an entry and exclusively less than the next entry, + * it falls in the first entry's range bucket and is classified accordingly + * by this method. */ function searchForProperty(codePoint: number, left: number, right: number): WordBreakProperty { // All items that are not found in the array are assigned the 'Other' property. diff --git a/common/models/wordbreakers/test/test-ascii-word-breaker.js b/common/models/wordbreakers/test/test-ascii-word-breaker.js index 598780ad95..55b9c3b8ba 100644 --- a/common/models/wordbreakers/test/test-ascii-word-breaker.js +++ b/common/models/wordbreakers/test/test-ascii-word-breaker.js @@ -1,5 +1,5 @@ var assert = require('chai').assert; -var breakASCIIWords = require('..').wordBreakers['ascii']; +var breakASCIIWords = require('../build').wordBreakers['ascii']; describe('The ASCII word breaker', function () { it('should break simple English sentences', function () { diff --git a/common/models/wordbreakers/test/test-default-word-breaker.js b/common/models/wordbreakers/test/test-default-word-breaker.js index 6cf2362695..f69bf528c6 100644 --- a/common/models/wordbreakers/test/test-default-word-breaker.js +++ b/common/models/wordbreakers/test/test-default-word-breaker.js @@ -2,22 +2,34 @@ * Smoke-test the default */ -const assert = require('chai').assert; -const breakWords = require('..').wordBreakers['default']; + const assert = require('chai').assert; + const breakWords = require('../build').wordBreakers['default']; -const SHY = '\u00AD'; + const SHY = '\u00AD'; // Other, Format. The "Soft HYphen" - usually invisible unless needed for word-wrapping. -describe('The default word breaker', function () { - it('should break multilingual text', function () { - let breaks = breakWords( - `Добрый день! ᑕᐻ᙮ — after working on ka${SHY}wen${SHY}non:${SHY}nis, - let's eat phở! 🥣` - ); - let words = breaks.map(span => span.text); - assert.deepEqual(words, [ - 'Добрый', 'день', '!', 'ᑕᐻ', '᙮', '—', 'after', - 'working', 'on', `ka${SHY}wen${SHY}non:${SHY}nis`, ',', - "let's", 'eat', 'phở', '!', '🥣' - ]); - }); -}); + describe('The default word breaker', function () { + it('should break multilingual text', function () { + let breaks = breakWords( + `Добрый день! ᑕᐻ᙮ — after working on ka${SHY}wen${SHY}non:${SHY}nis, + let's eat phở! 🥣` + ); + let words = breaks.map(span => span.text); + assert.deepEqual(words, [ + 'Добрый', 'день', '!', 'ᑕᐻ', '᙮', '—', 'after', + 'working', 'on', `ka${SHY}wen${SHY}non:${SHY}nis`, ',', + "let's", 'eat', 'phở', '!', '🥣' + ]); + }); + + it('handles heavily-punctuated English text', function() { + // This test case brought to you by http://unicode.org/reports/tr29/#Word_Boundaries, Figure 1. + let breaks = breakWords( + `The quick ("brown") fox can't jump 32.3 feet, right?` + ); + let words = breaks.map(span => span.text); + assert.deepEqual(words, [ + 'The', 'quick', '(', '"', 'brown', '"', ')', 'fox', "can't", + 'jump', '32.3', 'feet', ',', 'right', '?' + ]); + }); + }); diff --git a/common/models/wordbreakers/test/test-placeholder-word-breaker.js b/common/models/wordbreakers/test/test-placeholder-word-breaker.js index b23fd5d6bb..dbc1aef34e 100644 --- a/common/models/wordbreakers/test/test-placeholder-word-breaker.js +++ b/common/models/wordbreakers/test/test-placeholder-word-breaker.js @@ -1,5 +1,5 @@ const assert = require('chai').assert; -const breakWords = require('..').wordBreakers['placeholder']; +const breakWords = require('../build').wordBreakers['placeholder']; describe('The placeholder word breaker', function () { From 9634c7635340a93babd6bee323941d97d797a54d Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 6 Sep 2022 08:57:56 +0700 Subject: [PATCH 052/132] chore(common/models): suggested tweak from review --- common/web/lm-worker/src/transformUtils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/web/lm-worker/src/transformUtils.ts b/common/web/lm-worker/src/transformUtils.ts index 8512d4e9a2..3bc66f33ce 100644 --- a/common/web/lm-worker/src/transformUtils.ts +++ b/common/web/lm-worker/src/transformUtils.ts @@ -16,10 +16,10 @@ class TransformUtils { } static isBackspace(transform: Transform): boolean { - return transform.insert == "" && transform.deleteLeft > 0 && !transform.deleteRight; + return transform.insert == "" && transform.deleteLeft > 0 && transform.deleteRight == 0; } static isEmpty(transform: Transform): boolean { - return transform.insert == '' && transform.deleteLeft == 0; + return transform.insert == '' && transform.deleteLeft == 0 && transform.deleteRight == 0; } } \ No newline at end of file From a09cc5d8229c118f6134f48b7980719709517905 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 6 Sep 2022 09:48:52 +0700 Subject: [PATCH 053/132] fix(common/models): models/templates unit tests --- common/models/templates/package.json | 4 ++-- common/models/templates/test/helpers.js | 8 ++++--- common/models/templates/test/test-common.js | 8 +++---- .../templates/test/test-priority-queue.js | 6 ++--- .../templates/test/test-quote-behavior.js | 8 +++---- .../templates/test/test-tokenization.js | 14 +++++------ .../models/templates/test/test-trie-model.js | 6 ++--- .../templates/test/test-trie-traversal.js | 10 ++++---- common/models/templates/tsconfig.bundled.json | 21 +++++++++++++++++ common/models/wordbreakers/package.json | 2 +- common/predictive-text/unit_tests/test.sh | 12 ++++++++++ common/web/utils/package.json | 4 ++-- package-lock.json | 23 ++----------------- 13 files changed, 71 insertions(+), 55 deletions(-) create mode 100644 common/models/templates/tsconfig.bundled.json diff --git a/common/models/templates/package.json b/common/models/templates/package.json index d4eafa6f0c..3cc7ecebac 100644 --- a/common/models/templates/package.json +++ b/common/models/templates/package.json @@ -33,7 +33,7 @@ }, "scripts": { "build": "tsc", - "pretest": "npm run build", + "pretest": "tsc -b tsconfig.bundled.json", "test": "mocha -r test/helpers.js" }, "bugs": { @@ -47,7 +47,7 @@ "@types/node": "^14.0.4", "chai": "^4.3.4", "mocha": "^8.4.0", - "typescript": "^3.8.3" + "typescript": "^4.5.4" }, "dependencies": { "@keymanapp/models-wordbreakers": "*" diff --git a/common/models/templates/test/helpers.js b/common/models/templates/test/helpers.js index e3931d648a..268e24b7dd 100644 --- a/common/models/templates/test/helpers.js +++ b/common/models/templates/test/helpers.js @@ -7,8 +7,10 @@ var _ = global; // TODO: then mocha invocation is as follows: // TODO: mocha -r @keymanapp/models-test-helpers test/ -// KMW string must be included, so do it here: -require('@keymanapp/web-utils'); +// Ensure that we can successfully load the module & apply kmwLength, as it's +// needed for some of the unit tests. +require('../build/index.bundled.js'); + assert.ok('💩'.kmwLength); /** @@ -21,7 +23,7 @@ _.jsonFixture = function (name) { /** * Returns the Context of an empty buffer; no text, at both the start and * end of the buffer. - * + * * @returns {Context} */ _.emptyContext = function emptyContext() { diff --git a/common/models/templates/test/test-common.js b/common/models/templates/test/test-common.js index 299cb25443..7dfc565369 100644 --- a/common/models/templates/test/test-common.js +++ b/common/models/templates/test/test-common.js @@ -3,7 +3,7 @@ */ var assert = require('chai').assert; -var models = require('../').models; +var models = require('../build/index.bundled.js').models; describe('Common utility functions', function() { // TODO: unit tests for other common utility functions @@ -97,7 +97,7 @@ describe('Common utility functions', function() { let apple = { insert: 'apple', deleteLeft: 0, - deleteRight: 2 + deleteRight: 2 }; let banana = { @@ -143,7 +143,7 @@ describe('Common utility functions', function() { let apple = { insert: 'apple', deleteLeft: 0, - deleteRight: 2 + deleteRight: 2 }; let banana = { @@ -167,7 +167,7 @@ describe('Common utility functions', function() { let apple = { insert: 'apple', deleteLeft: 0, - deleteRight: 2 + deleteRight: 2 }; let banana = { diff --git a/common/models/templates/test/test-priority-queue.js b/common/models/templates/test/test-priority-queue.js index 9f2e48e1d2..b004027d84 100644 --- a/common/models/templates/test/test-priority-queue.js +++ b/common/models/templates/test/test-priority-queue.js @@ -3,12 +3,12 @@ */ var assert = require('chai').assert; -var PriorityQueue = require('../').models.PriorityQueue; +var PriorityQueue = require('../build/index.bundled.js').models.PriorityQueue; describe('Priority queue', function() { it('can act as a min-heap', function () { let input = [1, 10, 2, 9, 3, 8, 4, 7, 5, 6]; - + let queue = new PriorityQueue((a, b) => a - b); input.forEach((input) => queue.enqueue(input)); @@ -41,7 +41,7 @@ describe('Priority queue', function() { it('can act as a max-heap', function () { let input = [1, 10, 2, 9, 3, 8, 4, 7, 5, 6]; - + let queue = new PriorityQueue((a, b) => b - a); input.forEach((input) => queue.enqueue(input)); diff --git a/common/models/templates/test/test-quote-behavior.js b/common/models/templates/test/test-quote-behavior.js index eb169424be..dd5408f6a9 100644 --- a/common/models/templates/test/test-quote-behavior.js +++ b/common/models/templates/test/test-quote-behavior.js @@ -3,7 +3,7 @@ */ var assert = require('chai').assert; -var QuoteBehavior = require('../').models.QuoteBehavior; +var QuoteBehavior = require('../build/index.bundled.js').models.QuoteBehavior; describe('Quote behaviors', function() { describe('Script directionality', function() { @@ -144,15 +144,15 @@ describe('Quote behaviors', function() { quotesForKeepSuggestion: { open: `“`, close: `”`}, insertAfterWord: " " } - + assert.throws(function() { QuoteBehavior.apply(QuoteBehavior.default, "hello", englishPunctuation, QuoteBehavior.default); }); - + assert.throws(function() { QuoteBehavior.apply(QuoteBehavior.useQuotes, "hello", englishPunctuation, QuoteBehavior.default); }); - + assert.throws(function() { QuoteBehavior.apply(QuoteBehavior.noQuotes, "hello", englishPunctuation, QuoteBehavior.default); }); diff --git a/common/models/templates/test/test-tokenization.js b/common/models/templates/test/test-tokenization.js index 22b975a69c..4775064b38 100644 --- a/common/models/templates/test/test-tokenization.js +++ b/common/models/templates/test/test-tokenization.js @@ -3,7 +3,7 @@ */ var assert = require('chai').assert; -var models = require('../').models; +var models = require('../build/index.bundled.js').models; var wordBreakers = require('@keymanapp/models-wordbreakers').wordBreakers; describe('Tokenization functions', function() { @@ -132,7 +132,7 @@ describe('Tokenization functions', function() { left: '', startOfBuffer: true, right: '', endOfBuffer: true }; - + let tokenization = models.tokenize(wordBreakers.default, context); let expectedResult = { @@ -140,7 +140,7 @@ describe('Tokenization functions', function() { right: [], caretSplitsToken: false }; - + assert.deepEqual(tokenization, expectedResult); }); @@ -153,7 +153,7 @@ describe('Tokenization functions', function() { right: [], caretSplitsToken: false }; - + assert.deepEqual(tokenization, expectedResult); }); @@ -163,7 +163,7 @@ describe('Tokenization functions', function() { left: ' ', startOfBuffer: true, right: '', endOfBuffer: true }; - + let tokenization = models.tokenize(wordBreakers.default, context); let expectedResult = { @@ -171,7 +171,7 @@ describe('Tokenization functions', function() { right: [], caretSplitsToken: false }; - + assert.deepEqual(tokenization, expectedResult); }); @@ -233,7 +233,7 @@ describe('Tokenization functions', function() { case 'ស្រុកខ្មែរ': return [srok, shiftSpan(khmer, srok.length)]; // array of the two. case 'កខ្មែរ': - // I'd admittedly be at least somewhat surprised if a real wordbreaker got this + // I'd admittedly be at least somewhat surprised if a real wordbreaker got this // and similar situations perfectly right... but at least it gives us what // we need for a test. return [k, shiftSpan(khmer, k.length)]; diff --git a/common/models/templates/test/test-trie-model.js b/common/models/templates/test/test-trie-model.js index 36a5944b84..b48dc254ec 100644 --- a/common/models/templates/test/test-trie-model.js +++ b/common/models/templates/test/test-trie-model.js @@ -3,7 +3,7 @@ */ var assert = require('chai').assert; -var TrieModel = require('../').models.TrieModel; +var TrieModel = require('../build/index.bundled.js').models.TrieModel; describe('LMLayerWorker trie model for word lists', function() { describe('instantiation', function () { @@ -20,7 +20,7 @@ describe('LMLayerWorker trie model for word lists', function() { } } }) - + assert.equal(model.punctuation.insertAfterWord, spaceMark); assert.equal(model.punctuation.quotesForKeepSuggestion.open, openQuote); assert.equal(model.punctuation.quotesForKeepSuggestion.close, closeQuote); @@ -176,7 +176,7 @@ describe('LMLayerWorker trie model for word lists', function() { ); }); }); - + describe('The default key function', function () { it('uses the default key function', function () { var model = new TrieModel(jsonFixture('tries/accented')); diff --git a/common/models/templates/test/test-trie-traversal.js b/common/models/templates/test/test-trie-traversal.js index 09d1361525..31f52f854a 100644 --- a/common/models/templates/test/test-trie-traversal.js +++ b/common/models/templates/test/test-trie-traversal.js @@ -3,7 +3,7 @@ */ var assert = require('chai').assert; -var TrieModel = require('../').models.TrieModel; +var TrieModel = require('../build/index.bundled.js').models.TrieModel; // Useful for tests related to strings with supplementary pairs. var smpForUnicode = function(code){ @@ -52,7 +52,7 @@ describe('Trie traversal abstractions', function() { assert.isEmpty(child.traversal().entries); for(tChild of traversalInner1.children()) { - if(tChild.char == 'h') { + if(tChild.char == 'h') { hSuccess = true; let traversalInner2 = tChild.traversal(); assert.isDefined(traversalInner2); @@ -64,7 +64,7 @@ describe('Trie traversal abstractions', function() { eSuccess = true; let traversalInner3 = hChild.traversal(); assert.isDefined(traversalInner3); - + assert.isDefined(traversalInner3.entries); assert.equal(traversalInner3.entries[0], "the"); @@ -104,7 +104,7 @@ describe('Trie traversal abstractions', function() { assert.isEmpty(child.traversal().entries); for(tChild of traversalInner1.children()) { - if(tChild.char == 'r') { + if(tChild.char == 'r') { let traversalInner2 = tChild.traversal(); assert.isDefined(traversalInner2); assert.isArray(tChild.traversal().entries); @@ -185,7 +185,7 @@ describe('Trie traversal abstractions', function() { assert.isEmpty(child.traversal().entries); for(aChild of traversalInner1.children()) { - if(aChild.char == smpP) { + if(aChild.char == smpP) { pSuccess = true; let traversalInner2 = aChild.traversal(); assert.isDefined(traversalInner2); diff --git a/common/models/templates/tsconfig.bundled.json b/common/models/templates/tsconfig.bundled.json new file mode 100644 index 0000000000..fd8dc5f65a --- /dev/null +++ b/common/models/templates/tsconfig.bundled.json @@ -0,0 +1,21 @@ +{ + // This variant of the tsconfig.json exists to create a 'leaf', 'bundled' + // version of the models/templates build product. The same reference + // cannot be prepended twice in a composite tsc build, posing problems + // for certain down-line builds if the two tsconfigs are not differentiated. + "extends": "./tsconfig.json", + "compilerOptions": { + "outFile": "build/index.bundled.js", + }, + "references": [ + { "path": "../../web/keyman-version", "prepend": true}, + { "path": "../../web/utils", "prepend": true }, + { "path": "../types" } + ], + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "test" + ] +} diff --git a/common/models/wordbreakers/package.json b/common/models/wordbreakers/package.json index 38cf3a0175..a7fd87a1f0 100644 --- a/common/models/wordbreakers/package.json +++ b/common/models/wordbreakers/package.json @@ -14,7 +14,7 @@ ], "homepage": "https://github.com/keymanapp/keyman", "license": "MIT", - "main": "index.js", + "main": "build/index.js", "directories": { "lib": "lib", "test": "test" diff --git a/common/predictive-text/unit_tests/test.sh b/common/predictive-text/unit_tests/test.sh index ed8c787c3e..8a30532771 100755 --- a/common/predictive-text/unit_tests/test.sh +++ b/common/predictive-text/unit_tests/test.sh @@ -33,6 +33,18 @@ test-headless ( ) { _FLAGS="$_FLAGS --reporter mocha-teamcity-reporter" fi + pushd "$KEYMAN_ROOT/common/models/wordbreakers" + npm run test || fail "models/wordbreakers tests failed" + popd + + pushd "$KEYMAN_ROOT/common/models/templates" + npm run test || fail "models/templates tests failed" + popd + + pushd "$KEYMAN_ROOT/common/models/types" + npm run test || fail "models/types tests failed" + popd + npm run mocha -- --recursive $_FLAGS ./unit_tests/headless/*.js ./unit_tests/headless/**/*.js } diff --git a/common/web/utils/package.json b/common/web/utils/package.json index 680231d3b3..eee6e45040 100644 --- a/common/web/utils/package.json +++ b/common/web/utils/package.json @@ -1,8 +1,8 @@ { "name": "@keymanapp/web-utils", "description": "Common utility functions used throughout other Keyman packages", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", + "main": "./build/index.js", + "types": "./build/index.d.ts", "scripts": { "build": "gosh ./build.sh", "tsc": "tsc" diff --git a/package-lock.json b/package-lock.json index 03dd8ee2b0..32b6461065 100644 --- a/package-lock.json +++ b/package-lock.json @@ -117,7 +117,7 @@ "@types/node": "^14.0.4", "chai": "^4.3.4", "mocha": "^8.4.0", - "typescript": "^3.8.3" + "typescript": "^4.5.4" } }, "common/models/templates/node_modules/@types/node": { @@ -126,19 +126,6 @@ "integrity": "sha512-q4jlIR71hUpWTnGhXWcakgkZeHa3CCjcQcnuzU8M891BAWA2jHiziiWEPEkdS5pFsz7H9HJiy8BrK7tBRNrY7A==", "dev": true }, - "common/models/templates/node_modules/typescript": { - "version": "3.9.10", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.10.tgz", - "integrity": "sha512-w6fIxVE/H1PkLKcCPsFqKE7Kv7QUwhU8qQY2MueZXWx5cPZdwFupLgKK3vntcK98BtNHZtAF4LA/yl2a7k8R6Q==", - "dev": true, - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=4.2.0" - } - }, "common/models/types": { "name": "@keymanapp/models-types", "license": "MIT", @@ -7791,7 +7778,7 @@ "@types/node": "^14.0.4", "chai": "^4.3.4", "mocha": "^8.4.0", - "typescript": "^3.8.3" + "typescript": "^4.5.4" }, "dependencies": { "@types/node": { @@ -7799,12 +7786,6 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.12.tgz", "integrity": "sha512-q4jlIR71hUpWTnGhXWcakgkZeHa3CCjcQcnuzU8M891BAWA2jHiziiWEPEkdS5pFsz7H9HJiy8BrK7tBRNrY7A==", "dev": true - }, - "typescript": { - "version": "3.9.10", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.10.tgz", - "integrity": "sha512-w6fIxVE/H1PkLKcCPsFqKE7Kv7QUwhU8qQY2MueZXWx5cPZdwFupLgKK3vntcK98BtNHZtAF4LA/yl2a7k8R6Q==", - "dev": true } } }, From 413e88daa20719c68e69abff3087fe450d03afdb Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 6 Sep 2022 10:07:02 +0700 Subject: [PATCH 054/132] chore(web): where'd that extra indentation come from? --- .../test/test-default-word-breaker.js | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/common/models/wordbreakers/test/test-default-word-breaker.js b/common/models/wordbreakers/test/test-default-word-breaker.js index f69bf528c6..7898723de2 100644 --- a/common/models/wordbreakers/test/test-default-word-breaker.js +++ b/common/models/wordbreakers/test/test-default-word-breaker.js @@ -2,34 +2,34 @@ * Smoke-test the default */ - const assert = require('chai').assert; - const breakWords = require('../build').wordBreakers['default']; +const assert = require('chai').assert; +const breakWords = require('../build').wordBreakers['default']; - const SHY = '\u00AD'; // Other, Format. The "Soft HYphen" - usually invisible unless needed for word-wrapping. +const SHY = '\u00AD'; // Other, Format. The "Soft HYphen" - usually invisible unless needed for word-wrapping. - describe('The default word breaker', function () { - it('should break multilingual text', function () { - let breaks = breakWords( - `Добрый день! ᑕᐻ᙮ — after working on ka${SHY}wen${SHY}non:${SHY}nis, - let's eat phở! 🥣` - ); - let words = breaks.map(span => span.text); - assert.deepEqual(words, [ - 'Добрый', 'день', '!', 'ᑕᐻ', '᙮', '—', 'after', - 'working', 'on', `ka${SHY}wen${SHY}non:${SHY}nis`, ',', - "let's", 'eat', 'phở', '!', '🥣' - ]); - }); +describe('The default word breaker', function () { + it('should break multilingual text', function () { + let breaks = breakWords( + `Добрый день! ᑕᐻ᙮ — after working on ka${SHY}wen${SHY}non:${SHY}nis, + let's eat phở! 🥣` + ); + let words = breaks.map(span => span.text); + assert.deepEqual(words, [ + 'Добрый', 'день', '!', 'ᑕᐻ', '᙮', '—', 'after', + 'working', 'on', `ka${SHY}wen${SHY}non:${SHY}nis`, ',', + "let's", 'eat', 'phở', '!', '🥣' + ]); + }); - it('handles heavily-punctuated English text', function() { - // This test case brought to you by http://unicode.org/reports/tr29/#Word_Boundaries, Figure 1. - let breaks = breakWords( - `The quick ("brown") fox can't jump 32.3 feet, right?` - ); - let words = breaks.map(span => span.text); - assert.deepEqual(words, [ - 'The', 'quick', '(', '"', 'brown', '"', ')', 'fox', "can't", - 'jump', '32.3', 'feet', ',', 'right', '?' - ]); - }); - }); + it('handles heavily-punctuated English text', function() { + // This test case brought to you by http://unicode.org/reports/tr29/#Word_Boundaries, Figure 1. + let breaks = breakWords( + `The quick ("brown") fox can't jump 32.3 feet, right?` + ); + let words = breaks.map(span => span.text); + assert.deepEqual(words, [ + 'The', 'quick', '(', '"', 'brown', '"', ')', 'fox', "can't", + 'jump', '32.3', 'feet', ',', 'right', '?' + ]); + }); +}); From 9be8839685ae8d65fbae37ff5eb9ab836f863804 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 6 Sep 2022 10:47:51 +0700 Subject: [PATCH 055/132] fix(common/models): undefined != 0 --- common/web/lm-worker/src/transformUtils.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/web/lm-worker/src/transformUtils.ts b/common/web/lm-worker/src/transformUtils.ts index 3bc66f33ce..b91951722f 100644 --- a/common/web/lm-worker/src/transformUtils.ts +++ b/common/web/lm-worker/src/transformUtils.ts @@ -16,10 +16,10 @@ class TransformUtils { } static isBackspace(transform: Transform): boolean { - return transform.insert == "" && transform.deleteLeft > 0 && transform.deleteRight == 0; + return transform.insert == "" && transform.deleteLeft > 0 && !transform.deleteRight; } static isEmpty(transform: Transform): boolean { - return transform.insert == '' && transform.deleteLeft == 0 && transform.deleteRight == 0; + return transform.insert == '' && transform.deleteLeft == 0 && !transform.deleteRight; } } \ No newline at end of file From 91448c344f68ab7f8770dd355fa774f1545b7555 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 6 Sep 2022 08:56:09 +0700 Subject: [PATCH 056/132] feat(web): more default-breaker unit tests --- .../models/wordbreakers/src/default/index.ts | 2 +- .../test/test-default-word-breaker.js | 151 ++++++++++++++++++ 2 files changed, 152 insertions(+), 1 deletion(-) diff --git a/common/models/wordbreakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts index 357372b228..5f14165c13 100644 --- a/common/models/wordbreakers/src/default/index.ts +++ b/common/models/wordbreakers/src/default/index.ts @@ -14,7 +14,7 @@ namespace wordBreakers { return []; } - // All non-empty strings have at least TWO boundaries at the start and end of + // All non-empty strings have at least TWO boundaries: at the start and at the end of // the string. let spans = []; for (let i = 0; i < boundaries.length - 1; i++) { diff --git a/common/models/wordbreakers/test/test-default-word-breaker.js b/common/models/wordbreakers/test/test-default-word-breaker.js index 7898723de2..41995cdbcc 100644 --- a/common/models/wordbreakers/test/test-default-word-breaker.js +++ b/common/models/wordbreakers/test/test-default-word-breaker.js @@ -32,4 +32,155 @@ describe('The default word breaker', function () { 'jump', '32.3', 'feet', ',', 'right', '?' ]); }); + + // The way these two tests are written is a bit much on the "white-box" style, + // but they do decently cover the boundary rules mentioned. + it('Does not split empty contexts (WB1 + WB2)', function() { + let breaks = breakWords(''); + let words = breaks.map(span => span.text); + assert.deepEqual(words, []); + }); + + it('Does split at context boundaries (WB1 + WB2)', function() { + let breaks = breakWords('a'); + let words = breaks.map(span => span.text); + assert.deepEqual(words, ['a']); + }); + + // WB3, WB3a, WB3b are all handled internally, within the top-level function. + + // iff, as in "if and only if" + it('ignores the zero-width joiner iff appropriate (WB4)', function() { + const zwj = '\u200d'; + + let breaks = breakWords(`a${zwj}b\n${zwj}c${zwj}\nd`); + let words = breaks.map(span => span.text); + + // Does NOT ignore the zwj immediately after a newline - the notable exception + // (the reason for "iff", not "if"). + assert.deepEqual(words, [`a${zwj}b`, `${zwj}`, `c${zwj}`, `d`]); + }) + + it('ignores extend characters iff appropriate (WB4)', function() { + const comboGrave = '\u0300'; // The 'combining grave accent', as used in NFD. + + let breaks = breakWords(`a${comboGrave}e\n${comboGrave}i${comboGrave}\no`); + let words = breaks.map(span => span.text); + + // Does NOT ignore the zwj immediately after a newline - the notable exception + // (the reason for "iff", not "if"). + assert.deepEqual(words, [`a${comboGrave}e`, `${comboGrave}`, `i${comboGrave}`, `o`]); + }); + + it('ignores format characters iff appropriate (WB4)', function() { + // Re-uses `const SHY` from above. + let breaks = breakWords(`a${SHY}e\n${SHY}i${SHY}\no`); + let words = breaks.map(span => span.text); + + // Does NOT ignore the zwj immediately after a newline - the notable exception + // (the reason for "iff", not "if"). + assert.deepEqual(words, [`a${SHY}e`, `${SHY}`, `i${SHY}`, `o`]); + }); + + it('does not break between most alphabetic characters (WB5)', function() { + let breaks = breakWords(`aəσאБ лאʈγX`); // a mix of latin, hebrew, greek, cyrillic, and IPA chars + // for both "words". + let words = breaks.map(span => span.text); + + assert.deepEqual(words, [`aəσאБ`, `лאʈγX`]); + }); + + it('does not break letters across specific punctuation patterns (WB6, WB7)', function() { + // `'`: MidNumLetQ (from Single_Quote) + // '.': MidNumLet + // ':': MidLetter + let breaks = breakWords(`don't b.r.e.a.k t:h:e:s:e`); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, [`don't`, `b.r.e.a.k`, `t:h:e:s:e`]); + + let breaks2 = breakWords(`.drop: :the' 'extras.`); + let words2 = breaks2.map(span => span.text); + + assert.deepEqual(words2, [`.`, `drop`, `:`, `:`, `the`, `'`, `'`, `extras`, `.`]); + + // ',': MidNum (is NOT included by rule!) + let breaks3 = breakWords('do br,eak that'); + let words3 = breaks3.map(span => span.text); + + assert.deepEqual(words3, ['do', 'br', ',', 'eak', 'that']); + }); + + it('treats Hebrew properly (WB7a-c)', function() { + const aleph = 'א'; + const bet = 'ב'; + + // As Hebrew is RTL... this is probably the clearest way for us LTR people to + // clearly see what's going on without ordering mechanics messing up the render. + let breaks = breakWords(`${aleph}' ${aleph}" ${aleph}"${bet}`); + let words = breaks.map(span => span.text); + + // A lingering double-quote isn't cool, but one in the middle's fine. + // Lingering single-quote is fine regardless. + assert.deepEqual(words, [`${aleph}'`, `${aleph}`, `"`, `${aleph}"${bet}`]); + }); + + it(`doesn't break within digit + digit/letter sequences (WB8-10)`, function() { + let breaks = breakWords('a1b2c3 hunter2 ab12cd34 1234567890'); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, ['a1b2c3', 'hunter2', 'ab12cd34', '1234567890']); + }); + + it('does not break within formatted number sequences (WB11-12)', function() { + // Note: `'` fits "MidNumLetQ", part of the two rules! + let breaks = breakWords(`1.2.3 3,458.01 3.45'8,01`); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, [`1.2.3`, `3,458.01`, `3.45'8,01`]); + + let breaks2 = breakWords(`.1' ,3.`); + let words2 = breaks2.map(span => span.text); + + assert.deepEqual(words2, [`.`, `1`, `'`, `,`, `3`, `.`]); + }); + + it('does not break between Katakana (WB13)', function() { + const kataSmA = '\u30a2'; //ァ + const kataA = '\u30a2'; //ア + const kataSound = '\u309b'; // ゛ + + let breaks = breakWords(`${kataSound}${kataA} ${kataSmA}${kataSound}b ${kataA}${kataSound}${kataSmA}`); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, [ + `${kataSound}${kataA}`, + `${kataSmA}${kataSound}`, + 'b', + `${kataA}${kataSound}${kataSmA}` + ]); + }); + + it('does not break form extenders (WB13a-b)', function() { + // The `_` (underscore) fits the ExtendNumLet class this rule focuses on. + const kataA = '\u30a2'; //ア + + let breaks = breakWords(`${kataA}_a__0_b_${kataA} _${kataA} 1_ _c_ ____`); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, [ + `${kataA}_a__0_b_${kataA}`, + `_${kataA}`, + `1_`, + `_c_`, + `____` + ]); + }); + + it('handles emoji flag sequences properly (WB15-16)', function() { + let breaks = breakWords(`🇨🇦🇰🇭🇽🇽`); // May not render well within VSCode. + let words = breaks.map(span => span.text); + + assert.deepEqual(words, ['🇨🇦', '🇰🇭', '🇽🇽']); // the last is two emoji in a row. + }); }); From c98fb89b86db32425371d8ce01f4a5035946ab57 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 7 Sep 2022 05:56:01 +1000 Subject: [PATCH 057/132] fix(developer): suppress repeated warnings about unreachable code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #7216. The warning message 0x209A 'The rule will never be matched because its key code is never fired.' was being generated multiple times for a single line because the `JavaScript_Key` function it is generated by is used for various purposes. This PR keeps a cache of reported key rules to ensure that the message is reported only once for a given key rule, and also improves the reporting to clarify which specific key is unreachable, which makes it easier to diagnose when using `any(k)` style messages, for example: ``` lao_phonetic.kmn (237): Warning: 209A The rule will never be matched for key 'ñ' because its key code is never fired. ``` This also reduces the warning to a hint, as this should not be a blocking issue for a keyboard, rather just a place the keyboard author can tidy up. --- .../delphi/compiler/CompileErrorCodes.pas | 2 +- developer/src/kmcmpdll/comperr.h | 2 +- .../src/tike/compile/CompileKeymanWeb.pas | 73 ++++++++++++++++++- 3 files changed, 74 insertions(+), 3 deletions(-) diff --git a/developer/src/common/delphi/compiler/CompileErrorCodes.pas b/developer/src/common/delphi/compiler/CompileErrorCodes.pas index b61cdca9f6..be16128fc3 100644 --- a/developer/src/common/delphi/compiler/CompileErrorCodes.pas +++ b/developer/src/common/delphi/compiler/CompileErrorCodes.pas @@ -194,7 +194,7 @@ const CWARN_VisualKeyboardFileMissing = $2097; CWARN_ExtendedShiftFlagsNotSupportedInKeymanWeb = $2098; // I4118 CWARN_TouchLayoutUnidentifiedKey = $2099; // I4142 - CWARN_UnreachableKeyCode = $209A; // I4141 + CHINT_UnreachableKeyCode = $109A; // I4141 CWARN_CouldNotCopyJsonFile = $209B; // I4688 CWARN_PlatformNotInTargets = $209C; diff --git a/developer/src/kmcmpdll/comperr.h b/developer/src/kmcmpdll/comperr.h index 17f72bce88..98235c7018 100644 --- a/developer/src/kmcmpdll/comperr.h +++ b/developer/src/kmcmpdll/comperr.h @@ -191,7 +191,7 @@ #define CWARN_VisualKeyboardFileMissing 0x00002097 #define CWARN_ExtendedShiftFlagsNotSupportedInKeymanWeb 0x00002098 // I4118 #define CWARN_TouchLayoutUnidentifiedKey 0x00002099 -#define CWARN_UnreachableKeyCode 0x0000209A +#define CHINT_UnreachableKeyCode 0x0000109A #define CWARN_CouldNotCopyJsonFile 0x0000209B #define CWARN_PlatformNotInTargets 0x0000209C diff --git a/developer/src/tike/compile/CompileKeymanWeb.pas b/developer/src/tike/compile/CompileKeymanWeb.pas index a1e16dcc20..c15a0032a3 100644 --- a/developer/src/tike/compile/CompileKeymanWeb.pas +++ b/developer/src/tike/compile/CompileKeymanWeb.pas @@ -102,6 +102,7 @@ uses Winapi.Windows, System.Character, System.Classes, + System.Generics.Collections, System.UITypes, compile, @@ -214,6 +215,7 @@ type FTouchLayoutFont: string; FFix183_LadderLength: Integer; FCloseBrace: Boolean; // I4872 + FUnreachableKeys: TList; function JavaScript_String(ch: DWord): string; // I2242 @@ -254,6 +256,8 @@ type function IsKeyboardVersion15OrLater: Boolean; function WriteBeginStatement(const name: string; groupIndex: Integer): string; + function FormatKeyForErrorMessage(fkp: PFILE_KEY; + FMnemonic: Boolean): string; public function Compile(AOwnerProject: TProject; const InFile: string; const OutFile: string; Debug: Boolean; Callback: TCompilerCallbackW): Boolean; // I3681 // I4140 // I4688 // I4866 constructor Create; @@ -300,6 +304,8 @@ var WarnDeprecatedCode: Boolean; Data: string; begin + FUnreachableKeys.Clear; + FCallback := Callback; FInFile := InFile; FOutFile := OutFile; // I4140 // I4155 // I4154 @@ -372,6 +378,7 @@ end; constructor TCompileKeymanWeb.Create; begin + FUnreachableKeys := TList.Create; FillChar(fk, sizeof(fk), 0); FFix183_LadderLength := FKeymanDeveloperOptions.Fix183_LadderLength; // How frequently to break ladders end; @@ -379,6 +386,7 @@ end; destructor TCompileKeymanWeb.Destroy; begin // TODO: Free FK values + FUnreachableKeys.Free; inherited; end; @@ -797,6 +805,63 @@ const // I1585 - add space to conversion VK_NUMLOCK, // &H90 VK_SCROLL); // &H91 +function TCompileKeymanWeb.FormatKeyForErrorMessage(fkp: PFILE_KEY; FMnemonic: Boolean): string; + function FormatShift(ShiftFlags: DWord): string; + const + mask: array[0..13] of string = ( + 'LCTRL', // 0X0001 + 'RCTRL', // 0X0002 + 'LALT', // 0X0004 + 'RALT', // 0X0008 + + 'SHIFT', // 0X0010 + 'CTRL', // 0X0020 + 'ALT', // 0X0040 + + '???', // Reserved + + 'CAPS', // 0X0100 + 'NCAPS', // 0X0200 + + 'NUMLOCK', // 0X0400 + 'NNUMLOCK', // 0X0800 + + 'SCROLLLOCK', // 0X1000 + 'NSCROLLLOCK' // 0X2000 + ); + var + i: Integer; + begin + Result := ''; + for i := 0 to High(mask) do + begin + if ShiftFlags and (1 shl i) <> 0 then + begin + Result := Result + mask[i] + ' '; + end; + end; + end; +begin + if not FMnemonic then + begin + if (fkp.ShiftFlags and KMX_ISVIRTUALKEY) = KMX_ISVIRTUALKEY then + begin + if Ord(fkp.Key) < 256 + then Result := Format('[%s%s]', [FormatShift(fkp.ShiftFlags), VKeyNames[Ord(fkp.Key)]]) + else Result := Format('[%sK_%x]', [FormatShift(fkp.ShiftFlags), Ord(fkp.Key)]); + end + else + begin + Result := Format('''%s''', [fkp.Key]); + end; + end + else + begin + if (fkp.ShiftFlags and KMX_VIRTUALCHARKEY) = KMX_VIRTUALCHARKEY + then Result := Format('[%s''%s'']', [FormatShift(fkp.ShiftFlags), fkp.Key]) + else Result := Format('''%s''', [fkp.Key]); + end; +end; function TCompileKeymanWeb.JavaScript_Key(fkp: PFILE_KEY; FMnemonic: Boolean): Integer; var @@ -832,7 +897,13 @@ begin if (Result = 0) or (Result >= Ord(Low(TKeymanWebTouchStandardKey))) then // I4141 begin - ReportError(fkp.Line, CWARN_UnreachableKeyCode, 'The rule will never be matched because its key code is never fired.'); + if not FUnreachableKeys.Contains(fkp) then + begin + ReportError(fkp.Line, CHINT_UnreachableKeyCode, + 'The rule will never be matched for key '+ + FormatKeyForErrorMessage(fkp,FMnemonic)+' because its key code is never fired.'); + FUnreachableKeys.Add(fkp); + end; end; end; From 5104914672c11a91308644621d14d3f01bcbf2d0 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 7 Sep 2022 10:24:01 +1000 Subject: [PATCH 058/132] feat(core): add km_kbp_event API endpoint Adds infrastructure required to support caps lock control on keyboard activation per #5822. Keyboardprocessor implementation is currently a stub, and engines will need corresponding updates to support the new API. --- core/include/keyman/keyboardprocessor.h | 43 +++++++++++++++++++++++++ core/src/km_kbp_processevent_api.cpp | 39 ++++++++++++++++++++++ core/src/processor.hpp | 9 ++++++ 3 files changed, 91 insertions(+) diff --git a/core/include/keyman/keyboardprocessor.h b/core/include/keyman/keyboardprocessor.h index 3a6700c833..e49fa4b779 100644 --- a/core/include/keyman/keyboardprocessor.h +++ b/core/include/keyman/keyboardprocessor.h @@ -1118,6 +1118,49 @@ KMN_API km_kbp_status km_kbp_process_queued_actions(km_kbp_state *state); +/* +``` +### `km_kbp_event` +##### Description: +Tell the keyboard processor that an external event has occurred, such as a keyboard +being activated through the language switching UI. +##### Return status: +- `KM_KBP_STATUS_OK`: On success. +- `KM_KBP_STATUS_NO_MEM`: +In the event memory is unavailable to allocate internal buffers. +- `KM_KBP_STATUS_INVALID_ARGUMENT`: +In the event the `state` pointer is null or an invalid event or data is passed. + +The keyboard processor may generate actions which should be processed by the +consumer of the API. + +The action list will be cleared at the start of this call; options and context in +the state may also be modified. + +##### Parameters: +- __state__: A pointer to the opaque state object. +- __event__: The event to be processed, from KM_KBP_EVENT enumeration +- __data__: Additional event-specific data. Currently unused, must be nullptr. + +```c +*/ +KMN_API +km_kbp_status +km_kbp_event( + km_kbp_state *state, + uint32_t event, + void* data +); + +enum km_kbp_event_codes { + /** + * A keyboard has been activated by the user. The processor may use this + * event, for example, to switch caps lock state or provide other UX. + */ + KM_KBP_EVENT_KEYBOARD_ACTIVATED = 1, + //future: KM_KBP_EVENT_KEYBOARD_DEACTIVATED = 2, +}; + #if defined(__cplusplus) } // extern "C" #endif diff --git a/core/src/km_kbp_processevent_api.cpp b/core/src/km_kbp_processevent_api.cpp index 0a3df41f31..57b9310754 100644 --- a/core/src/km_kbp_processevent_api.cpp +++ b/core/src/km_kbp_processevent_api.cpp @@ -12,11 +12,42 @@ #include "processor.hpp" #include "state.hpp" +km_kbp_status +km_kbp_event( + km_kbp_state *state, + uint32_t event, + void* data +) { + assert(state != nullptr); + if(state == nullptr) { + return KM_KBP_STATUS_INVALID_ARGUMENT; + } + + // event: KM_KBP_EVENT_KEYBOARD_ACTIVATED; data should be nullptr + // future event: KM_KBP_EVENT_KEYBOARD_DEACTIVATED + switch(event) { + case KM_KBP_EVENT_KEYBOARD_ACTIVATED: + assert(data == nullptr); + if(data == nullptr) { + return KM_KBP_STATUS_INVALID_ARGUMENT; + } + break; + default: + return KM_KBP_STATUS_INVALID_ARGUMENT; + } + + return state->processor().external_event(state, event, data); +} + km_kbp_status km_kbp_process_event(km_kbp_state *state, km_kbp_virtual_key vk, uint16_t modifier_state, uint8_t is_key_down) { + assert(state != nullptr); + if(state == nullptr) { + return KM_KBP_STATUS_INVALID_ARGUMENT; + } return state->processor().process_event(state, vk, modifier_state, is_key_down); } @@ -24,11 +55,19 @@ km_kbp_status km_kbp_process_queued_actions( km_kbp_state *state ) { + assert(state != nullptr); + if(state == nullptr) { + return KM_KBP_STATUS_INVALID_ARGUMENT; + } return state->processor().process_queued_actions(state); } km_kbp_attr const * km_kbp_get_engine_attrs(km_kbp_state const *state) { + assert(state != nullptr); + if(state == nullptr) { + return nullptr; + } return &state->processor().attributes(); } diff --git a/core/src/processor.hpp b/core/src/processor.hpp index d68343ce96..e4095a013d 100644 --- a/core/src/processor.hpp +++ b/core/src/processor.hpp @@ -44,6 +44,15 @@ namespace kbp uint8_t is_key_down ) = 0; + virtual km_kbp_status + external_event( + km_kbp_state* state, + uint32_t event, + void* data + ) { + return KM_KBP_STATUS_OK; + } + virtual km_kbp_attr const & attributes() const = 0; virtual km_kbp_status validate() const = 0; From 8678a0d35b652822fae56dd05b13dc78afad6d63 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 7 Sep 2022 10:36:35 +1000 Subject: [PATCH 059/132] chore(core): mark unused parameters --- core/src/processor.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/processor.hpp b/core/src/processor.hpp index e4095a013d..7041a72100 100644 --- a/core/src/processor.hpp +++ b/core/src/processor.hpp @@ -46,9 +46,9 @@ namespace kbp virtual km_kbp_status external_event( - km_kbp_state* state, - uint32_t event, - void* data + km_kbp_state* _kmn_unused(state), + uint32_t _kmn_unused(event), + void* _kmn_unused(data) ) { return KM_KBP_STATUS_OK; } From f96e3493af6f0ae8d928f18f4d5eeb0be9da1538 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 7 Sep 2022 11:13:22 +1000 Subject: [PATCH 060/132] fix(developer): hide key-sizes when in desktop layout in touch layout editor Fixes #7028. Note that the desktop layout is not currently used by KeymanWeb. The designer has a number of additional issues, as the .keyman-touch-layout format is not well suited to describing a fixed hardware layout, but fixing this is outside the scope of this issue. --- developer/src/tike/xml/layoutbuilder/builder.css | 5 +++++ developer/src/tike/xml/layoutbuilder/builder.js | 5 +---- developer/src/tike/xml/layoutbuilder/constants.js | 3 ++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/developer/src/tike/xml/layoutbuilder/builder.css b/developer/src/tike/xml/layoutbuilder/builder.css index 60daff8f3c..6e40f5b5e1 100644 --- a/developer/src/tike/xml/layoutbuilder/builder.css +++ b/developer/src/tike/xml/layoutbuilder/builder.css @@ -613,6 +613,11 @@ body:not(.text-controls-in-toolbar) input#inpSubKeyCap { color: white; } +#kbd.desktop .key-size { + /* the layout is fixed on desktop so key size is not useful */ + display: none; +} + /* Position flick keys relative to the flick grid, by hand */ #flick .key { diff --git a/developer/src/tike/xml/layoutbuilder/builder.js b/developer/src/tike/xml/layoutbuilder/builder.js index 561c4d6d0a..a432f8cd97 100644 --- a/developer/src/tike/xml/layoutbuilder/builder.js +++ b/developer/src/tike/xml/layoutbuilder/builder.js @@ -12,10 +12,7 @@ $(function() { this.getPresentation = function () { - var platform = $('#selPlatformPresentation').val(); - //if(platform == 'tablet') return 'tablet-ipad'; - //if(platform == 'phone') return 'phone-iphone5'; - return platform; + return $('#selPlatformPresentation').val(); } this.saveSelection = function() { diff --git a/developer/src/tike/xml/layoutbuilder/constants.js b/developer/src/tike/xml/layoutbuilder/constants.js index 593c7e4ca0..807a8bd30a 100644 --- a/developer/src/tike/xml/layoutbuilder/constants.js +++ b/developer/src/tike/xml/layoutbuilder/constants.js @@ -604,7 +604,8 @@ $(function() { "tablet-ipad-landscape": { "x": 829, "y": 299, "name": "iPad (landscape)" }, // 829x622 = iPad tablet box size; (97,101)-(926,723) "tablet-ipad-portrait": { "x": 605, "y": 300, "name": "iPad (portrait)" }, // 605x806 = iPad tablet box size; (98,94)-(703,900) "phone-iphone5-landscape": { "x": 731, "y": 196, "name": "iPhone 5 (landscape)" }, // 731x412 = iPhone box size; (144,39)-(875,451) - "phone-iphone5-portrait": { "x": 526, "y": 266, "name": "iPhone 5 (portrait)"} // 528x936 = iPhone box size; (90,204)-(618,1040) + "phone-iphone5-portrait": { "x": 526, "y": 266, "name": "iPhone 5 (portrait)"}, // 528x936 = iPhone box size; (90,204)-(618,1040) + "desktop": { "x": 640, "y": 300, "name": "Desktop" }, }; this.keyMargin = 15; From fb0cc5086cd5f0abbf6d0cac4d166cd691fbca88 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 7 Sep 2022 14:57:53 +0700 Subject: [PATCH 061/132] fix(common/models): backspacing shouldn't make 'new' tokens --- .../web/lm-worker/src/correction/context-tracker.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/common/web/lm-worker/src/correction/context-tracker.ts b/common/web/lm-worker/src/correction/context-tracker.ts index fba8826168..b670f1dd4a 100644 --- a/common/web/lm-worker/src/correction/context-tracker.ts +++ b/common/web/lm-worker/src/correction/context-tracker.ts @@ -127,7 +127,6 @@ namespace correction { } else { this.searchSpace = []; } - token.newFlag = true; this.tokens.push(token); let state = this; @@ -428,6 +427,7 @@ namespace correction { pushedToken.transformDistributions = transformDistribution ? [transformDistribution] : []; } + pushedToken.newFlag = true; state.pushTail(pushedToken); } else { // We're editing the final context token. // TODO: Assumption: we didn't 'miss' any inputs somehow. @@ -448,6 +448,7 @@ namespace correction { let token = new TrackedContextToken(); token.raw = tokenizedContext[0]; token.transformDistributions = [transformDistribution]; + token.newFlag = true; state.pushTail(token); } else { // Edit the lone context token. // Consider backspace entry for this case? @@ -497,15 +498,11 @@ namespace correction { state.pushTail(token); } - for(let i = 0; i < state.tokens.length - 1; i++) { - state.tokens[i].newFlag = false; - } - const finalToken = state.tokens[state.tokens.length - 1]; const baseTransform = (transformDistribution && transformDistribution.length > 0) ? transformDistribution[0] : null; - if(!baseTransform || baseTransform.sample.insert != finalToken.raw) { - finalToken.newFlag = false; + if(baseTransform && baseTransform.sample.insert == finalToken.raw) { + finalToken.newFlag = true; } return state; From 16533c2e263f0c82fefac722abe29fd0d383bdd8 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Wed, 7 Sep 2022 14:01:22 -0400 Subject: [PATCH 062/132] auto: increment master version to 16.0.58 --- HISTORY.md | 4 ++++ VERSION.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 76eb2baed3..f51d7fbe71 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,9 @@ # Keyman Version History +## 16.0.57 alpha 2022-09-07 + +* fix(android/engine): Fix Backspace key to delete without errant subkeys (#7156) + ## 16.0.56 alpha 2022-09-05 * chore(linux): Fix ibus-keyman.postinst script (#7192) diff --git a/VERSION.md b/VERSION.md index 2eda1c465b..8e5a3227df 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.57 \ No newline at end of file +16.0.58 \ No newline at end of file From 3f36a07abb6b9ffdb9bf59b8170cd543e0ed0322 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 8 Sep 2022 09:11:26 +0700 Subject: [PATCH 063/132] change(common/models): tweak to flag-emoji test for clarity --- .../wordbreakers/test/test-default-word-breaker.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/common/models/wordbreakers/test/test-default-word-breaker.js b/common/models/wordbreakers/test/test-default-word-breaker.js index 41995cdbcc..b4db8cb1a3 100644 --- a/common/models/wordbreakers/test/test-default-word-breaker.js +++ b/common/models/wordbreakers/test/test-default-word-breaker.js @@ -178,9 +178,14 @@ describe('The default word breaker', function () { }); it('handles emoji flag sequences properly (WB15-16)', function() { - let breaks = breakWords(`🇨🇦🇰🇭🇽🇽`); // May not render well within VSCode. + // For clarity on what's being tested... + let CA_FLAG = '\u{1f1e8}\u{1f1e6}' // '🇨🇦' (canadian flag emoji); should not be broken. + let KH_FLAG = '\u{1f1f0}\u{1f1ed}' // '🇰🇭' (khmer flag emoji); same + let X_FLAG_PIECE = '\u{1f1fd}' // '🇽' (half of a flag emoji; '🇽🇽' doesn't match a flag) + let breaks = breakWords(`${CA_FLAG}${KH_FLAG}${X_FLAG_PIECE}${X_FLAG_PIECE}`); let words = breaks.map(span => span.text); - assert.deepEqual(words, ['🇨🇦', '🇰🇭', '🇽🇽']); // the last is two emoji in a row. + // Note that the emoji may not render well within VSCode, but they show up nicely on GitHub. + assert.deepEqual(words, ['🇨🇦', '🇰🇭', '🇽🇽']); }); }); From 52cb9aa19d944f53f15838dace9c156d81984ce7 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 8 Sep 2022 10:48:12 +0700 Subject: [PATCH 064/132] change(common/models): drops .isNew, replaces with tokenized context contrast logic --- .../src/correction/context-tracker.ts | 15 --- common/web/lm-worker/src/model-compositor.ts | 104 ++++++++++++------ 2 files changed, 71 insertions(+), 48 deletions(-) diff --git a/common/web/lm-worker/src/correction/context-tracker.ts b/common/web/lm-worker/src/correction/context-tracker.ts index b670f1dd4a..f6104ef921 100644 --- a/common/web/lm-worker/src/correction/context-tracker.ts +++ b/common/web/lm-worker/src/correction/context-tracker.ts @@ -27,16 +27,11 @@ namespace correction { export class TrackedContextToken { raw: string; replacementText: string; - newFlag: boolean = false; transformDistributions: Distribution[] = []; replacements: TrackedContextSuggestion[]; activeReplacementId: number = -1; - get isNew(): boolean { - return this.newFlag; - } - get currentText(): string { if(this.replacementText === undefined || this.replacementText === null) { return this.raw; @@ -190,7 +185,6 @@ namespace correction { } // Replace old token's raw-text with new token's raw-text. editedToken.raw = tokenText; - editedToken.newFlag = false; } toRawTokenization() { @@ -427,7 +421,6 @@ namespace correction { pushedToken.transformDistributions = transformDistribution ? [transformDistribution] : []; } - pushedToken.newFlag = true; state.pushTail(pushedToken); } else { // We're editing the final context token. // TODO: Assumption: we didn't 'miss' any inputs somehow. @@ -448,7 +441,6 @@ namespace correction { let token = new TrackedContextToken(); token.raw = tokenizedContext[0]; token.transformDistributions = [transformDistribution]; - token.newFlag = true; state.pushTail(token); } else { // Edit the lone context token. // Consider backspace entry for this case? @@ -498,13 +490,6 @@ namespace correction { state.pushTail(token); } - const finalToken = state.tokens[state.tokens.length - 1]; - const baseTransform = (transformDistribution && transformDistribution.length > 0) ? transformDistribution[0] : null; - - if(baseTransform && baseTransform.sample.insert == finalToken.raw) { - finalToken.newFlag = true; - } - return state; } diff --git a/common/web/lm-worker/src/model-compositor.ts b/common/web/lm-worker/src/model-compositor.ts index 8709ab0c6b..9a7da82a80 100644 --- a/common/web/lm-worker/src/model-compositor.ts +++ b/common/web/lm-worker/src/model-compositor.ts @@ -85,7 +85,7 @@ class ModelCompositor { // Used to restore whitespaces if operations would remove them. let prefixTransform: Transform; - let contextState: correction.TrackedContextState = null; + let postContextState: correction.TrackedContextState = null; // Section 1: determining 'prediction roots'. if(!this.contextTracker) { @@ -120,12 +120,15 @@ class ModelCompositor { // Running in bulk over all suggestions, duplicate entries may be possible. rawPredictions = this.predictFromCorrections(predictionRoots, context); } else { - contextState = this.contextTracker.analyzeState(this.lexicalModel, - postContext, - !TransformUtils.isEmpty(inputTransform) ? - transformDistribution: - null - ); + // Token replacement benefits greatly from knowledge of the prior context state. + let contextState = this.contextTracker.analyzeState(this.lexicalModel, context, null); + // Corrections and predictions are based upon the post-context state, though. + postContextState = this.contextTracker.analyzeState(this.lexicalModel, + postContext, + !TransformUtils.isEmpty(inputTransform) ? + transformDistribution: + null + ); // TODO: Should we filter backspaces & whitespaces out of the transform distribution? // Ideally, the answer (in the future) will be no, but leaving it in right now may pose an issue. @@ -134,20 +137,68 @@ class ModelCompositor { // let's just note that right now, there will only ever be one. // // The 'eventual' logic will be significantly more complex, though still manageable. - let searchSpace = contextState.searchSpace[0]; + let searchSpace = postContextState.searchSpace[0]; - let newToken = false; - // Detect if we're starting a new context state. - let contextTokens = contextState.tokens; - if(contextTokens.length == 0 || contextTokens[contextTokens.length - 1].isNew) { - // Always note if we have a new token (so that we don't try to delete existing context) - newToken = true; - // If the new token is due to whitespace, or if we had a context-reset trigger this (thus, no input...) - // (Lingering question: do we need the .isEmpty check here? Track `prefixTransform` and find out.) - if(TransformUtils.isEmpty(inputTransform) || TransformUtils.isWhitespace(inputTransform)) { + // No matter the prediction, once we know the root of the prediction, we'll always 'replace' the + // same amount of text. We can handle this before the big 'prediction root' loop. + let deleteLeft = 0; + + // The amount of text to 'replace' depends upon whatever sort of context change occurs + // from the received input. + let postContextLength = postContextState.tokens.length; + let contextLengthDelta = postContextState.tokens.length - contextState.tokens.length; + // If the context now has more tokens, the token we'll be 'predicting' didn't originally exist. + if(postContextLength == 0 || contextLengthDelta > 0) { + // As the word/token being corrected/predicted didn't originally exist, there's no + // part of it to 'replace'. + deleteLeft = 0; + + // If the new token is due to whitespace or due to a different input type that would + // likely imply a tokenization boundary... + if(TransformUtils.isWhitespace(inputTransform)) { + /* TODO: consider/implement: the second half of the comment above. + * For example: on input of a `'`, predict new words instead of replacing the `'`. + * (since after a letter, the `'` will be ignored, anyway) + * + * Idea: if the model's most likely prediction (with no root) would make a new + * token if appended to the current token, that's probably a good case. + * Keeps the check simple & quick. + * + * Might need a mixed mode, though: ';' is close enough that `l` is a reasonable + * fat-finger guess. So yeah, we're not addressing this idea right now. + * - so... consider multiple context behavior angles when building prediction roots? + * + * May need something similar to help handle contractions during their construction, + * but that'd be within `ContextTracker`. + * can' => [`can`, `'`] + * can't => [`can't`] (WB6, 7 of https://unicode.org/reports/tr29/#Word_Boundary_Rules) + * + * (Would also helps WB7b+c for Hebrew text) + */ + + // Infer 'new word' mode, even if we received new text when reaching + // this position. That new text didn't exist before, so still - nothing + // to 'replace'. prefixTransform = inputTransform; - context = postContext; // Ensure the whitespace token is preapplied! + context = postContext; // As far as predictions are concerned, the post-context state + // should not be replaced. Predictions are to be rooted on + // text "up for correction" - so we want a null root for this + // branch. + contextState = postContextState; } + // If the tokenized context length is shorter... sounds like a backspace (or similar). + } else if (contextLengthDelta < 0) { + /* Ooh, we've dropped context here. Almost certainly from a backspace. + * Even if we drop multiple tokens... well, we know exactly how many chars + * were actually deleted - `inputTransform.deleteLeft`. + * Since we replace a word being corrected/predicted, we take length of the remaining + * context's tail token in addition to however far was deleted to reach that state. + */ + deleteLeft = this.wordbreak(postContext).kmwLength() + inputTransform.deleteLeft; + } else { + // Suggestions are applied to the pre-input context, so get the token's original length. + // We're on the same token, so just delete its text for the replacement op. + deleteLeft = this.wordbreak(context).kmwLength(); } // TODO: whitespace, backspace filtering. Do it here. @@ -171,19 +222,6 @@ class ModelCompositor { finalInput = inputTransform; // A fallback measure. Greatly matters for empty contexts. } - let deleteLeft = 0; - // remove actual token string. If new token, there should be nothing to delete. - if(!newToken) { - // If this is triggered from a backspace, make sure to use its results - // and also include its left-deletions! It's the one post-input context case. - if(allowBksp) { - deleteLeft = this.wordbreak(postContext).kmwLength() + inputTransform.deleteLeft; - } else { - // Normal case - use the pre-input context. - deleteLeft = this.wordbreak(context).kmwLength(); - } - } - // Replace the existing context with the correction. let correctionTransform: Transform = { insert: correction, // insert correction string @@ -390,8 +428,8 @@ class ModelCompositor { // Store the suggestions on the final token of the current context state (if it exists). // Or, once phrase-level suggestions are possible, on whichever token serves as each prediction's root. - if(contextState) { - contextState.tail.replacements = suggestions.map(function(suggestion) { + if(postContextState) { + postContextState.tail.replacements = suggestions.map(function(suggestion) { return { suggestion: suggestion, tokenWidth: 1 From 3aed0bf39556a2f7431b1c17bc2655efa7bc152c Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 8 Sep 2022 10:48:23 +0700 Subject: [PATCH 065/132] feat(common/models): also, unit tests --- .../headless/worker-model-compositor.js | 43 ++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/common/predictive-text/unit_tests/headless/worker-model-compositor.js b/common/predictive-text/unit_tests/headless/worker-model-compositor.js index f7e9173cee..dc8a36e9dc 100644 --- a/common/predictive-text/unit_tests/headless/worker-model-compositor.js +++ b/common/predictive-text/unit_tests/headless/worker-model-compositor.js @@ -66,12 +66,53 @@ describe('ModelCompositor', function() { // Suggestions always delete the full root of the suggestion. // // After a backspace, that means the text 'the' - 3 chars. - // Char 4 is for the original backspace, as suggstions are built + // Char 4 is for the original backspace, as suggestions are built // based on the context state BEFORE the triggering input - // here, a backspace. assert.equal(suggestion.transform.deleteLeft, 4); }); }); + + it('properly handles suggestions for the first letter after a ` `', function() { + let compositor = new ModelCompositor(plainModel); + let context = { + left: 'the', startOfBuffer: true, endOfBuffer: true, + }; + + let inputTransform = { + insert: ' ', + deleteLeft: 0 + }; + + let suggestions = compositor.predict(inputTransform, context); + suggestions.forEach(function(suggestion) { + // After a space, predictions are based on a new, zero-length root. + // With nothing to replace, .deleteLeft should be zero. + assert.equal(suggestion.transform.deleteLeft, 0); + }); + }); + + it('properly handles suggestions for the first letter after a `\'`', function() { + let compositor = new ModelCompositor(plainModel); + let context = { + left: "the '", startOfBuffer: true, endOfBuffer: true, + }; + + // This results in a new word boundary (between the `'` and the `a`). + // Basically, an implied (but nonexistent) ` `. + let inputTransform = { + insert: "a", + deleteLeft: 0 + }; + + let suggestions = compositor.predict(inputTransform, context); + suggestions.forEach(function(suggestion) { + // Suggestions always delete the full root of the suggestion. + // Which, here, didn't exist before the input. Nothing to + // replace => nothing for the suggestion to delete. + assert.equal(suggestion.transform.deleteLeft, 0); + }); + }); }); describe('applySuggestionCasing', function() { From 75e0b27bf379f80813855d6fb35f12bb26f64c2a Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 8 Sep 2022 10:54:01 +0700 Subject: [PATCH 066/132] change(web): pushWhitespaceToTail tweak --- common/web/lm-worker/src/correction/context-tracker.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/web/lm-worker/src/correction/context-tracker.ts b/common/web/lm-worker/src/correction/context-tracker.ts index f6104ef921..528edc2ce8 100644 --- a/common/web/lm-worker/src/correction/context-tracker.ts +++ b/common/web/lm-worker/src/correction/context-tracker.ts @@ -137,6 +137,8 @@ namespace correction { // Will be needed for phrase-level correction/prediction. if(transformDistribution) { whitespaceToken.transformDistributions = [transformDistribution]; + } else { + whitespaceToken.transformDistributions = []; } whitespaceToken.raw = null; From 6c50de9e9a220f28c49050dae44e6f3e01ef7a18 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 8 Sep 2022 10:56:32 +0700 Subject: [PATCH 067/132] change(web): conciser version of last commit --- common/web/lm-worker/src/correction/context-tracker.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/common/web/lm-worker/src/correction/context-tracker.ts b/common/web/lm-worker/src/correction/context-tracker.ts index 528edc2ce8..e17e201d78 100644 --- a/common/web/lm-worker/src/correction/context-tracker.ts +++ b/common/web/lm-worker/src/correction/context-tracker.ts @@ -135,11 +135,7 @@ namespace correction { // Track the Transform that resulted in the whitespace 'token'. // Will be needed for phrase-level correction/prediction. - if(transformDistribution) { - whitespaceToken.transformDistributions = [transformDistribution]; - } else { - whitespaceToken.transformDistributions = []; - } + whitespaceToken.transformDistributions = transformDistribution ? [transformDistribution] : []; whitespaceToken.raw = null; this.tokens.push(whitespaceToken); From f836574cbc714084eaa5c8b72ec1337df1b1ad4b Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 8 Sep 2022 14:32:39 +1000 Subject: [PATCH 068/132] chore: use keyman.com instead of keyman-staging.com Fixes #7227. We want to use keyman.com for now even for alpha builds, as keyman-staging.com is going to be used for transition to Docker-based hosting, and it will be problematic to try and share that process with alpha builds. --- .../tavultesoft/kmea/data/CloudRepository.java | 2 +- .../com/tavultesoft/kmea/util/KMPLink.java | 2 +- .../windows/delphi/general/Upload_Settings.pas | 4 ++-- .../KeymanEngine/Classes/KeymanHosts.swift | 6 +++--- linux/keyman-config/keyman_config/__init__.py | 12 ++++++------ .../Keyman4MacIM/KMInputMethodAppDelegate.m | 18 +++++++++--------- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/data/CloudRepository.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/data/CloudRepository.java index 1baba069ed..a7f5974718 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/data/CloudRepository.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/data/CloudRepository.java @@ -39,7 +39,7 @@ public class CloudRepository { public static final String DOWNLOAD_IDENTIFIER_CATALOGUE = "catalogue"; public static final String API_PRODUCTION_HOST = "api.keyman.com"; - public static final String API_STAGING_HOST = "api.keyman-staging.com"; + public static final String API_STAGING_HOST = "api.keyman.com"; // #7227 disabling: "api.keyman-staging.com"; public static final String API_MODEL_LANGUAGE_FORMATSTR = "https://%s/model?q=bcp47:%s"; public static final String API_PACKAGE_VERSION_FORMATSTR = "https://%s/package-version?platform=android%s%s"; diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/KMPLink.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/KMPLink.java index ff234e85c7..19ad4aaa72 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/KMPLink.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/KMPLink.java @@ -21,7 +21,7 @@ import java.util.regex.Pattern; public final class KMPLink { public static final String KMP_PRODUCTION_HOST = "keyman.com"; - public static final String KMP_STAGING_HOST = "keyman-staging.com"; + public static final String KMP_STAGING_HOST = "keyman.com"; // #7227 disabling: "keyman-staging.com"; private static final String KMP_INSTALL_KEYBOARDS_PATTERN_FORMATSTR = "^http(s)?://(%s|%s)/keyboards/install/([^\\?/]+)(\\?(.+))?$"; private static final String installPatternFormatStr = KMString.format(KMP_INSTALL_KEYBOARDS_PATTERN_FORMATSTR, diff --git a/common/windows/delphi/general/Upload_Settings.pas b/common/windows/delphi/general/Upload_Settings.pas index 41355e9c0f..f38cc07890 100644 --- a/common/windows/delphi/general/Upload_Settings.pas +++ b/common/windows/delphi/general/Upload_Settings.pas @@ -108,8 +108,8 @@ const // Alpha versions will work against the staging server so that they // can access new APIs etc that will only be available there. The staging // servers have resource constraints but should be okay for limited use. - S_KeymanCom_Staging = 'https://keyman-staging.com'; - S_APIServer_Staging = 'api.keyman-staging.com'; + S_KeymanCom_Staging = 'https://keyman.com'; // #7227 disabling: 'https://keyman-staging.com'; + S_APIServer_Staging = 'api.keyman.com'; // #7227 disabling: 'api.keyman-staging.com'; const URLPath_PackageDownload_Format = '/go/package/download/%0:s?platform=windows&tier=%1:s&bcp47=%2:s&update=%3:d'; diff --git a/ios/engine/KMEI/KeymanEngine/Classes/KeymanHosts.swift b/ios/engine/KMEI/KeymanEngine/Classes/KeymanHosts.swift index cf312a8e0d..897b162a5e 100644 --- a/ios/engine/KMEI/KeymanEngine/Classes/KeymanHosts.swift +++ b/ios/engine/KMEI/KeymanEngine/Classes/KeymanHosts.swift @@ -30,7 +30,7 @@ public enum KeymanHosts { case .alpha: fallthrough case .beta: - return URL.init(string: "https://api.keyman-staging.com")! + return URL.init(string: "https://api.keyman.com")! // #7227 disabling: "https://api.keyman-staging.com")! case .stable: return URL.init(string: "https://api.keyman.com")! } @@ -54,7 +54,7 @@ public enum KeymanHosts { case .alpha: fallthrough case .beta: - return URL.init(string: "https://help.keyman-staging.com")! + return URL.init(string: "https://help.keyman.com")! // #7227 disabling: "https://help.keyman-staging.com")! case .stable: return URL.init(string: "https://help.keyman.com")! } @@ -78,7 +78,7 @@ public enum KeymanHosts { case .alpha: fallthrough case .beta: - return URL.init(string: "https://keyman-staging.com")! + return URL.init(string: "https://keyman.com")! // #7227 disabling: "https://keyman-staging.com")! case .stable: return URL.init(string: "https://keyman.com")! } diff --git a/linux/keyman-config/keyman_config/__init__.py b/linux/keyman-config/keyman_config/__init__.py index 7c3972c247..ee2326a86f 100644 --- a/linux/keyman-config/keyman_config/__init__.py +++ b/linux/keyman-config/keyman_config/__init__.py @@ -40,15 +40,15 @@ def secure_lookup(data, key1, key2 = None): gettext.bindtextdomain('keyman-config', '/usr/share/locale') gettext.textdomain('keyman-config') -if __tier__ == 'alpha' or __tier__ == 'beta': +#if __tier__ == 'alpha' or __tier__ == 'beta': // #7227 disabling: # Alpha and beta versions will work against the staging server so that they # can access new APIs etc that will only be available there. The staging # servers have resource constraints but should be okay for limited use. - KeymanComUrl = 'https://keyman-staging.com' - KeymanApiUrl = 'https://api.keyman-staging.com' -else: - KeymanComUrl = 'https://keyman.com' - KeymanApiUrl = 'https://api.keyman.com' +# KeymanComUrl = 'https://keyman-staging.com' +# KeymanApiUrl = 'https://api.keyman-staging.com' +#else: +KeymanComUrl = 'https://keyman.com' +KeymanApiUrl = 'https://api.keyman.com' # There's no staging site for downloads KeymanDownloadsUrl = 'https://downloads.keyman.com' diff --git a/mac/Keyman4MacIM/Keyman4MacIM/KMInputMethodAppDelegate.m b/mac/Keyman4MacIM/Keyman4MacIM/KMInputMethodAppDelegate.m index 61558b7ecf..8a4cc0734b 100644 --- a/mac/Keyman4MacIM/Keyman4MacIM/KMInputMethodAppDelegate.m +++ b/mac/Keyman4MacIM/Keyman4MacIM/KMInputMethodAppDelegate.m @@ -130,16 +130,16 @@ NSString* _keymanDataPath = nil; result.tier = [keymanInfo objectForKey:@"Tier"]; result.versionRelease = [keymanInfo objectForKey:@"VersionRelease"]; result.versionWithTag = [keymanInfo objectForKey:@"VersionWithTag"]; - if([result.tier isEqualToString:@"stable"]) { - result.keymanCom = @"keyman.com"; - result.helpKeymanCom = @"help.keyman.com"; - result.apiKeymanCom = @"api.keyman.com"; - } + // if([result.tier isEqualToString:@"stable"]) { // #7227 disabling: + result.keymanCom = @"keyman.com"; + result.helpKeymanCom = @"help.keyman.com"; + result.apiKeymanCom = @"api.keyman.com"; + /*} else { result.keymanCom = @"keyman-staging.com"; result.helpKeymanCom = @"help.keyman-staging.com"; result.apiKeymanCom = @"api.keyman-staging.com"; - } + }*/ return result; } @@ -632,11 +632,11 @@ CGEventRef eventTapFunction(CGEventTapProxy proxy, CGEventType type, CGEventRef NSString *path = [[self keymanDataPath] stringByAppendingPathComponent:packageFolder]; KMPackageInfo *packageInfo = [self.packageReader loadPackageInfo:path]; - + if (packageInfo) { packageName = packageInfo.packageName; } - + return packageName; } @@ -1149,7 +1149,7 @@ CGEventRef eventTapFunction(CGEventTapProxy proxy, CGEventType type, CGEventRef [self.receivedData writeToFile:filePath atomically:YES]; [self unzipFile:filePath]; [[NSFileManager defaultManager] removeItemAtPath:filePath error:nil]; - + [_downloadInfoView setMessageText:NSLocalizedString(@"message-keyboard-download-complete", nil)]; NSButton *button = (NSButton *)[_downloadInfoView.buttons objectAtIndex:0]; [button setTitle:NSLocalizedString(@"button-download-complete", nil)]; From 4a3aca8a05ca41433200f6d76dcdbd2fc5a10887 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 8 Sep 2022 14:47:44 +1000 Subject: [PATCH 069/132] chore(android): disable keyman-staging tests --- .../tavultesoft/kmea/util/KMPLinkTest.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/android/KMEA/app/src/test/java/com/tavultesoft/kmea/util/KMPLinkTest.java b/android/KMEA/app/src/test/java/com/tavultesoft/kmea/util/KMPLinkTest.java index 013da4132a..56743502ed 100644 --- a/android/KMEA/app/src/test/java/com/tavultesoft/kmea/util/KMPLinkTest.java +++ b/android/KMEA/app/src/test/java/com/tavultesoft/kmea/util/KMPLinkTest.java @@ -18,9 +18,9 @@ public class KMPLinkTest { Assert.assertFalse(KMPLink.isKeymanInstallLink("")); // Valid Keyman keyboard install links - Assert.assertTrue(KMPLink.isKeymanInstallLink("https://keyman-staging.com/keyboards/install/malar_malayalam")); - Assert.assertTrue(KMPLink.isKeymanInstallLink("https://keyman-staging.com/keyboards/install/malar_malayalam?bcp47=ml")); - Assert.assertTrue(KMPLink.isKeymanInstallLink("https://keyman-staging.com/keyboards/install/fv_sencoten?bcp47=str-Latn")); + // Assert.assertTrue(KMPLink.isKeymanInstallLink("https://keyman-staging.com/keyboards/install/malar_malayalam")); + // Assert.assertTrue(KMPLink.isKeymanInstallLink("https://keyman-staging.com/keyboards/install/malar_malayalam?bcp47=ml")); + // Assert.assertTrue(KMPLink.isKeymanInstallLink("https://keyman-staging.com/keyboards/install/fv_sencoten?bcp47=str-Latn")); Assert.assertTrue(KMPLink.isKeymanInstallLink("https://keyman.com/keyboards/install/lao%20unicode")); // legacy keyboard within download dialog Assert.assertTrue(KMPLink.isKeymanInstallLink("https://keyman.com/keyboards/install/khmer_angkor?_gid=112233&bcp47=lo")); // other query params appearing @@ -28,11 +28,11 @@ public class KMPLinkTest { Assert.assertTrue(KMPLink.isKeymanInstallLink("https://keyman.com/keyboards/install/malar_malayalam?bcp47=ml")); // Keyboard link is wrong - Assert.assertFalse(KMPLink.isKeymanInstallLink("https://keyman-staging.com/keyboard/install/malar_malayalam")); + // Assert.assertFalse(KMPLink.isKeymanInstallLink("https://keyman-staging.com/keyboard/install/malar_malayalam")); // link missing packageID - Assert.assertFalse(KMPLink.isKeymanInstallLink("https://keyman-staging.com/keyboards/install")); - Assert.assertFalse(KMPLink.isKeymanInstallLink("https://keyman-staging.com/keyboards/install/")); + // Assert.assertFalse(KMPLink.isKeymanInstallLink("https://keyman-staging.com/keyboards/install")); + // Assert.assertFalse(KMPLink.isKeymanInstallLink("https://keyman-staging.com/keyboards/install/")); Assert.assertFalse(KMPLink.isKeymanInstallLink("https://keyman.com/keyboards/install")); Assert.assertFalse(KMPLink.isKeymanInstallLink("https://keyman.com/keyboards/install/")); } @@ -43,14 +43,14 @@ public class KMPLinkTest { Assert.assertFalse(KMPLink.isKeymanDownloadLink("")); // Valid Keyman keyboard download links - Assert.assertTrue(KMPLink.isKeymanDownloadLink("https://keyman-staging.com/go/package/download/malar_malayalam?platform=android&tier=alpha")); - Assert.assertTrue(KMPLink.isKeymanDownloadLink("https://keyman-staging.com/go/package/download/malar_malayalam?platform=android&tier=alpha&bcp47=ml")); + // Assert.assertTrue(KMPLink.isKeymanDownloadLink("https://keyman-staging.com/go/package/download/malar_malayalam?platform=android&tier=alpha")); + // Assert.assertTrue(KMPLink.isKeymanDownloadLink("https://keyman-staging.com/go/package/download/malar_malayalam?platform=android&tier=alpha&bcp47=ml")); Assert.assertTrue(KMPLink.isKeymanDownloadLink("https://keyman.com/go/package/download/malar_malayalam?platform=android&tier=alpha")); Assert.assertTrue(KMPLink.isKeymanDownloadLink("https://keyman.com/go/package/download/malar_malayalam?platform=android&tier=alpha&bcp47=ml")); // link missing packageID - Assert.assertFalse(KMPLink.isKeymanDownloadLink("https://keyman-staging.com/go/package/download")); - Assert.assertFalse(KMPLink.isKeymanDownloadLink("https://keyman-staging.com/go/package/download/")); + // Assert.assertFalse(KMPLink.isKeymanDownloadLink("https://keyman-staging.com/go/package/download")); + // Assert.assertFalse(KMPLink.isKeymanDownloadLink("https://keyman-staging.com/go/package/download/")); Assert.assertFalse(KMPLink.isKeymanDownloadLink("https://keyman.com/go/package/download")); Assert.assertFalse(KMPLink.isKeymanDownloadLink("https://keyman.com/go/package/download/")); } From 9b29339594309048634226ef78d969164dc4b695 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 8 Sep 2022 13:13:58 +0700 Subject: [PATCH 070/132] fix(common/models): blocks full-text corrections --- .../src/correction/distance-modeler.ts | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/common/web/lm-worker/src/correction/distance-modeler.ts b/common/web/lm-worker/src/correction/distance-modeler.ts index 7b4456e4d8..47c7ad88eb 100644 --- a/common/web/lm-worker/src/correction/distance-modeler.ts +++ b/common/web/lm-worker/src/correction/distance-modeler.ts @@ -204,6 +204,15 @@ namespace correction { // TODO: might should also track diagonalWidth. return inputString + models.SENTINEL_CODE_UNIT + matchString; } + + get isFullReplacement(): boolean { + // If the known edit-distance cost is equal to the input length, this means + // that literally every input has been full-on replaced. Thus, this is + // likely not a good 'root' to use for predictions. + // + // Logic exception: 0 cost, 0 length != a "replacement". + return this.knownCost && this.knownCost == this.priorInput.length; + } } class SearchSpaceTier { @@ -590,6 +599,14 @@ namespace correction { // Build batches of same-cost entries. while(preprocessedQueue.count > 0) { let entry = preprocessedQueue.dequeue(); + + // Is the entry a reasonable result? + if(entry.isFullReplacement) { + // If the entry's 'match' fully replaces the input string, we consider it + // unreasonable and ignore it. + continue; + } + let batch = batcher.checkAndAdd(entry); if(batch) { @@ -626,6 +643,13 @@ namespace correction { if(newResult.type == 'none') { break; } else if(newResult.type == 'complete') { + // Is the entry a reasonable result? + if(newResult.finalNode.isFullReplacement) { + // If the entry's 'match' fully replaces the input string, we consider it + // unreasonable and ignore it. Also, if we've reached this point... + // we can(?) assume that everything thereafter is as well. + break; + } batch = batcher.checkAndAdd(newResult.finalNode); } From 239da832d5465d9e3491d9e5d4a49e2ef377cdbd Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Thu, 8 Sep 2022 09:36:57 +0700 Subject: [PATCH 071/132] fix(android/engine): Fix language tag in language-picker menu --- .../java/com/tavultesoft/kmea/packages/PackageProcessor.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/packages/PackageProcessor.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/packages/PackageProcessor.java index 9a06a1255f..db6ae9276e 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/packages/PackageProcessor.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/packages/PackageProcessor.java @@ -626,6 +626,9 @@ public class PackageProcessor { for (int j = 0; j < languages.length(); j++) { JSONObject language = languages.getJSONObject(j); String languageID = language.getString("id"); + if (languageID != null) { + languageID = languageID.toLowerCase(); + } String languageName = language.getString("name"); if (!excludeInstalledLanguages || !(KeyboardController.getInstance().keyboardExists(packageID, keyboardID, languageID))) { From f90ed759cf0ee906d34c868596c431f08a55ed66 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Thu, 8 Sep 2022 10:45:16 +0700 Subject: [PATCH 072/132] chore(android/engine): Revert packageprocessor change --- .../java/com/tavultesoft/kmea/packages/PackageProcessor.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/packages/PackageProcessor.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/packages/PackageProcessor.java index db6ae9276e..9a06a1255f 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/packages/PackageProcessor.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/packages/PackageProcessor.java @@ -626,9 +626,6 @@ public class PackageProcessor { for (int j = 0; j < languages.length(); j++) { JSONObject language = languages.getJSONObject(j); String languageID = language.getString("id"); - if (languageID != null) { - languageID = languageID.toLowerCase(); - } String languageName = language.getString("name"); if (!excludeInstalledLanguages || !(KeyboardController.getInstance().keyboardExists(packageID, keyboardID, languageID))) { From 1fb1c243efde99e89264fbf8f66ca575f57223ba Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Thu, 8 Sep 2022 10:45:40 +0700 Subject: [PATCH 073/132] fix(android/engine): Use BCP47.toggleLanguage --- .../kmapro/SelectLanguageFragment.java | 6 +--- .../java/com/tavultesoft/kmea/util/BCP47.java | 34 +++++++++++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/SelectLanguageFragment.java b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/SelectLanguageFragment.java index c1128f9966..80d5367d4c 100644 --- a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/SelectLanguageFragment.java +++ b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/SelectLanguageFragment.java @@ -199,11 +199,7 @@ public final class SelectLanguageFragment extends Fragment implements BlockingSt languageList = new ArrayList(); } String selectedLanguageID = k.getLanguageID(); - if (languageList.contains(selectedLanguageID)) { - languageList.remove(selectedLanguageID); - } else { - languageList.add(selectedLanguageID); - } + BCP47.toggleLanguage(languageList, selectedLanguageID); } else { // Otherwise, add the language association if (addKeyboardsList == null) { diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/BCP47.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/BCP47.java index 7e396dc551..c49b9b64c6 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/BCP47.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/BCP47.java @@ -4,6 +4,8 @@ package com.tavultesoft.kmea.util; +import java.util.ArrayList; + public final class BCP47 { /** * Utility to compare two language ID strings @@ -18,4 +20,36 @@ public final class BCP47 { return id1.equalsIgnoreCase(id2); } + + /** + * Utility to modify languageList. + * If languageID exists in the list, remove it. Otherwise, add languageID to the list. + * @param languageList + * @param languageID + */ + public static void toggleLanguage(ArrayList languageList, String languageID) { + if (languageList != null) { + if (languageList.size() == 0) { + languageList.add(languageID.toLowerCase()); + return; + } + + boolean languageExists = false; + int index = 0; + // See if languageID already exists in the languageList + for (String l: languageList) { + if (languageEquals(l, languageID)) { + languageExists = true; + break; + } + index++; + } + + if (languageExists && index < languageList.size()) { + languageList.remove(index); + } else { + languageList.add(languageID.toLowerCase()); + } + } + } } \ No newline at end of file From 04906cfe0d11d854497bd79c50bb382b4a5ae3e6 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Thu, 8 Sep 2022 13:28:12 +0700 Subject: [PATCH 074/132] fix(android/engine): Do similar toggle for keyboard list --- .../kmapro/SelectLanguageFragment.java | 41 ++++++++++++++++--- .../java/com/tavultesoft/kmea/util/BCP47.java | 2 +- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/SelectLanguageFragment.java b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/SelectLanguageFragment.java index 80d5367d4c..96c641b1a8 100644 --- a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/SelectLanguageFragment.java +++ b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/SelectLanguageFragment.java @@ -190,6 +190,41 @@ public final class SelectLanguageFragment extends Fragment implements BlockingSt listView.setOnItemClickListener(new AdapterView.OnItemClickListener() { + /** + * Utility to modify keyboardList. Based on BCP47.toggleLanguage + * If keyboard's languageID exists in the list, remove the keyboard. + * Otherwise, add the keyboard to the list. + * @param keyboardList + * @param k + */ + private void toggleLanguage(ArrayList keyboardList, Keyboard k) { + if (keyboardList != null) { + if (keyboardList.size() == 0) { + k.setLanguage(k.getLanguageID().toLowerCase(), k.getLanguageName()); + keyboardList.add(k); + return; + } + + boolean languageExists = false; + int index = 0; + // See if languageID already exists in the keyboardList + for (Keyboard l: keyboardList) { + if (BCP47.languageEquals(l.getLanguageID(), k.getLanguageID())) { + languageExists = true; + break; + } + index++; + } + + if (languageExists && index < keyboardList.size()) { + keyboardList.remove(index); + } else { + k.setLanguage(k.getLanguageID().toLowerCase(), k.getLanguageName()); + keyboardList.add(k); + } + } + } + @Override public void onItemClick(AdapterView parent, View view, int position, long id) { Keyboard k = availableKeyboardsList.get(position); @@ -205,11 +240,7 @@ public final class SelectLanguageFragment extends Fragment implements BlockingSt if (addKeyboardsList == null) { addKeyboardsList = new ArrayList(); } - if (addKeyboardsList.contains(k)) { - addKeyboardsList.remove(k); - } else { - addKeyboardsList.add(k); - } + toggleLanguage(addKeyboardsList, k); } // Disable install button if no languages selected or all languages already installed diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/BCP47.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/BCP47.java index c49b9b64c6..4342386a32 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/BCP47.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/BCP47.java @@ -52,4 +52,4 @@ public final class BCP47 { } } } -} \ No newline at end of file +} From 90851ad3112713fc351e09c17f2f9247aa560c4d Mon Sep 17 00:00:00 2001 From: Ross Date: Thu, 8 Sep 2022 22:28:35 +1000 Subject: [PATCH 075/132] fix(windows): TSF apps delete all of smp When the windows platform recieves the action for a backspace it will check to see if the character being delete is a surrogate pair. If it is and the app is TSF aware app it will need to backspace twice to remove both parts of the surrogate pair. For legacy apps this is not required. --- windows/src/engine/keyman32/kmprocessactions.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/windows/src/engine/keyman32/kmprocessactions.cpp b/windows/src/engine/keyman32/kmprocessactions.cpp index 6a5e03c123..1e07523925 100644 --- a/windows/src/engine/keyman32/kmprocessactions.cpp +++ b/windows/src/engine/keyman32/kmprocessactions.cpp @@ -30,8 +30,17 @@ static BOOL processAlert(AITIP* app) { static BOOL processBack(AITIP* app, const km_kbp_action_item* actionItem) { if (actionItem->backspace.expected_type == KM_KBP_BT_MARKER) { app->QueueAction(QIT_BACK, BK_DEADKEY); - } else /* actionItem->backspace.expected_type == KM_KBP_BT_CHAR, KM_KBP_BT_UNKNOWN */ { - app->QueueAction(QIT_BACK, 0); + } else if(actionItem->backspace.expected_type == KM_KBP_BT_CHAR) { + // If this is a TSF-aware app we need to set the BK_SURROGATE flag to delete + // both parts of the surrogate pair. Legacy apps receive a BKSP WM_KEYDOWN event + // which results in deleting both parts in one action. + if (!app->IsLegacy() && Uni_IsSMP(actionItem->backspace.expected_value)) { + app->QueueAction(QIT_BACK, BK_DEFAULT | BK_SURROGATE); + } else { + app->QueueAction(QIT_BACK, BK_DEFAULT); + } + } else { // KM_KBP_BT_UNKNOWN + app->QueueAction(QIT_BACK, BK_DEFAULT); } return TRUE; } From 531a911ca06ecc2ac4f015e751b0462be0aa42e2 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Thu, 8 Sep 2022 14:18:01 -0400 Subject: [PATCH 076/132] auto: increment master version to 16.0.59 --- HISTORY.md | 5 +++++ VERSION.md | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index f51d7fbe71..4be594dcfe 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,10 @@ # Keyman Version History +## 16.0.58 alpha 2022-09-08 + +* fix(common/models): reconnects unit tests for worker-internal submodules (#7215) +* feat(common/models): extra Unicode-based wordbreaker unit tests (#7217) + ## 16.0.57 alpha 2022-09-07 * fix(android/engine): Fix Backspace key to delete without errant subkeys (#7156) diff --git a/VERSION.md b/VERSION.md index 8e5a3227df..481187aab7 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.58 \ No newline at end of file +16.0.59 \ No newline at end of file From d27d77955b5ab8704cb391d7b2a637347ac8874e Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Fri, 9 Sep 2022 07:47:58 +0700 Subject: [PATCH 077/132] Apply suggestions from code review Co-authored-by: Marc Durdin --- .../kmapro/SelectLanguageFragment.java | 15 ++++-------- .../java/com/tavultesoft/kmea/util/BCP47.java | 24 +++++++------------ 2 files changed, 13 insertions(+), 26 deletions(-) diff --git a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/SelectLanguageFragment.java b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/SelectLanguageFragment.java index 96c641b1a8..93cd17543b 100644 --- a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/SelectLanguageFragment.java +++ b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/SelectLanguageFragment.java @@ -205,23 +205,16 @@ public final class SelectLanguageFragment extends Fragment implements BlockingSt return; } - boolean languageExists = false; - int index = 0; // See if languageID already exists in the keyboardList for (Keyboard l: keyboardList) { if (BCP47.languageEquals(l.getLanguageID(), k.getLanguageID())) { - languageExists = true; - break; + keyboardList.remove(l); + return; } - index++; } - if (languageExists && index < keyboardList.size()) { - keyboardList.remove(index); - } else { - k.setLanguage(k.getLanguageID().toLowerCase(), k.getLanguageName()); - keyboardList.add(k); - } + k.setLanguage(k.getLanguageID().toLowerCase(), k.getLanguageName()); + keyboardList.add(k); } } diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/BCP47.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/BCP47.java index 4342386a32..88a8428d52 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/BCP47.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/BCP47.java @@ -28,28 +28,22 @@ public final class BCP47 { * @param languageID */ public static void toggleLanguage(ArrayList languageList, String languageID) { - if (languageList != null) { - if (languageList.size() == 0) { - languageList.add(languageID.toLowerCase()); - return; - } + if (languageList == null) { + throw new InvalidParameterException("languageList must not be null"); + } + if (languageID == null) { + throw new InvalidParameterException("languageID must not be null"); + } - boolean languageExists = false; - int index = 0; // See if languageID already exists in the languageList for (String l: languageList) { if (languageEquals(l, languageID)) { - languageExists = true; - break; + languageList.remove(l); + return; } - index++; } - if (languageExists && index < languageList.size()) { - languageList.remove(index); - } else { - languageList.add(languageID.toLowerCase()); - } + languageList.add(languageID.toLowerCase()); } } } From 77997f447776fe501579a5709b609c1d3f30e361 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Fri, 9 Sep 2022 08:48:55 +0700 Subject: [PATCH 078/132] fix(android/engine): Cleanup toggleLanguage() --- .../kmapro/SelectLanguageFragment.java | 30 +++++++++---------- .../java/com/tavultesoft/kmea/util/BCP47.java | 16 +++++----- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/SelectLanguageFragment.java b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/SelectLanguageFragment.java index 93cd17543b..da53c2000f 100644 --- a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/SelectLanguageFragment.java +++ b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/SelectLanguageFragment.java @@ -32,6 +32,7 @@ import com.tavultesoft.kmea.util.KMLog; import org.json.JSONObject; import java.io.File; +import java.security.InvalidParameterException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -198,24 +199,23 @@ public final class SelectLanguageFragment extends Fragment implements BlockingSt * @param k */ private void toggleLanguage(ArrayList keyboardList, Keyboard k) { - if (keyboardList != null) { - if (keyboardList.size() == 0) { - k.setLanguage(k.getLanguageID().toLowerCase(), k.getLanguageName()); - keyboardList.add(k); + if (keyboardList == null) { + throw new InvalidParameterException("keyboardList must not be null"); + } + if (k == null) { + throw new InvalidParameterException("keyboard must not be null"); + } + + // See if languageID already exists in the keyboardList + for (Keyboard l: keyboardList) { + if (BCP47.languageEquals(l.getLanguageID(), k.getLanguageID())) { + keyboardList.remove(l); return; } - - // See if languageID already exists in the keyboardList - for (Keyboard l: keyboardList) { - if (BCP47.languageEquals(l.getLanguageID(), k.getLanguageID())) { - keyboardList.remove(l); - return; - } - } - - k.setLanguage(k.getLanguageID().toLowerCase(), k.getLanguageName()); - keyboardList.add(k); } + + k.setLanguage(k.getLanguageID().toLowerCase(), k.getLanguageName()); + keyboardList.add(k); } @Override diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/BCP47.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/BCP47.java index 88a8428d52..81542ca86d 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/BCP47.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/BCP47.java @@ -5,6 +5,7 @@ package com.tavultesoft.kmea.util; import java.util.ArrayList; +import java.security.InvalidParameterException; public final class BCP47 { /** @@ -35,15 +36,14 @@ public final class BCP47 { throw new InvalidParameterException("languageID must not be null"); } - // See if languageID already exists in the languageList - for (String l: languageList) { - if (languageEquals(l, languageID)) { - languageList.remove(l); - return; - } + // See if languageID already exists in the languageList + for (String l: languageList) { + if (languageEquals(l, languageID)) { + languageList.remove(l); + return; } - - languageList.add(languageID.toLowerCase()); } + + languageList.add(languageID.toLowerCase()); } } From 5a3dad029886dc553c6bdc7e0d56a34ef26fd60d Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Fri, 9 Sep 2022 15:32:48 +1000 Subject: [PATCH 079/132] chore(core): fixup km_kbp_event docs --- core/include/keyman/keyboardprocessor.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/include/keyman/keyboardprocessor.h b/core/include/keyman/keyboardprocessor.h index e49fa4b779..efac59d472 100644 --- a/core/include/keyman/keyboardprocessor.h +++ b/core/include/keyman/keyboardprocessor.h @@ -1139,7 +1139,7 @@ the state may also be modified. ##### Parameters: - __state__: A pointer to the opaque state object. -- __event__: The event to be processed, from KM_KBP_EVENT enumeration +- __event__: The event to be processed, from km_kbp_event_code enumeration - __data__: Additional event-specific data. Currently unused, must be nullptr. ```c @@ -1152,7 +1152,7 @@ km_kbp_event( void* data ); -enum km_kbp_event_codes { +enum km_kbp_event_code { /** * A keyboard has been activated by the user. The processor may use this * event, for example, to switch caps lock state or provide other UX. From fc57b3d8e4a28f3d0374bd5a47896a5e4312e046 Mon Sep 17 00:00:00 2001 From: Ross Date: Fri, 9 Sep 2022 16:10:57 +1000 Subject: [PATCH 080/132] fix(windows): fix includes and dep for unit test proj Also add a test for surrogate backspace --- .../src/engine/keyman32/keyman-engine.vcxproj | 4 +-- .../src/engine/keyman32/kmprocessactions.h | 2 +- .../keyman-engine-tests.vcxproj | 12 ++++++--- .../kmprocessactionstests.cpp | 26 +++++++++++++++++++ 4 files changed, 37 insertions(+), 7 deletions(-) diff --git a/windows/src/engine/keyman32/keyman-engine.vcxproj b/windows/src/engine/keyman32/keyman-engine.vcxproj index 6249823191..df421a3619 100644 --- a/windows/src/engine/keyman32/keyman-engine.vcxproj +++ b/windows/src/engine/keyman32/keyman-engine.vcxproj @@ -55,7 +55,7 @@ $(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\src;$(ProjectDir)..\..\..\..\core\build\rust\x86\$(Configuration);$(LibraryPath) - $(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\include;$(ProjectDir)..\..\..\..\core\include;$(IncludePath) + $(ProjectDir)..\..\..\..\common\include;$(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\include;$(ProjectDir)..\..\..\..\core\include;$(IncludePath) $(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\src;$(ProjectDir)..\..\..\..\core\build\rust\x86\$(Configuration);$(LibraryPath) @@ -402,4 +402,4 @@ - + \ No newline at end of file diff --git a/windows/src/engine/keyman32/kmprocessactions.h b/windows/src/engine/keyman32/kmprocessactions.h index 80827ea623..db4a95f758 100644 --- a/windows/src/engine/keyman32/kmprocessactions.h +++ b/windows/src/engine/keyman32/kmprocessactions.h @@ -6,7 +6,7 @@ * Keyman for Windows engine. */ #ifndef _KMPROCESSACTIONS_H -#define _KMPROCESSACTIONS_h +#define _KMPROCESSACTIONS_H /** * Process the actions queued in the core processor diff --git a/windows/src/engine/keyman32/tests/keyman-engine-tests/keyman-engine-tests.vcxproj b/windows/src/engine/keyman32/tests/keyman-engine-tests/keyman-engine-tests.vcxproj index 4980f99332..94ec1574c2 100644 --- a/windows/src/engine/keyman32/tests/keyman-engine-tests/keyman-engine-tests.vcxproj +++ b/windows/src/engine/keyman32/tests/keyman-engine-tests/keyman-engine-tests.vcxproj @@ -34,10 +34,10 @@ - $(ProjectDir)..\..\..\..\..\..\core\include;$(ProjectDir)..\..\..\..\..\..\core\build\x86\$(Configuration)\include;$(ProjectDir)..\..\..\..\global\inc;$(ProjectDir)..\..;$(IncludePath) + $(ProjectDir)..\..\..\..\..\..\common\include;$(ProjectDir)..\..\..\..\..\..\core\include;$(ProjectDir)..\..\..\..\..\..\core\build\x86\$(Configuration)\include;$(ProjectDir)..\..\..\..\global\inc;$(ProjectDir)..\..;$(IncludePath) $(ProjectDir)bin\$(Platform)\$(Configuration)\ $(ProjectDir)obj\$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\..\..\..\..\core\build\x86\$(Configuration)\src;$(ProjectDir)..\..\..\..\..\..\core\build\rust\x86\$(Configuration);$(LibraryPath) + $(ProjectDir)..\..\..\..\..\..\core\build\x86\$(Configuration)\src;$(ProjectDir)..\..\..\..\..\..\core\build\rust\x86\$(Configuration);$(ProjectDir)..\..\..\kmtip\bin\Win32\$(Configuration);$(LibraryPath) true @@ -46,7 +46,7 @@ $(ProjectDir)obj\$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\..\..\global\inc;$(ProjectDir)..\..;$(IncludePath) + $(ProjectDir)..\..\..\..\..\..\common\include;$(ProjectDir)..\..\..\..\..\..\core\build\x86\$(Configuration)\include;$(ProjectDir)..\..\..\..\..\..\core\include;$(ProjectDir)..\..\..\..\global\inc;$(ProjectDir)..\..;$(IncludePath) $(ProjectDir)bin\$(Platform)\$(Configuration)\ $(ProjectDir)obj\$(Platform)\$(Configuration)\ @@ -59,6 +59,7 @@ + @@ -99,6 +100,9 @@ Console version.lib;psapi.lib;imm32.lib;libkmnkbp0.a;%(AdditionalDependencies) + + false + @@ -153,4 +157,4 @@ - + \ No newline at end of file diff --git a/windows/src/engine/keyman32/tests/keyman-engine-tests/kmprocessactionstests.cpp b/windows/src/engine/keyman32/tests/keyman-engine-tests/kmprocessactionstests.cpp index 3df1d70be3..3960b64f2b 100644 --- a/windows/src/engine/keyman32/tests/keyman-engine-tests/kmprocessactionstests.cpp +++ b/windows/src/engine/keyman32/tests/keyman-engine-tests/kmprocessactionstests.cpp @@ -106,6 +106,32 @@ TEST_F(KMPROCESSACTIONS, processBackCharactertest) { EXPECT_STREQ(contextBuf, expectedContextFinal); } + +// KM_KBP_IT_BACK - processBack +// Press Backspace for a surrogate character. +// TODO: This test doesn't test for the PostKey handling +// of TSF and two backspaces being sent to the App. +TEST_F(KMPROCESSACTIONS, processBackSurrogateTSFtest) { + WCHAR callbuf[MAXCONTEXT]; + AITIP testApp; + WCHAR expectedContext[] = { 0 }; + WCHAR expectedStringSurrogate[] = {0xD801, 0xDC37, 0}; + + km_kbp_usv testSurrogateChar = Uni_SurrogateToUTF32(0xD801, 0xDC37); //𐐷'; + km_kbp_action_item itemAddChar = {KM_KBP_IT_CHAR, {0,}, {testSurrogateChar}}; + processUnicodeChar(&testApp, &itemAddChar); + + km_kbp_action_item itemBackSpace = {KM_KBP_IT_BACK}; + itemBackSpace.backspace.expected_type = KM_KBP_IT_CHAR; + itemBackSpace.backspace.expected_value = testSurrogateChar; + WCHAR *contextBuf = testApp.ContextBufMax(MAXCONTEXT); + EXPECT_STREQ(contextBuf, expectedStringSurrogate); + // backspace + processBack(&testApp, &itemBackSpace); + contextBuf = testApp.ContextBufMax(MAXCONTEXT); + EXPECT_STREQ(contextBuf, expectedContext); +} + // KM_KBP_IT_BACK - processBack // Press Backspace for a character doesn't match expected character // Note currently we don't check for a character match this should be updated From 666b63940407f2a945263dc9090ff25ecc2435f1 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 9 Sep 2022 15:35:12 +0700 Subject: [PATCH 081/132] fix(common/models): adds 'soft bound' for reasonable 100% fat-finger corrections --- common/web/lm-worker/src/model-compositor.ts | 42 +++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/common/web/lm-worker/src/model-compositor.ts b/common/web/lm-worker/src/model-compositor.ts index 329460776d..0010371e63 100644 --- a/common/web/lm-worker/src/model-compositor.ts +++ b/common/web/lm-worker/src/model-compositor.ts @@ -171,6 +171,16 @@ class ModelCompositor { } } + // Is the token under construction newly-constructed / is there no pre-existing root? + // If so, we want to strongly avoid overcorrection, even for 'nearby' keys. + // (Strong lexical frequency differences can easily cause overcorrection when only + // one key's available.) + // + // NOTE: we only want this applied word-initially, when any corrections 'correct' + // 100% of the word. Things are generally fine once it's not "all or nothing." + let tailToken = contextTokens[contextTokens.length - 1]; + const isTokenStart = tailToken.transformDistributions.length <= 1; + // TODO: whitespace, backspace filtering. Do it here. // Whitespace is probably fine, actually. Less sure about backspace. @@ -212,9 +222,39 @@ class ModelCompositor { id: inputTransform.id // The correction should always be based on the most recent external transform/transcription ID. } + let rootCost = match.totalCost; + + /* If we're dealing with the FIRST keystroke of a new sequence, we'll **dramatically** boost + * the exponent to ensure only VERY nearby corrections have a chance of winning, and only if + * there are significantly more likely words. We only need this to allow very minor fat-finger + * adjustments for 100% keystroke-sequence corrections in order to prevent finickiness on + * key borders. + * + * Technically, the probabilities this produces won't be normalized as-is... but there's no + * true NEED to do so for it, even if it'd be 'nice to have'. Consistently tracking when + * to apply it could become tricky, so it's simpler to leave out. + * + * Worst-case, it's possible to temporarily add normalization if a code deep-dive + * is needed in the future. + */ + if(isTokenStart) { + /* Suppose a key distribution: most likely with p=0.5,, second-most with 0.4 - a pretty + * ambiguous case that would only arise very near the center of the boundary between two keys. + * Raising (0.5/0.4)^16 ~= 35.53. + * That seems 'within reason' for correction very near boundaries. + * + * So, with the second-most-likely key being that close in probability, its best suggestion + * must be ~ 35.5x more likely than that of the truly-most-likely key to "win". So, it's not + * a HARD cutoff, but more of a 'soft' one. Keeping the principles in mind documented above, + * it's possible to tweak this to a more harsh or lenient setting if desired, rather than + * being totally "all or nothing" on which key is taken for highly-ambiguous keypresses. + */ + rootCost *= 16; + } + return { sample: correctionTransform, - p: Math.exp(-match.totalCost) + p: Math.exp(-rootCost) }; }, this); From 711dfb19b22b86d2486de0d2660ff93aec7cd555 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Fri, 9 Sep 2022 14:02:01 -0400 Subject: [PATCH 082/132] auto: increment master version to 16.0.60 --- HISTORY.md | 6 ++++++ VERSION.md | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 4be594dcfe..da243f3353 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,11 @@ # Keyman Version History +## 16.0.59 alpha 2022-09-09 + +* chore: use keyman.com instead of keyman-staging.com (#7233) +* feat(core): add `km_kbp_event` API endpoint (#7223) +* fix(windows): Delete both code units when deleting surrogate pairs in TSF-aware apps (#7243) + ## 16.0.58 alpha 2022-09-08 * fix(common/models): reconnects unit tests for worker-internal submodules (#7215) diff --git a/VERSION.md b/VERSION.md index 481187aab7..df00a2ed57 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.59 \ No newline at end of file +16.0.60 \ No newline at end of file From 30ccee63e800580a6412a19e579e9b65ac45aa56 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Sat, 10 Sep 2022 06:20:48 +1000 Subject: [PATCH 083/132] Update windows/src/engine/keyman32/tests/keyman-engine-tests/kmprocessactionstests.cpp Co-authored-by: Eberhard Beilharz --- .../tests/keyman-engine-tests/kmprocessactionstests.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/windows/src/engine/keyman32/tests/keyman-engine-tests/kmprocessactionstests.cpp b/windows/src/engine/keyman32/tests/keyman-engine-tests/kmprocessactionstests.cpp index 3960b64f2b..5b4e2bfff8 100644 --- a/windows/src/engine/keyman32/tests/keyman-engine-tests/kmprocessactionstests.cpp +++ b/windows/src/engine/keyman32/tests/keyman-engine-tests/kmprocessactionstests.cpp @@ -114,11 +114,11 @@ TEST_F(KMPROCESSACTIONS, processBackCharactertest) { TEST_F(KMPROCESSACTIONS, processBackSurrogateTSFtest) { WCHAR callbuf[MAXCONTEXT]; AITIP testApp; - WCHAR expectedContext[] = { 0 }; + WCHAR expectedContext[] = { 0 }; WCHAR expectedStringSurrogate[] = {0xD801, 0xDC37, 0}; km_kbp_usv testSurrogateChar = Uni_SurrogateToUTF32(0xD801, 0xDC37); //𐐷'; - km_kbp_action_item itemAddChar = {KM_KBP_IT_CHAR, {0,}, {testSurrogateChar}}; + km_kbp_action_item itemAddChar = {KM_KBP_IT_CHAR, {0,}, {testSurrogateChar}}; processUnicodeChar(&testApp, &itemAddChar); km_kbp_action_item itemBackSpace = {KM_KBP_IT_BACK}; From 1724778e7759cf53eec51dacaf45295b6e8eb19c Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Sat, 10 Sep 2022 06:23:26 +1000 Subject: [PATCH 084/132] chore: try disabling concurrency for browserstack tests We're still getting sporadic test failures with BrowserStack. So per one discussion online, attempting to run without concurrency to see if it improves stability. I note that the video from tests where we "fail to capture" shows a browser with an error: "Safari Can't Connect to the Server" "Safari can't open the page "localhost:9879/?id=75185158" because Safari can't connect to the server "localhost". https://automate.browserstack.com/dashboard/v2/builds/419c59decb8e12303e45e1f4bad1b945252df532/sessions/19712158833ce57592b66a50433965e4146fc972 --- common/test/resources/karma-browserstack-config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/test/resources/karma-browserstack-config.js b/common/test/resources/karma-browserstack-config.js index a6b3c95662..6fcae57bca 100644 --- a/common/test/resources/karma-browserstack-config.js +++ b/common/test/resources/karma-browserstack-config.js @@ -138,7 +138,7 @@ module.exports = function(baseConfigParams /* the project's base configuration // Concurrency level // We are alloted 5 total at once from BrowserStack. That said, note that we have multiple build configs that may // need BrowserStack-based testing simultaneously. It may be best to avoid bottlenecking on that limitation. - concurrency: 2, + concurrency: 1, customLaunchers: FINAL_LAUNCHER_DEFS, From e7dcf7fe08a989fbe6f59c6dde10bd8fe9d6a8a7 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Sat, 10 Sep 2022 14:02:03 -0400 Subject: [PATCH 085/132] auto: increment master version to 16.0.61 --- HISTORY.md | 7 +++++++ VERSION.md | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index da243f3353..48a2da1054 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,12 @@ # Keyman Version History +## 16.0.60 alpha 2022-09-10 + +* fix(web): enhanced timer for prediction algorithm (#7037) +* chore(core): fixup km_kbp_event docs (#7253) +* fix(windows) Update unit tests for tsf bkspace (#7254) +* fix(android): Standardize language ID in language picker menu (#7239) + ## 16.0.59 alpha 2022-09-09 * chore: use keyman.com instead of keyman-staging.com (#7233) diff --git a/VERSION.md b/VERSION.md index df00a2ed57..d6db1906b4 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.60 \ No newline at end of file +16.0.61 \ No newline at end of file From 4434cdc30df57a8546716d1b3372533af494f7c5 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Sun, 11 Sep 2022 07:19:18 +1000 Subject: [PATCH 086/132] chore(web): disable browserstack on non-web-specific builds Because BrowserStack tests are unreliable, I am disabling them for any builds that are not specifically targeting web. Two ways that the tests can be enabled: 1. Include '(web)' in the PR title 2. Include the label test-browserstack This only impacts builds in the CI test environment, not anything else. See build-utils-ci.inc.sh for helper functions for accessing PR metadata, and build-utils-ci.test.sh for unit tests for this functionality. --- resources/build/build-utils-ci.inc.sh | 62 ++++++++++++++++++++++++++ resources/build/build-utils-ci.test.sh | 57 +++++++++++++++++++++++ web/unit_tests/test.sh | 40 ++++++++++++++--- 3 files changed, 153 insertions(+), 6 deletions(-) create mode 100644 resources/build/build-utils-ci.inc.sh create mode 100644 resources/build/build-utils-ci.test.sh diff --git a/resources/build/build-utils-ci.inc.sh b/resources/build/build-utils-ci.inc.sh new file mode 100644 index 0000000000..6212f9720c --- /dev/null +++ b/resources/build/build-utils-ci.inc.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# +# This script gets CI / pull request details for builds, part of the build-utils +# builder_ suite of functions. All functions and variables in this file have the +# prefix builder_pull_. +# + +. "$KEYMAN_ROOT/resources/build/jq.inc.sh" + +# +# Returns 0 if current build is in CI and triggered from a pull request. If it +# returns 0, then a call is made to GitHub to get pull request details, and the +# PR details are added to $builder_pull_title, $builder_pull_number, and +# the $builder_pull_labels array. +# +# Note that the GitHub REST call is made with credentials if $GITHUB_TOKEN is +# available; without credentials it will be subject to IP-based rate limits. +# +builder_pull_get_details() { + builder_pull_title= + builder_pull_number= + builder_pull_labels=() + if [[ ! ${TEAMCITY_PR_NUMBER-} =~ ^[0-9]+$ ]]; then + return 1 + fi + + if [ -z "${GITHUB_TOKEN-}" ]; then + local pull_data=`curl -s https://api.github.com/repos/keymanapp/keyman/pulls/$TEAMCITY_PR_NUMBER` + else + local pull_data=`curl -H "Authorization: Bearer $GITHUB_TOKEN" -s https://api.github.com/repos/keymanapp/keyman/pulls/$TEAMCITY_PR_NUMBER` + fi + + builder_pull_title=`echo $pull_data | $JQ .title` + + # Simple poor bash test if data returned from API is valid + if [[ -z builder_pull_title ]]; then + return 1 + fi + + builder_pull_number=$TEAMCITY_PR_NUMBER + builder_pull_labels=(`echo $pull_data | $JQ -jr '.labels[].name|.," "'`) + + return 0 +} + + +# +# Returns 0 if the current PR has a particular label. Requires +# builder_pull_get_details to have successfully run first +# +# Usage: +# builder_pull_has_label "label-name" +# Parameters +# 1: $label label to test +# +function builder_pull_has_label() { + local label="$1" + if [[ " ${builder_pull_labels[*]} " =~ " $label " ]]; then + return 0 + fi + return 1 +} diff --git a/resources/build/build-utils-ci.test.sh b/resources/build/build-utils-ci.test.sh new file mode 100644 index 0000000000..b80be2d1b1 --- /dev/null +++ b/resources/build/build-utils-ci.test.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# +# This script tests build-utils-ci.sh +# + +set -eu + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")" +. "$(dirname "$THIS_SCRIPT")/build-utils.sh" +# END STANDARD BUILD SCRIPT INCLUDE + +. "$KEYMAN_ROOT/resources/build/build-utils-ci.inc.sh" +. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" + +# Tests + +echo "${COLOR_BLUE}## Testing: builder_pull_get_details with \$TEAMCITY_PR_NUMBER not set${COLOR_RESET}" +if ! builder_pull_get_details; then + echo "PASS: builder_pull_get_details should get no details if \$TEAMCITY_PR_NUMBER is not set" +else + fail "FAIL: builder_pull_get_details should get no details if \$TEAMCITY_PR_NUMBER is not set" +fi + + +echo "${COLOR_BLUE}## Testing: builder_pull_get_details with \$TEAMCITY_PR_NUMBER=master${COLOR_RESET}" +TEAMCITY_PR_NUMBER=master +if ! builder_pull_get_details; then + echo "PASS: builder_pull_get_details should get no details if \$TEAMCITY_PR_NUMBER=master" +else + fail "FAIL: builder_pull_get_details should get no details if \$TEAMCITY_PR_NUMBER=master" +fi + + +echo "${COLOR_BLUE}## Testing: builder_pull_get_details with \$TEAMCITY_PR_NUMBER=$TEAMCITY_PR_NUMBER${COLOR_RESET}" +TEAMCITY_PR_NUMBER=7248 +if ! builder_pull_get_details; then + fail "FAIL: builder_pull_get_details was unable to read and parse PR details for #$TEAMCITY_PR_NUMBER from GitHub" +fi + +# print metadata from PR +echo " pull number: $builder_pull_number" +echo " title: $builder_pull_title" +echo " labels: ${builder_pull_labels[*]}" + +echo "PASS: builder_pull_get_details with \$TEAMCITY_PR_NUMBER=$TEAMCITY_PR_NUMBER" + + +echo "${COLOR_BLUE}## Testing: builder_pull_has_label epic-ldml${COLOR_RESET}" +if ! builder_pull_has_label epic-ldml; then + fail "FAIL: builder_pull_has_label epic-ldml" +fi + +echo "PASS: builder_pull_has_label epic-ldml" + +echo "All tests passed." diff --git a/web/unit_tests/test.sh b/web/unit_tests/test.sh index e3f4edc1d9..e31d80893f 100755 --- a/web/unit_tests/test.sh +++ b/web/unit_tests/test.sh @@ -9,9 +9,11 @@ WORKING_DIRECTORY=`pwd` # adjust relative paths as necessary THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")" . "$(dirname "$THIS_SCRIPT")/../../resources/build/build-utils.sh" -. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" ## END STANDARD BUILD SCRIPT INCLUDE +. "$KEYMAN_ROOT/resources/build/build-utils-ci.inc.sh" +. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" + # A simple utility script to facilitate our different modes for unit-testing KMW. # It's rigged to be callable by NPM to facilitate testing during development when in other folders. @@ -134,10 +136,36 @@ pushd "$KEYMAN_ROOT/common/web/input-processor" # Once done, now we run the integrated (KeymanWeb) tests. popd -echo_heading "Running KeymanWeb integration test suite" -npm --no-color run modernizr -- -c unit_tests/modernizr.config.json -d unit_tests/modernizr.js -npm --no-color run karma -- start $FLAGS $BROWSERS unit_tests/$CONFIG +# Browserstack or CI-based tests -CODE=$? +DO_BROWSER_TEST_SUITE=true -exit $CODE \ No newline at end of file +if [[ $VERSION_ENVIRONMENT == test ]]; then + # Implied: CONFIG=CI.conf.js because `-CI` parameter is passed. + # + # If we are running a TeamCity test build, for now, only run BrowserStack + # tests when on a PR branch with a title including "(web)" or with the label + # test-browserstack. This is because the BrowserStack tests are currently + # unreliable, and the false positive failures are masking actual failures. + # + # We do not run BrowserStack tests on master, beta, or stable-x.y test + # builds. + DO_BROWSER_TEST_SUITE=false + if builder_pull_get_details; then + if [[ $builder_pull_title =~ \(web\) ]] || builder_pull_has_label test-browserstack; then + DO_BROWSER_TEST_SUITE=true + fi + fi +fi + +CODE=0 + +if $DO_BROWSER_TEST_SUITE; then + echo_heading "Running KeymanWeb integration test suite" + npm --no-color run modernizr -- -c unit_tests/modernizr.config.json -d unit_tests/modernizr.js + npm --no-color run karma -- start $FLAGS $BROWSERS unit_tests/$CONFIG + + CODE=$? +fi + +exit $CODE From 2aeb5435e8dd22d8d8982999f80cde6f17e5ae6a Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Sun, 11 Sep 2022 08:12:51 +1000 Subject: [PATCH 087/132] chore(web): disable browserstack for LM Layer tests --- common/predictive-text/unit_tests/test.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/common/predictive-text/unit_tests/test.sh b/common/predictive-text/unit_tests/test.sh index 8a30532771..8329e8c417 100755 --- a/common/predictive-text/unit_tests/test.sh +++ b/common/predictive-text/unit_tests/test.sh @@ -5,6 +5,7 @@ THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BA . "$(dirname "$THIS_SCRIPT")/../../../resources/build/build-utils.sh" ## END STANDARD BUILD SCRIPT INCLUDE +. "$KEYMAN_ROOT/resources/build/build-utils-ci.inc.sh" . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" SCRIPT_ROOT="$(dirname "$THIS_SCRIPT")" @@ -97,6 +98,24 @@ if (( RUN_HEADLESS )); then test-headless || fail "DOMless tests failed!" fi +if (( RUN_BROWSERS )); then + if [[ $VERSION_ENVIRONMENT == test ]]; then + # If we are running a TeamCity test build, for now, only run BrowserStack + # tests when on a PR branch with a title including "(web)" or with the label + # test-browserstack. This is because the BrowserStack tests are currently + # unreliable, and the false positive failures are masking actual failures. + # + # We do not run BrowserStack tests on master, beta, or stable-x.y test + # builds. + RUN_BROWSERS=0 + if builder_pull_get_details; then + if [[ $builder_pull_title =~ \(web\) ]] || builder_pull_has_label test-browserstack; then + RUN_BROWSERS=1 + fi + fi + fi +fi + # Run browser-based tests. if (( RUN_BROWSERS )); then test-browsers || fail "Browser-based tests failed!" From e406018267f378293e81db9f7745c4a212be4e69 Mon Sep 17 00:00:00 2001 From: Ross Date: Mon, 12 Sep 2022 10:20:49 +1000 Subject: [PATCH 088/132] docs(windows): add steps for using testhost debugging --- windows/src/engine/testhost/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/windows/src/engine/testhost/README.md b/windows/src/engine/testhost/README.md index d0c10e7ade..c8d2fd055e 100644 --- a/windows/src/engine/testhost/README.md +++ b/windows/src/engine/testhost/README.md @@ -5,3 +5,13 @@ This project is a debug host for keyman32 (and keyman64). It is setup to run a s The primary use of testhost is in interactive debugging of Keyman Core, and intermediate Engine code around Core. The final stage of characters reaching the document will not be identical to Keyman Engine in normal use, and there may be other interactive limitations as well. Before trying to run, make sure you have built keyman32 in Debug configuration. Testhost will attempt to load keyman32.dll from ../keyman32/bin/Win32/Debug/keyman32.dll, and if that is not present, from ../keyman32/bin/Win32/Release/keyman32.dll, and finally, from a hardcoded "C:\\Program Files (x86)\\Common Files\\Keyman\\Keyman Engine\\keyman32.dll" (which is hacky but probably suffices for now). + +The work flow that works well using testhost. +With Visual Studio(VS) +1. Build Keyman32 with Debug Configuration. +2. Build testhost with Debug Configuration. +3. Insure Keyman is not running and that language associated with keyboard you want to test is not selected in the language bar. +4. Press F5 to start debugging. +5. Now use the language bar to select the language associated with a keyman keyboard. +6. You should now be able to debug both the platform and keyman core code. +7. Once you stop the debug session you will need to got back to step 3 start a new session. [ or step 1 if you are making code changes you should not need to reboot windows] From f8a0397fb33698aceb6d09cc3893986c62c67423 Mon Sep 17 00:00:00 2001 From: rc-swag <58423624+rc-swag@users.noreply.github.com> Date: Mon, 12 Sep 2022 10:38:43 +1000 Subject: [PATCH 089/132] docs(windows): apply review suggestions Co-authored-by: Marc Durdin --- windows/src/engine/testhost/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/windows/src/engine/testhost/README.md b/windows/src/engine/testhost/README.md index c8d2fd055e..0f7c559010 100644 --- a/windows/src/engine/testhost/README.md +++ b/windows/src/engine/testhost/README.md @@ -10,8 +10,8 @@ The work flow that works well using testhost. With Visual Studio(VS) 1. Build Keyman32 with Debug Configuration. 2. Build testhost with Debug Configuration. -3. Insure Keyman is not running and that language associated with keyboard you want to test is not selected in the language bar. +3. Ensure Keyman is not running and that language associated with keyboard you want to test is not selected in the language bar. 4. Press F5 to start debugging. 5. Now use the language bar to select the language associated with a keyman keyboard. 6. You should now be able to debug both the platform and keyman core code. -7. Once you stop the debug session you will need to got back to step 3 start a new session. [ or step 1 if you are making code changes you should not need to reboot windows] +7. Once you stop the debug session you will need to go back to step 3 to start a new session. (or step 1 if you are making code changes; you should not need to reboot windows.) From fbb3375ed99d12e6dbbd7ecc3fd8b52424f6f5e1 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 12 Sep 2022 12:02:09 +1000 Subject: [PATCH 090/132] fix(developer): show more useful error if out of space during Setup Fixes #7261. --- common/windows/delphi/setup/SFX.pas | 45 ++++++++++++++++++----------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/common/windows/delphi/setup/SFX.pas b/common/windows/delphi/setup/SFX.pas index 13491802cb..5af3972261 100644 --- a/common/windows/delphi/setup/SFX.pas +++ b/common/windows/delphi/setup/SFX.pas @@ -82,27 +82,38 @@ var fs: TFileStream; ms: TMemoryStream; begin - fs := TFileStream.Create(ParamStr(0), fmOpenRead or fmShareDenyWrite); - ms := TMemoryStream.Create; try - if not FindFirstHeader(fs) then - Exit(False); - - fs.Seek(StartOfFile, TSeekOrigin.soBeginning); - - ms.CopyFrom(fs, fs.Size - StartOfFile); - ms.Position := 0; - - with TZipFile.Create do + fs := TFileStream.Create(ParamStr(0), fmOpenRead or fmShareDenyWrite); + ms := TMemoryStream.Create; try - Open(ms, zmRead); - ExtractAll(ExtPath); + if not FindFirstHeader(fs) then + Exit(False); + + fs.Seek(StartOfFile, TSeekOrigin.soBeginning); + + ms.CopyFrom(fs, fs.Size - StartOfFile); + ms.Position := 0; + + with TZipFile.Create do + try + Open(ms, zmRead); + ExtractAll(ExtPath); + finally + Free; + end; finally - Free; + fs.Free; + ms.Free; + end; + except + on E:Exception do + begin + raise Exception.Create( + 'Failed to extract setup archive. '+ + 'You may have run out of disk space or there may be a '+ + 'problem with the source files.'#13#10#13#10+ + 'The error received was: '+E.Message); end; - finally - fs.Free; - ms.Free; end; Result := True; end; From 6632d597d524e83a71001869582af698d09a8af4 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 12 Sep 2022 10:25:24 +0700 Subject: [PATCH 091/132] feat(common/models): adds unit test, minor branch polish --- .../headless/worker-model-compositor.js | 39 +++++++++++++++++++ common/web/lm-worker/src/model-compositor.ts | 23 +++++++++-- 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/common/predictive-text/unit_tests/headless/worker-model-compositor.js b/common/predictive-text/unit_tests/headless/worker-model-compositor.js index f7e9173cee..464ed7b203 100644 --- a/common/predictive-text/unit_tests/headless/worker-model-compositor.js +++ b/common/predictive-text/unit_tests/headless/worker-model-compositor.js @@ -50,6 +50,45 @@ describe('ModelCompositor', function() { }); }); + it('strongly avoids corrections for single-character roots', function() { + let compositor = new ModelCompositor(plainModel); + let context = { + left: '', startOfBuffer: true, endOfBuffer: true, + }; + + // The 'weights' involved imply that we have an edge-case fat finger on the bottom of + // the 'q' key, slightly in its favor. + let inputDistribution = [ + {sample: {insert: 'q', deleteLeft: 0}, p: 0.5}, // 'quite' (679) and 'question' (644) are included! + {sample: {insert: 'a', deleteLeft: 0}, p: 0.4} // but at lower weight than 'and' (998). + ]; + + compositor.predict({insert: '', deleteLeft: 0}, context); // Initialize context tracking first! + let suggestions = compositor.predict(inputDistribution, context); + + // remove the keep suggestion; we're not testing that here. + suggestions = suggestions.filter((suggestion) => suggestion.tag != 'keep'); + + suggestions.sort((a, b) => b.p - a.p); + + // There are only 4 suggestions in this limited test model that begin with 'q'. + // We expect more than that, since 'a' is indicated to be very close by. + assert.isAbove(suggestions.length, 4, "fat-finger style corrections needed for test comparisons are missing"); + + // Note: 'and' is (currently) modeled by the text-fixture model to have 9.3x the base probability + // that the worst 'q'-rooted suggestion ('quality') does. Without single-character correction + // avoidance logic, this test _will_ fail. + // + // In case a tweak to test parameters is desired, note that 'and' beats rank #3 - 'questions' - + // at 3.36x base. At the time of writing this test, upping 'a's probability to 0.45 will block + // 'quality' while the top three 'q's (ending with 'questions') remain in place. + let qRange = suggestions.slice(0, 4); + assert.isUndefined(qRange.find((suggestion) => suggestion.transform.insert.charAt(0) != 'q')); + + let aRange = suggestions.slice(4); + assert.isUndefined(aRange.find((suggestion) => suggestion.transform.insert.charAt(0) == 'q')); + }); + it('properly handles suggestions after a backspace', function() { let compositor = new ModelCompositor(plainModel); let context = { diff --git a/common/web/lm-worker/src/model-compositor.ts b/common/web/lm-worker/src/model-compositor.ts index 0010371e63..d048d8b246 100644 --- a/common/web/lm-worker/src/model-compositor.ts +++ b/common/web/lm-worker/src/model-compositor.ts @@ -6,6 +6,23 @@ class ModelCompositor { private static readonly MAX_SUGGESTIONS = 12; readonly punctuation: LexicalModelPunctuation; + /** + * Controls the strength of anti-corrective measures for single-character scenarios. + * The base key probability will be raised to this power for this specific case. + * + * Current selection's motivation: (0.5 / 0.4) ^ 16 ~= 35.5. + * - if the most likely has p=0.5 and second-most has p=0.4 - a highly-inaccurate key + * stroke - the net effect will apply a factor of 35.5 to the lexical probability of + * the best key's prediction roots, favoring it in this manner. + * - less extreme edge cases will have a significantly stronger factor, acting as a + * "soft threshold". + * - truly ambiguous, "coin flip" cases will have a lower factor and thus favor the + * more likely words from the pair. + * - Our OSK key-element borders aren't visible to the user, so the 'spot' where + * behavior changes might feel arbitrary to users if we used a hard threshold instead. + */ + private static readonly SINGLE_CHAR_KEY_PROB_EXPONENT = 16; + private SUGGESTION_ID_SEED = 0; constructor(lexicalModel: LexicalModel) { @@ -238,9 +255,9 @@ class ModelCompositor { * is needed in the future. */ if(isTokenStart) { - /* Suppose a key distribution: most likely with p=0.5,, second-most with 0.4 - a pretty + /* Suppose a key distribution: most likely with p=0.5, second-most with 0.4 - a pretty * ambiguous case that would only arise very near the center of the boundary between two keys. - * Raising (0.5/0.4)^16 ~= 35.53. + * Raising (0.5/0.4)^16 ~= 35.53. (At time of writing, SINGLE_CHAR_KEY_PROB_EXPONENT = 16.) * That seems 'within reason' for correction very near boundaries. * * So, with the second-most-likely key being that close in probability, its best suggestion @@ -249,7 +266,7 @@ class ModelCompositor { * it's possible to tweak this to a more harsh or lenient setting if desired, rather than * being totally "all or nothing" on which key is taken for highly-ambiguous keypresses. */ - rootCost *= 16; + rootCost *= ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT; // note the `Math.exp` below. } return { From 6e000d10743803a53a37f5287324c6cda3d52d05 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 8 Sep 2022 14:50:37 +0700 Subject: [PATCH 092/132] change(web): better touch-distrib weighting --- .../src/text/inputProcessor.ts | 3 ++- .../src/keyboards/activeLayout.ts | 26 ++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/common/web/input-processor/src/text/inputProcessor.ts b/common/web/input-processor/src/text/inputProcessor.ts index 9471c244fe..b45098e538 100644 --- a/common/web/input-processor/src/text/inputProcessor.ts +++ b/common/web/input-processor/src/text/inputProcessor.ts @@ -12,7 +12,7 @@ namespace com.keyman.text { /** * Indicates the device (platform) to be used for non-keystroke events, - * such as those sent to `begin postkeystroke` and `begin newcontext` + * such as those sent to `begin postkeystroke` and `begin newcontext` * entry points. */ private contextDevice: utils.DeviceSpec; @@ -273,6 +273,7 @@ namespace com.keyman.text { let totalMass = 0; // Tracks sum of non-error probabilities. for(let pair of keyDistribution) { if(pair.p < KEYSTROKE_EPSILON) { + totalMass += pair.p; break; } else if(timer && timer() >= TIMEOUT_THRESHOLD) { // Note: it's always possible that the thread _executing_ our JS diff --git a/common/web/keyboard-processor/src/keyboards/activeLayout.ts b/common/web/keyboard-processor/src/keyboards/activeLayout.ts index b1be473bc6..432b051f73 100644 --- a/common/web/keyboard-processor/src/keyboards/activeLayout.ts +++ b/common/web/keyboard-processor/src/keyboards/activeLayout.ts @@ -511,17 +511,41 @@ namespace com.keyman.keyboards { let keyProbs: {[keyId: string]: number} = {}; let totalMass = 0; + let bestKey = null; + let bestProb = Number.MIN_VALUE; // Should we wish to allow multiple different transforms for distance -> probability, use a function parameter in place // of the formula in the loop below. for(let key in keyDists) { - totalMass += keyProbs[key] = 1 / (keyDists[key] + 1e-6); // Prevent div-by-0 errors. + keyProbs[key] = 1 / (Math.pow(keyDists[key], 2) + 1e-6); // Prevent div-by-0 errors. + totalMass += keyProbs[key]; + + if(keyProbs[key] > bestProb) { + bestProb = keyProbs[key]; + bestKey = key; + } } for(let key in keyProbs) { keyProbs[key] /= totalMass; } + // To help ensure the highest probability key gets priority, we'll square-root its probability, + // then renormalize. (p <= 1) Has the largest effect when near the edge of the best key. + const originalBestProb = keyProbs[bestKey]; + const finalBestProb = Math.sqrt(keyProbs[bestKey]); + + const normDelta = finalBestProb - originalBestProb; // will be positive. + const renormalizer = 1 / (1 + normDelta); // as we're increasing the sum-total probability mass. + + for(let key in keyProbs) { + if(key == bestKey) { + keyProbs[key] = finalBestProb * renormalizer; // override with the adjusted value, renorm'd. + } else { + keyProbs[key] *= renormalizer; + } + } + return keyProbs; } From c05df0e323b4e1fc523d1b023cf8af40004832d8 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 9 Sep 2022 15:34:00 +0700 Subject: [PATCH 093/132] fix(web): fixes final-key-of-row dist mapping, drops arbitrary prob adjustment for best key --- .../src/keyboards/activeLayout.ts | 30 ++----------------- 1 file changed, 2 insertions(+), 28 deletions(-) diff --git a/common/web/keyboard-processor/src/keyboards/activeLayout.ts b/common/web/keyboard-processor/src/keyboards/activeLayout.ts index 432b051f73..9e8c84896f 100644 --- a/common/web/keyboard-processor/src/keyboards/activeLayout.ts +++ b/common/web/keyboard-processor/src/keyboards/activeLayout.ts @@ -337,14 +337,12 @@ namespace com.keyman.keyboards { // Allow for right OSK margin (15 layout units) let rightMargin = ActiveKey.DEFAULT_RIGHT_MARGIN/totalWidth; - totalPercent += rightMargin; // If a single key, and padding is negative, add padding to right align the key if(keys.length == 1 && parseInt(keys[0]['pad'],10) < 0) { keyPercent=parseInt(keys[0]['width'],10)/totalWidth; keys[0]['widthpc']=keyPercent; - totalPercent += keyPercent; - keys[0]['padpc']=1-totalPercent; + keys[0]['padpc']=1-(totalPercent + keyPercent + rightMargin); // compute center's default x-coord (used in headless modes) setProportions(keys[0] as ActiveKey, padPercent, keyPercent, totalPercent); @@ -352,8 +350,7 @@ namespace com.keyman.keyboards { let j=keys.length-1; padPercent=parseInt(keys[j]['pad'],10)/totalWidth; keys[j]['padpc']=padPercent; - totalPercent += padPercent; - keys[j]['widthpc'] = keyPercent = 1-totalPercent; + keys[j]['widthpc'] = keyPercent = 1-(totalPercent + padPercent + rightMargin); // compute center's default x-coord (used in headless modes) setProportions(keys[j] as ActiveKey, padPercent, keyPercent, totalPercent); @@ -511,41 +508,18 @@ namespace com.keyman.keyboards { let keyProbs: {[keyId: string]: number} = {}; let totalMass = 0; - let bestKey = null; - let bestProb = Number.MIN_VALUE; // Should we wish to allow multiple different transforms for distance -> probability, use a function parameter in place // of the formula in the loop below. for(let key in keyDists) { keyProbs[key] = 1 / (Math.pow(keyDists[key], 2) + 1e-6); // Prevent div-by-0 errors. totalMass += keyProbs[key]; - - if(keyProbs[key] > bestProb) { - bestProb = keyProbs[key]; - bestKey = key; - } } for(let key in keyProbs) { keyProbs[key] /= totalMass; } - // To help ensure the highest probability key gets priority, we'll square-root its probability, - // then renormalize. (p <= 1) Has the largest effect when near the edge of the best key. - const originalBestProb = keyProbs[bestKey]; - const finalBestProb = Math.sqrt(keyProbs[bestKey]); - - const normDelta = finalBestProb - originalBestProb; // will be positive. - const renormalizer = 1 / (1 + normDelta); // as we're increasing the sum-total probability mass. - - for(let key in keyProbs) { - if(key == bestKey) { - keyProbs[key] = finalBestProb * renormalizer; // override with the adjusted value, renorm'd. - } else { - keyProbs[key] *= renormalizer; - } - } - return keyProbs; } From b8dd9baf5b9863c0bac2fac468e9d9d437038fde Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 9 Sep 2022 15:51:42 +0700 Subject: [PATCH 094/132] chore(web): minor cleanup --- common/web/keyboard-processor/src/keyboards/activeLayout.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/common/web/keyboard-processor/src/keyboards/activeLayout.ts b/common/web/keyboard-processor/src/keyboards/activeLayout.ts index 9e8c84896f..cd4baa0a27 100644 --- a/common/web/keyboard-processor/src/keyboards/activeLayout.ts +++ b/common/web/keyboard-processor/src/keyboards/activeLayout.ts @@ -512,8 +512,7 @@ namespace com.keyman.keyboards { // Should we wish to allow multiple different transforms for distance -> probability, use a function parameter in place // of the formula in the loop below. for(let key in keyDists) { - keyProbs[key] = 1 / (Math.pow(keyDists[key], 2) + 1e-6); // Prevent div-by-0 errors. - totalMass += keyProbs[key]; + totalMass += keyProbs[key] = 1 / (Math.pow(keyDists[key], 2) + 1e-6); // Prevent div-by-0 errors. } for(let key in keyProbs) { From d91132bc0fdf1bc4dfad24d3dd487c73b7b55e3e Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 12 Sep 2022 10:39:41 +0700 Subject: [PATCH 095/132] fix(web): filters more non-output keys from fat-finger distrib --- .../web/keyboard-processor/src/keyboards/activeLayout.ts | 9 +++++++++ common/web/keyboard-processor/src/text/codes.ts | 5 ++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/common/web/keyboard-processor/src/keyboards/activeLayout.ts b/common/web/keyboard-processor/src/keyboards/activeLayout.ts index cd4baa0a27..93a00f84c9 100644 --- a/common/web/keyboard-processor/src/keyboards/activeLayout.ts +++ b/common/web/keyboard-processor/src/keyboards/activeLayout.ts @@ -37,6 +37,7 @@ namespace com.keyman.keyboards { layer: string; displayLayer: string; nextlayer: string; + sp?: ButtonClass; private baseKeyEvent: text.KeyEvent; isMnemonic: boolean = false; @@ -73,6 +74,12 @@ namespace com.keyman.keyboards { return this.id; } + public get isPadding(): boolean { + // Does not include 9 (class: blank) as that may be an intentional 'catch' for misplaced + // keystrokes. + return this['sp'] == 10; // Button class: hidden. + } + /** * A unique identifier based on both the key ID & the 'desktop layer' to be used for the key. * @@ -547,6 +554,8 @@ namespace com.keyman.keyboards { // Results in a more optimized distribution. if(text.Codes.isKnownOSKModifierKey(key.baseKeyID)) { return; + } else if(key.isPadding) { // to the user, blank / padding keys do not exist. + return; } } // These represent the within-key distance of the touch from the key's center. diff --git a/common/web/keyboard-processor/src/text/codes.ts b/common/web/keyboard-processor/src/text/codes.ts index 915ae7feed..48fde0d728 100644 --- a/common/web/keyboard-processor/src/text/codes.ts +++ b/common/web/keyboard-processor/src/text/codes.ts @@ -81,10 +81,13 @@ namespace com.keyman.text { case 'K_SHIFT': case 'K_LOPT': case 'K_ROPT': - case 'K_NUMLOCK': // Often used for numeric layers. + case 'K_NUMLOCK': // Often used for numeric layers. case 'K_CAPS': return true; default: + if(Codes.keyCodes[keyID] >= 50000) { // A few are used by `sil_euro_latin`. + return true; // is a 'K_' key defined for layer shifting or 'control' use. + } // Refer to text/codes.ts - these are Keyman-custom "keycodes" used for // layer shifting keys. To be safe, we currently let K_TABBACK and // K_TABFWD through, though we might be able to drop them too. From 5cd48a578493f2c2c15236e1a9accc844b9a1482 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 12 Sep 2022 11:05:27 +0700 Subject: [PATCH 096/132] fix(web): new key prop needs enumerable --- common/web/keyboard-processor/src/keyboards/activeLayout.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/common/web/keyboard-processor/src/keyboards/activeLayout.ts b/common/web/keyboard-processor/src/keyboards/activeLayout.ts index 93a00f84c9..7f0f2f935f 100644 --- a/common/web/keyboard-processor/src/keyboards/activeLayout.ts +++ b/common/web/keyboard-processor/src/keyboards/activeLayout.ts @@ -74,6 +74,7 @@ namespace com.keyman.keyboards { return this.id; } + @Enumerable public get isPadding(): boolean { // Does not include 9 (class: blank) as that may be an intentional 'catch' for misplaced // keystrokes. From 415201bf15d7cdb429df44a8f86bf2984ba82399 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 12 Sep 2022 15:03:48 +0700 Subject: [PATCH 097/132] fix(common/models): isWhitespace, adds related unit tests --- .../unit_tests/headless/transform-utils.js | 58 +++++++++++++++++++ common/web/lm-worker/src/transformUtils.ts | 10 +--- 2 files changed, 61 insertions(+), 7 deletions(-) create mode 100644 common/predictive-text/unit_tests/headless/transform-utils.js diff --git a/common/predictive-text/unit_tests/headless/transform-utils.js b/common/predictive-text/unit_tests/headless/transform-utils.js new file mode 100644 index 0000000000..b00a24a123 --- /dev/null +++ b/common/predictive-text/unit_tests/headless/transform-utils.js @@ -0,0 +1,58 @@ +var assert = require('chai').assert; + +let TransformUtils = require('../../../web/lm-worker/build/intermediate.js').TransformUtils; + +describe('TransformUtils', function () { + describe('isWhitespace', function () { + it("should not match a string containing standard alphabetic characters", function () { + let testTransforms = [{ + insert: "a ", + deleteLeft: 0 + }, { + insert: " a", + deleteLeft: 0 + }, { + insert: "ab", + deleteLeft: 0 + }]; + + testTransforms.forEach((transform) => assert.isFalse(TransformUtils.isWhitespace(transform), `failed with: '${transform.insert}'`)); + }); + + it("should match a simple ' ' transform", function() { + transform = { + insert: " ", + deleteLeft: 0 + }; + + assert.isTrue(TransformUtils.isWhitespace(transform)); + }); + + it("should match a simple ' ' transform with delete-left", function() { + transform = { + insert: " ", + deleteLeft: 1 + }; + + assert.isTrue(TransformUtils.isWhitespace(transform)); + }); + + it("should match a transform consisting of multiple characters of only whitespace", function() { + transform = { + insert: " \n\r\u00a0\t\u2000 ", + deleteLeft: 0 + }; + + assert.isTrue(TransformUtils.isWhitespace(transform)); + }); + + it("stress tests", function() { + transform = { + insert: " \n\r\u00a0\ta\u2000 ", // the 'a' should cause failure. + deleteLeft: 0 + }; + + assert.isFalse(TransformUtils.isWhitespace(transform)); + }); + }); +}); diff --git a/common/web/lm-worker/src/transformUtils.ts b/common/web/lm-worker/src/transformUtils.ts index b91951722f..8c82f16b3b 100644 --- a/common/web/lm-worker/src/transformUtils.ts +++ b/common/web/lm-worker/src/transformUtils.ts @@ -1,18 +1,14 @@ class TransformUtils { static isWhitespace(transform: Transform): boolean { // Matches prefixed text + any instance of a character with Unicode general property Z* or the following: CR, LF, and Tab. - let whitespaceRemover = /.*[\u0009\u000A\u000D\u0020\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b\u2028\u2029\u202f\u205f\u3000]/i; + const whitespaceRemover = /^[\u0009\u000A\u000D\u0020\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b\u2028\u2029\u202f\u205f\u3000]+$/i; // Filter out null-inserts; their high probability can cause issues. - if(transform.insert == '') { // Can actually register as 'whitespace'. + if(transform.insert == '') { return false; } - let insert = transform.insert; - - insert = insert.replace(whitespaceRemover, ''); - - return insert == ''; + return transform.insert.match(whitespaceRemover) != null; } static isBackspace(transform: Transform): boolean { From 665b149b096735a361ae4dc79f6678198abd872c Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 12 Sep 2022 15:04:17 +0700 Subject: [PATCH 098/132] fix(common/models): needed export for unit tests --- common/web/lm-worker/src/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/web/lm-worker/src/index.ts b/common/web/lm-worker/src/index.ts index 7cc3bcbde1..e3c16fa2c8 100644 --- a/common/web/lm-worker/src/index.ts +++ b/common/web/lm-worker/src/index.ts @@ -32,6 +32,7 @@ /// /// /// +/// /** * Encapsulates all the state required for the LMLayer's worker thread. @@ -407,6 +408,7 @@ if (typeof module !== 'undefined' && typeof module.exports !== 'undefined') { module.exports['wordBreakers'] = wordBreakers; /// XXX: export the ModelCompositor for testing. module.exports['ModelCompositor'] = ModelCompositor; + module.exports['TransformUtils'] = TransformUtils; } else if (typeof self !== 'undefined' && 'postMessage' in self && 'importScripts' in self) { // Automatically install if we're in a Web Worker. LMLayerWorker.install(self as any); // really, 'as typeof globalThis', but we're currently getting TS errors from use of that. From 5bad267e11f4d76078ef9e394aa7331c90b053f2 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 7 Sep 2022 09:52:37 +0700 Subject: [PATCH 099/132] refactor(common/models): default wordbreaker state tracking --- .../models/wordbreakers/src/default/index.ts | 235 +++++++++++++----- 1 file changed, 166 insertions(+), 69 deletions(-) diff --git a/common/models/wordbreakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts index 5f14165c13..415d0b0046 100644 --- a/common/models/wordbreakers/src/default/index.ts +++ b/common/models/wordbreakers/src/default/index.ts @@ -61,6 +61,121 @@ namespace wordBreakers { } } + /** + * Provides a useful presentation for wordbreaker's context for use in word-breaking rules. + * + * @see https://unicode.org/reports/tr29/#Word_Boundary_Rules + */ + export class BreakerContext { + private readonly text: string; + + /** + * Represents the property of character immediately preceding `left`'s character. + */ + readonly lookbehind: WordBreakProperty = WordBreakProperty.sot; + + /** + * Represents the property of the character immediately preceding the potential word boundary. + */ + readonly left: WordBreakProperty = WordBreakProperty.sot; + + /** + * Represents the property of the character immediately following the potential word boundary. + */ + readonly right: WordBreakProperty = WordBreakProperty.sot; + + /** + * Represents the property of the character immediately following `right`'s character. + */ + readonly lookahead: WordBreakProperty; + + /** + * Initializes the word-breaking context at the start of the word-breaker's boundary-detection + * algorithm. + * @param text The text to be word-broken + * @param lookaheadPos The position corresponding to `lookahead`. + */ + constructor(text: string, lookaheadPos: number); + /** + * Used internally by the boundary-detection algorithm during context-shifting operations. + * @param text + * @param lookbehind + * @param left + * @param right + * @param lookahead + */ + constructor(text: string, + lookbehind: WordBreakProperty, + left: WordBreakProperty, + right: WordBreakProperty, + lookahead: WordBreakProperty); + constructor(text: string, + prop1: WordBreakProperty | number, + prop2?: WordBreakProperty, + prop3?: WordBreakProperty, + prop4?: WordBreakProperty) { + this.text = text; + + if(arguments.length == 2) { + this.lookahead = this.wordbreakPropertyAt(prop1);// prop1; + } else /*if(arguments.length == 5)*/ { + this.lookbehind = prop1 as WordBreakProperty; + this.left = prop2 as WordBreakProperty; + this.right = prop3 as WordBreakProperty; + this.lookahead = prop4 as WordBreakProperty; + } + } + + /** + * The general use-case when shifting boundary-check position if WB4 is not active. + * @param lookahead The WordBreakProperty for the character to become `lookahead`. + * @returns + */ + public next(lookaheadPos: number): BreakerContext { + let newLookahead = this.wordbreakPropertyAt(lookaheadPos); + return new BreakerContext(this.text, this.left, this.right, this.lookahead, newLookahead); + } + + /** + * Used for WB4: when ignoring characters before an intervening linebreak, we + * replace `right` with the current `lookahead`, without affecting `lookbehind` + * or `left`. A new `lookahead` is then needed. + * @param lookahead + * @returns + */ + public ignoringRight(lookaheadPos: number) { + let newLookahead = this.wordbreakPropertyAt(lookaheadPos); + return new BreakerContext(this.text, this.lookbehind, this.left, this.lookahead, newLookahead); + } + + /** + * Used for WB4: when ignoring characters after an intervening linebreak, it's + * `lookahead` that gets replaced without shifting the other tracked properties. + * @param lookahead + * @returns + */ + public ignoringLookahead(lookaheadPos: number) { + let newLookahead = this.wordbreakPropertyAt(lookaheadPos); + return new BreakerContext(this.text, this.lookbehind, this.left, this.right, newLookahead); + } + + /** + * Return the value of the Word_Break property at the given string index. + * @param pos position in the text. + */ + private wordbreakPropertyAt(pos: number) { + if (pos < 0) { + return WordBreakProperty.sot; // Always "start of string" before the string starts! + } else if (pos >= this.text.length) { + return WordBreakProperty.eot; // Always "end of string" after the string ends! + } else if (isStartOfSurrogatePair(this.text[pos])) { + // Surrogate pairs the next TWO items from the string! + return property(this.text[pos] + this.text[pos + 1]); + } + return property(this.text[pos]); + } + } + /** * Returns true when the chunk does not solely consist of whitespace. * @@ -112,10 +227,7 @@ namespace wordBreakers { let rightPos: number; let lookaheadPos = 0; // lookahead, one scalar value to the right of right. // Before the start of the string is also the start of the string. - let lookbehind: WordBreakProperty; - let left = WordBreakProperty.sot; - let right = WordBreakProperty.sot; - let lookahead = wordbreakPropertyAt(0); + let state = new BreakerContext(text, lookaheadPos); // Count RIs to make sure we're not splitting emoji flags: let nConsecutiveRegionalIndicators = 0; @@ -124,34 +236,33 @@ namespace wordBreakers { rightPos = lookaheadPos; lookaheadPos = positionAfter(lookaheadPos); // Shift all properties, one scalar value to the right. - [lookbehind, left, right, lookahead] = - [left, right, lookahead, wordbreakPropertyAt(lookaheadPos)]; + state = state.next(lookaheadPos); // Break at the start and end of text, unless the text is empty. // WB1: Break at start of text... - if (left === WordBreakProperty.sot) { + if (state.left === WordBreakProperty.sot) { boundaries.push(rightPos); continue; } // WB2: Break at the end of text... - if (right === WordBreakProperty.eot) { + if (state.right === WordBreakProperty.eot) { boundaries.push(rightPos); break; // Reached the end of the string. We're done! } // WB3: Do not break within CRLF: - if (left === WordBreakProperty.CR && right === WordBreakProperty.LF) + if (state.left === WordBreakProperty.CR && state.right === WordBreakProperty.LF) continue; // WB3b: Otherwise, break after... - if (left === WordBreakProperty.Newline || - left === WordBreakProperty.CR || - left === WordBreakProperty.LF) { + if (state.left === WordBreakProperty.Newline || + state.left === WordBreakProperty.CR || + state.left === WordBreakProperty.LF) { boundaries.push(rightPos); continue; } // WB3a: ...and before newlines - if (right === WordBreakProperty.Newline || - right === WordBreakProperty.CR || - right === WordBreakProperty.LF) { + if (state.right === WordBreakProperty.Newline || + state.right === WordBreakProperty.CR || + state.right === WordBreakProperty.LF) { boundaries.push(rightPos); continue; } @@ -163,7 +274,7 @@ namespace wordBreakers { // https://www.unicode.org/Public/emoji/12.0/emoji-zwj-sequences.txt // WB3d: Keep horizontal whitespace together - if (left === WordBreakProperty.WSegSpace && right == WordBreakProperty.WSegSpace) + if (state.left === WordBreakProperty.WSegSpace && state.right == WordBreakProperty.WSegSpace) continue; // WB4: Ignore format and extend characters @@ -171,99 +282,99 @@ namespace wordBreakers { // See: Section 6.2: https://unicode.org/reports/tr29/#Grapheme_Cluster_and_Format_Rules // N.B.: The rule about "except after sot, CR, LF, and // Newline" already been by WB1, WB2, WB3a, and WB3b above. - while (right === WordBreakProperty.Format || - right === WordBreakProperty.Extend || - right === WordBreakProperty.ZWJ) { + while (state.right === WordBreakProperty.Format || + state.right === WordBreakProperty.Extend || + state.right === WordBreakProperty.ZWJ) { // Continue advancing in the string, as if these // characters do not exist. DO NOT update left and // lookbehind however! [rightPos, lookaheadPos] = [lookaheadPos, positionAfter(lookaheadPos)]; - [right, lookahead] = [lookahead, wordbreakPropertyAt(lookaheadPos)]; + state = state.ignoringRight(lookaheadPos); } // In ignoring the characters in the previous loop, we could // have fallen off the end of the string, so end the loop // prematurely if that happens! - if (right === WordBreakProperty.eot) { + if (state.right === WordBreakProperty.eot) { boundaries.push(rightPos); break; } // WB4 (continued): Lookahead must ALSO ignore these format, // extend, ZWJ characters! - while (lookahead === WordBreakProperty.Format || - lookahead === WordBreakProperty.Extend || - lookahead === WordBreakProperty.ZWJ) { + while (state.lookahead === WordBreakProperty.Format || + state.lookahead === WordBreakProperty.Extend || + state.lookahead === WordBreakProperty.ZWJ) { // Continue advancing in the string, as if these // characters do not exist. DO NOT update left and right, // however! lookaheadPos = positionAfter(lookaheadPos); - lookahead = wordbreakPropertyAt(lookaheadPos); + state = state.ignoringLookahead(lookaheadPos); } // WB5: Do not break between most letters. - if (isAHLetter(left) && isAHLetter(right)) + if (isAHLetter(state.left) && isAHLetter(state.right)) continue; // Do not break across certain punctuation // WB6: (Don't break before apostrophes in contractions) - if (isAHLetter(left) && isAHLetter(lookahead) && - (right === WordBreakProperty.MidLetter || isMidNumLetQ(right))) + if (isAHLetter(state.left) && isAHLetter(state.lookahead) && + (state.right === WordBreakProperty.MidLetter || isMidNumLetQ(state.right))) continue; // WB7: (Don't break after apostrophes in contractions) - if (isAHLetter(lookbehind) && isAHLetter(right) && - (left === WordBreakProperty.MidLetter || isMidNumLetQ(left))) + if (isAHLetter(state.lookbehind) && isAHLetter(state.right) && + (state.left === WordBreakProperty.MidLetter || isMidNumLetQ(state.left))) continue; // WB7a - if (left === WordBreakProperty.Hebrew_Letter && right === WordBreakProperty.Single_Quote) + if (state.left === WordBreakProperty.Hebrew_Letter && state.right === WordBreakProperty.Single_Quote) continue; // WB7b - if (left === WordBreakProperty.Hebrew_Letter && right === WordBreakProperty.Double_Quote && - lookahead === WordBreakProperty.Hebrew_Letter) + if (state.left === WordBreakProperty.Hebrew_Letter && state.right === WordBreakProperty.Double_Quote && + state.lookahead === WordBreakProperty.Hebrew_Letter) continue; // WB7c - if (lookbehind === WordBreakProperty.Hebrew_Letter && left === WordBreakProperty.Double_Quote && - right === WordBreakProperty.Hebrew_Letter) + if (state.lookbehind === WordBreakProperty.Hebrew_Letter && state.left === WordBreakProperty.Double_Quote && + state.right === WordBreakProperty.Hebrew_Letter) continue; // Do not break within sequences of digits, or digits adjacent to letters. // e.g., "3a" or "A3" // WB8 - if (left === WordBreakProperty.Numeric && right === WordBreakProperty.Numeric) + if (state.left === WordBreakProperty.Numeric && state.right === WordBreakProperty.Numeric) continue; // WB9 - if (isAHLetter(left) && right === WordBreakProperty.Numeric) + if (isAHLetter(state.left) && state.right === WordBreakProperty.Numeric) continue; // WB10 - if (left === WordBreakProperty.Numeric && isAHLetter(right)) + if (state.left === WordBreakProperty.Numeric && isAHLetter(state.right)) continue; // Do not break within sequences, such as 3.2, 3,456.789 // WB11 - if (lookbehind === WordBreakProperty.Numeric && right === WordBreakProperty.Numeric && - (left === WordBreakProperty.MidNum || isMidNumLetQ(left))) + if (state.lookbehind === WordBreakProperty.Numeric && state.right === WordBreakProperty.Numeric && + (state.left === WordBreakProperty.MidNum || isMidNumLetQ(state.left))) continue; // WB12 - if (left === WordBreakProperty.Numeric && lookahead === WordBreakProperty.Numeric && - (right === WordBreakProperty.MidNum || isMidNumLetQ(right))) + if (state.left === WordBreakProperty.Numeric && state.lookahead === WordBreakProperty.Numeric && + (state.right === WordBreakProperty.MidNum || isMidNumLetQ(state.right))) continue; // WB13: Do not break between Katakana - if (left === WordBreakProperty.Katakana && right === WordBreakProperty.Katakana) + if (state.left === WordBreakProperty.Katakana && state.right === WordBreakProperty.Katakana) continue; // Do not break from extenders (e.g., U+202F NARROW NO-BREAK SPACE) // WB13a - if ((isAHLetter(left) || - left === WordBreakProperty.Numeric || - left === WordBreakProperty.Katakana || - left === WordBreakProperty.ExtendNumLet) && - right === WordBreakProperty.ExtendNumLet) + if ((isAHLetter(state.left) || + state.left === WordBreakProperty.Numeric || + state.left === WordBreakProperty.Katakana || + state.left === WordBreakProperty.ExtendNumLet) && + state.right === WordBreakProperty.ExtendNumLet) continue; // WB13b - if ((isAHLetter(right) || - right === WordBreakProperty.Numeric || - right === WordBreakProperty.Katakana) && left === WordBreakProperty.ExtendNumLet) + if ((isAHLetter(state.right) || + state.right === WordBreakProperty.Numeric || + state.right === WordBreakProperty.Katakana) && state.left === WordBreakProperty.ExtendNumLet) continue; // WB15 & WB16: // Do not break within emoji flag sequences. That is, do not break between // regional indicator (RI) symbols if there is an odd number of RI // characters before the break point. - if (right === WordBreakProperty.Regional_Indicator) { + if (state.right === WordBreakProperty.Regional_Indicator) { // Emoji flags are actually composed of TWO scalar values, each being a // "regional indicator". These indicators correspond to Latin letters. Put // two of them together, and they spell out an ISO 3166-1-alpha-2 country @@ -301,22 +412,6 @@ namespace wordBreakers { return pos + 1; } - /** - * Return the value of the Word_Break property at the given string index. - * @param pos position in the text. - */ - function wordbreakPropertyAt(pos: number) { - if (pos < 0) { - return WordBreakProperty.sot; // Always "start of string" before the string starts! - } else if (pos >= text.length) { - return WordBreakProperty.eot; // Always "end of string" after the string ends! - } else if (isStartOfSurrogatePair(text[pos])) { - // Surrogate pairs the next TWO items from the string! - return property(text[pos] + text[pos + 1]); - } - return property(text[pos]); - } - // Word_Break rule macros // See: https://unicode.org/reports/tr29/#WB_Rule_Macros function isAHLetter(prop: WordBreakProperty): boolean { @@ -340,7 +435,7 @@ namespace wordBreakers { * Note that * @param character a scalar value */ - function property(character: string): WordBreakProperty { + export function property(character: string): WordBreakProperty { // This MUST be a scalar value. // TODO: remove dependence on character.codepointAt()? let codepoint = character.codePointAt(0) as number; @@ -383,6 +478,8 @@ namespace wordBreakers { // implementing a namespace, BUT we can manually make the // assignment and **declare** it as part of the namespace. wordBreakers['default'] = wordBreakers.default_; +wordBreakers['unicodeProperty'] = wordBreakers.property; declare namespace wordBreakers { export { default_ as default }; + export { property as unicodeProperty }; } From 991a5bc271500334b1e48444a12ffce10527c98e Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 7 Sep 2022 10:33:50 +0700 Subject: [PATCH 100/132] feat(common/models): wordbreaker context .match method --- .../models/wordbreakers/src/default/index.ts | 136 +++++++++++------- 1 file changed, 81 insertions(+), 55 deletions(-) diff --git a/common/models/wordbreakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts index 415d0b0046..4f86efc507 100644 --- a/common/models/wordbreakers/src/default/index.ts +++ b/common/models/wordbreakers/src/default/index.ts @@ -87,7 +87,7 @@ namespace wordBreakers { /** * Represents the property of the character immediately following `right`'s character. */ - readonly lookahead: WordBreakProperty; + readonly lookahead: WordBreakProperty; // Always initialized by constructor. /** * Initializes the word-breaking context at the start of the word-breaker's boundary-detection @@ -174,6 +174,25 @@ namespace wordBreakers { } return property(this.text[pos]); } + + /** + * Returns `true` if and only if each member of the context has a property included within + * its corresponding set (when specified). Any set may be replaced with null to disable + * a check against its corresponding property. + * @param lookbehindSet + * @param leftSet + * @param rightSet + * @param lookaheadSet + */ + public match(lookbehindSet: WordBreakProperty[] | null, + leftSet: WordBreakProperty[] | null, + rightSet: WordBreakProperty[] | null, + lookaheadSet: WordBreakProperty[] | null) : boolean { + let result: boolean = lookbehindSet?.includes(this.lookbehind) ?? true; + result = result && (leftSet?.includes(this.left) ?? true); + result = result && (rightSet?.includes(this.right) ?? true); + return result && (lookaheadSet?.includes(this.lookahead) ?? true); + } } /** @@ -240,29 +259,28 @@ namespace wordBreakers { // Break at the start and end of text, unless the text is empty. // WB1: Break at start of text... - if (state.left === WordBreakProperty.sot) { + if (state.match(null, [WordBreakProperty.sot], null, null)) { boundaries.push(rightPos); continue; } // WB2: Break at the end of text... - if (state.right === WordBreakProperty.eot) { + if (state.match(null, null, [WordBreakProperty.eot], null)) { boundaries.push(rightPos); break; // Reached the end of the string. We're done! } // WB3: Do not break within CRLF: - if (state.left === WordBreakProperty.CR && state.right === WordBreakProperty.LF) + if (state.match(null, [WordBreakProperty.CR], [WordBreakProperty.LF], null)) { continue; + } + // WB3b: Otherwise, break after... - if (state.left === WordBreakProperty.Newline || - state.left === WordBreakProperty.CR || - state.left === WordBreakProperty.LF) { + const NEWLINE_SET = [WordBreakProperty.Newline, WordBreakProperty.CR, WordBreakProperty.LF]; + if(state.match(null, NEWLINE_SET, null, null)) { boundaries.push(rightPos); continue; } // WB3a: ...and before newlines - if (state.right === WordBreakProperty.Newline || - state.right === WordBreakProperty.CR || - state.right === WordBreakProperty.LF) { + if (state.match(null, null, NEWLINE_SET, null)) { boundaries.push(rightPos); continue; } @@ -274,17 +292,17 @@ namespace wordBreakers { // https://www.unicode.org/Public/emoji/12.0/emoji-zwj-sequences.txt // WB3d: Keep horizontal whitespace together - if (state.left === WordBreakProperty.WSegSpace && state.right == WordBreakProperty.WSegSpace) + if (state.match(null, [WordBreakProperty.WSegSpace], [WordBreakProperty.WSegSpace], null)) { continue; + } // WB4: Ignore format and extend characters // This is to keep grapheme clusters together! // See: Section 6.2: https://unicode.org/reports/tr29/#Grapheme_Cluster_and_Format_Rules // N.B.: The rule about "except after sot, CR, LF, and // Newline" already been by WB1, WB2, WB3a, and WB3b above. - while (state.right === WordBreakProperty.Format || - state.right === WordBreakProperty.Extend || - state.right === WordBreakProperty.ZWJ) { + const SET_WB4_IGNORE = [WordBreakProperty.Format, WordBreakProperty.Extend, WordBreakProperty.ZWJ]; + while (state.match(null, null, SET_WB4_IGNORE, null)) { // Continue advancing in the string, as if these // characters do not exist. DO NOT update left and // lookbehind however! @@ -300,9 +318,7 @@ namespace wordBreakers { } // WB4 (continued): Lookahead must ALSO ignore these format, // extend, ZWJ characters! - while (state.lookahead === WordBreakProperty.Format || - state.lookahead === WordBreakProperty.Extend || - state.lookahead === WordBreakProperty.ZWJ) { + while (state.match(null, null, null, SET_WB4_IGNORE)) { // Continue advancing in the string, as if these // characters do not exist. DO NOT update left and right, // however! @@ -310,65 +326,87 @@ namespace wordBreakers { state = state.ignoringLookahead(lookaheadPos); } + // See: https://unicode.org/reports/tr29/#WB_Rule_Macros + const SET_AHLETTER = [WordBreakProperty.ALetter, WordBreakProperty.Hebrew_Letter]; + const SET_MIDNUMLETQ = [WordBreakProperty.MidNumLet, WordBreakProperty.Single_Quote]; + // WB5: Do not break between most letters. - if (isAHLetter(state.left) && isAHLetter(state.right)) + // if (isAHLetter(state.left) && isAHLetter(state.right)) + if(state.match(null, SET_AHLETTER, SET_AHLETTER, null)) { continue; + } // Do not break across certain punctuation // WB6: (Don't break before apostrophes in contractions) - if (isAHLetter(state.left) && isAHLetter(state.lookahead) && - (state.right === WordBreakProperty.MidLetter || isMidNumLetQ(state.right))) + const SET_ALL_MIDLETTER = [WordBreakProperty.MidLetter, ...SET_MIDNUMLETQ]; + if(state.match(null, SET_AHLETTER, SET_ALL_MIDLETTER, SET_AHLETTER)) { continue; + } // WB7: (Don't break after apostrophes in contractions) - if (isAHLetter(state.lookbehind) && isAHLetter(state.right) && - (state.left === WordBreakProperty.MidLetter || isMidNumLetQ(state.left))) + if(state.match(SET_AHLETTER, SET_ALL_MIDLETTER, SET_AHLETTER, null)) { continue; + } + // WB7a - if (state.left === WordBreakProperty.Hebrew_Letter && state.right === WordBreakProperty.Single_Quote) + if(state.match(null, [WordBreakProperty.Hebrew_Letter], [WordBreakProperty.Single_Quote], null)) { continue; + } // WB7b - if (state.left === WordBreakProperty.Hebrew_Letter && state.right === WordBreakProperty.Double_Quote && - state.lookahead === WordBreakProperty.Hebrew_Letter) + if(state.match(null, + [WordBreakProperty.Hebrew_Letter], + [WordBreakProperty.Double_Quote], + [WordBreakProperty.Hebrew_Letter])) { continue; + } // WB7c - if (state.lookbehind === WordBreakProperty.Hebrew_Letter && state.left === WordBreakProperty.Double_Quote && - state.right === WordBreakProperty.Hebrew_Letter) + if(state.match([WordBreakProperty.Hebrew_Letter], + [WordBreakProperty.Double_Quote], + [WordBreakProperty.Hebrew_Letter], + null)) { continue; + } // Do not break within sequences of digits, or digits adjacent to letters. // e.g., "3a" or "A3" // WB8 - if (state.left === WordBreakProperty.Numeric && state.right === WordBreakProperty.Numeric) + if(state.match(null, [WordBreakProperty.Numeric], [WordBreakProperty.Numeric], null)) { continue; + } // WB9 - if (isAHLetter(state.left) && state.right === WordBreakProperty.Numeric) + if(state.match(null, SET_AHLETTER, [WordBreakProperty.Numeric], null)) { continue; + } // WB10 - if (state.left === WordBreakProperty.Numeric && isAHLetter(state.right)) + if(state.match(null, [WordBreakProperty.Numeric], SET_AHLETTER, null)) { continue; + } // Do not break within sequences, such as 3.2, 3,456.789 // WB11 - if (state.lookbehind === WordBreakProperty.Numeric && state.right === WordBreakProperty.Numeric && - (state.left === WordBreakProperty.MidNum || isMidNumLetQ(state.left))) + const SET_ALL_MIDNUM = [WordBreakProperty.MidNum, ...SET_MIDNUMLETQ]; + if(state.match([WordBreakProperty.Numeric], SET_ALL_MIDNUM, [WordBreakProperty.Numeric], null)) { continue; + } // WB12 - if (state.left === WordBreakProperty.Numeric && state.lookahead === WordBreakProperty.Numeric && - (state.right === WordBreakProperty.MidNum || isMidNumLetQ(state.right))) + if(state.match(null, [WordBreakProperty.Numeric], SET_ALL_MIDNUM, [WordBreakProperty.Numeric])) { continue; + } // WB13: Do not break between Katakana - if (state.left === WordBreakProperty.Katakana && state.right === WordBreakProperty.Katakana) + if(state.match(null, [WordBreakProperty.Katakana], [WordBreakProperty.Katakana], null)) { continue; + } // Do not break from extenders (e.g., U+202F NARROW NO-BREAK SPACE) // WB13a - if ((isAHLetter(state.left) || - state.left === WordBreakProperty.Numeric || - state.left === WordBreakProperty.Katakana || - state.left === WordBreakProperty.ExtendNumLet) && - state.right === WordBreakProperty.ExtendNumLet) + const SET_NUM_KAT_LET = [WordBreakProperty.Katakana, + WordBreakProperty.Numeric, + ...SET_AHLETTER]; + if(state.match(null, SET_NUM_KAT_LET, [WordBreakProperty.ExtendNumLet], null)) { continue; + } + if(state.match(null, [WordBreakProperty.ExtendNumLet], [WordBreakProperty.ExtendNumLet], null)) { + continue; + } // WB13b - if ((isAHLetter(state.right) || - state.right === WordBreakProperty.Numeric || - state.right === WordBreakProperty.Katakana) && state.left === WordBreakProperty.ExtendNumLet) + if(state.match(null, [WordBreakProperty.ExtendNumLet], SET_NUM_KAT_LET, null)) { continue; + } // WB15 & WB16: // Do not break within emoji flag sequences. That is, do not break between @@ -411,18 +449,6 @@ namespace wordBreakers { } return pos + 1; } - - // Word_Break rule macros - // See: https://unicode.org/reports/tr29/#WB_Rule_Macros - function isAHLetter(prop: WordBreakProperty): boolean { - return prop === WordBreakProperty.ALetter || - prop === WordBreakProperty.Hebrew_Letter; - } - - function isMidNumLetQ(prop: WordBreakProperty): boolean { - return prop === WordBreakProperty.MidNumLet || - prop === WordBreakProperty.Single_Quote; - } } function isStartOfSurrogatePair(character: string) { From 51caf533a874643822683e247ea5e9f8217287bc Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 7 Sep 2022 11:52:32 +0700 Subject: [PATCH 101/132] feat(common/models): interface for custom breaker rules --- .../models/wordbreakers/src/default/data.ts | 2 +- .../models/wordbreakers/src/default/index.ts | 48 +++++++++++++++++-- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/common/models/wordbreakers/src/default/data.ts b/common/models/wordbreakers/src/default/data.ts index ffb6b8e79a..7401fd667e 100644 --- a/common/models/wordbreakers/src/default/data.ts +++ b/common/models/wordbreakers/src/default/data.ts @@ -4,7 +4,7 @@ export namespace data { /** * Valid values for a word break property. */ -export const enum WordBreakProperty { +export const enum WordBreakProperty { // Scary bit: this does not exist as an object at run-time! Other, LF, Newline, diff --git a/common/models/wordbreakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts index 4f86efc507..9e4d4a27ee 100644 --- a/common/models/wordbreakers/src/default/index.ts +++ b/common/models/wordbreakers/src/default/index.ts @@ -61,13 +61,35 @@ namespace wordBreakers { } } + /** + * An abstraction supporting custom wordbreaker boundary rules. While this doesn't provide + * support for more complex rules like WB4, WB15, or WB16, this is sufficient for all other + * default word-breaking rules. Thus, this should cover the majority of cases requiring + * custom handling for specific languages. + * + * @see https://unicode.org/reports/tr29/#WB_Rule_Macros + */ + export interface WordbreakerRule { + /** + * Indicates whether or not the rule applies in the specified context. + * @param context + */ + match(context: BreakerContext): boolean; + + /** + * Indicates whether or not the rule indicates a word boundary at the context's site when it matches. + */ + breakIfMatch: boolean; + } + /** * Provides a useful presentation for wordbreaker's context for use in word-breaking rules. * * @see https://unicode.org/reports/tr29/#Word_Boundary_Rules */ export class BreakerContext { - private readonly text: string; + // Referenced by this object in order to facilitate `lookahead` maintenance. + private readonly text: string; /** * Represents the property of character immediately preceding `left`'s character. @@ -163,7 +185,7 @@ namespace wordBreakers { * Return the value of the Word_Break property at the given string index. * @param pos position in the text. */ - private wordbreakPropertyAt(pos: number) { + private wordbreakPropertyAt(pos: number) { if (pos < 0) { return WordBreakProperty.sot; // Always "start of string" before the string starts! } else if (pos >= this.text.length) { @@ -216,13 +238,17 @@ namespace wordBreakers { * * @param text Text to find word boundaries in. */ - function findBoundaries(text: string): number[] { + function findBoundaries(text: string, options?: WordbreakerRule[]): number[] { // WB1 and WB2: no boundaries if given an empty string. if (text.length === 0) { // There are no boundaries in an empty string! return []; } + if(!options) { + options = []; + } + // This algorithm works by maintaining a sliding window of four SCALAR VALUES. // // - Scalar values? JavaScript strings are NOT actually a string of @@ -330,6 +356,22 @@ namespace wordBreakers { const SET_AHLETTER = [WordBreakProperty.ALetter, WordBreakProperty.Hebrew_Letter]; const SET_MIDNUMLETQ = [WordBreakProperty.MidNumLet, WordBreakProperty.Single_Quote]; + // Start: Custom rules + let customMatch: boolean = false; + for(const rule of options) { + customMatch = rule.match(state); + if(customMatch) { + if(rule.breakIfMatch) { + boundaries.push(rightPos); + } + break; // as customMatch == true here, this will trigger the `continue` that follows. + } + } + if(customMatch) { + continue; + } + // End: Custom rules + // WB5: Do not break between most letters. // if (isAHLetter(state.left) && isAHLetter(state.right)) if(state.match(null, SET_AHLETTER, SET_AHLETTER, null)) { From 075147ba30c945c70d68bc985990bc23b2f978d6 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Mon, 12 Sep 2022 14:04:02 -0400 Subject: [PATCH 102/132] auto: increment master version to 16.0.62 --- HISTORY.md | 8 ++++++++ VERSION.md | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 48a2da1054..3431ee2692 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,13 @@ # Keyman Version History +## 16.0.61 alpha 2022-09-12 + +* docs(windows): add steps for using testhost debugging (#7263) +* fix(developer): compiler mismatch on currentLine (#7190) +* fix(developer): suppress repeated warnings about unreachable code (#7219) +* chore: try disabling concurrency for browserstack tests (#7258) +* chore(web): disable browserstack on non-web-specific builds (#7260) + ## 16.0.60 alpha 2022-09-10 * fix(web): enhanced timer for prediction algorithm (#7037) diff --git a/VERSION.md b/VERSION.md index d6db1906b4..f4737a994e 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.61 \ No newline at end of file +16.0.62 \ No newline at end of file From 8e076fe3fdd94598af53fb12e9aca425d480cbc4 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 13 Sep 2022 10:18:24 +0700 Subject: [PATCH 103/132] feat(common/models): actual wordbreaker customization --- .../models/wordbreakers/src/default/data.ts | 26 + .../models/wordbreakers/src/default/index.ts | 152 ++++-- .../test/test-default-word-breaker.js | 476 ++++++++++++------ 3 files changed, 457 insertions(+), 197 deletions(-) diff --git a/common/models/wordbreakers/src/default/data.ts b/common/models/wordbreakers/src/default/data.ts index 7401fd667e..16a4d9beb3 100644 --- a/common/models/wordbreakers/src/default/data.ts +++ b/common/models/wordbreakers/src/default/data.ts @@ -28,6 +28,32 @@ export const enum WordBreakProperty { // Scary bit: this does not exist as an o eot }; +// Not currently built by the auto-generator tool, but it easily could be. +// If and when we import the data.ts rebuilder, we can add this in. +export const propertyMap = [ + "Other", + "LF", + "Newline", + "CR", + "WSegSpace", + "Double_Quote", + "Single_Quote", + "MidNum", + "MidNumLet", + "Numeric", + "MidLetter", + "ALetter", + "ExtendNumLet", + "Format", + "Extend", + "Hebrew_Letter", + "ZWJ", + "Katakana", + "Regional_Indicator", + "sot", + "eot" +]; + /** * Constants for indexing values in WORD_BREAK_PROPERTY. */ diff --git a/common/models/wordbreakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts index 9e4d4a27ee..cfe251fbf2 100644 --- a/common/models/wordbreakers/src/default/index.ts +++ b/common/models/wordbreakers/src/default/index.ts @@ -1,6 +1,34 @@ // Include the word-breaking data here: /// namespace wordBreakers { + /** + * A set of options used to customize and extend the behavior of the default + * Unicode wordbreaker. + */ + export interface DefaultWordBreakerOptions { + /** + * Allows addition of custom wordbreaking rules, which will be applied + * after WB1-WB4 and before all other default wordbreaking rules. + * + * @see `WordbreakerRule` + */ + rules?: WordbreakerRule[]; + + /** + * Allows assignment of characters to different word-breaking properties than + * their standard word-breaking assignment, including to custom properties + * specified within `customProperties`. + * @param char + */ + propertyMapping?(char: string): string; + + /** + * Allows definition of extra word-breaking properties for use with custom + * rules. + */ + customProperties?: [string]; + } + /** * Word breaker based on Unicode Standard Annex #29, Section 4.1: * Default Word Boundary Specification. @@ -8,8 +36,8 @@ namespace wordBreakers { * @see http://unicode.org/reports/tr29/#Word_Boundaries * @see https://github.com/eddieantonio/unicode-default-word-boundary/tree/v12.0.0 */ - export function default_(text: string): Span[] { - let boundaries = findBoundaries(text); + export function default_(text: string, options?: DefaultWordBreakerOptions): Span[] { + let boundaries = findBoundaries(text, options); if (boundaries.length == 0) { return []; } @@ -22,7 +50,7 @@ namespace wordBreakers { let end = boundaries[i + 1]; let span = new LazySpan(text, start, end); - if (isNonSpace(span.text)) { + if (isNonSpace(span.text, options)) { spans.push(span); // Preserve a sequence-final space if it exists. Needed to signal "end of word". } else if (i == boundaries.length - 2) { // if "we just checked the final boundary"... @@ -64,8 +92,7 @@ namespace wordBreakers { /** * An abstraction supporting custom wordbreaker boundary rules. While this doesn't provide * support for more complex rules like WB4, WB15, or WB16, this is sufficient for all other - * default word-breaking rules. Thus, this should cover the majority of cases requiring - * custom handling for specific languages. + * default word-breaking rules and can be used to define custom rules of similar structure. * * @see https://unicode.org/reports/tr29/#WB_Rule_Macros */ @@ -90,6 +117,7 @@ namespace wordBreakers { export class BreakerContext { // Referenced by this object in order to facilitate `lookahead` maintenance. private readonly text: string; + readonly options?: DefaultWordBreakerOptions; /** * Represents the property of character immediately preceding `left`'s character. @@ -117,7 +145,7 @@ namespace wordBreakers { * @param text The text to be word-broken * @param lookaheadPos The position corresponding to `lookahead`. */ - constructor(text: string, lookaheadPos: number); + constructor(text: string, options: DefaultWordBreakerOptions | undefined, lookaheadPos: number); /** * Used internally by the boundary-detection algorithm during context-shifting operations. * @param text @@ -127,20 +155,23 @@ namespace wordBreakers { * @param lookahead */ constructor(text: string, + options: DefaultWordBreakerOptions | undefined, lookbehind: WordBreakProperty, left: WordBreakProperty, right: WordBreakProperty, lookahead: WordBreakProperty); constructor(text: string, + options: DefaultWordBreakerOptions | undefined, prop1: WordBreakProperty | number, prop2?: WordBreakProperty, prop3?: WordBreakProperty, prop4?: WordBreakProperty) { this.text = text; + this.options = options; - if(arguments.length == 2) { + if(arguments.length == 3) { this.lookahead = this.wordbreakPropertyAt(prop1);// prop1; - } else /*if(arguments.length == 5)*/ { + } else /*if(arguments.length == 6)*/ { this.lookbehind = prop1 as WordBreakProperty; this.left = prop2 as WordBreakProperty; this.right = prop3 as WordBreakProperty; @@ -155,7 +186,7 @@ namespace wordBreakers { */ public next(lookaheadPos: number): BreakerContext { let newLookahead = this.wordbreakPropertyAt(lookaheadPos); - return new BreakerContext(this.text, this.left, this.right, this.lookahead, newLookahead); + return new BreakerContext(this.text, this.options, this.left, this.right, this.lookahead, newLookahead); } /** @@ -167,7 +198,7 @@ namespace wordBreakers { */ public ignoringRight(lookaheadPos: number) { let newLookahead = this.wordbreakPropertyAt(lookaheadPos); - return new BreakerContext(this.text, this.lookbehind, this.left, this.lookahead, newLookahead); + return new BreakerContext(this.text, this.options, this.lookbehind, this.left, this.lookahead, newLookahead); } /** @@ -178,7 +209,7 @@ namespace wordBreakers { */ public ignoringLookahead(lookaheadPos: number) { let newLookahead = this.wordbreakPropertyAt(lookaheadPos); - return new BreakerContext(this.text, this.lookbehind, this.left, this.right, newLookahead); + return new BreakerContext(this.text, this.options, this.lookbehind, this.left, this.right, newLookahead); } /** @@ -194,7 +225,7 @@ namespace wordBreakers { // Surrogate pairs the next TWO items from the string! return property(this.text[pos] + this.text[pos + 1]); } - return property(this.text[pos]); + return property(this.text[pos], this.options); } /** @@ -215,15 +246,38 @@ namespace wordBreakers { result = result && (rightSet?.includes(this.right) ?? true); return result && (lookaheadSet?.includes(this.lookahead) ?? true); } - } + + /** + * Returns `true` if and only if each member of the context has a property included within + * its corresponding set (when specified). Any set may be replaced with null to disable + * a check against its corresponding property. + * + * Names should match those found at https://unicode.org/reports/tr29/#Word_Boundary_Rules + * or defined in the word-breaker customization options; matching is case-insensitive. + * @param lookbehindSet + * @param leftSet + * @param rightSet + * @param lookaheadSet + */ + public propertyMatch(lookbehindSet: string[] | null, + leftSet: string[] | null, + rightSet: string[] | null, + lookaheadSet: string[] | null) : boolean { + const propMapper = (name: string) => propertyVal(name, this.options); + return this.match(lookbehindSet?.map(propMapper) as WordBreakProperty[] | null, + leftSet?.map(propMapper) as WordBreakProperty[] | null, + rightSet?.map(propMapper) as WordBreakProperty[] | null, + lookaheadSet?.map(propMapper) as WordBreakProperty[] | null); + } + } /** * Returns true when the chunk does not solely consist of whitespace. * * @param chunk a chunk of text. Starts and ends at word boundaries. */ - function isNonSpace(chunk: string): boolean { - return !Array.from(chunk).map(property).every(wb => ( + function isNonSpace(chunk: string, options?: DefaultWordBreakerOptions): boolean { + return !Array.from(chunk).map((char) => property(char, options)).every(wb => ( wb === WordBreakProperty.CR || wb === WordBreakProperty.LF || wb === WordBreakProperty.Newline || @@ -238,15 +292,15 @@ namespace wordBreakers { * * @param text Text to find word boundaries in. */ - function findBoundaries(text: string, options?: WordbreakerRule[]): number[] { + function findBoundaries(text: string, options?: DefaultWordBreakerOptions): number[] { // WB1 and WB2: no boundaries if given an empty string. if (text.length === 0) { // There are no boundaries in an empty string! return []; } - if(!options) { - options = []; + if(options && !options.rules) { + options.rules = []; } // This algorithm works by maintaining a sliding window of four SCALAR VALUES. @@ -272,7 +326,7 @@ namespace wordBreakers { let rightPos: number; let lookaheadPos = 0; // lookahead, one scalar value to the right of right. // Before the start of the string is also the start of the string. - let state = new BreakerContext(text, lookaheadPos); + let state = new BreakerContext(text, options, lookaheadPos); // Count RIs to make sure we're not splitting emoji flags: let nConsecutiveRegionalIndicators = 0; @@ -356,21 +410,22 @@ namespace wordBreakers { const SET_AHLETTER = [WordBreakProperty.ALetter, WordBreakProperty.Hebrew_Letter]; const SET_MIDNUMLETQ = [WordBreakProperty.MidNumLet, WordBreakProperty.Single_Quote]; - // Start: Custom rules - let customMatch: boolean = false; - for(const rule of options) { - customMatch = rule.match(state); - if(customMatch) { - if(rule.breakIfMatch) { - boundaries.push(rightPos); + // Custom rules may override the base ruleset aside from the first few fundamental ones. + if(options?.rules) { + let customMatch: boolean = false; + for(const rule of options.rules) { + customMatch = rule.match(state); + if(customMatch) { + if(rule.breakIfMatch) { + boundaries.push(rightPos); + } + break; // as customMatch == true here, this will trigger the `continue` that follows. } - break; // as customMatch == true here, this will trigger the `continue` that follows. + } + if(customMatch) { + continue; } } - if(customMatch) { - continue; - } - // End: Custom rules // WB5: Do not break between most letters. // if (isAHLetter(state.left) && isAHLetter(state.right)) @@ -503,13 +558,43 @@ namespace wordBreakers { * Note that * @param character a scalar value */ - export function property(character: string): WordBreakProperty { + function property(character: string, options?: DefaultWordBreakerOptions): WordBreakProperty { + // If there is a customized mapping for the character, prioritize that. + if(options?.propertyMapping) { + let propName = options.propertyMapping(character); + if(propName) { + return propertyVal(propName, options); + } + } + // This MUST be a scalar value. // TODO: remove dependence on character.codepointAt()? let codepoint = character.codePointAt(0) as number; + return searchForProperty(codepoint, 0, WORD_BREAK_PROPERTY.length - 1); } + function propertyVal(propName: string, options?: DefaultWordBreakerOptions) { + const matcher = (name: string) => name.toLowerCase() == propName.toLowerCase() + + const customIndex = options?.customProperties?.findIndex(matcher) ?? -1; + return customIndex != -1 ? -customIndex - 1 : data.propertyMap.findIndex(matcher); + } + + // /** + // * Provides the word-breaking property name for the specified character based on the property + // * values used by https://unicode.org/reports/tr29/#Word_Boundary_Rules. + // * @param character + // * @returns + // */ + // export function unicodeProperty(character: string): string { + // // Since we use a const enum for property names, the TS compiler optimizes it away + // // and does not provide a reverse lookup for us. So, we do that here. + // const enumVal = property(character); + + // return data.propertyMap[enumVal]; + // } + /** * Binary search for the word break property of a given CODE POINT. * @@ -546,8 +631,7 @@ namespace wordBreakers { // implementing a namespace, BUT we can manually make the // assignment and **declare** it as part of the namespace. wordBreakers['default'] = wordBreakers.default_; -wordBreakers['unicodeProperty'] = wordBreakers.property; +// wordBreakers['unicodeProperty'] = wordBreakers.unicodeProperty; declare namespace wordBreakers { export { default_ as default }; - export { property as unicodeProperty }; } diff --git a/common/models/wordbreakers/test/test-default-word-breaker.js b/common/models/wordbreakers/test/test-default-word-breaker.js index b4db8cb1a3..85129b836c 100644 --- a/common/models/wordbreakers/test/test-default-word-breaker.js +++ b/common/models/wordbreakers/test/test-default-word-breaker.js @@ -8,184 +8,334 @@ const breakWords = require('../build').wordBreakers['default']; const SHY = '\u00AD'; // Other, Format. The "Soft HYphen" - usually invisible unless needed for word-wrapping. describe('The default word breaker', function () { - it('should break multilingual text', function () { - let breaks = breakWords( - `Добрый день! ᑕᐻ᙮ — after working on ka${SHY}wen${SHY}non:${SHY}nis, - let's eat phở! 🥣` - ); - let words = breaks.map(span => span.text); - assert.deepEqual(words, [ - 'Добрый', 'день', '!', 'ᑕᐻ', '᙮', '—', 'after', - 'working', 'on', `ka${SHY}wen${SHY}non:${SHY}nis`, ',', - "let's", 'eat', 'phở', '!', '🥣' - ]); + describe('default configuration', function() { + it('should break multilingual text', function () { + let breaks = breakWords( + `Добрый день! ᑕᐻ᙮ — after working on ka${SHY}wen${SHY}non:${SHY}nis, + let's eat phở! 🥣` + ); + let words = breaks.map(span => span.text); + assert.deepEqual(words, [ + 'Добрый', 'день', '!', 'ᑕᐻ', '᙮', '—', 'after', + 'working', 'on', `ka${SHY}wen${SHY}non:${SHY}nis`, ',', + "let's", 'eat', 'phở', '!', '🥣' + ]); + }); + + it('handles heavily-punctuated English text', function() { + // This test case brought to you by http://unicode.org/reports/tr29/#Word_Boundaries, Figure 1. + let breaks = breakWords( + `The quick ("brown") fox can't jump 32.3 feet, right?` + ); + let words = breaks.map(span => span.text); + assert.deepEqual(words, [ + 'The', 'quick', '(', '"', 'brown', '"', ')', 'fox', "can't", + 'jump', '32.3', 'feet', ',', 'right', '?' + ]); + }); + + // The way these two tests are written is a bit much on the "white-box" style, + // but they do decently cover the boundary rules mentioned. + it('Does not split empty contexts (WB1 + WB2)', function() { + let breaks = breakWords(''); + let words = breaks.map(span => span.text); + assert.deepEqual(words, []); + }); + + it('Does split at context boundaries (WB1 + WB2)', function() { + let breaks = breakWords('a'); + let words = breaks.map(span => span.text); + assert.deepEqual(words, ['a']); + }); + + // WB3, WB3a, WB3b are all handled internally, within the top-level function. + + // iff, as in "if and only if" + it('ignores the zero-width joiner iff appropriate (WB4)', function() { + const zwj = '\u200d'; + + let breaks = breakWords(`a${zwj}b\n${zwj}c${zwj}\nd`); + let words = breaks.map(span => span.text); + + // Does NOT ignore the zwj immediately after a newline - the notable exception + // (the reason for "iff", not "if"). + assert.deepEqual(words, [`a${zwj}b`, `${zwj}`, `c${zwj}`, `d`]); + }) + + it('ignores extend characters iff appropriate (WB4)', function() { + const comboGrave = '\u0300'; // The 'combining grave accent', as used in NFD. + + let breaks = breakWords(`a${comboGrave}e\n${comboGrave}i${comboGrave}\no`); + let words = breaks.map(span => span.text); + + // Does NOT ignore the zwj immediately after a newline - the notable exception + // (the reason for "iff", not "if"). + assert.deepEqual(words, [`a${comboGrave}e`, `${comboGrave}`, `i${comboGrave}`, `o`]); + }); + + it('ignores format characters iff appropriate (WB4)', function() { + // Re-uses `const SHY` from above. + let breaks = breakWords(`a${SHY}e\n${SHY}i${SHY}\no`); + let words = breaks.map(span => span.text); + + // Does NOT ignore the zwj immediately after a newline - the notable exception + // (the reason for "iff", not "if"). + assert.deepEqual(words, [`a${SHY}e`, `${SHY}`, `i${SHY}`, `o`]); + }); + + it('does not break between most alphabetic characters (WB5)', function() { + let breaks = breakWords(`aəσאБ лאʈγX`); // a mix of latin, hebrew, greek, cyrillic, and IPA chars + // for both "words". + let words = breaks.map(span => span.text); + + assert.deepEqual(words, [`aəσאБ`, `лאʈγX`]); + }); + + it('does not break letters across specific punctuation patterns (WB6, WB7)', function() { + // `'`: MidNumLetQ (from Single_Quote) + // '.': MidNumLet + // ':': MidLetter + let breaks = breakWords(`don't b.r.e.a.k t:h:e:s:e`); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, [`don't`, `b.r.e.a.k`, `t:h:e:s:e`]); + + let breaks2 = breakWords(`.drop: :the' 'extras.`); + let words2 = breaks2.map(span => span.text); + + assert.deepEqual(words2, [`.`, `drop`, `:`, `:`, `the`, `'`, `'`, `extras`, `.`]); + + // ',': MidNum (is NOT included by rule!) + let breaks3 = breakWords('do br,eak that'); + let words3 = breaks3.map(span => span.text); + + assert.deepEqual(words3, ['do', 'br', ',', 'eak', 'that']); + }); + + it('treats Hebrew properly (WB7a-c)', function() { + const aleph = 'א'; + const bet = 'ב'; + + // As Hebrew is RTL... this is probably the clearest way for us LTR people to + // clearly see what's going on without ordering mechanics messing up the render. + let breaks = breakWords(`${aleph}' ${aleph}" ${aleph}"${bet}`); + let words = breaks.map(span => span.text); + + // A lingering double-quote isn't cool, but one in the middle's fine. + // Lingering single-quote is fine regardless. + assert.deepEqual(words, [`${aleph}'`, `${aleph}`, `"`, `${aleph}"${bet}`]); + }); + + it(`doesn't break within digit + digit/letter sequences (WB8-10)`, function() { + let breaks = breakWords('a1b2c3 hunter2 ab12cd34 1234567890'); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, ['a1b2c3', 'hunter2', 'ab12cd34', '1234567890']); + }); + + it('does not break within formatted number sequences (WB11-12)', function() { + // Note: `'` fits "MidNumLetQ", part of the two rules! + let breaks = breakWords(`1.2.3 3,458.01 3.45'8,01`); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, [`1.2.3`, `3,458.01`, `3.45'8,01`]); + + let breaks2 = breakWords(`.1' ,3.`); + let words2 = breaks2.map(span => span.text); + + assert.deepEqual(words2, [`.`, `1`, `'`, `,`, `3`, `.`]); + }); + + it('does not break between Katakana (WB13)', function() { + const kataSmA = '\u30a2'; //ァ + const kataA = '\u30a2'; //ア + const kataSound = '\u309b'; // ゛ + + let breaks = breakWords(`${kataSound}${kataA} ${kataSmA}${kataSound}b ${kataA}${kataSound}${kataSmA}`); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, [ + `${kataSound}${kataA}`, + `${kataSmA}${kataSound}`, + 'b', + `${kataA}${kataSound}${kataSmA}` + ]); + }); + + it('does not break form extenders (WB13a-b)', function() { + // The `_` (underscore) fits the ExtendNumLet class this rule focuses on. + const kataA = '\u30a2'; //ア + + let breaks = breakWords(`${kataA}_a__0_b_${kataA} _${kataA} 1_ _c_ ____`); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, [ + `${kataA}_a__0_b_${kataA}`, + `_${kataA}`, + `1_`, + `_c_`, + `____` + ]); + }); + + it('handles emoji flag sequences properly (WB15-16)', function() { + // For clarity on what's being tested... + let CA_FLAG = '\u{1f1e8}\u{1f1e6}' // '🇨🇦' (canadian flag emoji); should not be broken. + let KH_FLAG = '\u{1f1f0}\u{1f1ed}' // '🇰🇭' (khmer flag emoji); same + let X_FLAG_PIECE = '\u{1f1fd}' // '🇽' (half of a flag emoji; '🇽🇽' doesn't match a flag) + let breaks = breakWords(`${CA_FLAG}${KH_FLAG}${X_FLAG_PIECE}${X_FLAG_PIECE}`); + let words = breaks.map(span => span.text); + + // Note that the emoji may not render well within VSCode, but they show up nicely on GitHub. + assert.deepEqual(words, ['🇨🇦', '🇰🇭', '🇽🇽']); + }); + + it('breaks hyphenated words by default', function() { + let breaks = breakWords('Smith-Jones'); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, ['Smith', '-', 'Jones']); + }); }); - it('handles heavily-punctuated English text', function() { - // This test case brought to you by http://unicode.org/reports/tr29/#Word_Boundaries, Figure 1. - let breaks = breakWords( - `The quick ("brown") fox can't jump 32.3 feet, right?` - ); - let words = breaks.map(span => span.text); - assert.deepEqual(words, [ - 'The', 'quick', '(', '"', 'brown', '"', ')', 'fox', "can't", - 'jump', '32.3', 'feet', ',', 'right', '?' - ]); - }); + describe('customization', function() { + // Refer to https://unicode.org/reports/tr29/#Word_Boundary_Rules, third bullet point. + it('custom prop, rule: do not break on letter-adjacent hyphens', function() { + let customization = { + rules: [{ + match: (context) => { + if(context.propertyMatch(null, ["ALetter"], ["Hyphen"], ["ALetter"])) { + return true; + } else if(context.propertyMatch(["ALetter"], ["Hyphen"], ["ALetter"], null)) { + return true; + } else { + return false; + } + }, + breakIfMatch: false + }], + propertyMapping: (char) => { + const validHyphenCodes = [ + '\u002d', '\u2010', '\u058a', '\u30a0' + ]; + if(validHyphenCodes.includes(char)) { + return "Hyphen"; + } - // The way these two tests are written is a bit much on the "white-box" style, - // but they do decently cover the boundary rules mentioned. - it('Does not split empty contexts (WB1 + WB2)', function() { - let breaks = breakWords(''); - let words = breaks.map(span => span.text); - assert.deepEqual(words, []); - }); + return null; + }, + customProperties: ["Hyphen"] + } - it('Does split at context boundaries (WB1 + WB2)', function() { - let breaks = breakWords('a'); - let words = breaks.map(span => span.text); - assert.deepEqual(words, ['a']); - }); + let breaks = breakWords('Smith-Jones', customization); + let words = breaks.map(span => span.text); - // WB3, WB3a, WB3b are all handled internally, within the top-level function. + assert.deepEqual(words, ['Smith-Jones']); + }); - // iff, as in "if and only if" - it('ignores the zero-width joiner iff appropriate (WB4)', function() { - const zwj = '\u200d'; + it('mid-word hyphen via reassignment to MidLetter', function() { + let customization = { + propertyMapping: (char) => { + const validHyphenCodes = [ + '\u002d', '\u2010', '\u058a', '\u30a0' + ]; + if(validHyphenCodes.includes(char)) { + return "MidLetter"; + } - let breaks = breakWords(`a${zwj}b\n${zwj}c${zwj}\nd`); - let words = breaks.map(span => span.text); + return null; + } + } - // Does NOT ignore the zwj immediately after a newline - the notable exception - // (the reason for "iff", not "if"). - assert.deepEqual(words, [`a${zwj}b`, `${zwj}`, `c${zwj}`, `d`]); - }) + let breaks = breakWords('Smith-Jones', customization); + let words = breaks.map(span => span.text); - it('ignores extend characters iff appropriate (WB4)', function() { - const comboGrave = '\u0300'; // The 'combining grave accent', as used in NFD. + assert.deepEqual(words, ['Smith-Jones']); + }); - let breaks = breakWords(`a${comboGrave}e\n${comboGrave}i${comboGrave}\no`); - let words = breaks.map(span => span.text); + // Useful for some regional minority languages that prefer word-breaking spaces. + it('character reassignment: Khmer letters as ALetter', function() { + let customization = { + propertyMapping: (char) => { + if(char >= '\u1780' && char <= '\u17b3') { + return "ALetter"; + } else { + // The other Khmer characters already have useful word-breaking + // property assignments. + return null; + } + } + } - // Does NOT ignore the zwj immediately after a newline - the notable exception - // (the reason for "iff", not "if"). - assert.deepEqual(words, [`a${comboGrave}e`, `${comboGrave}`, `i${comboGrave}`, `o`]); - }); + let breaks = breakWords('ស្រុក ខ្មែរ', customization); + let words = breaks.map(span => span.text); - it('ignores format characters iff appropriate (WB4)', function() { - // Re-uses `const SHY` from above. - let breaks = breakWords(`a${SHY}e\n${SHY}i${SHY}\no`); - let words = breaks.map(span => span.text); + assert.deepEqual(words, ['ស្រុក', 'ខ្មែរ']); + }); - // Does NOT ignore the zwj immediately after a newline - the notable exception - // (the reason for "iff", not "if"). - assert.deepEqual(words, [`a${SHY}e`, `${SHY}`, `i${SHY}`, `o`]); - }); + // See: suggested language-specific WB5a from the spec's notes. + it("french/italian apostrophe / vowel boundaries", function() { + let customization = { + rules: [ + // WB5, but with differentiated consonants (ALetter) and vowels (AVowel) + { + match: (context) => { + if(context.propertyMatch(null, ["ALetter", "AVowel"], ["ALetter", "AVowel"], null)) { + return true; + } else { + return false; + } + }, + breakIfMatch: false + }, + // Proposed WB5a + { + match: (context) => { + if(context.propertyMatch(null, ["Single_Quote"], ["AVowel"], null)) { + return true; + } else { + return false; + } + }, + breakIfMatch: true + }, + // WB6, 7 + { + match: (context) => { + if(context.propertyMatch(null, + ["ALetter", "AVowel"], + ["MidLetter", "MidNumLet", "Single_Quote"], + ["ALetter", "AVowel"])) { + return true; + } else if(context.propertyMatch(["ALetter", "AVowel"], + ["MidLetter", "MidNumLet", "Single_Quote"], + ["ALetter", "AVowel"], + null)) { + return true; + } else { + return false; + } + }, + breakIfMatch: false + } + // Similar extensions to WB9, 10, 13a, and 13b would also be needed for robustness. + // And I kind of left the Hebrew_Letter out of the WB5, 6, and 7 rewrites. + ], + propertyMapping: (char) => { + const vowels = ['a', 'e', 'i', 'o', 'u']; + if(vowels.includes(char)) { + return "AVowel"; + } - it('does not break between most alphabetic characters (WB5)', function() { - let breaks = breakWords(`aəσאБ лאʈγX`); // a mix of latin, hebrew, greek, cyrillic, and IPA chars - // for both "words". - let words = breaks.map(span => span.text); + return null; + }, + customProperties: ["AVowel"] + } - assert.deepEqual(words, [`aəσאБ`, `лאʈγX`]); - }); + let breaks = breakWords("l'objectif aujourd'hui", customization); + let words = breaks.map(span => span.text); - it('does not break letters across specific punctuation patterns (WB6, WB7)', function() { - // `'`: MidNumLetQ (from Single_Quote) - // '.': MidNumLet - // ':': MidLetter - let breaks = breakWords(`don't b.r.e.a.k t:h:e:s:e`); - let words = breaks.map(span => span.text); - - assert.deepEqual(words, [`don't`, `b.r.e.a.k`, `t:h:e:s:e`]); - - let breaks2 = breakWords(`.drop: :the' 'extras.`); - let words2 = breaks2.map(span => span.text); - - assert.deepEqual(words2, [`.`, `drop`, `:`, `:`, `the`, `'`, `'`, `extras`, `.`]); - - // ',': MidNum (is NOT included by rule!) - let breaks3 = breakWords('do br,eak that'); - let words3 = breaks3.map(span => span.text); - - assert.deepEqual(words3, ['do', 'br', ',', 'eak', 'that']); - }); - - it('treats Hebrew properly (WB7a-c)', function() { - const aleph = 'א'; - const bet = 'ב'; - - // As Hebrew is RTL... this is probably the clearest way for us LTR people to - // clearly see what's going on without ordering mechanics messing up the render. - let breaks = breakWords(`${aleph}' ${aleph}" ${aleph}"${bet}`); - let words = breaks.map(span => span.text); - - // A lingering double-quote isn't cool, but one in the middle's fine. - // Lingering single-quote is fine regardless. - assert.deepEqual(words, [`${aleph}'`, `${aleph}`, `"`, `${aleph}"${bet}`]); - }); - - it(`doesn't break within digit + digit/letter sequences (WB8-10)`, function() { - let breaks = breakWords('a1b2c3 hunter2 ab12cd34 1234567890'); - let words = breaks.map(span => span.text); - - assert.deepEqual(words, ['a1b2c3', 'hunter2', 'ab12cd34', '1234567890']); - }); - - it('does not break within formatted number sequences (WB11-12)', function() { - // Note: `'` fits "MidNumLetQ", part of the two rules! - let breaks = breakWords(`1.2.3 3,458.01 3.45'8,01`); - let words = breaks.map(span => span.text); - - assert.deepEqual(words, [`1.2.3`, `3,458.01`, `3.45'8,01`]); - - let breaks2 = breakWords(`.1' ,3.`); - let words2 = breaks2.map(span => span.text); - - assert.deepEqual(words2, [`.`, `1`, `'`, `,`, `3`, `.`]); - }); - - it('does not break between Katakana (WB13)', function() { - const kataSmA = '\u30a2'; //ァ - const kataA = '\u30a2'; //ア - const kataSound = '\u309b'; // ゛ - - let breaks = breakWords(`${kataSound}${kataA} ${kataSmA}${kataSound}b ${kataA}${kataSound}${kataSmA}`); - let words = breaks.map(span => span.text); - - assert.deepEqual(words, [ - `${kataSound}${kataA}`, - `${kataSmA}${kataSound}`, - 'b', - `${kataA}${kataSound}${kataSmA}` - ]); - }); - - it('does not break form extenders (WB13a-b)', function() { - // The `_` (underscore) fits the ExtendNumLet class this rule focuses on. - const kataA = '\u30a2'; //ア - - let breaks = breakWords(`${kataA}_a__0_b_${kataA} _${kataA} 1_ _c_ ____`); - let words = breaks.map(span => span.text); - - assert.deepEqual(words, [ - `${kataA}_a__0_b_${kataA}`, - `_${kataA}`, - `1_`, - `_c_`, - `____` - ]); - }); - - it('handles emoji flag sequences properly (WB15-16)', function() { - // For clarity on what's being tested... - let CA_FLAG = '\u{1f1e8}\u{1f1e6}' // '🇨🇦' (canadian flag emoji); should not be broken. - let KH_FLAG = '\u{1f1f0}\u{1f1ed}' // '🇰🇭' (khmer flag emoji); same - let X_FLAG_PIECE = '\u{1f1fd}' // '🇽' (half of a flag emoji; '🇽🇽' doesn't match a flag) - let breaks = breakWords(`${CA_FLAG}${KH_FLAG}${X_FLAG_PIECE}${X_FLAG_PIECE}`); - let words = breaks.map(span => span.text); - - // Note that the emoji may not render well within VSCode, but they show up nicely on GitHub. - assert.deepEqual(words, ['🇨🇦', '🇰🇭', '🇽🇽']); + assert.deepEqual(words, ["l'", "objectif", "aujourd'hui"]); + }); }); }); From 261e25202615add0198a8ed3c78f5f1fc06bb869 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 13 Sep 2022 10:38:34 +0700 Subject: [PATCH 104/132] docs(common/models): adds minor doc re 'sot', 'eot' --- common/models/wordbreakers/src/default/index.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/common/models/wordbreakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts index cfe251fbf2..228e19a62e 100644 --- a/common/models/wordbreakers/src/default/index.ts +++ b/common/models/wordbreakers/src/default/index.ts @@ -254,6 +254,9 @@ namespace wordBreakers { * * Names should match those found at https://unicode.org/reports/tr29/#Word_Boundary_Rules * or defined in the word-breaker customization options; matching is case-insensitive. + * Also includes two extra properties: + * - `sot` - start of text + * - `eot` - end of text * @param lookbehindSet * @param leftSet * @param rightSet From 94face23d0b27735d51873a62391897ccf12dadf Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Tue, 13 Sep 2022 11:06:54 +0700 Subject: [PATCH 105/132] chore(common/models): minor cleanup --- common/models/wordbreakers/src/default/index.ts | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/common/models/wordbreakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts index 228e19a62e..13081e7bc2 100644 --- a/common/models/wordbreakers/src/default/index.ts +++ b/common/models/wordbreakers/src/default/index.ts @@ -584,20 +584,6 @@ namespace wordBreakers { return customIndex != -1 ? -customIndex - 1 : data.propertyMap.findIndex(matcher); } - // /** - // * Provides the word-breaking property name for the specified character based on the property - // * values used by https://unicode.org/reports/tr29/#Word_Boundary_Rules. - // * @param character - // * @returns - // */ - // export function unicodeProperty(character: string): string { - // // Since we use a const enum for property names, the TS compiler optimizes it away - // // and does not provide a reverse lookup for us. So, we do that here. - // const enumVal = property(character); - - // return data.propertyMap[enumVal]; - // } - /** * Binary search for the word break property of a given CODE POINT. * @@ -634,7 +620,6 @@ namespace wordBreakers { // implementing a namespace, BUT we can manually make the // assignment and **declare** it as part of the namespace. wordBreakers['default'] = wordBreakers.default_; -// wordBreakers['unicodeProperty'] = wordBreakers.unicodeProperty; declare namespace wordBreakers { export { default_ as default }; } From e6403c3092580a79fba4c741499720caa13ad6a5 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Tue, 13 Sep 2022 11:20:37 +0200 Subject: [PATCH 106/132] chore(linux): Update debian changelog (cherry picked from commit 6c31e9ed837c90c0b976744b345ae138b6378166) --- linux/debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/linux/debian/changelog b/linux/debian/changelog index a6a93469c5..ab682e3d73 100644 --- a/linux/debian/changelog +++ b/linux/debian/changelog @@ -1,3 +1,10 @@ +keyman (15.0.270-1) unstable; urgency=medium + + * New upstream release. + * Re-release to Debian + + -- Eberhard Beilharz Tue, 13 Sep 2022 11:20:25 +0200 + keyman (15.0.269-1) unstable; urgency=medium * New upstream release. From 34b8e136b015737b6117c8de22f5ae2955037e21 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Tue, 13 Sep 2022 16:01:10 +0200 Subject: [PATCH 107/132] fix(linux): Fix ignored error Previously we ignored an error that showed in the build logs because the eval line got executed before calling reconf.sh which created the version file. I think despite this error things worked, but it didn't look nice to have and ignore a build error in the build log. Fixes #7275. --- linux/keyman-config/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/linux/keyman-config/Makefile b/linux/keyman-config/Makefile index ee01e2bfc9..cb0fe8ab51 100644 --- a/linux/keyman-config/Makefile +++ b/linux/keyman-config/Makefile @@ -55,8 +55,10 @@ deb: dist man: ./build-help.sh --man --no-reconf -version: +version_reconf: cd .. && ./scripts/reconf.sh keyman-config + +version: version_reconf $(eval VERSION := $(shell python3 -c "from keyman_config import __releaseversion__; print(__releaseversion__)")) # i18n From 2e66c99a832fedd5de656e5be8d091f5270e7f7c Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Tue, 13 Sep 2022 14:01:39 -0400 Subject: [PATCH 108/132] auto: increment master version to 16.0.63 --- HISTORY.md | 5 +++++ VERSION.md | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 3431ee2692..d7897516bf 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,10 @@ # Keyman Version History +## 16.0.62 alpha 2022-09-13 + +* fix(developer): hide key-sizes when in desktop layout in touch layout editor (#7225) +* fix(developer): show more useful error if out of space during Setup (#7267) + ## 16.0.61 alpha 2022-09-12 * docs(windows): add steps for using testhost debugging (#7263) diff --git a/VERSION.md b/VERSION.md index f4737a994e..923642153a 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.62 \ No newline at end of file +16.0.63 \ No newline at end of file From 62aee6d11bcb63052005d6d39bd453f400b9e2a4 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 14 Sep 2022 10:28:30 +1000 Subject: [PATCH 109/132] chore: improve auto labeling --- .github/multi-labeler.yml | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/.github/multi-labeler.yml b/.github/multi-labeler.yml index 1278db2a28..97fd20f4f5 100644 --- a/.github/multi-labeler.yml +++ b/.github/multi-labeler.yml @@ -11,6 +11,8 @@ version: v1 # labels: + # conventional commit / semantic PR styles + - label: 'feat' matcher: title: '^feat(\(|:)' @@ -20,26 +22,38 @@ labels: - label: 'chore' matcher: title: '^chore(\(|:)' - # "change", - label: 'docs' matcher: title: '^docs(\(|:)' - # "style", - label: 'refactor' matcher: title: '^refactor(\(|:)' - # "test", - label: 'auto' matcher: title: '^auto(\(|:)' - # Below are the scopes that we look for in the PR title + # additional meta flags + + # note, this does not pick up chained PRs automatically + - label: 'stable' + matcher: + baseBranch: '^stable-\\d+\\.' + + - label: 'cherry-pick' + matcher: + title: '(🍒|:cherries:)' + + # Scopes that we look for in the PR title + - label: 'android/' matcher: title: '\(.*android.*\):' - label: 'common/' matcher: title: '\(.*common.*\):' + - label: 'core/' + matcher: + title: '\(.*core.*\):' - label: 'developer/' matcher: title: '\(.*developer.*\):' @@ -62,6 +76,8 @@ labels: matcher: title: '\(.*windows.*\):' - - label: 'cherry-pick' + # epics -- we will add/remove these as we work on new epics each release + + - label: 'epic-ldml' matcher: - title: '(🍒|:cherries:)' + branch: '.*epic-ldml.*' # anywhere in the branch name, e.g. feat/epic-ldml/developer/... or feat/developer/foo-epic-ldml From 747d3776713992957e234668b7d406b48647aed6 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Wed, 14 Sep 2022 09:52:27 +0700 Subject: [PATCH 110/132] fix(android/engine): Switch keyboard if uninstalling current one --- .../main/java/com/tavultesoft/kmea/KeyboardPickerActivity.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardPickerActivity.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardPickerActivity.java index 50a478dd30..ec32425899 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardPickerActivity.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardPickerActivity.java @@ -433,7 +433,7 @@ public final class KeyboardPickerActivity extends BaseActivity { if(adapter != null) { adapter.notifyDataSetChanged(); } - if (position == curKbPos && listView != null) { + if (position == curKbPos) { switchKeyboard(0,false); } else if(listView != null) { // A bit of a hack, since LanguageSettingsActivity calls this method too. curKbPos = KeyboardController.getInstance().getKeyboardIndex(KMKeyboard.currentKeyboard()); From 0845210f2000dc99ebfe10edf099359c268dc062 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 14 Sep 2022 09:57:29 +0700 Subject: [PATCH 111/132] fix(common/models): max prediction wait check --- common/web/lm-worker/src/correction/distance-modeler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/web/lm-worker/src/correction/distance-modeler.ts b/common/web/lm-worker/src/correction/distance-modeler.ts index b0cd2179a9..1d23c14b1e 100644 --- a/common/web/lm-worker/src/correction/distance-modeler.ts +++ b/common/web/lm-worker/src/correction/distance-modeler.ts @@ -653,7 +653,7 @@ namespace correction { shouldTimeout(): boolean { const now = Date.now(); - if(this.start - now > this.maxTrueTime) { + if(now - this.start > this.maxTrueTime) { return true; } From 6fbe0892441fbf5eb3888b7b7f8246f0e94fc3d6 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 14 Sep 2022 10:13:29 +0700 Subject: [PATCH 112/132] change(web): tweaks max wait time threshold to mitigate lag --- common/web/lm-worker/src/correction/distance-modeler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/web/lm-worker/src/correction/distance-modeler.ts b/common/web/lm-worker/src/correction/distance-modeler.ts index 1d23c14b1e..d29b9048a0 100644 --- a/common/web/lm-worker/src/correction/distance-modeler.ts +++ b/common/web/lm-worker/src/correction/distance-modeler.ts @@ -710,7 +710,7 @@ namespace correction { let batcher = new BatchingAssistant(); - const timer = new ExecutionTimer(maxTime*3, maxTime); + const timer = new ExecutionTimer(maxTime*1.5, maxTime); // Stage 1 - if we already have extracted results, build a queue just for them and iterate over it first. let returnedValues = Object.values(this.returnedValues); From be947eb76abf17c61f64e194781655da564f028b Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 14 Sep 2022 11:58:10 +1000 Subject: [PATCH 113/132] chore: improve label match for stable --- .github/multi-labeler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/multi-labeler.yml b/.github/multi-labeler.yml index 97fd20f4f5..76b2cbb647 100644 --- a/.github/multi-labeler.yml +++ b/.github/multi-labeler.yml @@ -37,7 +37,7 @@ labels: # note, this does not pick up chained PRs automatically - label: 'stable' matcher: - baseBranch: '^stable-\\d+\\.' + baseBranch: '^stable-\\d+\\.\\d+' - label: 'cherry-pick' matcher: From 68819c6b6c18559ab1dfb0b7258aa8fe8f2fae90 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 14 Sep 2022 11:59:46 +1000 Subject: [PATCH 114/132] chore: improve label match for stable --- .github/multi-labeler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/multi-labeler.yml b/.github/multi-labeler.yml index 76b2cbb647..f5f509cd3d 100644 --- a/.github/multi-labeler.yml +++ b/.github/multi-labeler.yml @@ -37,7 +37,7 @@ labels: # note, this does not pick up chained PRs automatically - label: 'stable' matcher: - baseBranch: '^stable-\\d+\\.\\d+' + baseBranch: '^stable-.+' - label: 'cherry-pick' matcher: From 59ad38eea3b0f407df55d7c897bbd5f137f53c6c Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 14 Sep 2022 13:49:18 +1000 Subject: [PATCH 115/132] chore: upgrade multi-labeler --- .github/workflows/labeler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 1c8704da25..6faa005e70 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -12,7 +12,7 @@ jobs: repo-token: "${{ secrets.GITHUB_TOKEN }}" - name: Update labels based on PR title id: labeler - uses: fuxingloh/multi-labeler@8afa186ed03230c98fe24ebf9fe35093072ad46e # v1.4.0 + uses: fuxingloh/multi-labeler@fb9bc28b2d65e406ffd208384c5095793c3fd59a # v1.8.0 with: github-token: ${{secrets.GITHUB_TOKEN}} config-path: .github/multi-labeler.yml From ec865c911d35747a5a4940a447e812c4da37440f Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 14 Sep 2022 14:08:01 +1000 Subject: [PATCH 116/132] chore: add feature- branches to labeler --- .github/multi-labeler.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/multi-labeler.yml b/.github/multi-labeler.yml index f5f509cd3d..ffab4f5572 100644 --- a/.github/multi-labeler.yml +++ b/.github/multi-labeler.yml @@ -11,7 +11,9 @@ version: v1 # labels: + # # conventional commit / semantic PR styles + # - label: 'feat' matcher: @@ -32,18 +34,28 @@ labels: matcher: title: '^auto(\(|:)' + # # additional meta flags + # - # note, this does not pick up chained PRs automatically + # stable-targeted patches; note, this does not pick up chained PRs automatically - label: 'stable' matcher: baseBranch: '^stable-.+' + # PRs marked as cherry-picks by title - label: 'cherry-pick' matcher: title: '(🍒|:cherries:)' + # long-lived feature branches + - label: 'feature-branch' + matcher: + branch: '^feature-.+' + + # # Scopes that we look for in the PR title + # - label: 'android/' matcher: @@ -76,7 +88,9 @@ labels: matcher: title: '\(.*windows.*\):' + # # epics -- we will add/remove these as we work on new epics each release + # - label: 'epic-ldml' matcher: From a390c2fb2ffce809ed4f8cd5289bff172a2b062b Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Wed, 14 Sep 2022 13:32:14 +0700 Subject: [PATCH 117/132] chore(common/models): Apply suggestions from code review Co-authored-by: Marc Durdin --- common/web/lm-worker/src/transformUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/web/lm-worker/src/transformUtils.ts b/common/web/lm-worker/src/transformUtils.ts index 8c82f16b3b..b5aa598b65 100644 --- a/common/web/lm-worker/src/transformUtils.ts +++ b/common/web/lm-worker/src/transformUtils.ts @@ -1,6 +1,6 @@ class TransformUtils { static isWhitespace(transform: Transform): boolean { - // Matches prefixed text + any instance of a character with Unicode general property Z* or the following: CR, LF, and Tab. + // Matches a string that is entirely one or more characters with Unicode general property Z* or the following: CR, LF, and Tab. const whitespaceRemover = /^[\u0009\u000A\u000D\u0020\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b\u2028\u2029\u202f\u205f\u3000]+$/i; // Filter out null-inserts; their high probability can cause issues. From fee6acabed3ae666aa43b33a5c4d37ccf4c36582 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Wed, 14 Sep 2022 14:01:46 -0400 Subject: [PATCH 118/132] auto: increment master version to 16.0.64 --- HISTORY.md | 5 +++++ VERSION.md | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index d7897516bf..b4c028256e 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,10 @@ # Keyman Version History +## 16.0.63 alpha 2022-09-14 + +* chore(linux): Update debian changelog (#7281) +* fix(linux): Fix ignored error (#7284) + ## 16.0.62 alpha 2022-09-13 * fix(developer): hide key-sizes when in desktop layout in touch layout editor (#7225) diff --git a/VERSION.md b/VERSION.md index 923642153a..f6b5bf34d8 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.63 \ No newline at end of file +16.0.64 \ No newline at end of file From 917fce0750ae35026d1dd6c1c0f24eb7e35e4bcd Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 15 Sep 2022 08:20:36 +0700 Subject: [PATCH 119/132] chore(common/models): final requested tweak --- common/web/lm-worker/src/transformUtils.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/common/web/lm-worker/src/transformUtils.ts b/common/web/lm-worker/src/transformUtils.ts index b5aa598b65..cbb2c151fe 100644 --- a/common/web/lm-worker/src/transformUtils.ts +++ b/common/web/lm-worker/src/transformUtils.ts @@ -2,12 +2,6 @@ class TransformUtils { static isWhitespace(transform: Transform): boolean { // Matches a string that is entirely one or more characters with Unicode general property Z* or the following: CR, LF, and Tab. const whitespaceRemover = /^[\u0009\u000A\u000D\u0020\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b\u2028\u2029\u202f\u205f\u3000]+$/i; - - // Filter out null-inserts; their high probability can cause issues. - if(transform.insert == '') { - return false; - } - return transform.insert.match(whitespaceRemover) != null; } From 26fb6da3abdd1df7c6a9f36ed9aaa0797468da9d Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 15 Sep 2022 13:23:21 +0700 Subject: [PATCH 120/132] feat(common/models): adds tokenization tests for wordbreaks near the caret --- .../templates/test/test-tokenization.js | 76 +++++++++++++++++++ .../models/wordbreakers/src/default/index.ts | 2 +- 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/common/models/templates/test/test-tokenization.js b/common/models/templates/test/test-tokenization.js index 4775064b38..a7ae0835e0 100644 --- a/common/models/templates/test/test-tokenization.js +++ b/common/models/templates/test/test-tokenization.js @@ -289,6 +289,82 @@ describe('Tokenization functions', function() { assert.deepEqual(tokenization, expectedResult); }); + + let midLetterNonbreaker = (text) => { + let customization = { + rules: [{ + match: (context) => { + if(context.propertyMatch(null, ["ALetter"], ["MidLetter"], ["eot"])) { + return true; + } else { + return false; + } + }, + breakIfMatch: false + }], + propertyMapping: (char) => { + let hyphens = ['\u002d', '\u2010', '\u058a', '\u30a0']; + if(hyphens.includes(char)) { + return "MidLetter"; + } else { + return null; + } + } + }; + + return wordBreakers.default(text, customization); + } + + it('treats caret as `eot` for pre-caret text', function() { + let context = { + left: "don-", // We use a hyphen here b/c single-quote is hardcoded. + right: " worry", + endOfBuffer: true, + startOfBuffer: true + }; + + let tokenization = models.tokenize(wordBreakers.default, context); + + assert.deepEqual(tokenization, { + left: ["don", "-"], + right: ["worry"], + caretSplitsToken: false + }); + + tokenization = models.tokenize(midLetterNonbreaker, context); + + assert.deepEqual(tokenization, { + left: ["don-"], + right: ["worry"], + caretSplitsToken: false + }); + }); + + it('handles mid-contraction tokenization', function() { + let context = { + left: "don:", + right: "t worry", + endOfBuffer: true, + startOfBuffer: true + }; + + let tokenization = models.tokenize(wordBreakers.default, context); + + assert.deepEqual(tokenization, { + left: ["don", ":"], // This particular case feels like a possible issue. + right: ["t", "worry"], // It'd be a three-way split token, as "don:t" would + // be a single token were it not for the caret in the middle. + caretSplitsToken: false + }) + + tokenization = models.tokenize(midLetterNonbreaker, context); + + assert.deepEqual(tokenization, { + left: ["don:"], + right: ["t", "worry"], + caretSplitsToken: true + }); + }); }); describe('getLastPreCaretToken', function() { diff --git a/common/models/wordbreakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts index 13081e7bc2..715b722932 100644 --- a/common/models/wordbreakers/src/default/index.ts +++ b/common/models/wordbreakers/src/default/index.ts @@ -26,7 +26,7 @@ namespace wordBreakers { * Allows definition of extra word-breaking properties for use with custom * rules. */ - customProperties?: [string]; + customProperties?: string[]; } /** From af6c6db49ac8e3eacfdbad8d80e760fb064e0354 Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Thu, 15 Sep 2022 18:53:44 +0200 Subject: [PATCH 121/132] chore(linux): Remove unused IBusLookupTable --- linux/ibus-keyman/src/engine.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/linux/ibus-keyman/src/engine.c b/linux/ibus-keyman/src/engine.c index bfc267486c..b0c03bb439 100644 --- a/linux/ibus-keyman/src/engine.c +++ b/linux/ibus-keyman/src/engine.c @@ -70,7 +70,6 @@ struct _IBusKeymanEngine { gboolean lalt_pressed; gboolean ralt_pressed; gboolean emitting_keystroke; - IBusLookupTable *table; IBusProperty *status_prop; IBusPropList *prop_list; #ifdef GDK_WINDOWING_X11 @@ -262,8 +261,6 @@ ibus_keyman_engine_init(IBusKeymanEngine *keyman) { g_object_ref_sink(keyman->prop_list); ibus_prop_list_append(keyman->prop_list, keyman->status_prop); - keyman->table = ibus_lookup_table_new(9, 0, TRUE, TRUE); - g_object_ref_sink(keyman->table); keyman->state = NULL; #ifdef GDK_WINDOWING_X11 keyman->xdisplay = NULL; @@ -469,11 +466,6 @@ ibus_keyman_engine_destroy (IBusKeymanEngine *keyman) keyman->status_prop = NULL; } - if (keyman->table) { - g_debug("DAR: unref keyman->table"); - g_object_unref (keyman->table); - keyman->table = NULL; - } if (keyman->state) { km_kbp_state_dispose(keyman->state); keyman->state = NULL; From 6b847e3ec7c19a1575b69fb5b9ba3a64c3b708b9 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Thu, 15 Sep 2022 14:02:06 -0400 Subject: [PATCH 122/132] auto: increment master version to 16.0.65 --- HISTORY.md | 6 ++++++ VERSION.md | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index b4c028256e..ec8afd279d 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,11 @@ # Keyman Version History +## 16.0.64 alpha 2022-09-15 + +* fix(android/engine): Switch keyboard if uninstalling current one (#7291) +* fix(common/models): fixes quote-adjacent pred-text suggestions (#7205) +* fix(common/models): max prediction wait check (#7290) + ## 16.0.63 alpha 2022-09-14 * chore(linux): Update debian changelog (#7281) diff --git a/VERSION.md b/VERSION.md index f6b5bf34d8..c3bbc393bb 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.64 \ No newline at end of file +16.0.65 \ No newline at end of file From 11afaf03f80bf3db795c927c9472939428d825d7 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Fri, 16 Sep 2022 11:39:42 +0700 Subject: [PATCH 123/132] test(android): Add final keyboard to test K_ENTER --- .../app/src/main/assets/final.kmp | Bin 0 -> 31745 bytes .../tests/keyboardHarness/MainActivity.java | 16 ++++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 android/Tests/KeyboardHarness/app/src/main/assets/final.kmp diff --git a/android/Tests/KeyboardHarness/app/src/main/assets/final.kmp b/android/Tests/KeyboardHarness/app/src/main/assets/final.kmp new file mode 100644 index 0000000000000000000000000000000000000000..80d024623b2672ec2d9c80eb0efc1a42f9c599be GIT binary patch literal 31745 zcmZ6yV|Z;#(=Hs_wr$%sS8Ut%ifw1bwrz98wrxB4vfuqY-#PF3F~{hheOGnW-8HVB zv#N)JG%yGX000C8K&mQ(3flPLwF(db06r)H05|{ufR&9sorSHbOTwnyx&T7(HSr-X zjh_F3nCov6?Z2Q!jV^y?AcTvk%F87?SEQFZX0I~zw@U3+Z}BC=yiBfd+0BN{%os-& zv5=oSw%^AZccuE>cb>i2>8+Es*p{hg(*%3mMi{c|rnMPn9y00#EgO985H%KDhmGan zoEj&9wSy~^mVlna@FTl5gkZx0JxX8;{rDA$2zxmMM(*dLb`fKYQ(#JXKAyQal?( zFpZ6^fs=J%r$gAzWnuL)y)uYWcg40zC^!{%J0IOSgTwld=u#MwrW5Kv@4~c}R}84* zE@uIc967Z?1*FzHIo6_2seJW=itBLLtk%tfc1BdXX5{tubM7)7P4i5R39!tZI&4BP z&8r%>&YeYXG8(DhkpbF*n7Ugt`v*m4y!H(oX2<19enQy?^Luj&=ZBte(nRDebGmJ<63JJI_JJxU3Z5&EM`t~F016AMM zGzt(R^}2(wR($Rv2VZZa>cgnKyGoWZe|SSwDraJQp8;{44f-NzmF=jaBlNS$T_)9X zyC{MOwY5Azw}o^31XuW9k(X2a>(SvS@b?Q`D&3$qSC$~Et8sDBp|ng)K+BSFN;=z^k}B!PS^NbZIxAVy-)*3 zoou|{9Yd$PY{;&?#Bc#MQ$mU*WYH@nqT?Tm&7gOF!`o76o6%J<`Y1;b3Q=0ocb#@Q7T(Y$!!wMQqt!6yc)q~hMgg}`R1~U~#2!JieNUNH7vt$i^{W+N9=?dq zLhs_WEw4!z-u2S5QJ~Ch&po%aA_Qfnm^8F=IesWltC`tQskfYw?QXb@qwcZ{?3&av zAlxSQ*2Fl`P*?P+#m0`rnO?zRPSt~hm5GMb^R6gP<}tdBF^QJDR;0xOIw!n9)893I z!Azrlf+sA#;i~+15@J$HfhU0yACLK><#j^E?Z3iO$<2yx3VYlV6Cyui9KhOp3CrM$ zIk{iVzJYL{=FFH3q0tW0oY#Q^hqbBcxX6(i=Fb=>1v?phKL=pI0Vg*HKA^uwUy>}P zWgBi7_rEUWd#!b}_KugO)rtkhY*ZplLaFE}j|H}#lio|{_!nnU{%(xt}S?^BCKgiDW-f!&l} zhBRZdsNgupHFd4em}|uBxGVVzsI=*Z8qKzE@w3~wem#Zc`v&~qVTm%UY?u!Q0Dy%C z008+DmZlcA2G(>|HtvUMKISSe7$bi3DnxA4sLY7*CZ^5KlSLDMt!ksQlOCmT+T ze1eaWkEflZ_jP=M-Qtzgd46O|=rhl;kB5-A`%5#O80T6@{Z6$u{GV~^e(}EMl|541 z>`U8))2@%Ud4Go^>WV2Juc6{J4I~)M?4tN{WcLZ-Kw;N$|Lw9 zS~i@3-@PYSi<}0XzY;cS*jEqlInukZKD+_Jr)G3L+K$D$*w=^MXgRei zyooQ6mnT2WI)2wkPLuFKp26R~4FC^8-+=W6mKl%1`iGLPY79A1+Ecm6b+WjlJhr=I z-l*RXVS}3~6%0)TIF+#vP6XZ`Vz|*83j$KwcLz4X%T>jAdlBfNxQs8*cHv{H`qX0Ke=~0ynAR+QA=c*g$+Kt;{)$ z=_5DJqIE;!qdhlk%h_%C5zlJ4E~t$3INpt12&!H#Xamthq#re0>s>CR8Qc7A7jD+J zUVCmcPmz5N#!s7l=-JyfH=zHER=T z1y>2YSdYey!da#Jz|QVM&0bOesi3#qnl~nB zEacw(R%hLfd%5t_WRMe3x^Sj69gk?|Z!&y37-P8Ezd>e^W^GC@T>?i4@>BlDKOLx6 zsyI#&TSjf-?8CF2ke`7zG528i6fH=0aHi1Fre3R`X*Jz_2%N$;z}yqvQ`TYp5Jz!{ zT2Qy>vJYK*^+116mqBhns@Ph2|4~KXsmC*c`y2kB0o)MopJ<4^k@x>g=qSFs|HB-J z_dgR}v3QH|Ok@T7K^w!xzXNv@WFOM?rvj%7A5`})|IFy}V*1aFv@bF4DNfq^q+&lX zFRZJdMf2@s^mo6A7Qw-*q6nBCd|@ zuJ@##?YBH`gZlAhR^H*0VUX z;rVU(i6ZwRv*k=^I4AAFtH zE-}mSol}oo^3L5SK(kP@!THJg_z|k%bVxchnzUG`pAwB?jjD}u4@R?|itnV(vNa0W zBB^0)5p<|JWbN9{!6zlU1KVq6x_`7=NpH|W#=2{yV7^0#e08F1zAI!-+9Tzh;calU z#Is*4ZMpZD$deRTpUcxMZFMd)q{l}awXie6Dj^vN5+Wx->5F-KocN`?^?lyz^vywygTC^MwXL+)hmpRh zWBv@IF~*Q=J0uS>xe^Quv>poK7z%))Z&R&|Z)S~J5ZTf?=g2RUWSwgv&N>>cHcwNFAIXYmy=W1O7k31v zNXtxkF~lVwDR3oUy-Az@L`phOYHZgL?$v*G1}w#HJNcJT4J&#gBS$%4*)hl@HtlcLGj*pz5AZrMFh z&o3@}i*RM;C)QYBVXZFcZHLss)d|ND;}=I(gp~o8_AlyV($}cPQOBzUC=Q?v#MF=M z+3xNBt)b7h0$~lr24)rb*axN`st;Qmi|^oyYz^TGwFJZs%nkmIdI3}ktO^pjTX>oS z@P>Ne~h>yU7OR37x)SD1bG3uf~1Zk z1IP{J6_K~~y9>19_vPZzu!_TQ8<;I(f0Cbhr}$kxb}a8J-HPPOP?seVlL+y$Tq??ns4;xssSs8|>VesD6%tzl39Igt^^BQ@)twPcmJ@?O2G{3oa+BhfTR=BZo z=`BTE9*1aJ)6-KcuBQB^}hx^VRK1Awk_sI-#^A z=i5XNyV4tOQ#i2fCu^WrT>(Em>}McGPD+pUTgXXAXepkNfz&J{vGMQcHUl0S*>7EI z%%YYOU+(74XI=RVpG_@1+>T|+ji`TQ+( zrRVHTzV7Fuwemgwm}Bsran3*TwOswJc2U#z1%0t?|A~9y*U-}x3MK&aECKZ>0l(LP z$XW40pmjMv+r|v0QsQ=mTi>kfVtZ-NfStvBP^3d{1=;f17yT9Ud9D48`jo2k4f%xM z`XznAuJg_KWY_+M_q43zx6D24vLGs%!GqFJzwC8P}RC=R*># zury+}r4FVDa+t=_*+jLX31xnFJ4B7XxhM({ED>1k5ZSa-f0 zQe2X<@iml7ZnWO)Nsf$O(%I9Xyh>A}<{dxYa9=_^#q3SKx$g@ez4GkhG?6n^&EXP$ zRux@kv%Ra^;<<0JZo9@~bq#?wBVP7EB>$M1ws_;cMwd*@BF6?GE=`;{exz$M5^G)r z4(;O*ewCFqed8rD`3IoP1Dk*qHOJytN?|dLq^w3lhGXr*1&G9??(c<8__&NufHIAr zTNOar>K9!ZC8)%t?eB$3FbP>!0ZZMVp5_k>E-vHeW{sc(Y~>iT)H;#fu_$(2J{bzo z!BU|?3&9P*fifWHq8ptn6n>T|@8AP>~fFhLNN)xOQ?mZq) z@C(}en52d21uf&64&Js*blK^o0F86r>yz}9LnQ^#g~fYc={OIuQi2rU#bY3Lr2N@i z`A9L|qYxpVkTs<7coch_@bp!xJR!DTD&YxzJY~9{Y@hPM?i1Sl{zZiD0X`pIlkS@V zDDQ9)TrF&fCJ(f~Em-5%8J{#Ua8*UOWFVpmCpWZmGFfyZGo>UqGBfRANl|N(oz!bl zd5~W6T~RCLkaR{;jpS`T3Vq@`V*FkYDdH+=xQ?VD`ACBj4|Uyy$wgRApsnQu48JI;dTWhJF*NNiv3p{o|Z7jfuKOLO(a} zbxp2YJR7jHN)G+xHBhOPhKee@G!_@NqgxZ{qS@9cW6^m=Rq@q$f@RlOhIlaxvCMGP zOwnTkpC0{YaubzF9_?hK$SG%D+Os+YO-l3EG&||aya?g{mS5-iFv-)t~_@7Utw~!I{-?!KIE-Qb6#f}uNH@;oa z4!*s5LigU-|7KzBIiIwF*;NY4GYzLWheS5Nm3H`*qU(%nVQZPopgUYVNi*2Oh~0`= zpx0@6XNfLdX~&9y$e5+ znas#)9uyxOr(m78k?Qe5Q)(g){d&*lrGCDI`L40$ZGS|1|LlRhJ6STic=iTAsL{&# zdAgR6FT~pqG2bOx_Eut@rEilEBwJ$a73Qi+zNI(YzdIPe+8^1y%SaVksv$Sq3C$e% z^wAnz&O<$|#OA6{aqc||LJ+EJw%d*&epTb|`ux!ve}pd5o`yX1+`aTZ-SQ?5?JPd} z9O02~Tj<}`Zo^`a+{Ord-e;C%Ejzw}w{wji7=E~4ol*Y)F-WRS8hyM-{;>FkI<+_X zoLMIT3P~|FQuR{rB=<1=E%LJq%B(LJJGY1EcD5n%SF|t2Rhya7TIlMl z0EL*8?nNf3s_>0310aM-NgW%AKoFe*!S5q_Fv<`qdHI$WC8naL0dDT`B%YRjd4eL4f}S*-_#V z{;NRzzhhqiqaEB$`Ne++5lxt9U?C0NNrDf;gbC7nk5ANK=@6`G#AGQ>{-6QaGzTYs z`ZI0u(+NPLJI>@f>5m{WEi&E@7pnP2|M4kYjsoww|Mz{ny(a>`sVG;`%m2^-yz~zr z_U`s>e`y26)qhyikes6U*BV%k|2R$E$@U*hr+SqB^&-Pv@IRIshW>|;LAw9C!~0(z z-~VwQ4*n{}L*ND?@@J8(cJ2Pl$J#%QslAvEqpy4TGs83j;WNYk;W~ipKT8DaBJ=MY z&}XFos-`K``;TpcUH?Bw=m)fWvt!%gNA4apgh21&&~mkm0^xR!_mHuU5~Q(iH(Jx0 z9{lGpj2$GaJAW}pwfvAEAV9^SmjDQb(GT`(MPbP3+7NDE`(&ttCK-N@|vLa4A zMNHO8WZ+8JCqx&?zC`!(w>O_2QoP@bi}G`Rn0S2HJ28tTmO)}MB%4)+L1Zy#v!$lN zpJLc%t2KkrVqjNGE`!u!Xfjr8dxd)_I19GjT2ts4^JW9724Hd4n%y*0^cbtBouDIF z8LO>+$w|JU3gjc&Y6dP=}BbwqOwqaR{33eS>TT0P}^s*e(lz~u}ocjXsOMGUt+ zx+|=cR_VNbyx%b$FkDxTC#}@E`@Vpk99rRZnPS6CBr#-+5y@Z~qGj_gU?vwMTH|6Jra-myRmz z9qs@8A$IsFbv4zs!gBsB>(kE%>L;`7XkuV&V?t-{Y~!OiAveH)(0NQ9VsASy0+Giy z3F=uI1cKCM@0=iOxVzG~6($x2zsLTVnGz-fVWOy5%N5&<#u#(j|>hprLxq#5RFP+t}oM-hZ0C0*v6=sm^|4D z5CFzq+=!(nz8Y*;(76~L+dIfG_3M{(fo0)xk0w_WLkdkz!)fnZH-LhksD+{s2!#})g(4A+0DvGQ1W-f)4@4k^ zfzCoB2$X7t65ywEB5T1jh$wNoEY5{)$xm6iTV6TuyxVVI-rpV?8kG%lQ`x-CUQd7R zrX>e$Y*;O!*r0%bfGk}&U<3sO9uQl%2#JBd0vQ*#KtS?w&lp5~rf-{mxsbf;T*ri0 zh_rz`GkZnRRwqrbJ4(R_co9G*4tLftD89{Y_jI=6A9mirj_-*!HD^|xWaPkKXE zpP{rMXD)c;wA^C2(xle((@Bv+Vt}oHlpc6o*fz-epY!3%8OeW6K+KAKjlkZAf+O=g z-tiT#&LL6*;V0K{*4SsyW8d+;k4T&>;unSUrf|!0b8|y((qDDsjr($}`@#Xa>JkhZ z4^T&00qvWBv50!MpqRO%X(?TD8YyT!;Vs~5X=%Otke*P(UvGApnJ0YQkrv_eXP{IQ zV8}y_*qd==BP&dp!X-|(#PufOOm{%IS10d?c$u=Z+cA@^iOIfCz1iwgq^b}IaZFI@ zV~lOC?CNUM<0RKGdDqgjp1dEC1Z|dof4tt8^)2rD`zI&LN+H297yx2|tMekENkCnL zjg*#~h>nQYM~{Sa3mP|`{C3CGWDyDOIK&A8`6N!m4Ds#IUVJKyO$Y{)!RUK^cB3=Q z-3LRQ1Bo5%??kr6f?85<3Y8B|;AVj5)XxSSTusykL;wWygsTaU40neJ0-}>B+A_#2a^_mXIDl`ao_QbJ#huAW=br!5YuyL)alsPlA#&ZBUik7DCwq0J}r_BHrf*~LdO4hS)1#~M;tVi!eU z7cB#iOQAE&;mcNhA)2`1evt#sCs3*lSE`7bJ)k_0HV>;XBsynEV1=!#`@&9;0|<=e zmdu|EsGqjtHvvK|jH0RD1NMP#v_=tmgIbHt*%z<@#Raka6()KM83B#=b7uBV_^*dQ zB_%oeSGVy6Q+=oyUp(N7vC!Z_2&R4s0Hkw7$XPH@q0Tn%@`Iw>C$K|3bAu?W3zOyZ z9-Xz#r_-t5F&f;)vhr=ucAJu=3vbNZtvVm63J}n+fe}hJBs(Q*J*?5A{at&*p-EhQ z+mUuq!ORKM&`5&4_q5+QHC1<`eTCKE8SW1IDN>t50|EAZ{v#vFM!_oBtkl<>ITVo| zC1_U5>J1kYJQTQC29c6zV5l&Bkmz<8&6HGlj6tB$YR}7D>}m8vGVZmU&>6j(v6J4M z5f0s+pr23kiR%H=*PX@{@!m-Yg5i{I+<4PnAx5fDnFjll8MRJ!YTmTBp$ZoRlo_;K zS|hH3xYY6dgpZz^{hRiRGm6INXro@PJ~lPUz?0R=XDa&X(yOuKSY`NMYez<}53P1m zLW)<(tgP_X^kQHs3F!-q<6`r5)MwyV!e6^hMA5u0+D$zn2eK_gY|z3g1*ZJ6p3n8q zTuNn}JdE?`Z1 zR27$tY*XL>wYT$o4DDI!_@%H30q5}caV0~~SO%~yptGpRH{La{J@ue;7*E^~oH;Eh zSzv`)xZ6#5zdnu>BvuGesWgOdWaJhdsR}Qeep%qY%$$`wzG%_W>UP$}Nvw!R_Ijyr ze9f2pSi`gfos!&9_>ki;a{wt%q&YA6RER%vMhgKomMd&g#ogEn_DeL0WIG;&Gec>0 zHasqAoX&=c>SEVCSNu(VKBu23RHj2dQkuCpfiA60CBEohf)>l|lIAGA<`#_!2Zp}24}v1 zj&;S`k^PZvdOU57^Lwi)&mnKR{x3CB@h+BQq&5sn-i1R@bGV=oS%@&5E6*1OmVAaa zL;4*Ba-wLI%3m3hi5kZuGTN)Q``|8+VF!x3)(ZzNwp!zC>m(7xM4hg- zClyu$DBR7!5r0-{Vkd5viQMH3k`l{xD%1UI}cs=CT@v!OYF3dML0(QWVQGo*GN0 z58yF$0-X^#C79tyX8L)AI%1OU>22E&?s*iJFcW~Q|?9XS~!i1 zGED^JSN_pfe`QznBXwN$d0E~1Nyf?wZAsbER zGqg_fxmGb!`~QKY!;UGywV+6|dC>8Bc6RQGSguerB#`O2pE@cz)AD0sOuY~JKrl7T z!yJH*8nhW6w-DU^CJb2lq7W@g8P%gpyAjo!p^K*nxdb-@z6#>*v5w(ulIClkMJ52^ju8rd<0_fUb6sN$)iq%xokl>(YAuFhpn3h>zeNAzGGkeqzJC zF7oQES#3J%NoM)-ndF$RX`^4FMi(Hv-iU4_er06y2nl(*j=kw2lC9b}rl*zK{~3n( zu-dL6escz?{n=)9WHLSruZSK?fChbJpp{&Z+widNk{ms}8HDzANU>05;kjC|Keb%$ zh8K8!$bw_sqR=P9SuXzPgAwlvTP!qn%?|N&gFnoToaixBe_I1j0cygc(S~4@)BJ*y z_wnxlQB?^^jsC}gGTy?;?8Z^QdC+%M!WH+Reejh659%y1YZe59J{i@zdh4lI$8!Fl zpHeh#D?28+=9<22*bNN6m=CdQK<5p)`PQ^6n8C+T! zrXJ6W|DIr$&s{J`EEhRllhfnkFI;ZludkBl2|EkR6gI+$B56=I|N6rESFHmNEe8v* zD%Gq^f6wnGkqz;c!JNFjJnmq%3>P6Xgx|XQow|hAr48q^s!Qh9pISlV| zZ|zuq^3!rt-bTBrdCP@`X|c9}fSGg0Bg^bXW%ps~tvtz^WNY(Za9N`Yu4d2iX=kU?i1Wjo-=e@N{X+3#}inUO#QyIIfn9uxc0dG|f29UQ4QYH9? zP@svR{$n7~qJm~56Tj*Q(f$}Ge0K4S{*Fc1qN6X}^{^aHYgtskNQ zs8TLne6JRa^q4bD8n2+DT6;RioLOh?!RP9c;c2hVHUYouqFBf85_l2uH`jlK0lSs1 zfU9l}daE1EhKFnPfHee}jFyu9v3-L(%_&WrTvsOC1!bD=(bpDMz9xa1{EWtXozZgE{Hb*{#Kf z6ah+dA1D50w$aj?;UUh-f|oK8`=@RgZpA2&J>qQ^_~~!j9z?)*{^+9_fB52VxCSkk z(Gyq11Lm=yK_hJgX0LtP@{Kz$-~%>8EiT3kLjw<5MpoBqv}cR-Jr#c3tj8PwS#}nF zgriQY6%_0!nsM54$!Pc6IwX4O00~evX4_Jz@AKD97NQBAtlnVN@4Q%hL=r*m!Fe>q z5I?G z+ExwK(I->W%YB&9o1;4VFXpA)`5C`lPtI}I+$bxBGZD_mt|yFk;w?tmwyUy>O0klTwOfNY zer930ZC{J}hGn6$EMHi)XD73Vxt@jAqVhGA&62ZnDlh`Y^#cJA9-z?q& z;oYz^))e&uaB5pv95Hjk@wgKNocg{#ATrv%@WbgPLv zq1k)RBOHiKQq?8Hg9W2za7(Dgqlv^$@y*?vudF{FJpDXPWm|IO-$F2H=I~U1f0~=105tP67N_Tb4 zRF+0&9N^G^TK2&QIg_Y`&)8g_2;lH9U;eX0iGlrC6~1{0W-}TO57czJ&x8U>ls0<)khVh@{fWlAC%A7)bo$VH_Y6c*EP9o17Iq@M#ivR3mF+xK;M zm`f2)?lFds8_=x)R^lnJsR`8NiA@tEnLS{@RFAa7;oL3Vv&v<4CIf_p84xtKa; zQQ(CTnNQPJSo(jWZ|a7Z?7}9)G`y#mUd}Zr>OBQdkeopgEKM-ed zxx{uI?4mBounqo>w`$9z1fhgt3X(Uq&`!Y{i|SypM{cv3dU2_$D*2weAQttg>D3Yu){9srAMH4!Q7=?JGJf+J7#FGno z7H!97a6lNfgq$2JF~g4l>{D@dvIbS4C@PG_!5?&zbfhNHHS$JET8kaVdFp!56K z^TDo8Ob42j^8Sv$6QTBRwKYq+@0FYDSk@t;c8gd5*Y6S zVRa0_RRpy?=iznjfKH$%`(_r2x;_&DVlwJ%N||&!EUr$+WK< zj=Sff?zh5MEd-g3k%Uub1?v;dsqlq^{qv8V;|Mmd)YbJd@cVHmNXzqL;PMxSIrM*_ z;Hn$O5}cW_5{ufc&g(gNy*gM0M-d6w);l0DDnKdm*<+#;RM;t!R3dH4r520TZMmoYXo&l;9R*Gfps4$5*YIF(^h0+HTUf zhP_adD6k_4{>|H)lxQcC6&_y7`3NH)4APIzymr&u)~LLjiJ;4hBYPAGAEN|>(|zpGyjU32LirH3BB6xva@9JjNc z`|%^Z{Ue+lVrOLP?c<0(Pu|%_n#*90@7)c+y6KfSM=jHpL4c!YE!()%34j-@%aZU_ zGO5KOo|I$yFcW)@8_J`-Y50oln%~>KIO3CpAsT%2youTjAbqlr02JjJ;RSw=-t`qXOnDhVuW4Gs0WbD0gRt&MeQE7Nj*au&lhG#y0HhiypAkk8x{7K)6M z-)~l7QR^CVZ02#_KQc^>B#gJ2f$D$?P?CfESkh2t(G7bMvvfhR+pig$HS%{+UZY?v zm7WIxz*9ti%h}7*k?Nq&#k9#NLM9mI15q;yL1=2A^X|0LRcb zFqNjy>r9*F^KljE5ln6L9V^bOqTZdZ$HJ-Wx=T%4 z{Fy}f3+3IIWJ+qh1yqCWl|pVDFj&YQEy$01Ys&5j)q z$8Fp!cb})K=*o(ycoV+Q>O-xPV0U`IW>QxIBaHD@WW_1r&P4ebOg6UTIrI(#q5 zN#i8#T6U5_HeoR~~~{`3ximV{W;t_E4+C3Cu=Dvx4d-j@kWHlN8mN-d*FN zg1^HrZ`df*FFD0rr(KAN^i`i#uC1~O_vO1q6j1XCV5S-j?ID%fhaX%L8}Ex3uD;Jx<=Fnk9paVc%P*tcIqwtYa_U5N^xP}K4`^P za{!wAH1n2d_#;(QerwYWQz~5~YXr#@`IF>o4OhPkQE#k2K`ut@Z{5SD$@#2*8!#d7 zo@c;hBX0R@^GpH#W$Yr6g9&-lBCcWX&eiCdXaU-)k`bQpy8wXN_tLL+j1i(JqAonU zpva0w#MEdu{F2Tq4wP1h>M(Zl3L8pY;8XC`qzTK&@f{QT8Dti&yYz>ihU?=y;yo&8 zSa-gsD8mlr+uceVxGuH({dUY>-g$=uDQ9Uq$&I;j?C=n7;wZQRKU;cB>W}(8n#&tBGvD{K13sv`klo>Om=^Dp71_v99H3&ju=KM8m0cp5 zp1N%ZQM#CcVL;20%ctTcIn~Ouk>JcTNybBv?Pn&o7}Q?f&N?O zbDbRqtmtK+^yBq)X>t7a#BJR^73zloz{G`Ikg7X6(xE-irPSm#NAZg&sn_B7oNlb- zdU}P{pJT;MOu7Smj$l!6%PPUHdcNZ%!*k@D8Wom{UlVNehAq&L(eu$N(2SXW?kEyP zGtJ?`g_$WvPl`&Lv93kvbX`}Ky2L*@CtBiq9Fl^*+9VX$}%fL-IkI?=eCBz=~lz1Rqjtg5uaac`2>eaM~%!O zA2)j~y@cEO2emx%4|HK{mkwG#9ygY}U=i6WJ(OclaJpi5wf{)QwicX$DyhY}%j)q^ zM=S;OsTqCn#XPI2oOXo0U;R+XAE6BmA!!omp~}MIxbfD>HE4I zA$wzSFD)%0G@n)goDaEKpvX7rd!S?pdADwR2VO0;0@DNDzC`*WlWRNPjZ(6uZU{+K zKVG6H-6Q5@B=&tHSb0=4U}iGGyZJ%NT8B0#jjlZHlv$XUr{kJhy=cl6U}jwH4u;B~ zszQqTQnjD3ZUb$i&{N9z>;*6>{<%HpWb+Yp$)81O)VE}o0SRbNPHP;dgwBW_%+_Kh zWzb(c-EJujpPVn&rzQ(ZK^IUDLzq{7LIZ_`RNWHN%E6F8%6;Qgme1`+^Uz$N9^tGn z!g5miSeRmR*2{>B0@W`N94%%VK0?FKf7d6)bI7Prr3Zg%W#jn`DxjZe@!3IkuHNTj zRT8p0gBrIx1{oC$T7qz%?(a8S(!`jN+Njx9Q4wQhdvimA&3`FOG(u=m9GZPy5$@Zt z8!PK}m(NgxGe=b@BB;uwkx}=Y%x*kfWN--xxldyCO=+&0Igl@L`+^Gbw&qzWv(9Eh z?TD$i7m$Ua_K48O)B8ekX~mM{#aD{gt55(zF_4P}G01_0Uz^r$(mVEE?Lh7<6U;hL z00|KKpGf*_N4K{Q*2g3#3pjZ44OOR4SpKXt6MQ99Q6cQ2w84AEv7xqhN21d|=~RLH z^yn0lV=k*{2KYUOR4{^Zh$BvO?j`jaR=^~!2pi?_-kXIN$_Zy z$wiLz`Ig?ULqG!$e1l)^rgUcDL6Px9W;q`vxaV9fdWltr#`8Y@aW@?Dmm2m%vJzB3 zht_|Q;l{hl80wHLNMr<#$&v`p(UGG=!}4Bd4hu++c!)9rQo34H zMEd2L&3x!^o-_)6TDMa!-tc|Mcx7c2F zshNW^c{XwLQ2B-UcU=A)rU?X~cMdfX#py7RV?{Ty7Y;t>%gh0M{-Qc*RWBS%RNMpXPgu->)U?k z4a8L}2U$V0_U0yjt9E2^MWMQOx!@=DLMn6R31$T@-+jQUAqdy0Jz`n6)(`b36+W^{ zHq%TC?*;jg9e|VNPWwwlmpvs5fJc)W!jlD|%U#&P|WqPfcJ7te|VY9)jz6 zy!E)bXmS5!Pdt2h6^*V^TJ-}7V+MTU>K7H<Vjm3ZPt)SDJ0+d zE4AidpG)eupEF4w|4CuX8mAn|R#|o1W_~D6H9YkAMA~9`y1=4cVObWmzb_9{wqZ~~ ziYr=E5~C3<1DsH8WSuvVycQg|9Py%Ozlw?)d!w_M1^EdQS4dJ^<+xJZ)R9Klk?E~a z%%OHYR-Wad#p@+ zLDPND7o%~LD}!N@L9ewL{U6lt+#(?{IQ794{??m@u-JJMIU`0g?o#yiBTz**Mu|}IZ7J$0@b1I*S=OMw7M0|&%Kqp^ zRr8OeCp4jlOMdR9!$!RmqEGW4G1=GSxqXY!HH(W`FSzRS>#JwF12*C+5y6~2Rr;e_ zAF|gbh2&Ts+D7Xn9q)zrU4NsXt~JtWT*!i7_!irUUDQDf#w(U*(CNaHw=uaT3!6kH zv$S81%?UNAwGV?dOvE4{^5!Rsef1D}{k8WirUc#K#Mt@C*OXQ?xc$Al$3bX~x~y(< z{(EANX|^^NOb^&(y*rq^SkLa&MG6(sV74lnXrH^|?Nbp-OQTz~ZQoA!2Uwj;nh-`Z z=P;qj`aH&5L#P4{*9*xtT-@7+y_hB8#VrcZX{4eV;Mb;pm!IBnJ0V{A1PwTvm)SXF zmzJF*f1tCdcV|4Gqc}SkR~i!e_D(1GhT$N5o;%ysc({R+H6QW)i0%XTCc8_%qOMUu z0VfyMx6WOHJMQ4zf@eQCK@i;p(U&RZts5H zF)I5`Hk&m1UQrv8LmGMvhzU8+K_ck6dCp2!0i*3kQHt|n!yPHkdO& zzcK}ahiUZ7M8H-1qfd{pB=wBeEobGKy8CpKBlY8x1Jl;Q6B+cX)JN zh@M%urufu-k~7}7oxtKQy>2Hre<)nT)nn)?k1{bY{>d>A9~G~+;Oz|4N9Y0x2rf?p zNTSb7Zn(b>71xci=1(Qu$&oln2~Z;E1S)eb^>rb18Zchjt`g|Aw$1s7&BV#(x$(G)A?YT;OyR3|y8w+R5Njt~*~d9Zw8V-sT!c zSRr9 zJp28tk*FIO&uy&2yX!H$>C07LTRra{g>Ufga8S5?J#VDNkg=X9>W_ocz{%rGynW1= zDQCDreu-oaGkf&~;FU6xH|&PQAl3R)oA*``_leb=%&gPf=M0yI3Kkl@do?{RP193s zUO)xGX1;UIl;i_-jn4`dGuTfgLr>5cERd4kEwI`up5a)PVh(=&2Oh4TZ;g6%*Q@=fv4kNi=&%aizBXCoeRp z1dGyx!=@(Imsi~`6``6SZ57pGYgLoa^QArg%2)0qc2d7nBJJ}eM6Y(;xtZCer2_u# zO!@5i`Mv%LiI7+^kXwzTx!$l_dpFCB&31(6H_D+wO=!1GpF&Ig;9QsfZN6?m@}`US zFQ(d~q}pzlP4lipL5I1#yb;{F!W?_!>qEWd(!DvGIQ6N*9>1lWS$kCeI$LP$ zW0}-o*;W9KY=Q56RG8#=w249wiCSk=-{38QOv}=M)Tz6fL#M{^06Ul9p!<+vVt9{$ zPqMI%OaLXE48%dKySI;RVzZUkW6z|JZ<>(T$90##}bk>x=fJQdDM7L#x9@TS+R-+cP?_1`DrY|_6M}=>$u;>VPz>1 z)RbWucWn27EI<%>Om?4c{lBeT?|yoRCy(;*ysr!xPow%d_wogS*(DHt8itiS#Hfu)KtAIHJDBP3FXA-joG#dGjNbY1Ch6jaYj6(62d(dmL^?8g?3J~CAr-F6OvpYocg(G%v zAoKC38PmuBtQ}tk>vkXu`5=4A`#Bj9+sYGIQy4^jH3>dK{bdUMa|&fyK)4EMKPcs} zgGIltvc3X%R~|O*))=#t5CpnE)7n4v#Dl}BehKqCuqM&01m7#Fw75+OrTQSysZ648a{}j*3xni;B4#>!wmeii81`Jh5 z_Ine6AJ$0Au-!~CufFj+hhb51n{e&gPU2^?rL+M#SS)CI>}^awCnrVMO;%TTWXcW| zdGMSdU+~BE2G23%bihVN()1!lW62}PO-4Vk%OS)J-w_-5o&^((^$uNmd8yiMPOJ*8 zdnBS+BZ7|r^Mr&uoO0BkBV~DeQLIxRW>wH+_&Q*%%V@@1`dFdi7YUg^_ZheX&h4%N zz(*jRc6#aF7=nKO=x8rD_js+IOpHfwD1Z2(* z1oT}F{_l*g^1nj52kquunYvG3vPk5fV+JHa;VJp7(1BE}WWQ$epEe<8Q1hb&qL7HC z8h_~(L^4UugN5UmxXkd*e6o%tpZ&Q!`|R@46Nur)i>Z5B-EzKYTKcT?x%?Gor9cCt z_OF00jp=WYO#kgZ^YtxW6HJG0;hTj&U|>cqN$C5HokL5rWXM9gUjWcey?PGqgO>g& z6dTTMR5So9xjKHr>@8^#zs@DIxw2{7M508(F%lsyT(y3z!jBMq@|`}Ik0uONi`G6CntMkrC0I+8@^XhMijE`{u>|}qv(kPKG^hOfc+@hBof%M- zzkD@NO9z;-LL4>g&>D348te@c9@E96CObU6?v(CTXxrId9NvHM8$lwA>*R$St8{XD zAXmdzWama(t-VlSo6gZVgEu_IZivk*bxkhC+r_nb2y?4J6p)_y{AbK_<3HE6HsB z?sKWB*{)~084$iDuPLSY;SW(Zjt&`^DDuMCJ$gH6y_r7BJ z8U#s8uhT`!%ZcIT2_YD&hB?ofcrp%ra}INL_Io+#^LrQ3$24QE6mF^1e_UwD{tkYG zHhb+|C?P8y1?eA+f~~l)sEQ81jmJV}Q3;%wF%kOX{3w<2&$)BYd>Sxm0GiKgtYGIt zEP8b62$NCLGSV5df?XVxwa^F5baj9X!bQQ_O&+| z65Gm#ZfO>aS)D|+Tu(zoc}cV!QIu=;Rc8@hjB~F|&>cO0UU@;JdjJ zSA6dk-NU6$A5x(3{?-@kN1zaw@sAn~W`=f=PgLFN{2rKkzYM03{p(pu%!C2WP2-{) zOnjHLMcnk%Gx4WG);_eo&lz!vj%Yyrqg{vvp@EfFZHlP9VeguJ`D%Q>X=4mee!bEaT zIG^N90E9d?2cT)Gwje-DUr3xa;;?p%p=UY6tBEI*P$UYHlCS8NZqt9@EVGmv{ppYh zJwRznbgfEv0F*N1d+-F-T$aH+KQjY>imTmMO>|tv&xBuo4V`^tg&OG%n@$EZ;iON~ z*O1BFWDzr?BS5Fo9%H+i_igMCjE%4p4^DbBcCuS0dJaWheh_3MSrV&u2zXp9TPt!F zk5?yDWxJ4gZNrao)>E7LQCf&8E{_#WTp>Fr7A3qLHXV@5wH&SG_nhVovWGx?(!l$eLEkg5XCSwbf%Brsv6c|cn$qh7mdYt+3w)x2e2I=y1Z zWExLj^Sl?YQFdM)TV)f@yxNZQ)UfSMcV7{Y+$D*c^oc2yht7iX%n4*8)X9|v1@-QBV5wL} zy-CM7^*d9_ArdDzzD$bZzw5(3x(3NqPo?K!SVVUotH&urr0@qp9)wn(S2#n2MuN#@ zr%q$JMX~EaF?rz{Kn`d=M!2WFG7fBzymeQz^1)GBfK-{@R30oUn+mgsAeMW-7qz?R49I z2=$yP5*dm(ap?w&@a^%2BHeS2KR@N8ED-Y=PT&ph`#Yc}H83E>n0QXyfN0Ip92pUGK1 z&$q{5pv3G-<&rN>Zsk4;tG3h1&ht)7$j{up={ggigPk@{s(zNzTa$V+U(9|-mK=4y z9+M`Z^At|wyYa9*MAAP+I0J`EAXg86mdU7LZwG*iG1TrXJ*U+R#dO}^fEeXxnP0}L zQ67z@z)*!wc9-}$Rwl9=7xdwNaDq$jWWt1TytBWt^0U1%gadAAdSJS)%={6)oU?+` zya`j<>Dhk$H66Sz4*^%NE=PkR)r#913GW#aR;K$TcpGv_aHCUa)ag%TA%QNcmB< zcd14p2VOM=qu`LxhrD;j3YU9;!i-X@Ugc(3T7>Chog1f1jvcEc6;MK}8WKXDJ)Ys{bR{ zokZ`lNn?5}*49l{Y=Jje^>7Ucxojj8u68uCYTLrIFeMM0?!F~j_GQ>8 zz}m+oRj(6Xk|yU8)I({d1JHRO&eYwG^R&U472-+pc?x0^`M!vP(O+V?0#KliPb4c% zpy}15wrF|dg2F8!f#M6#xOtss8FTG(Gwvy_W*BLTdf3!yib@9DH}`E1$vadgWtFNn z7-(FtEfG?K4t5b94c;#MS;5U%RJnus^n2%A)-15%53=jEmUq^*@b+kb?Z>C4o|_i0 zbAv=EpYOM{mLGRMy5G63{;CHhmj)jdp}&)nJKP^l_)FM(Y<&O9g)gZlA=6O07tXP9 zRcYog+~BmUwi|gj*mCx#h<#BS7S;%NaaKMz6xyLiHU}DX=?n?lJTizB*=H(06%SNO z(z(EJWd@0o(iI=k;_JM}LBUS*A(?qg%;O^Rq-hNiuqjT}8iMwOKDv|53u~s?msZkx zL3v2w%UQoBYxKMBy`$Jr#AIK%C?#E}o4poumaiiM_`qC(yl6B}w>R?u9x=vPRCJoTTT2 zhw)H;v&bw?6d}RI<7saMV&iuL3Kj22RKaO+Sx)J78B+XC@NbfVJ)RJyS|ZcrOF!T{lK@$B2&ul4L@dKy{l45{Qab}rKdv_CbmZ$ z^BLx@)64J0o?TmV#xXvqa@3)c_1%B_I8)Z3``KeX1yM4Fzx4xj$%}{t%xNeRnabBg z?2o;%j9d`=@XtH~hS_-T)ACK%QC#L0kc1j^n_MXfcw&Z#PqseDPzqAL;9?_d+1ZdO zaqle?KUxEUB6D)|4~9c-X|baDDDhDfc_U>TujVp0H&eMgai3%&_Yg-$XTt=#{E=~Z z`I$Ggq*eYxcu)7&S@k|VcMHb}$o0ygaBI73t&;uyyoc+h(Y(LBS%lMZ8at>UiaYd1 z<#E{aIU^-;(%7ZY%VE*W87xJDuvYi8gJriU|40BB0SUDYq>7s=(ADO^l)011S$4b=l3PNXI#;zy`Lb-uX1PDvw z_g#~EDuOlnsq!2!V;`!~GKVsiWTU3mfj0fRQH983DWHVWHY2%9{5lx+X9hPdAssI_ zGQVr;aC*f+jXCOOp=PvD+;`aBNyAG~r4T{qK%+8g+`#fY<=u6|PRoK}*i8zV`{+k z09|L7cn$x9ztNQbQlWhf*#{pUnJ>_x|MWLuP~WxAs8?V{-c- zE`OW5L5mFMYb1Z6HWOxt?@eRI*W4brG%=b>j7`WcQp?%N(TD^bOxw&ZSpi4d7aiXpaUxc0Q&A(o9b?%%n`)<< zb);(=&A9yA1Q$KffB~&4FL|$91}nIbejK_b^(z5>{w{=Ecm8`9_)PJ#y*xQcCEV_8 z69al=N-G7_IPo$DI7E41GC7_Yg}LtLITlj?5i5DQ!E(Q98ned0^?X=y9JePj+(zGo zq`zMkdmYRnf`E=aNeCfJqh?;U2^N&mnOqZ2Av9bMgDodk>86XANz<4-(hmlz#E1D3y7NxI& z`kxsesjR-y8TFeNZE)Z5~9y<|-caeZXqy1rYG5aBhW#70dajhE87me~#i>Kf1>AM-}T*RJfQNt zXz-o(vF>k!srOUqV0bRYLLYPXvL{#;n_1N=9<^UnOf76BMSt8AhNMjL1KUk8L^t8+b z{ou0a+{*$#v9QczZ{g<;!=8L2u0*Kl;xMXADg>4US%Q8~EenwhsEz}Cd&+pc%sqoI zu;P2kl#;T%p|SUs?ySTTICrY*xm!>~%~;=lM4Y??ikU%sQ~i}%P8TS4D6)2Fcgql6 zPk^Jz6V~`evl~atNwkwP73u2U6#zmRuI{~-!sOpBXC0r_o;0_qAN%S+hB*^n18Xos^a<6>1_Cum*P?5|* zF~R9~cfZ52UIP0XJ5eq3NYSYrW4pqNK@*{29s`uin4f;B>GIPc`=kNM%d8NMdPf06 zY;}isjxa3XIyE>FD@N%2?4((YmXRI)a~sLlH-*TCs~Nm^EZ#TD-Q(0C&I!Rti%T4@ zT?`($cv0z)B~KKR4uOqa1+B{qq5d+xxP5a&2n7g0kzgJ4b~&h$Zq`SHf5fe$QA z1C(t~1vQUQGs;<4>5|hmYrMi_g1gboFN1GB;nMH0WpBFd`s?#%kY4xc2yo?%A)ufT z%HN+`W%i>rY+x0+tFrA3A4;RjD8W2iyf6^QdUqU_p$b#~d+_fq3A^pytwMN>6J^Zo zYU*-9&(RQ`SHmbck!B*TES^ap!$)_ih@%7(7It8kN2qeca#m?bUy{?$1aLDV&!sU; z3SD{jCPQ6IXq%{iwvbH7l=Ao8=kkKotlO(qtjGSO@n|=*PieXFj2Q(yo2EMYe1UUK zEo!Um)-*J{S;?-;YH4MD&px^J8SdxKCXLHzi#^QI&7l(jZzcJPhT0w>jTIWy@w_~? zLCt{Mb|0eArB(gM@QRyIbT3?<}TyF)P z0+=-qeX~b+HQ+gjdh*#2j7}vo8=v~?bt{9AkkZ3c6v|GuE=_cBiO;q`z<(f(-B=i z4Jy~_9L_bBD!Z$CM?WD$Z?6n^^WzOd(csZsE7;-Fm!+jC`tkAFuBs4FJ!0Q_o3W~& zt3-X?ErM`72vVfKvfiYvdjUo$v zBnS?T&pKKSvvXKI$n#ABVB}kzO>ww0KAIO&a5)rKO6*65-1(3)suH~2Z0Xu&3uBAE zfDXI1yjKE^US!LN4s*A*zn4Q{Uf^rVorRLB9dDg+w-ZWzCi+$8XJ%$@8EVv@do_y2 zh!5Hs>1aj|(^BA;v14aB&WR{20%*=Abb7_!HfSV+)-Tc~pcmz|*Gc^zq5^+py29)3 zY&#y(FtiCLO(Aq_2N|bxD%HPJRlzIzWF3^Fb6AZ)_7#!8z<@I6><={p{`|alPv28b z-^=jaO%L^h?q!cIWucN$b}~9F;h^!h6u#pILqa{R+N%^$H{Um=-*AD8d#VHO*ev7Oky@^rr9V?tGyb0qr) zA_SWvXk378nghILSiGfXyzpTtp=>hm2_0j)Sw5j2fb|~Vu{D!uDipmgDU{voZM#5{4VRk0>n|fsQhfq#7--=Q>x-Zz7y~q&P~yl=E#RiW-qv(LgDFNWUW3N@y`??n3*kE z%Deg_p6$J$wz%e4ZWM1 zY7i3)57W=?ZokF9jgk&kyOh)eg_l?9^K_-hh+j=*9h+~WQA-`=v!#1@EJ&oU#$ob% zM0~@Ms8Sml82KgUa7z33hoX5|x4-n(oL$}cMw0MqYG3Gp%F*C)%nl*9?jn7dIu zMZ@AvWOWuy6LA=?kAC04_m6Y?7J;hwYtjAls>J6PbN{KaOM$CS`3&B$+r~;!I7n{Fd&dQg!;vzjug7Q5=PR(^vq<)1Aj9Ww<(uV}& z8RMgbQi<@Ic=P1xSoYRsoY^eVCRr)@z}Ow4O`aeDpG5n@=b&Q@&z7r)@tKbHt%F^v z1fo7D@Rubm1Z?^{@k~UmZRBt%T$O}!1NKtMb;!6%`ef$jnI~n2z1p}rhXwDD_?Hg# zGM(g{KaRSgMZqUGb4&i8zBnDiE|R2%k0pGPx=wpP9R{b4v+C6v+u>+=k zgic~nqiNzV1b$(nu`G&n(F?z^h9#pe4}2l`qAxeLUpGUY2KjJ%ByO$PB}GVIM+2zP z>FkPSVIbjoyo+uQELPRc4R-^e-szOKCq)c~(51aLZ3~J$5m&>rn@k7bJs(@k-MGg( z$KPof(9cEfyt5_k0GN3cf{UG6y(~% zPVD(-`V3gYSRLHC!5Vigg`r7XW89q@G9G`pA~mhF@4r4D&o zx%5*1$ZbqxJvDP}&=f-@?f7t+I2rpJD5LDANb2NbEqY zqKP1zA|QXd)+M!o$M^hzMu+VqS+xrL4hQgfy&w!_3eWx-hGidm$Y!qNe_LcVF3h6? zwG(w$j+&Z6euM^=p67p`jlXpOCfAa4H=rfK8yyLBs2%2e*O(J2)8bG5`8@S-iGTNLGU^#n7w^ZRSr9*-=3LVL-Z6iHj00$3hsx&WlE zu@1Vg2h&(@vL!V$2!M(e;t%~~I4{y;apZ4aN$Ri7bj-gT#iq+fEt zy&4mK^kcN*km>+IWt(4}<=iXoJHB4qt+m#h`iq4B!H*9~XGHhHU|7ZVMi={$T~?`3 z@3ITdCy4fxXVyCtUxGnbmn~M<{OT<>^C0`Dm&EtA

5upaBi&K7%P%^;29U6%4JeX!gL`;vN0X zT@HL5q!K0ODT~fi3J9wu8t`c-tSxgMfXL6?u5K^Vs&+E(?dD+FbM@c%!M4q>eQK2r;nEtn)QFjTtsDJCiYIOHYvn=RkQ-yoQ zWGJp`_#421nU@@p%3dH6PQtNInIwf=*$Bs@3J2gnxRh!XJ(db+z{qY)9t^&iJ9{0H zjqZ`f5e!@2>_+(bm)4~KJJ^rLyLjjFNKyDSa$Pw#jXR3VL!ae^H5sy@d1rzfA{pGR zPb#&ja|$leV}eW4wP+v+%lnMf9_X=!Y;O7bJRR2R z+3#J$j8!)xSFrb|Pqc{MklY14O-<2U$4@&!=Psr=2ikV(Zu*l}vzYgXL?!fR?7t-m zkMkYUgq>j#-|RBGiWC(NR<$(#9uJ4f*$L%nK++ivzd*ah#S(y<_x(WbDLMx!6LAP* zrbe5GdN|zr5=S4*Ts-sK#y(3hfY~uG4j$d{#?|NW1|DL4zRSzB%Q4sT6O6be4eP;^ zd&j+iy?)Wp4a)gR!@QktE?UI-6i%VE6;NSm9{|>m8Px4}T}cm>)1{nE`*2<2o-VxF zn)Ja9^}_JAtIlV$t0W*l4tpWo4{Bw+olJ{tZ^p8=wg#Y#;1qhAjexf|tY@}o>WY~2@AvlbxFu(p9Jql;;iXEn`EO~g0DLDsrBWU1P&1C_6 z$L;Xia4sI~|4OnosHLr)CnTyfE0CuF@=TNJL!vZ+k@L_HuXU zitmaVFk6*6*;W8-T`KEuHI?B!P;%&<5|UCFWQUV+DNt+ym|jHOKdq`^l`)F+?39L%sJFD#cOWVS+(fMdSy7g`8);Ac7D6bqzyP)ALhhy zlcgombJuzYG~*JyhNatoa6fk0!5gODY-I~<1>m=v3hJE8^A5<7$%^r&^dnxmOzdFu zcv_wF(Ubn|S2%4WOF-izd^#$Rz^HO!KUhOqYSRlOF zJZgm;+`EsVWo2!Mxs3cZ8r*|t&`IZ8|9oA2B1QgQ?lW|T#hu|OU_a|!EU22zCCIq* z#Y#=I;Z<6=+h;o6^x;PK1-@jE6RFjXytN2aI=LK`=u-n4vi(9%fbAFVh+Qql?xxB0 zAebM37>z45FO5Pif0<2IO0u#YMr!NK)`bK6X>bi1LpR#J7YFJs`wZ~e4|Jy7Eqqxg zR|*$AdgmhYyxX8q4Q`y7N(t)0oACp)jDe90Sn=$}nwCC1wpN~cIg~?;9QjO5?3&tw zc+L1(&EZ*;sOG4Rp13XD)qTR3opX>E521QwY15+^yg%mlhaiJq->n=e!O)fEXb7@s zvj}%$sE);#RB=cY5}n_k$_CySpAnYdjP4V2t+xglT_?7GkpNDH*EF-+mAakO4dgDP zn1jyp@vSYYR%5?}EL!@R^4bEx|JExQP4qptc)M{nnxUYd*tXCaFi&t>nbOL;-Mx^% z5w<#}yu`-M=*SH|{K2SkG_=RJLRfP9qAG{~Xmj#$&;cLl7WRQTXf3pb0m&4+z3xYZ zr-6Gkb?GvbqtBOFjHSOC3SS+e7<%(A&#quVeHOo5Im>XF5wB;=Hn2S>6+%3I^4UTz zr!#?mi7h~_LWCvfNA_ah*e`8NgYZ{}dV}MW+hiS|r4D%}y#RO@_Os=)>bDsa|0MhS zv|n2&NZQ1!I?YRJDlnDTwIY>c%#Pq}suKyrsVp)0aogCvqurMpZm>@UnmtJ3h1&a; zaD?whpT(`#jY_QTdGBTg5@QzFF?kq&O6yqYI9p`%yI4U3kPPUdT0R%=)y5wzzF0?i zRq!;1B}A;MbH!p}Fxp;=&ii-p2qU%Cb7wVQDINo^VvjSIoE*7r z#(ej8GP>CFg5!2{b|Es!MdFnkK+TVf%b$kflhElgm_I)%3%Nx{*D5}G7}U~pR!r$~ zo}?huBAops6cWn626^h+I<+x2fwAKJ|dI;zWP~njblaZI?q;WKJXXR2z}cI57yV^NZ#j z7U93k`T?~58o)#$L+4vSu&Bf3F#ed3!57J}>bHPnw6#kEq0_?L?ByJ?2@=$Gm`KiY z22%DjB65q(UXG}K`R>DVO+1?rq+bZghCNK#uE`93A(1(32(kInG2MM(f@AkDwx9|h z>}Zmnnq^61ntZ8^8Rg^^DeMulzv<&`=GTGM0nr|BrJ7P+%MN4hmN7OWDTOb~5L%#EJLFcFQ~#wu0|}I6b7GCX9x$kRgoS9 z*4F%u(?3R2eejyGV_Y4wpE4*7q%^Bd5o&GqWqZ84QwA6#==Xrr9RDqxZ!j=6Ul`l7 zEbf-$ncz218Vlzyj>6J|=I4CPLTp!MFqm#hWBaw&d;4xzIUhtAlD&JF_DNl-Yrk`+ z3IUB@$qpt^gd7I$!GS*+R$q|4F8vq+mWTRB0rcW?6u)z+PJ)4vah6)6$f@~#PUL|( z;yB|b`V#4{bwcr0Oc!k7iM6UotZLNu+DC07~lgvC~~GBVYsvK(|E0AXpexs~m+kygNPNohbyDtA?D@6878J zM|cAYd~3X{Zdf3uA0ya})^9iQ1trno+)V(eJa7|XcM!E~UapUGl+xR)tiQbRg`QWx zM^)wYq<}dKZ!IZ^@-$%F^gyCoIGbZBk7^9QiL$N{Pm~62f9O}6X2nK{pR7>u7i=J4PulL*m&Yt#*07=e-?yu)tFIBb z^Tx3VA7V;%`~yb0>Ezu%=ko88mb%*3 z)-4)OG=BC)Zi-c_eAjo?`^MKK^wg?l z7VTLU6_yJs3cO=Y{)Vppu zpBC+2c6=8JS?#awj79kznOA16KMX@O1BXqk&Q99W2UKb_75%G!efGi%xSIM$GaoeSlQ@0VsosKmzU+P~T2ifK>Kkm2lOugj^gp z`56G_IV3$=xqdQt=6n9>Phz;po(R28E*OXN(aIf0ri5BYtikNk(5tZw@PazRcp-Q7 zmN+9H4t8wA@0P-`QRUr4tB9nsc@KjcthipY1`!Ew4cOs?XV;8z??y>5fI8SUxbuRN z*jUvBRix%gi7s3LP#$dTL<{|l8dQd>nW$>vDWfqAP^nSXH10(CX^KZahX5u(gmSzE zxFxWH*i%D_^30~t7k+kD9P1_NN)R5oqxRSc&}Z&T;#hqxJn?@Y03?xgv8DawNr3?kOhv*?iVN9eUJ=?9H6B zHfjre*F~mJx{w-PBI{JP$ukL)G^9Sb2ZBRb*l!urP{erAN0`Z^_jd{quzm{}tX5`^ z3p&Y2U4y+Srit6P`B%0kLG|0JsWhA}Bn!RqmtKh#>X_ccx+oi_kcgp{Hj)OP{4A_pw|xHvqJ(Z7)EE;Z|UF}i&x>2n2M{5AQ|GXFLV?A z(-jAv_fw~e@lpRH^^33iR`I}*sJG-hgtzo5%#z}S`e_>4)+BrfI%Ee{^5Z7^H93_M z9+(iNxYPn$FD!I)RrfB)j5_nR>jHmvVAGqkaT6oY1YuQe{X>h2Ua_(|gq%v&`L-88 z=13q3zITj~_+_E3?<+04D-5u5U}*BjC2qf`IsZ@@Eet$-S7YheHcMugnnEeLgDK9; z^S*`Gs^3w$WX}nxqH#=JLZbYqo!aN;V7|$!*G{D0C8XKvXae}n8Jmon{O%RhWaek` zmibEQf^p3#UKYcw#)c_3g}|C>w<^L-h9D@0laU<(PS=dPe08(dRp|+62f@(@D;V-8nV`@AXbBj9Po-Z~^24@#&!jVe z0s+B(w*=fwZH(>z-&T6G@`PO`6Ouq`$?sWd=VjAW71c&y{d7FF=y@p{1b#74xaIP? zGX7FwRl?Gm8On+qs{$%5u>+%h`@^@ifZdCURQWJ_eS6{CTrnj&NDDTfv;@gi*giEf zpaG$%X$hC?RvF4mHFZlcjXDZNW}Z;58X3eameR;`C_3_NbJ?Cd3DeR0Va+`s8wji& zOqa!LgyxL+9sgMbcld5nsup<*TLA1@?35HvMb%nw2jY%px;57H!)T?|m`xb|{@z|V zxnFNWcVB%YnkF0GtohKhnC+S-nIX=av+!Tr9`Sg*!rSYt!_|Ci;?9jU2#=T#Z=a%qZp*d34LKpP&V2LGmvvs|p2u~`fpujAR7*VDU_r^*wJ+bpt^Ny^` zOx)cF%|WT;VdDcwiMY`XSzNy-#BPh@Qg=gnGTHvVjkuTb6hD&Wro!?>fC%%Pj@~mg zZwOZYoGZy!5Jij@zZ$zwVg?!V~xws`5XV0Nt;hkuoySml88GM3mV!Eeftm&%b&D8u(@CBqG4Ge+?^#Auhe}Bo} z52FA3{NEPne?tFM_Wr+j{|oZ`7Ww{59sMtX??0)38dd*IA%9D!|J$$nPwb!mx_@J< z-#^L!vf2HU`lq|$-xM15e^J)1*8hq9Z*G8pV`n)3X$tsH?th!S|2HRt_wRDezq&R5 zB>!2&`8Rp}^WWtEt>yeD`oEpdf1?=g|KqGGNP~YXEdC|2fChT_KF*IG|9bU*04sOA AW&i*H literal 0 HcmV?d00001 diff --git a/android/Tests/KeyboardHarness/app/src/main/java/com/keyman/android/tests/keyboardHarness/MainActivity.java b/android/Tests/KeyboardHarness/app/src/main/java/com/keyman/android/tests/keyboardHarness/MainActivity.java index 9454e940b7..c09a70c0fe 100644 --- a/android/Tests/KeyboardHarness/app/src/main/java/com/keyman/android/tests/keyboardHarness/MainActivity.java +++ b/android/Tests/KeyboardHarness/app/src/main/java/com/keyman/android/tests/keyboardHarness/MainActivity.java @@ -79,6 +79,22 @@ public class MainActivity extends AppCompatActivity implements OnKeyboardEventLi KMManager.KMDefault_KeyboardFont, KMManager.KMDefault_KeyboardFont); KMManager.addKeyboard(this, platformtestKBbInfo); + + // Final K_ENTER test keyboard + Keyboard finalKBInfo = new Keyboard( + "final", + "final", + "final Keyboard", + "en", + "English", + "1.0", + "", + "", + true, + KMManager.KMDefault_KeyboardFont, + KMManager.KMDefault_KeyboardFont); + KMManager.addKeyboard(this, finalKBInfo); + } @Override From b5708a92515d8415b0f311c08fcd1adfce43f7f6 Mon Sep 17 00:00:00 2001 From: Darcy Wong Date: Fri, 16 Sep 2022 11:42:16 +0700 Subject: [PATCH 124/132] chore(common): Update to Unicode 15.0 Update to Unicode 15.0 characters from https://www.unicode.org/Public/15.0.0/ucd/ --- .../unicode-character-database/Blocks.txt | 21 +- .../UnicodeData.txt | 300 +++++++++++++++++- 2 files changed, 313 insertions(+), 8 deletions(-) diff --git a/resources/standards-data/unicode-character-database/Blocks.txt b/resources/standards-data/unicode-character-database/Blocks.txt index cc5d61988b..12684594c9 100644 --- a/resources/standards-data/unicode-character-database/Blocks.txt +++ b/resources/standards-data/unicode-character-database/Blocks.txt @@ -1,10 +1,10 @@ -# Blocks-14.0.0.txt -# Date: 2021-01-22, 23:29:00 GMT [KW] -# © 2021 Unicode®, Inc. -# For terms of use, see http://www.unicode.org/terms_of_use.html +# Blocks-15.0.0.txt +# Date: 2022-01-28, 20:58:00 GMT [KW] +# © 2022 Unicode®, Inc. +# For terms of use, see https://www.unicode.org/terms_of_use.html # # Unicode Character Database -# For documentation, see http://www.unicode.org/reports/tr44/ +# For documentation, see https://www.unicode.org/reports/tr44/ # # Format: # Start Code..End Code; Block Name @@ -15,7 +15,7 @@ # and underbars are ignored. # For example, "Latin Extended-A" and "latin extended a" are equivalent. # For more information on the comparison of property values, -# see UAX #44: http://www.unicode.org/reports/tr44/ +# see UAX #44: https://www.unicode.org/reports/tr44/ # # All block ranges start with a value where (cp MOD 16) = 0, # and end with a value where (cp MOD 16) = 15. In other words, @@ -241,6 +241,7 @@ FFF0..FFFF; Specials 10D00..10D3F; Hanifi Rohingya 10E60..10E7F; Rumi Numeral Symbols 10E80..10EBF; Yezidi +10EC0..10EFF; Arabic Extended-C 10F00..10F2F; Old Sogdian 10F30..10F6F; Sogdian 10F70..10FAF; Old Uyghur @@ -272,11 +273,13 @@ FFF0..FFFF; Specials 11A50..11AAF; Soyombo 11AB0..11ABF; Unified Canadian Aboriginal Syllabics Extended-A 11AC0..11AFF; Pau Cin Hau +11B00..11B5F; Devanagari Extended-A 11C00..11C6F; Bhaiksuki 11C70..11CBF; Marchen 11D00..11D5F; Masaram Gondi 11D60..11DAF; Gunjala Gondi 11EE0..11EFF; Makasar +11F00..11F5F; Kawi 11FB0..11FBF; Lisu Supplement 11FC0..11FFF; Tamil Supplement 12000..123FF; Cuneiform @@ -284,7 +287,7 @@ FFF0..FFFF; Specials 12480..1254F; Early Dynastic Cuneiform 12F90..12FFF; Cypro-Minoan 13000..1342F; Egyptian Hieroglyphs -13430..1343F; Egyptian Hieroglyph Format Controls +13430..1345F; Egyptian Hieroglyph Format Controls 14400..1467F; Anatolian Hieroglyphs 16800..16A3F; Bamum Supplement 16A40..16A6F; Mro @@ -309,6 +312,7 @@ FFF0..FFFF; Specials 1D000..1D0FF; Byzantine Musical Symbols 1D100..1D1FF; Musical Symbols 1D200..1D24F; Ancient Greek Musical Notation +1D2C0..1D2DF; Kaktovik Numerals 1D2E0..1D2FF; Mayan Numerals 1D300..1D35F; Tai Xuan Jing Symbols 1D360..1D37F; Counting Rod Numerals @@ -316,9 +320,11 @@ FFF0..FFFF; Specials 1D800..1DAAF; Sutton SignWriting 1DF00..1DFFF; Latin Extended-G 1E000..1E02F; Glagolitic Supplement +1E030..1E08F; Cyrillic Extended-D 1E100..1E14F; Nyiakeng Puachue Hmong 1E290..1E2BF; Toto 1E2C0..1E2FF; Wancho +1E4D0..1E4FF; Nag Mundari 1E7E0..1E7FF; Ethiopic Extended-B 1E800..1E8DF; Mende Kikakui 1E900..1E95F; Adlam @@ -348,6 +354,7 @@ FFF0..FFFF; Specials 2CEB0..2EBEF; CJK Unified Ideographs Extension F 2F800..2FA1F; CJK Compatibility Ideographs Supplement 30000..3134F; CJK Unified Ideographs Extension G +31350..323AF; CJK Unified Ideographs Extension H E0000..E007F; Tags E0100..E01EF; Variation Selectors Supplement F0000..FFFFF; Supplementary Private Use Area-A diff --git a/resources/standards-data/unicode-character-database/UnicodeData.txt b/resources/standards-data/unicode-character-database/UnicodeData.txt index b5abef7ed4..ea963a7162 100644 --- a/resources/standards-data/unicode-character-database/UnicodeData.txt +++ b/resources/standards-data/unicode-character-database/UnicodeData.txt @@ -2975,6 +2975,7 @@ 0CEF;KANNADA DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 0CF1;KANNADA SIGN JIHVAMULIYA;Lo;0;L;;;;;N;;;;; 0CF2;KANNADA SIGN UPADHMANIYA;Lo;0;L;;;;;N;;;;; +0CF3;KANNADA SIGN COMBINING ANUSVARA ABOVE RIGHT;Mc;0;L;;;;;N;;;;; 0D00;MALAYALAM SIGN COMBINING ANUSVARA ABOVE;Mn;0;NSM;;;;;N;;;;; 0D01;MALAYALAM SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;; 0D02;MALAYALAM SIGN ANUSVARA;Mc;0;L;;;;;N;;;;; @@ -3339,6 +3340,7 @@ 0ECB;LAO TONE MAI CATAWA;Mn;122;NSM;;;;;N;;;;; 0ECC;LAO CANCELLATION MARK;Mn;0;NSM;;;;;N;;;;; 0ECD;LAO NIGGAHITA;Mn;0;NSM;;;;;N;;;;; +0ECE;LAO YAMAKKAN;Mn;0;NSM;;;;;N;;;;; 0ED0;LAO DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; 0ED1;LAO DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; 0ED2;LAO DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; @@ -19393,6 +19395,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 10EAD;YEZIDI HYPHENATION MARK;Pd;0;R;;;;;N;;;;; 10EB0;YEZIDI LETTER LAM WITH DOT ABOVE;Lo;0;R;;;;;N;;;;; 10EB1;YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE;Lo;0;R;;;;;N;;;;; +10EFD;ARABIC SMALL LOW WORD SAKTA;Mn;220;NSM;;;;;N;;;;; +10EFE;ARABIC SMALL LOW WORD QASR;Mn;220;NSM;;;;;N;;;;; +10EFF;ARABIC SMALL LOW WORD MADDA;Mn;220;NSM;;;;;N;;;;; 10F00;OLD SOGDIAN LETTER ALEPH;Lo;0;R;;;;;N;;;;; 10F01;OLD SOGDIAN LETTER FINAL ALEPH;Lo;0;R;;;;;N;;;;; 10F02;OLD SOGDIAN LETTER BETH;Lo;0;R;;;;;N;;;;; @@ -20058,6 +20063,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1123C;KHOJKI DOUBLE SECTION MARK;Po;0;L;;;;;N;;;;; 1123D;KHOJKI ABBREVIATION SIGN;Po;0;L;;;;;N;;;;; 1123E;KHOJKI SIGN SUKUN;Mn;0;NSM;;;;;N;;;;; +1123F;KHOJKI LETTER QA;Lo;0;L;;;;;N;;;;; +11240;KHOJKI LETTER SHORT I;Lo;0;L;;;;;N;;;;; +11241;KHOJKI VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;; 11280;MULTANI LETTER A;Lo;0;L;;;;;N;;;;; 11281;MULTANI LETTER I;Lo;0;L;;;;;N;;;;; 11282;MULTANI LETTER U;Lo;0;L;;;;;N;;;;; @@ -21256,6 +21264,16 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11AF6;PAU CIN HAU LOW-FALLING TONE LONG FINAL;Lo;0;L;;;;;N;;;;; 11AF7;PAU CIN HAU LOW-FALLING TONE FINAL;Lo;0;L;;;;;N;;;;; 11AF8;PAU CIN HAU GLOTTAL STOP FINAL;Lo;0;L;;;;;N;;;;; +11B00;DEVANAGARI HEAD MARK;Po;0;L;;;;;N;;;;; +11B01;DEVANAGARI HEAD MARK WITH HEADSTROKE;Po;0;L;;;;;N;;;;; +11B02;DEVANAGARI SIGN BHALE;Po;0;L;;;;;N;;;;; +11B03;DEVANAGARI SIGN BHALE WITH HOOK;Po;0;L;;;;;N;;;;; +11B04;DEVANAGARI SIGN EXTENDED BHALE;Po;0;L;;;;;N;;;;; +11B05;DEVANAGARI SIGN EXTENDED BHALE WITH HOOK;Po;0;L;;;;;N;;;;; +11B06;DEVANAGARI SIGN WESTERN FIVE-LIKE BHALE;Po;0;L;;;;;N;;;;; +11B07;DEVANAGARI SIGN WESTERN NINE-LIKE BHALE;Po;0;L;;;;;N;;;;; +11B08;DEVANAGARI SIGN REVERSED NINE-LIKE BHALE;Po;0;L;;;;;N;;;;; +11B09;DEVANAGARI SIGN MINDU;Po;0;L;;;;;N;;;;; 11C00;BHAIKSUKI LETTER A;Lo;0;L;;;;;N;;;;; 11C01;BHAIKSUKI LETTER AA;Lo;0;L;;;;;N;;;;; 11C02;BHAIKSUKI LETTER I;Lo;0;L;;;;;N;;;;; @@ -21584,6 +21602,92 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11EF6;MAKASAR VOWEL SIGN O;Mc;0;L;;;;;N;;;;; 11EF7;MAKASAR PASSIMBANG;Po;0;L;;;;;N;;;;; 11EF8;MAKASAR END OF SECTION;Po;0;L;;;;;N;;;;; +11F00;KAWI SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;; +11F01;KAWI SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; +11F02;KAWI SIGN REPHA;Lo;0;L;;;;;N;;;;; +11F03;KAWI SIGN VISARGA;Mc;0;L;;;;;N;;;;; +11F04;KAWI LETTER A;Lo;0;L;;;;;N;;;;; +11F05;KAWI LETTER AA;Lo;0;L;;;;;N;;;;; +11F06;KAWI LETTER I;Lo;0;L;;;;;N;;;;; +11F07;KAWI LETTER II;Lo;0;L;;;;;N;;;;; +11F08;KAWI LETTER U;Lo;0;L;;;;;N;;;;; +11F09;KAWI LETTER UU;Lo;0;L;;;;;N;;;;; +11F0A;KAWI LETTER VOCALIC R;Lo;0;L;;;;;N;;;;; +11F0B;KAWI LETTER VOCALIC RR;Lo;0;L;;;;;N;;;;; +11F0C;KAWI LETTER VOCALIC L;Lo;0;L;;;;;N;;;;; +11F0D;KAWI LETTER VOCALIC LL;Lo;0;L;;;;;N;;;;; +11F0E;KAWI LETTER E;Lo;0;L;;;;;N;;;;; +11F0F;KAWI LETTER AI;Lo;0;L;;;;;N;;;;; +11F10;KAWI LETTER O;Lo;0;L;;;;;N;;;;; +11F12;KAWI LETTER KA;Lo;0;L;;;;;N;;;;; +11F13;KAWI LETTER KHA;Lo;0;L;;;;;N;;;;; +11F14;KAWI LETTER GA;Lo;0;L;;;;;N;;;;; +11F15;KAWI LETTER GHA;Lo;0;L;;;;;N;;;;; +11F16;KAWI LETTER NGA;Lo;0;L;;;;;N;;;;; +11F17;KAWI LETTER CA;Lo;0;L;;;;;N;;;;; +11F18;KAWI LETTER CHA;Lo;0;L;;;;;N;;;;; +11F19;KAWI LETTER JA;Lo;0;L;;;;;N;;;;; +11F1A;KAWI LETTER JHA;Lo;0;L;;;;;N;;;;; +11F1B;KAWI LETTER NYA;Lo;0;L;;;;;N;;;;; +11F1C;KAWI LETTER TTA;Lo;0;L;;;;;N;;;;; +11F1D;KAWI LETTER TTHA;Lo;0;L;;;;;N;;;;; +11F1E;KAWI LETTER DDA;Lo;0;L;;;;;N;;;;; +11F1F;KAWI LETTER DDHA;Lo;0;L;;;;;N;;;;; +11F20;KAWI LETTER NNA;Lo;0;L;;;;;N;;;;; +11F21;KAWI LETTER TA;Lo;0;L;;;;;N;;;;; +11F22;KAWI LETTER THA;Lo;0;L;;;;;N;;;;; +11F23;KAWI LETTER DA;Lo;0;L;;;;;N;;;;; +11F24;KAWI LETTER DHA;Lo;0;L;;;;;N;;;;; +11F25;KAWI LETTER NA;Lo;0;L;;;;;N;;;;; +11F26;KAWI LETTER PA;Lo;0;L;;;;;N;;;;; +11F27;KAWI LETTER PHA;Lo;0;L;;;;;N;;;;; +11F28;KAWI LETTER BA;Lo;0;L;;;;;N;;;;; +11F29;KAWI LETTER BHA;Lo;0;L;;;;;N;;;;; +11F2A;KAWI LETTER MA;Lo;0;L;;;;;N;;;;; +11F2B;KAWI LETTER YA;Lo;0;L;;;;;N;;;;; +11F2C;KAWI LETTER RA;Lo;0;L;;;;;N;;;;; +11F2D;KAWI LETTER LA;Lo;0;L;;;;;N;;;;; +11F2E;KAWI LETTER WA;Lo;0;L;;;;;N;;;;; +11F2F;KAWI LETTER SHA;Lo;0;L;;;;;N;;;;; +11F30;KAWI LETTER SSA;Lo;0;L;;;;;N;;;;; +11F31;KAWI LETTER SA;Lo;0;L;;;;;N;;;;; +11F32;KAWI LETTER HA;Lo;0;L;;;;;N;;;;; +11F33;KAWI LETTER JNYA;Lo;0;L;;;;;N;;;;; +11F34;KAWI VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +11F35;KAWI VOWEL SIGN ALTERNATE AA;Mc;0;L;;;;;N;;;;; +11F36;KAWI VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +11F37;KAWI VOWEL SIGN II;Mn;0;NSM;;;;;N;;;;; +11F38;KAWI VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +11F39;KAWI VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;; +11F3A;KAWI VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;; +11F3E;KAWI VOWEL SIGN E;Mc;0;L;;;;;N;;;;; +11F3F;KAWI VOWEL SIGN AI;Mc;0;L;;;;;N;;;;; +11F40;KAWI VOWEL SIGN EU;Mn;0;NSM;;;;;N;;;;; +11F41;KAWI SIGN KILLER;Mc;9;L;;;;;N;;;;; +11F42;KAWI CONJOINER;Mn;9;NSM;;;;;N;;;;; +11F43;KAWI DANDA;Po;0;L;;;;;N;;;;; +11F44;KAWI DOUBLE DANDA;Po;0;L;;;;;N;;;;; +11F45;KAWI PUNCTUATION SECTION MARKER;Po;0;L;;;;;N;;;;; +11F46;KAWI PUNCTUATION ALTERNATE SECTION MARKER;Po;0;L;;;;;N;;;;; +11F47;KAWI PUNCTUATION FLOWER;Po;0;L;;;;;N;;;;; +11F48;KAWI PUNCTUATION SPACE FILLER;Po;0;L;;;;;N;;;;; +11F49;KAWI PUNCTUATION DOT;Po;0;L;;;;;N;;;;; +11F4A;KAWI PUNCTUATION DOUBLE DOT;Po;0;L;;;;;N;;;;; +11F4B;KAWI PUNCTUATION TRIPLE DOT;Po;0;L;;;;;N;;;;; +11F4C;KAWI PUNCTUATION CIRCLE;Po;0;L;;;;;N;;;;; +11F4D;KAWI PUNCTUATION FILLED CIRCLE;Po;0;L;;;;;N;;;;; +11F4E;KAWI PUNCTUATION SPIRAL;Po;0;L;;;;;N;;;;; +11F4F;KAWI PUNCTUATION CLOSING SPIRAL;Po;0;L;;;;;N;;;;; +11F50;KAWI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +11F51;KAWI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +11F52;KAWI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +11F53;KAWI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +11F54;KAWI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +11F55;KAWI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +11F56;KAWI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +11F57;KAWI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +11F58;KAWI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +11F59;KAWI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 11FB0;LISU LETTER YHA;Lo;0;L;;;;;N;;;;; 11FC0;TAMIL FRACTION ONE THREE-HUNDRED-AND-TWENTIETH;No;0;L;;;;1/320;N;;;;; 11FC1;TAMIL FRACTION ONE ONE-HUNDRED-AND-SIXTIETH;No;0;L;;;;1/160;N;;;;; @@ -24040,6 +24144,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1342C;EGYPTIAN HIEROGLYPH AA030;Lo;0;L;;;;;N;;;;; 1342D;EGYPTIAN HIEROGLYPH AA031;Lo;0;L;;;;;N;;;;; 1342E;EGYPTIAN HIEROGLYPH AA032;Lo;0;L;;;;;N;;;;; +1342F;EGYPTIAN HIEROGLYPH V011D;Lo;0;L;;;;;N;;;;; 13430;EGYPTIAN HIEROGLYPH VERTICAL JOINER;Cf;0;L;;;;;N;;;;; 13431;EGYPTIAN HIEROGLYPH HORIZONTAL JOINER;Cf;0;L;;;;;N;;;;; 13432;EGYPTIAN HIEROGLYPH INSERT AT TOP START;Cf;0;L;;;;;N;;;;; @@ -24049,6 +24154,35 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 13436;EGYPTIAN HIEROGLYPH OVERLAY MIDDLE;Cf;0;L;;;;;N;;;;; 13437;EGYPTIAN HIEROGLYPH BEGIN SEGMENT;Cf;0;L;;;;;N;;;;; 13438;EGYPTIAN HIEROGLYPH END SEGMENT;Cf;0;L;;;;;N;;;;; +13439;EGYPTIAN HIEROGLYPH INSERT AT MIDDLE;Cf;0;L;;;;;N;;;;; +1343A;EGYPTIAN HIEROGLYPH INSERT AT TOP;Cf;0;L;;;;;N;;;;; +1343B;EGYPTIAN HIEROGLYPH INSERT AT BOTTOM;Cf;0;L;;;;;N;;;;; +1343C;EGYPTIAN HIEROGLYPH BEGIN ENCLOSURE;Cf;0;L;;;;;N;;;;; +1343D;EGYPTIAN HIEROGLYPH END ENCLOSURE;Cf;0;L;;;;;N;;;;; +1343E;EGYPTIAN HIEROGLYPH BEGIN WALLED ENCLOSURE;Cf;0;L;;;;;N;;;;; +1343F;EGYPTIAN HIEROGLYPH END WALLED ENCLOSURE;Cf;0;L;;;;;N;;;;; +13440;EGYPTIAN HIEROGLYPH MIRROR HORIZONTALLY;Mn;0;NSM;;;;;N;;;;; +13441;EGYPTIAN HIEROGLYPH FULL BLANK;Lo;0;L;;;;;N;;;;; +13442;EGYPTIAN HIEROGLYPH HALF BLANK;Lo;0;L;;;;;N;;;;; +13443;EGYPTIAN HIEROGLYPH LOST SIGN;Lo;0;L;;;;;N;;;;; +13444;EGYPTIAN HIEROGLYPH HALF LOST SIGN;Lo;0;L;;;;;N;;;;; +13445;EGYPTIAN HIEROGLYPH TALL LOST SIGN;Lo;0;L;;;;;N;;;;; +13446;EGYPTIAN HIEROGLYPH WIDE LOST SIGN;Lo;0;L;;;;;N;;;;; +13447;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START;Mn;0;NSM;;;;;N;;;;; +13448;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM START;Mn;0;NSM;;;;;N;;;;; +13449;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT START;Mn;0;NSM;;;;;N;;;;; +1344A;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP END;Mn;0;NSM;;;;;N;;;;; +1344B;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP;Mn;0;NSM;;;;;N;;;;; +1344C;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM START AND TOP END;Mn;0;NSM;;;;;N;;;;; +1344D;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT START AND TOP;Mn;0;NSM;;;;;N;;;;; +1344E;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM END;Mn;0;NSM;;;;;N;;;;; +1344F;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START AND BOTTOM END;Mn;0;NSM;;;;;N;;;;; +13450;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM;Mn;0;NSM;;;;;N;;;;; +13451;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT START AND BOTTOM;Mn;0;NSM;;;;;N;;;;; +13452;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT END;Mn;0;NSM;;;;;N;;;;; +13453;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP AND END;Mn;0;NSM;;;;;N;;;;; +13454;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM AND END;Mn;0;NSM;;;;;N;;;;; +13455;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED;Mn;0;NSM;;;;;N;;;;; 14400;ANATOLIAN HIEROGLYPH A001;Lo;0;L;;;;;N;;;;; 14401;ANATOLIAN HIEROGLYPH A002;Lo;0;L;;;;;N;;;;; 14402;ANATOLIAN HIEROGLYPH A003;Lo;0;L;;;;;N;;;;; @@ -27289,9 +27423,11 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1B120;KATAKANA LETTER ARCHAIC YI;Lo;0;L;;;;;N;;;;; 1B121;KATAKANA LETTER ARCHAIC YE;Lo;0;L;;;;;N;;;;; 1B122;KATAKANA LETTER ARCHAIC WU;Lo;0;L;;;;;N;;;;; +1B132;HIRAGANA LETTER SMALL KO;Lo;0;L;;;;;N;;;;; 1B150;HIRAGANA LETTER SMALL WI;Lo;0;L;;;;;N;;;;; 1B151;HIRAGANA LETTER SMALL WE;Lo;0;L;;;;;N;;;;; 1B152;HIRAGANA LETTER SMALL WO;Lo;0;L;;;;;N;;;;; +1B155;KATAKANA LETTER SMALL KO;Lo;0;L;;;;;N;;;;; 1B164;KATAKANA LETTER SMALL WI;Lo;0;L;;;;;N;;;;; 1B165;KATAKANA LETTER SMALL WE;Lo;0;L;;;;;N;;;;; 1B166;KATAKANA LETTER SMALL WO;Lo;0;L;;;;;N;;;;; @@ -28573,6 +28709,26 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1D243;COMBINING GREEK MUSICAL TETRASEME;Mn;230;NSM;;;;;N;;;;; 1D244;COMBINING GREEK MUSICAL PENTASEME;Mn;230;NSM;;;;;N;;;;; 1D245;GREEK MUSICAL LEIMMA;So;0;ON;;;;;N;;;;; +1D2C0;KAKTOVIK NUMERAL ZERO;No;0;L;;;;0;N;;;;; +1D2C1;KAKTOVIK NUMERAL ONE;No;0;L;;;;1;N;;;;; +1D2C2;KAKTOVIK NUMERAL TWO;No;0;L;;;;2;N;;;;; +1D2C3;KAKTOVIK NUMERAL THREE;No;0;L;;;;3;N;;;;; +1D2C4;KAKTOVIK NUMERAL FOUR;No;0;L;;;;4;N;;;;; +1D2C5;KAKTOVIK NUMERAL FIVE;No;0;L;;;;5;N;;;;; +1D2C6;KAKTOVIK NUMERAL SIX;No;0;L;;;;6;N;;;;; +1D2C7;KAKTOVIK NUMERAL SEVEN;No;0;L;;;;7;N;;;;; +1D2C8;KAKTOVIK NUMERAL EIGHT;No;0;L;;;;8;N;;;;; +1D2C9;KAKTOVIK NUMERAL NINE;No;0;L;;;;9;N;;;;; +1D2CA;KAKTOVIK NUMERAL TEN;No;0;L;;;;10;N;;;;; +1D2CB;KAKTOVIK NUMERAL ELEVEN;No;0;L;;;;11;N;;;;; +1D2CC;KAKTOVIK NUMERAL TWELVE;No;0;L;;;;12;N;;;;; +1D2CD;KAKTOVIK NUMERAL THIRTEEN;No;0;L;;;;13;N;;;;; +1D2CE;KAKTOVIK NUMERAL FOURTEEN;No;0;L;;;;14;N;;;;; +1D2CF;KAKTOVIK NUMERAL FIFTEEN;No;0;L;;;;15;N;;;;; +1D2D0;KAKTOVIK NUMERAL SIXTEEN;No;0;L;;;;16;N;;;;; +1D2D1;KAKTOVIK NUMERAL SEVENTEEN;No;0;L;;;;17;N;;;;; +1D2D2;KAKTOVIK NUMERAL EIGHTEEN;No;0;L;;;;18;N;;;;; +1D2D3;KAKTOVIK NUMERAL NINETEEN;No;0;L;;;;19;N;;;;; 1D2E0;MAYAN NUMERAL ZERO;No;0;L;;;;0;N;;;;; 1D2E1;MAYAN NUMERAL ONE;No;0;L;;;;1;N;;;;; 1D2E2;MAYAN NUMERAL TWO;No;0;L;;;;2;N;;;;; @@ -30404,6 +30560,12 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1DF1C;LATIN SMALL LETTER TESH DIGRAPH WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;; 1DF1D;LATIN SMALL LETTER C WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;; 1DF1E;LATIN SMALL LETTER S WITH CURL;Ll;0;L;;;;;N;;;;; +1DF25;LATIN SMALL LETTER D WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; +1DF26;LATIN SMALL LETTER L WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; +1DF27;LATIN SMALL LETTER N WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; +1DF28;LATIN SMALL LETTER R WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; +1DF29;LATIN SMALL LETTER S WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; +1DF2A;LATIN SMALL LETTER T WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; 1E000;COMBINING GLAGOLITIC LETTER AZU;Mn;230;NSM;;;;;N;;;;; 1E001;COMBINING GLAGOLITIC LETTER BUKY;Mn;230;NSM;;;;;N;;;;; 1E002;COMBINING GLAGOLITIC LETTER VEDE;Mn;230;NSM;;;;;N;;;;; @@ -30442,6 +30604,69 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1E028;COMBINING GLAGOLITIC LETTER BIG YUS;Mn;230;NSM;;;;;N;;;;; 1E029;COMBINING GLAGOLITIC LETTER IOTATED BIG YUS;Mn;230;NSM;;;;;N;;;;; 1E02A;COMBINING GLAGOLITIC LETTER FITA;Mn;230;NSM;;;;;N;;;;; +1E030;MODIFIER LETTER CYRILLIC SMALL A;Lm;0;L; 0430;;;;N;;;;; +1E031;MODIFIER LETTER CYRILLIC SMALL BE;Lm;0;L; 0431;;;;N;;;;; +1E032;MODIFIER LETTER CYRILLIC SMALL VE;Lm;0;L; 0432;;;;N;;;;; +1E033;MODIFIER LETTER CYRILLIC SMALL GHE;Lm;0;L; 0433;;;;N;;;;; +1E034;MODIFIER LETTER CYRILLIC SMALL DE;Lm;0;L; 0434;;;;N;;;;; +1E035;MODIFIER LETTER CYRILLIC SMALL IE;Lm;0;L; 0435;;;;N;;;;; +1E036;MODIFIER LETTER CYRILLIC SMALL ZHE;Lm;0;L; 0436;;;;N;;;;; +1E037;MODIFIER LETTER CYRILLIC SMALL ZE;Lm;0;L; 0437;;;;N;;;;; +1E038;MODIFIER LETTER CYRILLIC SMALL I;Lm;0;L; 0438;;;;N;;;;; +1E039;MODIFIER LETTER CYRILLIC SMALL KA;Lm;0;L; 043A;;;;N;;;;; +1E03A;MODIFIER LETTER CYRILLIC SMALL EL;Lm;0;L; 043B;;;;N;;;;; +1E03B;MODIFIER LETTER CYRILLIC SMALL EM;Lm;0;L; 043C;;;;N;;;;; +1E03C;MODIFIER LETTER CYRILLIC SMALL O;Lm;0;L; 043E;;;;N;;;;; +1E03D;MODIFIER LETTER CYRILLIC SMALL PE;Lm;0;L; 043F;;;;N;;;;; +1E03E;MODIFIER LETTER CYRILLIC SMALL ER;Lm;0;L; 0440;;;;N;;;;; +1E03F;MODIFIER LETTER CYRILLIC SMALL ES;Lm;0;L; 0441;;;;N;;;;; +1E040;MODIFIER LETTER CYRILLIC SMALL TE;Lm;0;L; 0442;;;;N;;;;; +1E041;MODIFIER LETTER CYRILLIC SMALL U;Lm;0;L; 0443;;;;N;;;;; +1E042;MODIFIER LETTER CYRILLIC SMALL EF;Lm;0;L; 0444;;;;N;;;;; +1E043;MODIFIER LETTER CYRILLIC SMALL HA;Lm;0;L; 0445;;;;N;;;;; +1E044;MODIFIER LETTER CYRILLIC SMALL TSE;Lm;0;L; 0446;;;;N;;;;; +1E045;MODIFIER LETTER CYRILLIC SMALL CHE;Lm;0;L; 0447;;;;N;;;;; +1E046;MODIFIER LETTER CYRILLIC SMALL SHA;Lm;0;L; 0448;;;;N;;;;; +1E047;MODIFIER LETTER CYRILLIC SMALL YERU;Lm;0;L; 044B;;;;N;;;;; +1E048;MODIFIER LETTER CYRILLIC SMALL E;Lm;0;L; 044D;;;;N;;;;; +1E049;MODIFIER LETTER CYRILLIC SMALL YU;Lm;0;L; 044E;;;;N;;;;; +1E04A;MODIFIER LETTER CYRILLIC SMALL DZZE;Lm;0;L; A689;;;;N;;;;; +1E04B;MODIFIER LETTER CYRILLIC SMALL SCHWA;Lm;0;L; 04D9;;;;N;;;;; +1E04C;MODIFIER LETTER CYRILLIC SMALL BYELORUSSIAN-UKRAINIAN I;Lm;0;L; 0456;;;;N;;;;; +1E04D;MODIFIER LETTER CYRILLIC SMALL JE;Lm;0;L; 0458;;;;N;;;;; +1E04E;MODIFIER LETTER CYRILLIC SMALL BARRED O;Lm;0;L; 04E9;;;;N;;;;; +1E04F;MODIFIER LETTER CYRILLIC SMALL STRAIGHT U;Lm;0;L; 04AF;;;;N;;;;; +1E050;MODIFIER LETTER CYRILLIC SMALL PALOCHKA;Lm;0;L; 04CF;;;;N;;;;; +1E051;CYRILLIC SUBSCRIPT SMALL LETTER A;Lm;0;L; 0430;;;;N;;;;; +1E052;CYRILLIC SUBSCRIPT SMALL LETTER BE;Lm;0;L; 0431;;;;N;;;;; +1E053;CYRILLIC SUBSCRIPT SMALL LETTER VE;Lm;0;L; 0432;;;;N;;;;; +1E054;CYRILLIC SUBSCRIPT SMALL LETTER GHE;Lm;0;L; 0433;;;;N;;;;; +1E055;CYRILLIC SUBSCRIPT SMALL LETTER DE;Lm;0;L; 0434;;;;N;;;;; +1E056;CYRILLIC SUBSCRIPT SMALL LETTER IE;Lm;0;L; 0435;;;;N;;;;; +1E057;CYRILLIC SUBSCRIPT SMALL LETTER ZHE;Lm;0;L; 0436;;;;N;;;;; +1E058;CYRILLIC SUBSCRIPT SMALL LETTER ZE;Lm;0;L; 0437;;;;N;;;;; +1E059;CYRILLIC SUBSCRIPT SMALL LETTER I;Lm;0;L; 0438;;;;N;;;;; +1E05A;CYRILLIC SUBSCRIPT SMALL LETTER KA;Lm;0;L; 043A;;;;N;;;;; +1E05B;CYRILLIC SUBSCRIPT SMALL LETTER EL;Lm;0;L; 043B;;;;N;;;;; +1E05C;CYRILLIC SUBSCRIPT SMALL LETTER O;Lm;0;L; 043E;;;;N;;;;; +1E05D;CYRILLIC SUBSCRIPT SMALL LETTER PE;Lm;0;L; 043F;;;;N;;;;; +1E05E;CYRILLIC SUBSCRIPT SMALL LETTER ES;Lm;0;L; 0441;;;;N;;;;; +1E05F;CYRILLIC SUBSCRIPT SMALL LETTER U;Lm;0;L; 0443;;;;N;;;;; +1E060;CYRILLIC SUBSCRIPT SMALL LETTER EF;Lm;0;L; 0444;;;;N;;;;; +1E061;CYRILLIC SUBSCRIPT SMALL LETTER HA;Lm;0;L; 0445;;;;N;;;;; +1E062;CYRILLIC SUBSCRIPT SMALL LETTER TSE;Lm;0;L; 0446;;;;N;;;;; +1E063;CYRILLIC SUBSCRIPT SMALL LETTER CHE;Lm;0;L; 0447;;;;N;;;;; +1E064;CYRILLIC SUBSCRIPT SMALL LETTER SHA;Lm;0;L; 0448;;;;N;;;;; +1E065;CYRILLIC SUBSCRIPT SMALL LETTER HARD SIGN;Lm;0;L; 044A;;;;N;;;;; +1E066;CYRILLIC SUBSCRIPT SMALL LETTER YERU;Lm;0;L; 044B;;;;N;;;;; +1E067;CYRILLIC SUBSCRIPT SMALL LETTER GHE WITH UPTURN;Lm;0;L; 0491;;;;N;;;;; +1E068;CYRILLIC SUBSCRIPT SMALL LETTER BYELORUSSIAN-UKRAINIAN I;Lm;0;L; 0456;;;;N;;;;; +1E069;CYRILLIC SUBSCRIPT SMALL LETTER DZE;Lm;0;L; 0455;;;;N;;;;; +1E06A;CYRILLIC SUBSCRIPT SMALL LETTER DZHE;Lm;0;L; 045F;;;;N;;;;; +1E06B;MODIFIER LETTER CYRILLIC SMALL ES WITH DESCENDER;Lm;0;L; 04AB;;;;N;;;;; +1E06C;MODIFIER LETTER CYRILLIC SMALL YERU WITH BACK YER;Lm;0;L; A651;;;;N;;;;; +1E06D;MODIFIER LETTER CYRILLIC SMALL STRAIGHT U WITH STROKE;Lm;0;L; 04B1;;;;N;;;;; +1E08F;COMBINING CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I;Mn;230;NSM;;;;;N;;;;; 1E100;NYIAKENG PUACHUE HMONG LETTER MA;Lo;0;L;;;;;N;;;;; 1E101;NYIAKENG PUACHUE HMONG LETTER TSA;Lo;0;L;;;;;N;;;;; 1E102;NYIAKENG PUACHUE HMONG LETTER NTA;Lo;0;L;;;;;N;;;;; @@ -30603,6 +30828,48 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1E2F8;WANCHO DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; 1E2F9;WANCHO DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 1E2FF;WANCHO NGUN SIGN;Sc;0;ET;;;;;N;;;;; +1E4D0;NAG MUNDARI LETTER O;Lo;0;L;;;;;N;;;;; +1E4D1;NAG MUNDARI LETTER OP;Lo;0;L;;;;;N;;;;; +1E4D2;NAG MUNDARI LETTER OL;Lo;0;L;;;;;N;;;;; +1E4D3;NAG MUNDARI LETTER OY;Lo;0;L;;;;;N;;;;; +1E4D4;NAG MUNDARI LETTER ONG;Lo;0;L;;;;;N;;;;; +1E4D5;NAG MUNDARI LETTER A;Lo;0;L;;;;;N;;;;; +1E4D6;NAG MUNDARI LETTER AJ;Lo;0;L;;;;;N;;;;; +1E4D7;NAG MUNDARI LETTER AB;Lo;0;L;;;;;N;;;;; +1E4D8;NAG MUNDARI LETTER ANY;Lo;0;L;;;;;N;;;;; +1E4D9;NAG MUNDARI LETTER AH;Lo;0;L;;;;;N;;;;; +1E4DA;NAG MUNDARI LETTER I;Lo;0;L;;;;;N;;;;; +1E4DB;NAG MUNDARI LETTER IS;Lo;0;L;;;;;N;;;;; +1E4DC;NAG MUNDARI LETTER IDD;Lo;0;L;;;;;N;;;;; +1E4DD;NAG MUNDARI LETTER IT;Lo;0;L;;;;;N;;;;; +1E4DE;NAG MUNDARI LETTER IH;Lo;0;L;;;;;N;;;;; +1E4DF;NAG MUNDARI LETTER U;Lo;0;L;;;;;N;;;;; +1E4E0;NAG MUNDARI LETTER UC;Lo;0;L;;;;;N;;;;; +1E4E1;NAG MUNDARI LETTER UD;Lo;0;L;;;;;N;;;;; +1E4E2;NAG MUNDARI LETTER UK;Lo;0;L;;;;;N;;;;; +1E4E3;NAG MUNDARI LETTER UR;Lo;0;L;;;;;N;;;;; +1E4E4;NAG MUNDARI LETTER E;Lo;0;L;;;;;N;;;;; +1E4E5;NAG MUNDARI LETTER ENN;Lo;0;L;;;;;N;;;;; +1E4E6;NAG MUNDARI LETTER EG;Lo;0;L;;;;;N;;;;; +1E4E7;NAG MUNDARI LETTER EM;Lo;0;L;;;;;N;;;;; +1E4E8;NAG MUNDARI LETTER EN;Lo;0;L;;;;;N;;;;; +1E4E9;NAG MUNDARI LETTER ETT;Lo;0;L;;;;;N;;;;; +1E4EA;NAG MUNDARI LETTER ELL;Lo;0;L;;;;;N;;;;; +1E4EB;NAG MUNDARI SIGN OJOD;Lm;0;L;;;;;N;;;;; +1E4EC;NAG MUNDARI SIGN MUHOR;Mn;232;NSM;;;;;N;;;;; +1E4ED;NAG MUNDARI SIGN TOYOR;Mn;232;NSM;;;;;N;;;;; +1E4EE;NAG MUNDARI SIGN IKIR;Mn;220;NSM;;;;;N;;;;; +1E4EF;NAG MUNDARI SIGN SUTUH;Mn;230;NSM;;;;;N;;;;; +1E4F0;NAG MUNDARI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +1E4F1;NAG MUNDARI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +1E4F2;NAG MUNDARI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +1E4F3;NAG MUNDARI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +1E4F4;NAG MUNDARI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +1E4F5;NAG MUNDARI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +1E4F6;NAG MUNDARI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +1E4F7;NAG MUNDARI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +1E4F8;NAG MUNDARI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +1E4F9;NAG MUNDARI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 1E7E0;ETHIOPIC SYLLABLE HHYA;Lo;0;L;;;;;N;;;;; 1E7E1;ETHIOPIC SYLLABLE HHYU;Lo;0;L;;;;;N;;;;; 1E7E2;ETHIOPIC SYLLABLE HHYI;Lo;0;L;;;;;N;;;;; @@ -32678,6 +32945,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F6D5;HINDU TEMPLE;So;0;ON;;;;;N;;;;; 1F6D6;HUT;So;0;ON;;;;;N;;;;; 1F6D7;ELEVATOR;So;0;ON;;;;;N;;;;; +1F6DC;WIRELESS;So;0;ON;;;;;N;;;;; 1F6DD;PLAYGROUND SLIDE;So;0;ON;;;;;N;;;;; 1F6DE;WHEEL;So;0;ON;;;;;N;;;;; 1F6DF;RING BUOY;So;0;ON;;;;;N;;;;; @@ -32823,6 +33091,14 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F771;ALCHEMICAL SYMBOL FOR MONTH;So;0;ON;;;;;N;;;;; 1F772;ALCHEMICAL SYMBOL FOR HALF DRAM;So;0;ON;;;;;N;;;;; 1F773;ALCHEMICAL SYMBOL FOR HALF OUNCE;So;0;ON;;;;;N;;;;; +1F774;LOT OF FORTUNE;So;0;ON;;;;;N;;;;; +1F775;OCCULTATION;So;0;ON;;;;;N;;;;; +1F776;LUNAR ECLIPSE;So;0;ON;;;;;N;;;;; +1F77B;HAUMEA;So;0;ON;;;;;N;;;;; +1F77C;MAKEMAKE;So;0;ON;;;;;N;;;;; +1F77D;GONGGONG;So;0;ON;;;;;N;;;;; +1F77E;QUAOAR;So;0;ON;;;;;N;;;;; +1F77F;ORCUS;So;0;ON;;;;;N;;;;; 1F780;BLACK LEFT-POINTING ISOSCELES RIGHT TRIANGLE;So;0;ON;;;;;N;;;;; 1F781;BLACK UP-POINTING ISOSCELES RIGHT TRIANGLE;So;0;ON;;;;;N;;;;; 1F782;BLACK RIGHT-POINTING ISOSCELES RIGHT TRIANGLE;So;0;ON;;;;;N;;;;; @@ -32912,6 +33188,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F7D6;NEGATIVE CIRCLED TRIANGLE;So;0;ON;;;;;N;;;;; 1F7D7;CIRCLED SQUARE;So;0;ON;;;;;N;;;;; 1F7D8;NEGATIVE CIRCLED SQUARE;So;0;ON;;;;;N;;;;; +1F7D9;NINE POINTED WHITE STAR;So;0;ON;;;;;N;;;;; 1F7E0;LARGE ORANGE CIRCLE;So;0;ON;;;;;N;;;;; 1F7E1;LARGE YELLOW CIRCLE;So;0;ON;;;;;N;;;;; 1F7E2;LARGE GREEN CIRCLE;So;0;ON;;;;;N;;;;; @@ -33434,6 +33711,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FA72;BRIEFS;So;0;ON;;;;;N;;;;; 1FA73;SHORTS;So;0;ON;;;;;N;;;;; 1FA74;THONG SANDAL;So;0;ON;;;;;N;;;;; +1FA75;LIGHT BLUE HEART;So;0;ON;;;;;N;;;;; +1FA76;GREY HEART;So;0;ON;;;;;N;;;;; +1FA77;PINK HEART;So;0;ON;;;;;N;;;;; 1FA78;DROP OF BLOOD;So;0;ON;;;;;N;;;;; 1FA79;ADHESIVE BANDAGE;So;0;ON;;;;;N;;;;; 1FA7A;STETHOSCOPE;So;0;ON;;;;;N;;;;; @@ -33446,6 +33726,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FA84;MAGIC WAND;So;0;ON;;;;;N;;;;; 1FA85;PINATA;So;0;ON;;;;;N;;;;; 1FA86;NESTING DOLLS;So;0;ON;;;;;N;;;;; +1FA87;MARACAS;So;0;ON;;;;;N;;;;; +1FA88;FLUTE;So;0;ON;;;;;N;;;;; 1FA90;RINGED PLANET;So;0;ON;;;;;N;;;;; 1FA91;CHAIR;So;0;ON;;;;;N;;;;; 1FA92;RAZOR;So;0;ON;;;;;N;;;;; @@ -33475,6 +33757,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAAA;IDENTIFICATION CARD;So;0;ON;;;;;N;;;;; 1FAAB;LOW BATTERY;So;0;ON;;;;;N;;;;; 1FAAC;HAMSA;So;0;ON;;;;;N;;;;; +1FAAD;FOLDING HAND FAN;So;0;ON;;;;;N;;;;; +1FAAE;HAIR PICK;So;0;ON;;;;;N;;;;; +1FAAF;KHANDA;So;0;ON;;;;;N;;;;; 1FAB0;FLY;So;0;ON;;;;;N;;;;; 1FAB1;WORM;So;0;ON;;;;;N;;;;; 1FAB2;BEETLE;So;0;ON;;;;;N;;;;; @@ -33486,12 +33771,18 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAB8;CORAL;So;0;ON;;;;;N;;;;; 1FAB9;EMPTY NEST;So;0;ON;;;;;N;;;;; 1FABA;NEST WITH EGGS;So;0;ON;;;;;N;;;;; +1FABB;HYACINTH;So;0;ON;;;;;N;;;;; +1FABC;JELLYFISH;So;0;ON;;;;;N;;;;; +1FABD;WING;So;0;ON;;;;;N;;;;; +1FABF;GOOSE;So;0;ON;;;;;N;;;;; 1FAC0;ANATOMICAL HEART;So;0;ON;;;;;N;;;;; 1FAC1;LUNGS;So;0;ON;;;;;N;;;;; 1FAC2;PEOPLE HUGGING;So;0;ON;;;;;N;;;;; 1FAC3;PREGNANT MAN;So;0;ON;;;;;N;;;;; 1FAC4;PREGNANT PERSON;So;0;ON;;;;;N;;;;; 1FAC5;PERSON WITH CROWN;So;0;ON;;;;;N;;;;; +1FACE;MOOSE;So;0;ON;;;;;N;;;;; +1FACF;DONKEY;So;0;ON;;;;;N;;;;; 1FAD0;BLUEBERRIES;So;0;ON;;;;;N;;;;; 1FAD1;BELL PEPPER;So;0;ON;;;;;N;;;;; 1FAD2;OLIVE;So;0;ON;;;;;N;;;;; @@ -33502,6 +33793,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAD7;POURING LIQUID;So;0;ON;;;;;N;;;;; 1FAD8;BEANS;So;0;ON;;;;;N;;;;; 1FAD9;JAR;So;0;ON;;;;;N;;;;; +1FADA;GINGER ROOT;So;0;ON;;;;;N;;;;; +1FADB;PEA POD;So;0;ON;;;;;N;;;;; 1FAE0;MELTING FACE;So;0;ON;;;;;N;;;;; 1FAE1;SALUTING FACE;So;0;ON;;;;;N;;;;; 1FAE2;FACE WITH OPEN EYES AND HAND OVER MOUTH;So;0;ON;;;;;N;;;;; @@ -33510,6 +33803,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAE5;DOTTED LINE FACE;So;0;ON;;;;;N;;;;; 1FAE6;BITING LIP;So;0;ON;;;;;N;;;;; 1FAE7;BUBBLES;So;0;ON;;;;;N;;;;; +1FAE8;SHAKING FACE;So;0;ON;;;;;N;;;;; 1FAF0;HAND WITH INDEX FINGER AND THUMB CROSSED;So;0;ON;;;;;N;;;;; 1FAF1;RIGHTWARDS HAND;So;0;ON;;;;;N;;;;; 1FAF2;LEFTWARDS HAND;So;0;ON;;;;;N;;;;; @@ -33517,6 +33811,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAF4;PALM UP HAND;So;0;ON;;;;;N;;;;; 1FAF5;INDEX POINTING AT THE VIEWER;So;0;ON;;;;;N;;;;; 1FAF6;HEART HANDS;So;0;ON;;;;;N;;;;; +1FAF7;LEFTWARDS PUSHING HAND;So;0;ON;;;;;N;;;;; +1FAF8;RIGHTWARDS PUSHING HAND;So;0;ON;;;;;N;;;;; 1FB00;BLOCK SEXTANT-1;So;0;ON;;;;;N;;;;; 1FB01;BLOCK SEXTANT-2;So;0;ON;;;;;N;;;;; 1FB02;BLOCK SEXTANT-12;So;0;ON;;;;;N;;;;; @@ -33732,7 +34028,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 20000;;Lo;0;L;;;;;N;;;;; 2A6DF;;Lo;0;L;;;;;N;;;;; 2A700;;Lo;0;L;;;;;N;;;;; -2B738;;Lo;0;L;;;;;N;;;;; +2B739;;Lo;0;L;;;;;N;;;;; 2B740;;Lo;0;L;;;;;N;;;;; 2B81D;;Lo;0;L;;;;;N;;;;; 2B820;;Lo;0;L;;;;;N;;;;; @@ -34283,6 +34579,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 2FA1D;CJK COMPATIBILITY IDEOGRAPH-2FA1D;Lo;0;L;2A600;;;;N;;;;; 30000;;Lo;0;L;;;;;N;;;;; 3134A;;Lo;0;L;;;;;N;;;;; +31350;;Lo;0;L;;;;;N;;;;; +323AF;;Lo;0;L;;;;;N;;;;; E0001;LANGUAGE TAG;Cf;0;BN;;;;;N;;;;; E0020;TAG SPACE;Cf;0;BN;;;;;N;;;;; E0021;TAG EXCLAMATION MARK;Cf;0;BN;;;;;N;;;;; From 8c2fb269b674a3cd1b7fa884f56199ddde3cadbc Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 16 Sep 2022 14:26:28 +0700 Subject: [PATCH 125/132] fix(web): fixes unintended auto-acceptance of suggestion after reverting --- web/source/osk/banner.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/web/source/osk/banner.ts b/web/source/osk/banner.ts index e71a2e8f32..c4f28a6e19 100644 --- a/web/source/osk/banner.ts +++ b/web/source/osk/banner.ts @@ -560,13 +560,17 @@ namespace com.keyman.osk { private doAccept(suggestion: BannerSuggestion) { let _this = this; + // Selecting a suggestion or a reversion should both clear selection + // and clear the reversion-displaying state of the banner. + this.selected = null; + this.doRevert = false; + this.revertAcceptancePromise = suggestion.apply(); if(!this.revertAcceptancePromise) { // We get here either if suggestion acceptance fails or if it was a reversion. if(suggestion.suggestion && suggestion.suggestion.tag == 'revert') { // Reversion state management this.recentAccept = false; - this.doRevert = false; this.recentRevert = true; this.doUpdate(); @@ -581,9 +585,7 @@ namespace com.keyman.osk { } }); - this.selected = null; this.recentAccept = true; - this.doRevert = false; this.recentRevert = false; this.swallowPrediction = true; From 42675227e46150cb428f52088408f267b534efb1 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Fri, 16 Sep 2022 14:01:24 -0400 Subject: [PATCH 126/132] auto: increment master version to 16.0.66 --- HISTORY.md | 4 ++++ VERSION.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index ec8afd279d..cf756b6b07 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,9 @@ # Keyman Version History +## 16.0.65 alpha 2022-09-16 + +* chore(linux): Remove unused IBusLookupTable (#7296) + ## 16.0.64 alpha 2022-09-15 * fix(android/engine): Switch keyboard if uninstalling current one (#7291) diff --git a/VERSION.md b/VERSION.md index c3bbc393bb..039dedcefb 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.65 \ No newline at end of file +16.0.66 \ No newline at end of file From 93ec6106e144b0e8879aebe2c54604bd35a6e2dc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Sep 2022 21:06:23 +0000 Subject: [PATCH 127/132] chore(deps): bump nanoid and mocha Bumps [nanoid](https://github.com/ai/nanoid) and [mocha](https://github.com/mochajs/mocha). These dependencies needed to be updated together. Removes `nanoid` Updates `mocha` from 8.4.0 to 10.0.0 - [Release notes](https://github.com/mochajs/mocha/releases) - [Changelog](https://github.com/mochajs/mocha/blob/master/CHANGELOG.md) - [Commits](https://github.com/mochajs/mocha/compare/v8.4.0...v10.0.0) --- updated-dependencies: - dependency-name: nanoid dependency-type: indirect - dependency-name: mocha dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- common/models/templates/package.json | 2 +- common/models/wordbreakers/package.json | 2 +- common/predictive-text/package.json | 2 +- common/web/input-processor/package.json | 2 +- common/web/keyboard-processor/package.json | 2 +- common/web/lm-worker/package.json | 2 +- developer/src/kmlmc/package.json | 2 +- developer/src/server/package.json | 2 +- package-lock.json | 879 ++++++++------------- package.json | 2 +- web/package.json | 2 +- 11 files changed, 357 insertions(+), 542 deletions(-) diff --git a/common/models/templates/package.json b/common/models/templates/package.json index 3cc7ecebac..b9b84c6492 100644 --- a/common/models/templates/package.json +++ b/common/models/templates/package.json @@ -46,7 +46,7 @@ "@types/mocha": "^7.0.2", "@types/node": "^14.0.4", "chai": "^4.3.4", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "typescript": "^4.5.4" }, "dependencies": { diff --git a/common/models/wordbreakers/package.json b/common/models/wordbreakers/package.json index a7fd87a1f0..a4b97fc973 100644 --- a/common/models/wordbreakers/package.json +++ b/common/models/wordbreakers/package.json @@ -43,7 +43,7 @@ "@types/chai": "^4.2.11", "@types/mocha": "^7.0.2", "chai": "^4.3.4", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "ts-node": "^9.1.1", "typescript": "^4.5.4" }, diff --git a/common/predictive-text/package.json b/common/predictive-text/package.json index 81a92a4fa9..7557082ad5 100644 --- a/common/predictive-text/package.json +++ b/common/predictive-text/package.json @@ -37,7 +37,7 @@ "karma-mocha-reporter": "^2.2.5", "karma-safari-launcher": "^1.0.0", "karma-teamcity-reporter": "^1.1.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "sinon": "^7.1.1", "ts-node": "^9.1.1", diff --git a/common/web/input-processor/package.json b/common/web/input-processor/package.json index 9c9fc2d3ab..bf9e13cd41 100644 --- a/common/web/input-processor/package.json +++ b/common/web/input-processor/package.json @@ -21,7 +21,7 @@ "@keymanapp/resources-gosh": "*", "@types/node": "^11.9.4", "chai": "^4.3.4", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "ts-node": "^9.1.1", "typescript": "^4.5.4" diff --git a/common/web/keyboard-processor/package.json b/common/web/keyboard-processor/package.json index 95f7992a20..253715b3b0 100644 --- a/common/web/keyboard-processor/package.json +++ b/common/web/keyboard-processor/package.json @@ -19,7 +19,7 @@ "devDependencies": { "@keymanapp/resources-gosh": "*", "chai": "^4.3.4", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "ts-node": "^9.1.1", "typescript": "^4.5.4" diff --git a/common/web/lm-worker/package.json b/common/web/lm-worker/package.json index f0ebe95036..a93712c564 100644 --- a/common/web/lm-worker/package.json +++ b/common/web/lm-worker/package.json @@ -29,7 +29,7 @@ "karma-mocha-reporter": "^2.2.5", "karma-safari-launcher": "^1.0.0", "karma-teamcity-reporter": "^1.1.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "sinon": "^7.1.1", "ts-node": "^9.1.1", diff --git a/developer/src/kmlmc/package.json b/developer/src/kmlmc/package.json index 91b2583b72..843b1ea82a 100644 --- a/developer/src/kmlmc/package.json +++ b/developer/src/kmlmc/package.json @@ -52,7 +52,7 @@ "chai": "^4.3.4", "chalk": "^2.4.2", "jszip": "^3.7.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "ts-node": "^9.1.1" }, "mocha": { diff --git a/developer/src/server/package.json b/developer/src/server/package.json index ee9efbaa58..d6db75b7d6 100644 --- a/developer/src/server/package.json +++ b/developer/src/server/package.json @@ -34,7 +34,7 @@ "@types/ws": "^8.2.2", "chai": "^4.3.4", "copyfiles": "^2.4.1", - "mocha": "^9.1.4", + "mocha": "^10.0.0", "ts-node": "^10.4.0", "tsc-watch": "^4.5.0", "typescript": "^4.5.4" diff --git a/package-lock.json b/package-lock.json index 32b6461065..aa292021a4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ ], "devDependencies": { "chai": "^4.3.4", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "ts-node": "^9.1.1", "typescript": "^4.5.4" @@ -116,7 +116,7 @@ "@types/mocha": "^7.0.2", "@types/node": "^14.0.4", "chai": "^4.3.4", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "typescript": "^4.5.4" } }, @@ -144,7 +144,7 @@ "@types/chai": "^4.2.11", "@types/mocha": "^7.0.2", "chai": "^4.3.4", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "ts-node": "^9.1.1", "typescript": "^4.5.4" } @@ -184,7 +184,7 @@ "karma-mocha-reporter": "^2.2.5", "karma-safari-launcher": "^1.0.0", "karma-teamcity-reporter": "^1.1.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "sinon": "^7.1.1", "ts-node": "^9.1.1", @@ -222,7 +222,7 @@ "@keymanapp/resources-gosh": "*", "@types/node": "^11.9.4", "chai": "^4.3.4", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "ts-node": "^9.1.1", "typescript": "^4.5.4" @@ -240,7 +240,7 @@ "devDependencies": { "@keymanapp/resources-gosh": "*", "chai": "^4.3.4", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "ts-node": "^9.1.1", "typescript": "^4.5.4" @@ -308,7 +308,7 @@ "karma-mocha-reporter": "^2.2.5", "karma-safari-launcher": "^1.0.0", "karma-teamcity-reporter": "^1.1.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "sinon": "^7.1.1", "ts-node": "^9.1.1", @@ -454,7 +454,7 @@ "chai": "^4.3.4", "chalk": "^2.4.2", "jszip": "^3.7.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "ts-node": "^9.1.1" }, "engines": { @@ -558,7 +558,7 @@ "@types/ws": "^8.2.2", "chai": "^4.3.4", "copyfiles": "^2.4.1", - "mocha": "^9.1.4", + "mocha": "^10.0.0", "ts-node": "^10.4.0", "tsc-watch": "^4.5.0", "typescript": "^4.5.4" @@ -568,18 +568,6 @@ "node-windows-trayicon": "git+ssh://git@github.com/keymanapp/node-windows-trayicon.git#1e46786082213f3edcddd5953e33f5abdc7ea05f" } }, - "developer/src/server/node_modules/@cspotcode/source-map-support": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", - "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", - "dev": true, - "dependencies": { - "@jridgewell/trace-mapping": "0.3.9" - }, - "engines": { - "node": ">=12" - } - }, "developer/src/server/node_modules/@types/mocha": { "version": "9.1.1", "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-9.1.1.tgz", @@ -592,207 +580,6 @@ "integrity": "sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==", "dev": true }, - "developer/src/server/node_modules/chokidar": { - "version": "3.5.3", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", - "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", - "dev": true, - "funding": [ - { - "type": "individual", - "url": "https://paulmillr.com/funding/" - } - ], - "dependencies": { - "anymatch": "~3.1.2", - "braces": "~3.0.2", - "glob-parent": "~5.1.2", - "is-binary-path": "~2.1.0", - "is-glob": "~4.0.1", - "normalize-path": "~3.0.0", - "readdirp": "~3.6.0" - }, - "engines": { - "node": ">= 8.10.0" - }, - "optionalDependencies": { - "fsevents": "~2.3.2" - } - }, - "developer/src/server/node_modules/debug": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", - "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==", - "dev": true, - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "developer/src/server/node_modules/escape-string-regexp": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", - "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "developer/src/server/node_modules/glob": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", - "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", - "dev": true, - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "developer/src/server/node_modules/log-symbols": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", - "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", - "dev": true, - "dependencies": { - "chalk": "^4.1.0", - "is-unicode-supported": "^0.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "developer/src/server/node_modules/mocha": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-9.2.2.tgz", - "integrity": "sha512-L6XC3EdwT6YrIk0yXpavvLkn8h+EU+Y5UcCHKECyMbdUIxyMuZj4bX4U9e1nvnvUUvQVsV2VHQr5zLdcUkhW/g==", - "dev": true, - "dependencies": { - "@ungap/promise-all-settled": "1.1.2", - "ansi-colors": "4.1.1", - "browser-stdout": "1.3.1", - "chokidar": "3.5.3", - "debug": "4.3.3", - "diff": "5.0.0", - "escape-string-regexp": "4.0.0", - "find-up": "5.0.0", - "glob": "7.2.0", - "growl": "1.10.5", - "he": "1.2.0", - "js-yaml": "4.1.0", - "log-symbols": "4.1.0", - "minimatch": "4.2.1", - "ms": "2.1.3", - "nanoid": "3.3.1", - "serialize-javascript": "6.0.0", - "strip-json-comments": "3.1.1", - "supports-color": "8.1.1", - "which": "2.0.2", - "workerpool": "6.2.0", - "yargs": "16.2.0", - "yargs-parser": "20.2.4", - "yargs-unparser": "2.0.0" - }, - "bin": { - "_mocha": "bin/_mocha", - "mocha": "bin/mocha" - }, - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/mochajs" - } - }, - "developer/src/server/node_modules/mocha/node_modules/minimatch": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-4.2.1.tgz", - "integrity": "sha512-9Uq1ChtSZO+Mxa/CL1eGizn2vRn3MlLgzhT0Iz8zaY8NdvxvB0d5QdPFmCKf7JKA9Lerx5vRrnwO03jsSfGG9g==", - "dev": true, - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": ">=10" - } - }, - "developer/src/server/node_modules/mocha/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true - }, - "developer/src/server/node_modules/nanoid": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.1.tgz", - "integrity": "sha512-n6Vs/3KGyxPQd6uO0eH4Bv0ojGSUvuLlIHtC3Y0kEO23YRge8H9x1GCzLn28YX0H66pMkxuaeESFq4tKISKwdw==", - "dev": true, - "bin": { - "nanoid": "bin/nanoid.cjs" - }, - "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" - } - }, - "developer/src/server/node_modules/readdirp": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", - "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "dev": true, - "dependencies": { - "picomatch": "^2.2.1" - }, - "engines": { - "node": ">=8.10.0" - } - }, - "developer/src/server/node_modules/serialize-javascript": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz", - "integrity": "sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==", - "dev": true, - "dependencies": { - "randombytes": "^2.1.0" - } - }, - "developer/src/server/node_modules/supports-color": { - "version": "8.1.1", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", - "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", - "dev": true, - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/supports-color?sponsor=1" - } - }, "developer/src/server/node_modules/ts-node": { "version": "10.8.1", "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.8.1.tgz", @@ -845,12 +632,6 @@ "node": ">=0.3.1" } }, - "developer/src/server/node_modules/workerpool": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.2.0.tgz", - "integrity": "sha512-Rsk5qQHJ9eowMH28Jwhe8HEbmdYDX4lwoMWshiCXugjtHqMD9ZbiqSDLxcsfdqsETPzVUtX5s1Z5kStiIM6l4A==", - "dev": true - }, "node_modules/@actions/core": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.9.1.tgz", @@ -895,6 +676,18 @@ "node": ">=0.1.90" } }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/@jridgewell/resolve-uri": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.0.7.tgz", @@ -1812,7 +1605,7 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", - "devOptional": true, + "optional": true, "engines": { "node": ">=0.10.0" } @@ -2362,7 +2155,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", - "devOptional": true, + "optional": true, "engines": { "node": ">=0.10.0" } @@ -3527,15 +3320,6 @@ "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", "devOptional": true }, - "node_modules/growl": { - "version": "1.10.5", - "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", - "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", - "dev": true, - "engines": { - "node": ">=4.x" - } - }, "node_modules/har-schema": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", @@ -3751,7 +3535,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", - "devOptional": true, + "optional": true, "dependencies": { "number-is-nan": "^1.0.0" }, @@ -4314,18 +4098,6 @@ "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==" }, - "node_modules/log-symbols": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.0.0.tgz", - "integrity": "sha512-FN8JBzLx6CzeMrB0tg6pqlGU1wCrXW+ZXGH481kfsBqer0hToTIiHdjH4Mq8xJUbvATujKCvaREGWpGUionraA==", - "dev": true, - "dependencies": { - "chalk": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/log4js": { "version": "6.4.4", "resolved": "https://registry.npmjs.org/log4js/-/log4js-6.4.4.tgz", @@ -4546,43 +4318,40 @@ } }, "node_modules/mocha": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-8.4.0.tgz", - "integrity": "sha512-hJaO0mwDXmZS4ghXsvPVriOhsxQ7ofcpQdm8dE+jISUOKopitvnXFQmpRR7jd2K6VBG6E26gU3IAbXXGIbu4sQ==", + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.0.0.tgz", + "integrity": "sha512-0Wl+elVUD43Y0BqPZBzZt8Tnkw9CMUdNYnUsTfOM1vuhJVZL+kiesFYsqwBkEEuEixaiPe5ZQdqDgX2jddhmoA==", "dev": true, "dependencies": { "@ungap/promise-all-settled": "1.1.2", "ansi-colors": "4.1.1", "browser-stdout": "1.3.1", - "chokidar": "3.5.1", - "debug": "4.3.1", + "chokidar": "3.5.3", + "debug": "4.3.4", "diff": "5.0.0", "escape-string-regexp": "4.0.0", "find-up": "5.0.0", - "glob": "7.1.6", - "growl": "1.10.5", + "glob": "7.2.0", "he": "1.2.0", - "js-yaml": "4.0.0", - "log-symbols": "4.0.0", - "minimatch": "3.0.4", + "js-yaml": "4.1.0", + "log-symbols": "4.1.0", + "minimatch": "5.0.1", "ms": "2.1.3", - "nanoid": "3.1.20", - "serialize-javascript": "5.0.1", + "nanoid": "3.3.3", + "serialize-javascript": "6.0.0", "strip-json-comments": "3.1.1", "supports-color": "8.1.1", - "which": "2.0.2", - "wide-align": "1.1.3", - "workerpool": "6.1.0", + "workerpool": "6.2.1", "yargs": "16.2.0", "yargs-parser": "20.2.4", "yargs-unparser": "2.0.0" }, "bin": { "_mocha": "bin/_mocha", - "mocha": "bin/mocha" + "mocha": "bin/mocha.js" }, "engines": { - "node": ">= 10.12.0" + "node": ">= 14.0.0" }, "funding": { "type": "opencollective", @@ -4601,6 +4370,56 @@ "mocha": ">=6" } }, + "node_modules/mocha/node_modules/chokidar": { + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", + "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + ], + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/mocha/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/mocha/node_modules/debug/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, "node_modules/mocha/node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -4613,16 +4432,73 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/mocha/node_modules/js-yaml": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", - "integrity": "sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==", + "node_modules/mocha/node_modules/glob": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", "dev": true, "dependencies": { - "argparse": "^2.0.1" + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" }, - "bin": { - "js-yaml": "bin/js-yaml.js" + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/mocha/node_modules/glob/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/mocha/node_modules/log-symbols": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", + "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", + "dev": true, + "dependencies": { + "chalk": "^4.1.0", + "is-unicode-supported": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mocha/node_modules/minimatch": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.0.1.tgz", + "integrity": "sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==", + "dev": true, + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/mocha/node_modules/minimatch/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0" } }, "node_modules/mocha/node_modules/ms": { @@ -4631,6 +4507,39 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true }, + "node_modules/mocha/node_modules/nanoid": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.3.tgz", + "integrity": "sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==", + "dev": true, + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/mocha/node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/mocha/node_modules/serialize-javascript": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz", + "integrity": "sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==", + "dev": true, + "dependencies": { + "randombytes": "^2.1.0" + } + }, "node_modules/mocha/node_modules/supports-color": { "version": "8.1.1", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", @@ -4646,6 +4555,12 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, + "node_modules/mocha/node_modules/workerpool": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.2.1.tgz", + "integrity": "sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw==", + "dev": true + }, "node_modules/modernizr": { "version": "3.12.0", "resolved": "https://registry.npmjs.org/modernizr/-/modernizr-3.12.0.tgz", @@ -4864,18 +4779,6 @@ "node": ">= 0.10.0" } }, - "node_modules/nanoid": { - "version": "3.1.20", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.20.tgz", - "integrity": "sha512-a1cQNyczgKbLX9jwbS/+d7W8fX/RfgYR7lVWwWOGIPNgK2m0MWvrGF6/m4kk6U3QcFMnZf3RIhL0v2Jgh/0Uxw==", - "dev": true, - "bin": { - "nanoid": "bin/nanoid.cjs" - }, - "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" - } - }, "node_modules/negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -5138,7 +5041,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", - "devOptional": true, + "optional": true, "engines": { "node": ">=0.10.0" } @@ -5739,15 +5642,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, - "node_modules/serialize-javascript": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz", - "integrity": "sha512-SaaNal9imEO737H2c05Og0/8LUXG7EnsZyMa8MzkmuHoELfT6txuj0cMqRj6zfPKnmQ1yasR4PCJc8x+M4JSPA==", - "dev": true, - "dependencies": { - "randombytes": "^2.1.0" - } - }, "node_modules/serve-static": { "version": "1.14.2", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.2.tgz", @@ -6044,7 +5938,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", - "devOptional": true, + "optional": true, "dependencies": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -6068,7 +5962,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", - "devOptional": true, + "optional": true, "dependencies": { "ansi-regex": "^2.0.0" }, @@ -6540,7 +6434,7 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", - "devOptional": true, + "optional": true, "dependencies": { "string-width": "^1.0.2 || 2" } @@ -6559,12 +6453,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/workerpool": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.1.0.tgz", - "integrity": "sha512-toV7q9rWNYha963Pl/qyeZ6wG+3nnsyvolaNUS8+R5Wtw6qJPTxIlOP1ZSvcGhEJw+l3HMMmtiNo9Gl61G4GVg==", - "dev": true - }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -7101,7 +6989,7 @@ "karma-mocha-reporter": "^2.2.5", "karma-safari-launcher": "^1.0.0", "karma-teamcity-reporter": "^1.1.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "modernizr": "^3.11.7", "ts-node": "^9.1.1", "typescript": "^4.5.4" @@ -7152,6 +7040,15 @@ "integrity": "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==", "dev": true }, + "@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "requires": { + "@jridgewell/trace-mapping": "0.3.9" + } + }, "@jridgewell/resolve-uri": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.0.7.tgz", @@ -7345,7 +7242,7 @@ "copyfiles": "^2.4.1", "express": "^4.17.2", "hetrodo-node-hide-console-window-napi": "git+ssh://git@github.com/keymanapp/hetrodo-node-hide-console-window-napi.git#858b23036a9963b40ad6ff3c5bacd421e5839b92", - "mocha": "^9.1.4", + "mocha": "^10.0.0", "multer": "^1.4.4", "ngrok": "^4.2.2", "node-windows-trayicon": "git+ssh://git@github.com/keymanapp/node-windows-trayicon.git#1e46786082213f3edcddd5953e33f5abdc7ea05f", @@ -7356,15 +7253,6 @@ "ws": "^8.3.0" }, "dependencies": { - "@cspotcode/source-map-support": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", - "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", - "dev": true, - "requires": { - "@jridgewell/trace-mapping": "0.3.9" - } - }, "@types/mocha": { "version": "9.1.1", "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-9.1.1.tgz", @@ -7377,143 +7265,6 @@ "integrity": "sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==", "dev": true }, - "chokidar": { - "version": "3.5.3", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", - "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", - "dev": true, - "requires": { - "anymatch": "~3.1.2", - "braces": "~3.0.2", - "fsevents": "~2.3.2", - "glob-parent": "~5.1.2", - "is-binary-path": "~2.1.0", - "is-glob": "~4.0.1", - "normalize-path": "~3.0.0", - "readdirp": "~3.6.0" - } - }, - "debug": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", - "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, - "escape-string-regexp": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", - "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", - "dev": true - }, - "glob": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", - "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", - "dev": true, - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, - "log-symbols": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", - "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", - "dev": true, - "requires": { - "chalk": "^4.1.0", - "is-unicode-supported": "^0.1.0" - } - }, - "mocha": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-9.2.2.tgz", - "integrity": "sha512-L6XC3EdwT6YrIk0yXpavvLkn8h+EU+Y5UcCHKECyMbdUIxyMuZj4bX4U9e1nvnvUUvQVsV2VHQr5zLdcUkhW/g==", - "dev": true, - "requires": { - "@ungap/promise-all-settled": "1.1.2", - "ansi-colors": "4.1.1", - "browser-stdout": "1.3.1", - "chokidar": "3.5.3", - "debug": "4.3.3", - "diff": "5.0.0", - "escape-string-regexp": "4.0.0", - "find-up": "5.0.0", - "glob": "7.2.0", - "growl": "1.10.5", - "he": "1.2.0", - "js-yaml": "4.1.0", - "log-symbols": "4.1.0", - "minimatch": "4.2.1", - "ms": "2.1.3", - "nanoid": "3.3.1", - "serialize-javascript": "6.0.0", - "strip-json-comments": "3.1.1", - "supports-color": "8.1.1", - "which": "2.0.2", - "workerpool": "6.2.0", - "yargs": "16.2.0", - "yargs-parser": "20.2.4", - "yargs-unparser": "2.0.0" - }, - "dependencies": { - "minimatch": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-4.2.1.tgz", - "integrity": "sha512-9Uq1ChtSZO+Mxa/CL1eGizn2vRn3MlLgzhT0Iz8zaY8NdvxvB0d5QdPFmCKf7JKA9Lerx5vRrnwO03jsSfGG9g==", - "dev": true, - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true - } - } - }, - "nanoid": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.1.tgz", - "integrity": "sha512-n6Vs/3KGyxPQd6uO0eH4Bv0ojGSUvuLlIHtC3Y0kEO23YRge8H9x1GCzLn28YX0H66pMkxuaeESFq4tKISKwdw==", - "dev": true - }, - "readdirp": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", - "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "dev": true, - "requires": { - "picomatch": "^2.2.1" - } - }, - "serialize-javascript": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz", - "integrity": "sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==", - "dev": true, - "requires": { - "randombytes": "^2.1.0" - } - }, - "supports-color": { - "version": "8.1.1", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", - "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", - "dev": true, - "requires": { - "has-flag": "^4.0.0" - } - }, "ts-node": { "version": "10.8.1", "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.8.1.tgz", @@ -7542,12 +7293,6 @@ "dev": true } } - }, - "workerpool": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.2.0.tgz", - "integrity": "sha512-Rsk5qQHJ9eowMH28Jwhe8HEbmdYDX4lwoMWshiCXugjtHqMD9ZbiqSDLxcsfdqsETPzVUtX5s1Z5kStiIM6l4A==", - "dev": true } } }, @@ -7564,7 +7309,7 @@ "@types/node": "^11.9.4", "chai": "^4.3.4", "eventemitter3": "^4.0.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "ts-node": "^9.1.1", "typescript": "^4.5.4" @@ -7579,7 +7324,7 @@ "@keymanapp/web-utils": "*", "@types/node": "^11.9.4", "chai": "^4.3.4", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "ts-node": "^9.1.1", "typescript": "^4.5.4" @@ -7606,7 +7351,7 @@ "chalk": "^2.4.2", "commander": "^3.0.0", "jszip": "^3.7.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "ts-node": "^9.1.1", "typescript": "^4.5.4", "xml2js": "^0.4.19" @@ -7702,7 +7447,7 @@ "karma-mocha-reporter": "^2.2.5", "karma-safari-launcher": "^1.0.0", "karma-teamcity-reporter": "^1.1.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "sinon": "^7.1.1", "string.prototype.codepointat": "^0.2.1", @@ -7750,7 +7495,7 @@ "karma-mocha-reporter": "^2.2.5", "karma-safari-launcher": "^1.0.0", "karma-teamcity-reporter": "^1.1.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "sinon": "^7.1.1", "string.prototype.codepointat": "^0.2.1", @@ -7777,7 +7522,7 @@ "@types/mocha": "^7.0.2", "@types/node": "^14.0.4", "chai": "^4.3.4", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "typescript": "^4.5.4" }, "dependencies": { @@ -7803,7 +7548,7 @@ "@types/mocha": "^7.0.2", "@types/node": "^14.0.3", "chai": "^4.3.4", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "ts-node": "^9.1.1", "typescript": "^4.5.4" }, @@ -8585,7 +8330,7 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", - "devOptional": true + "optional": true }, "ansi-styles": { "version": "4.3.0", @@ -9050,7 +8795,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", - "devOptional": true + "optional": true }, "color-convert": { "version": "2.0.1", @@ -9969,12 +9714,6 @@ "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", "devOptional": true }, - "growl": { - "version": "1.10.5", - "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", - "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", - "dev": true - }, "har-schema": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", @@ -10134,7 +9873,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", - "devOptional": true, + "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -10564,7 +10303,7 @@ "karma-mocha-reporter": "^2.2.5", "karma-safari-launcher": "^1.0.0", "karma-teamcity-reporter": "^1.1.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "modernizr": "^3.11.7", "ts-node": "^9.1.1", "typescript": "^4.5.4" @@ -10632,15 +10371,6 @@ "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==" }, - "log-symbols": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.0.0.tgz", - "integrity": "sha512-FN8JBzLx6CzeMrB0tg6pqlGU1wCrXW+ZXGH481kfsBqer0hToTIiHdjH4Mq8xJUbvATujKCvaREGWpGUionraA==", - "dev": true, - "requires": { - "chalk": "^4.0.0" - } - }, "log4js": { "version": "6.4.4", "resolved": "https://registry.npmjs.org/log4js/-/log4js-6.4.4.tgz", @@ -10804,51 +10534,127 @@ } }, "mocha": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-8.4.0.tgz", - "integrity": "sha512-hJaO0mwDXmZS4ghXsvPVriOhsxQ7ofcpQdm8dE+jISUOKopitvnXFQmpRR7jd2K6VBG6E26gU3IAbXXGIbu4sQ==", + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.0.0.tgz", + "integrity": "sha512-0Wl+elVUD43Y0BqPZBzZt8Tnkw9CMUdNYnUsTfOM1vuhJVZL+kiesFYsqwBkEEuEixaiPe5ZQdqDgX2jddhmoA==", "dev": true, "requires": { "@ungap/promise-all-settled": "1.1.2", "ansi-colors": "4.1.1", "browser-stdout": "1.3.1", - "chokidar": "3.5.1", - "debug": "4.3.1", + "chokidar": "3.5.3", + "debug": "4.3.4", "diff": "5.0.0", "escape-string-regexp": "4.0.0", "find-up": "5.0.0", - "glob": "7.1.6", - "growl": "1.10.5", + "glob": "7.2.0", "he": "1.2.0", - "js-yaml": "4.0.0", - "log-symbols": "4.0.0", - "minimatch": "3.0.4", + "js-yaml": "4.1.0", + "log-symbols": "4.1.0", + "minimatch": "5.0.1", "ms": "2.1.3", - "nanoid": "3.1.20", - "serialize-javascript": "5.0.1", + "nanoid": "3.3.3", + "serialize-javascript": "6.0.0", "strip-json-comments": "3.1.1", "supports-color": "8.1.1", - "which": "2.0.2", - "wide-align": "1.1.3", - "workerpool": "6.1.0", + "workerpool": "6.2.1", "yargs": "16.2.0", "yargs-parser": "20.2.4", "yargs-unparser": "2.0.0" }, "dependencies": { + "chokidar": { + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", + "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", + "dev": true, + "requires": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "fsevents": "~2.3.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + } + }, + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + }, + "dependencies": { + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + } + } + }, "escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", "dev": true }, - "js-yaml": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", - "integrity": "sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==", + "glob": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", "dev": true, "requires": { - "argparse": "^2.0.1" + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "dependencies": { + "minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "requires": { + "brace-expansion": "^1.1.7" + } + } + } + }, + "log-symbols": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", + "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", + "dev": true, + "requires": { + "chalk": "^4.1.0", + "is-unicode-supported": "^0.1.0" + } + }, + "minimatch": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.0.1.tgz", + "integrity": "sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==", + "dev": true, + "requires": { + "brace-expansion": "^2.0.1" + }, + "dependencies": { + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0" + } + } } }, "ms": { @@ -10857,6 +10663,30 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true }, + "nanoid": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.3.tgz", + "integrity": "sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==", + "dev": true + }, + "readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "requires": { + "picomatch": "^2.2.1" + } + }, + "serialize-javascript": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz", + "integrity": "sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==", + "dev": true, + "requires": { + "randombytes": "^2.1.0" + } + }, "supports-color": { "version": "8.1.1", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", @@ -10865,6 +10695,12 @@ "requires": { "has-flag": "^4.0.0" } + }, + "workerpool": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.2.1.tgz", + "integrity": "sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw==", + "dev": true } } }, @@ -11047,12 +10883,6 @@ "xtend": "^4.0.0" } }, - "nanoid": { - "version": "3.1.20", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.20.tgz", - "integrity": "sha512-a1cQNyczgKbLX9jwbS/+d7W8fX/RfgYR7lVWwWOGIPNgK2m0MWvrGF6/m4kk6U3QcFMnZf3RIhL0v2Jgh/0Uxw==", - "dev": true - }, "negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -11264,7 +11094,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", - "devOptional": true + "optional": true }, "oauth-sign": { "version": "0.9.0", @@ -11711,15 +11541,6 @@ } } }, - "serialize-javascript": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz", - "integrity": "sha512-SaaNal9imEO737H2c05Og0/8LUXG7EnsZyMa8MzkmuHoELfT6txuj0cMqRj6zfPKnmQ1yasR4PCJc8x+M4JSPA==", - "dev": true, - "requires": { - "randombytes": "^2.1.0" - } - }, "serve-static": { "version": "1.14.2", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.2.tgz", @@ -11955,7 +11776,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", - "devOptional": true, + "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -11976,7 +11797,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", - "devOptional": true, + "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -12324,7 +12145,7 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", - "devOptional": true, + "optional": true, "requires": { "string-width": "^1.0.2 || 2" } @@ -12337,12 +12158,6 @@ "execa": "^1.0.0" } }, - "workerpool": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.1.0.tgz", - "integrity": "sha512-toV7q9rWNYha963Pl/qyeZ6wG+3nnsyvolaNUS8+R5Wtw6qJPTxIlOP1ZSvcGhEJw+l3HMMmtiNo9Gl61G4GVg==", - "dev": true - }, "wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", diff --git a/package.json b/package.json index 291c69d225..df60de59f8 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "private": true, "devDependencies": { "chai": "^4.3.4", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "ts-node": "^9.1.1", "typescript": "^4.5.4" diff --git a/web/package.json b/web/package.json index 13622917c3..8cd671bf35 100644 --- a/web/package.json +++ b/web/package.json @@ -36,7 +36,7 @@ "karma-mocha-reporter": "^2.2.5", "karma-safari-launcher": "^1.0.0", "karma-teamcity-reporter": "^1.1.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "modernizr": "^3.11.7", "ts-node": "^9.1.1", "typescript": "^4.5.4" From df78d25bb333cbbd8bbca6b9dd6fab1362dc271f Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Sat, 17 Sep 2022 14:01:38 -0400 Subject: [PATCH 128/132] auto: increment master version to 16.0.67 --- HISTORY.md | 4 ++++ VERSION.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index cf756b6b07..caaddcb686 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,9 @@ # Keyman Version History +## 16.0.66 alpha 2022-09-17 + +* chore: improve auto labeling (#7288) + ## 16.0.65 alpha 2022-09-16 * chore(linux): Remove unused IBusLookupTable (#7296) diff --git a/VERSION.md b/VERSION.md index 039dedcefb..b90f7346b3 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.66 \ No newline at end of file +16.0.67 \ No newline at end of file From 744354e2a1f75811acf8c7e9add04c6ecc0d3152 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 19 Sep 2022 08:52:50 +0700 Subject: [PATCH 129/132] fix(common/models): fixes reference dropped by git merge --- common/web/lm-worker/src/model-compositor.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/common/web/lm-worker/src/model-compositor.ts b/common/web/lm-worker/src/model-compositor.ts index 5a314f0644..95c2e390fd 100644 --- a/common/web/lm-worker/src/model-compositor.ts +++ b/common/web/lm-worker/src/model-compositor.ts @@ -162,8 +162,9 @@ class ModelCompositor { // The amount of text to 'replace' depends upon whatever sort of context change occurs // from the received input. - let postContextLength = postContextState.tokens.length; - let contextLengthDelta = postContextState.tokens.length - contextState.tokens.length; + const postContextTokens = postContextState.tokens; + let postContextLength = postContextTokens.length; + let contextLengthDelta = postContextTokens.length - contextState.tokens.length; // If the context now has more tokens, the token we'll be 'predicting' didn't originally exist. if(postContextLength == 0 || contextLengthDelta > 0) { // As the word/token being corrected/predicted didn't originally exist, there's no @@ -225,7 +226,7 @@ class ModelCompositor { // // NOTE: we only want this applied word-initially, when any corrections 'correct' // 100% of the word. Things are generally fine once it's not "all or nothing." - let tailToken = contextTokens[contextTokens.length - 1]; + let tailToken = postContextTokens[postContextTokens.length - 1]; const isTokenStart = tailToken.transformDistributions.length <= 1; // TODO: whitespace, backspace filtering. Do it here. From 466b58768f583ef45f1142f213a41e610ea7b206 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 19 Sep 2022 06:39:41 +1000 Subject: [PATCH 130/132] chore(common): update auto labeler configuration --- .github/labeler.yml | 39 +++++++++++++-------------------------- 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index e0abc29f2f..d831af12b6 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -5,15 +5,15 @@ # common ones. The others are commented out. There is still some variance between # folder names and labels; consider this documentation of that ;-) +docs: docs/** + # # Add labels based on changed files using actions/labeler # android/: android/** android/app/: android/KMAPro/** -#android/browser/: android/engine/: android/KMEA/** -#android/resources/: android/samples/: android/Samples/** common/: @@ -33,41 +33,34 @@ core/: developer/: - developer/** - - windows/src/developer/** developer/compilers/: - - windows/src/developer/kmcomp/** - - developer/src/kmlmc/** + - developer/src/kmcomp/** + - developer/src/kmcmpdll/** + - developer/src/kmc/** + - developer/src/kmc-*/** developer/ide/: - - windows/src/developer/TIKE/** + - developer/src/server/** + - developer/src/tike/** -# developer/resources/ -# developer/tools/ ios/: ios/** ios/app/: ios/keyman/** -# ios/browser/ ios/engine/: ios/engine/** -# ios/resources/ ios/samples/: ios/samples/** linux/: linux/** linux/config/: linux/keyman-config/** linux/engine/: - linux/ibus-keyman/** - - linux/ibus-kmfl/** - - linux/kmflcomp/** - - linux/libkmfl/** - - linux/scim_kmfl_imengine/** -# linux/resources/ -# linux/samples/ + - linux/legacy/ibus-kmfl/** + - linux/legacy/kmflcomp/** + - linux/legacy/libkmfl/** mac/: mac/** # mac/config/: -mac/engine/: mac/** -# mac/resources/ -# mac/samples/ +# mac/engine/: mac/** oem/: oem/** oem/fv/: oem/firstvoices/** @@ -78,15 +71,9 @@ oem/fv/windows/: oem/firstvoices/windows/** web/: web/** # web/bookmarklet/ web/engine/: web/source/** -# web/resources/ web/ui/: web/source/kmwui* web/samples/: web/samples/** -# Somewhat messy since we try and exclude Developer :) -windows/: -- any: ['windows/**', '!windows/src/developer/**'] - +windows/: windows/** windows/config/: windows/src/desktop/** windows/engine/: windows/src/engine/** -# windows/resources/ -# windows/samples/ From a4ca7968b9b0800568d9e52d3a3e304e6ddfd681 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 19 Sep 2022 06:39:13 +1000 Subject: [PATCH 131/132] chore(common): make scripts executable and add pre-commit test --- common/windows/cef-checkout.sh | 0 common/windows/mkver.sh | 0 resources/build/build-utils-ci.test.sh | 0 resources/devbox/iis-https/setup-iis-https.sh | 0 resources/devbox/macos/keyman.macos.env.sh | 0 resources/git-hooks/pre-commit | 19 +++++++++++++++++++ windows/src/desktop/locale/crowdin-control.sh | 0 .../i18n-check-unused-strings.sh | 0 8 files changed, 19 insertions(+) mode change 100644 => 100755 common/windows/cef-checkout.sh mode change 100644 => 100755 common/windows/mkver.sh mode change 100644 => 100755 resources/build/build-utils-ci.test.sh mode change 100644 => 100755 resources/devbox/iis-https/setup-iis-https.sh mode change 100644 => 100755 resources/devbox/macos/keyman.macos.env.sh create mode 100644 resources/git-hooks/pre-commit mode change 100644 => 100755 windows/src/desktop/locale/crowdin-control.sh mode change 100644 => 100755 windows/src/test/manual-tests/i18n-check-unused-strings/i18n-check-unused-strings.sh diff --git a/common/windows/cef-checkout.sh b/common/windows/cef-checkout.sh old mode 100644 new mode 100755 diff --git a/common/windows/mkver.sh b/common/windows/mkver.sh old mode 100644 new mode 100755 diff --git a/resources/build/build-utils-ci.test.sh b/resources/build/build-utils-ci.test.sh old mode 100644 new mode 100755 diff --git a/resources/devbox/iis-https/setup-iis-https.sh b/resources/devbox/iis-https/setup-iis-https.sh old mode 100644 new mode 100755 diff --git a/resources/devbox/macos/keyman.macos.env.sh b/resources/devbox/macos/keyman.macos.env.sh old mode 100644 new mode 100755 diff --git a/resources/git-hooks/pre-commit b/resources/git-hooks/pre-commit new file mode 100644 index 0000000000..3fd01c7d02 --- /dev/null +++ b/resources/git-hooks/pre-commit @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# +# We want to make sure that .sh files are executable; however, .inc.sh need not +# be as they are always source-included in scripts. +# +SH_NON_EXECUTABLE=`git ls-files --stage | grep "\\.sh$" | grep -v "\\.inc\\.sh$" | grep -v 755 | cut -f 2 -` + +if [ ! -z "$SH_NON_EXECUTABLE" ]; then + echo "ERROR: The following scripts are not marked as executable:" + echo + echo "$SH_NON_EXECUTABLE" + echo + echo "You can mark them as executable with:" + echo " git add --chmod=+x" + exit 1 +fi + +exit 0 diff --git a/windows/src/desktop/locale/crowdin-control.sh b/windows/src/desktop/locale/crowdin-control.sh old mode 100644 new mode 100755 diff --git a/windows/src/test/manual-tests/i18n-check-unused-strings/i18n-check-unused-strings.sh b/windows/src/test/manual-tests/i18n-check-unused-strings/i18n-check-unused-strings.sh old mode 100644 new mode 100755 From 9168d896e7cac18cc60c69ad4188cfbd864ca0b0 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 19 Sep 2022 14:47:37 +1000 Subject: [PATCH 132/132] chore(common): remove unused ncc dependency The `build:dist` script was never used; increment-version.sh uses only `build:ts`. --- package-lock.json | 26 ++++---------------------- resources/build/version/package.json | 7 +------ 2 files changed, 5 insertions(+), 28 deletions(-) diff --git a/package-lock.json b/package-lock.json index 32b6461065..59a165ca28 100644 --- a/package-lock.json +++ b/package-lock.json @@ -564,8 +564,8 @@ "typescript": "^4.5.4" }, "optionalDependencies": { - "hetrodo-node-hide-console-window-napi": "git+ssh://git@github.com/keymanapp/hetrodo-node-hide-console-window-napi.git#858b23036a9963b40ad6ff3c5bacd421e5839b92", - "node-windows-trayicon": "git+ssh://git@github.com/keymanapp/node-windows-trayicon.git#1e46786082213f3edcddd5953e33f5abdc7ea05f" + "hetrodo-node-hide-console-window-napi": "keymanapp/hetrodo-node-hide-console-window-napi#keyman-15.0", + "node-windows-trayicon": "keymanapp/node-windows-trayicon#keyman-15.0" } }, "developer/src/server/node_modules/@cspotcode/source-map-support": { @@ -1723,16 +1723,6 @@ "integrity": "sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q==", "dev": true }, - "node_modules/@zeit/ncc": { - "version": "0.21.1", - "resolved": "https://registry.npmjs.org/@zeit/ncc/-/ncc-0.21.1.tgz", - "integrity": "sha512-M9WzgquSOt2nsjRkYM9LRylBLmmlwNCwYbm3Up3PDEshfvdmIfqpFNSK8EJvR18NwZjGHE5z2avlDtYQx2JQnw==", - "deprecated": "@zeit/ncc is no longer maintained. Please use @vercel/ncc instead.", - "dev": true, - "bin": { - "ncc": "dist/ncc/cli.js" - } - }, "node_modules/abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -6863,7 +6853,6 @@ "devDependencies": { "@types/node": "^13.7.0", "@types/semver": "^7.1.0", - "@zeit/ncc": "^0.21.0", "semver": "^7.1.2", "ts-node": "^8.6.2" }, @@ -7181,7 +7170,6 @@ "@actions/github": "^2.1.0", "@types/node": "^13.7.0", "@types/semver": "^7.1.0", - "@zeit/ncc": "^0.21.0", "semver": "^7.1.2", "ts-node": "^8.6.2", "typescript": "^4.5.4", @@ -7344,11 +7332,11 @@ "chalk": "^4.1.2", "copyfiles": "^2.4.1", "express": "^4.17.2", - "hetrodo-node-hide-console-window-napi": "git+ssh://git@github.com/keymanapp/hetrodo-node-hide-console-window-napi.git#858b23036a9963b40ad6ff3c5bacd421e5839b92", + "hetrodo-node-hide-console-window-napi": "keymanapp/hetrodo-node-hide-console-window-napi#keyman-15.0", "mocha": "^9.1.4", "multer": "^1.4.4", "ngrok": "^4.2.2", - "node-windows-trayicon": "git+ssh://git@github.com/keymanapp/node-windows-trayicon.git#1e46786082213f3edcddd5953e33f5abdc7ea05f", + "node-windows-trayicon": "keymanapp/node-windows-trayicon#keyman-15.0", "open": "^8.4.0", "ts-node": "^10.4.0", "tsc-watch": "^4.5.0", @@ -8522,12 +8510,6 @@ "integrity": "sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q==", "dev": true }, - "@zeit/ncc": { - "version": "0.21.1", - "resolved": "https://registry.npmjs.org/@zeit/ncc/-/ncc-0.21.1.tgz", - "integrity": "sha512-M9WzgquSOt2nsjRkYM9LRylBLmmlwNCwYbm3Up3PDEshfvdmIfqpFNSK8EJvR18NwZjGHE5z2avlDtYQx2JQnw==", - "dev": true - }, "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", diff --git a/resources/build/version/package.json b/resources/build/version/package.json index 33698434fb..5902dce8cc 100644 --- a/resources/build/version/package.json +++ b/resources/build/version/package.json @@ -9,7 +9,6 @@ "devDependencies": { "@types/node": "^13.7.0", "@types/semver": "^7.1.0", - "@zeit/ncc": "^0.21.0", "semver": "^7.1.2", "ts-node": "^8.6.2" }, @@ -17,20 +16,16 @@ "node": "16" }, "files": [ - "dist", "src" ], "license": "MIT", - "main": "dist/index.js", + "main": "lib/index.js", "name": "@keymanapp/auto-history-action", "private": true, "scripts": { - "build": "npm run clean:dist; npm run build:dist", - "build:dist": "ncc build src/index.ts --minify --source-map --v8-cache", "build:ts": "tsc --project tsconfig.production.json", "build:ts:watch": "tsc --project tsconfig.production.json --watch", "clean": "run-p clean:*", - "clean:dist": "rm -rf dist", "clean:lib": "rm -rf lib", "types": "tsc --noEmit" }