diff --git a/.github/labeler.yml b/.github/labeler.yml index 70068cd1d5..d831af12b6 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -5,20 +5,19 @@ # common ones. The others are commented out. There is still some variance between # folder names and labels; consider this documentation of that ;-) +docs: docs/** + # # Add labels based on changed files using actions/labeler # android/: android/** android/app/: android/KMAPro/** -#android/browser/: android/engine/: android/KMEA/** -#android/resources/: android/samples/: android/Samples/** common/: - common/** - - core/** - resources/** common/models/: common/models/** @@ -29,46 +28,39 @@ common/models/wordbreakers/: common/models/wordbreakers/** common/resources/: resources/** common/web/: common/web/** -common/core/: +core/: - core/** developer/: - developer/** - - windows/src/developer/** developer/compilers/: - - windows/src/developer/kmcomp/** - - developer/src/kmlmc/** + - developer/src/kmcomp/** + - developer/src/kmcmpdll/** + - developer/src/kmc/** + - developer/src/kmc-*/** developer/ide/: - - windows/src/developer/TIKE/** + - developer/src/server/** + - developer/src/tike/** -# developer/resources/ -# developer/tools/ ios/: ios/** ios/app/: ios/keyman/** -# ios/browser/ ios/engine/: ios/engine/** -# ios/resources/ ios/samples/: ios/samples/** linux/: linux/** linux/config/: linux/keyman-config/** linux/engine/: - linux/ibus-keyman/** - - linux/ibus-kmfl/** - - linux/kmflcomp/** - - linux/libkmfl/** - - linux/scim_kmfl_imengine/** -# linux/resources/ -# linux/samples/ + - linux/legacy/ibus-kmfl/** + - linux/legacy/kmflcomp/** + - linux/legacy/libkmfl/** mac/: mac/** # mac/config/: -mac/engine/: mac/** -# mac/resources/ -# mac/samples/ +# mac/engine/: mac/** oem/: oem/** oem/fv/: oem/firstvoices/** @@ -79,15 +71,9 @@ oem/fv/windows/: oem/firstvoices/windows/** web/: web/** # web/bookmarklet/ web/engine/: web/source/** -# web/resources/ web/ui/: web/source/kmwui* web/samples/: web/samples/** -# Somewhat messy since we try and exclude Developer :) -windows/: -- any: ['windows/**', '!windows/src/developer/**'] - +windows/: windows/** windows/config/: windows/src/desktop/** windows/engine/: windows/src/engine/** -# windows/resources/ -# windows/samples/ diff --git a/.github/multi-labeler.yml b/.github/multi-labeler.yml index 1278db2a28..ffab4f5572 100644 --- a/.github/multi-labeler.yml +++ b/.github/multi-labeler.yml @@ -11,6 +11,10 @@ version: v1 # labels: + # + # conventional commit / semantic PR styles + # + - label: 'feat' matcher: title: '^feat(\(|:)' @@ -20,26 +24,48 @@ labels: - label: 'chore' matcher: title: '^chore(\(|:)' - # "change", - label: 'docs' matcher: title: '^docs(\(|:)' - # "style", - label: 'refactor' matcher: title: '^refactor(\(|:)' - # "test", - label: 'auto' matcher: title: '^auto(\(|:)' - # Below are the scopes that we look for in the PR title + # + # additional meta flags + # + + # stable-targeted patches; note, this does not pick up chained PRs automatically + - label: 'stable' + matcher: + baseBranch: '^stable-.+' + + # PRs marked as cherry-picks by title + - label: 'cherry-pick' + matcher: + title: '(๐Ÿ’|:cherries:)' + + # long-lived feature branches + - label: 'feature-branch' + matcher: + branch: '^feature-.+' + + # + # Scopes that we look for in the PR title + # + - label: 'android/' matcher: title: '\(.*android.*\):' - label: 'common/' matcher: title: '\(.*common.*\):' + - label: 'core/' + matcher: + title: '\(.*core.*\):' - label: 'developer/' matcher: title: '\(.*developer.*\):' @@ -62,6 +88,10 @@ labels: matcher: title: '\(.*windows.*\):' - - label: 'cherry-pick' + # + # epics -- we will add/remove these as we work on new epics each release + # + + - label: 'epic-ldml' matcher: - title: '(๐Ÿ’|:cherries:)' + branch: '.*epic-ldml.*' # anywhere in the branch name, e.g. feat/epic-ldml/developer/... or feat/developer/foo-epic-ldml diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 1c8704da25..6faa005e70 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -12,7 +12,7 @@ jobs: repo-token: "${{ secrets.GITHUB_TOKEN }}" - name: Update labels based on PR title id: labeler - uses: fuxingloh/multi-labeler@8afa186ed03230c98fe24ebf9fe35093072ad46e # v1.4.0 + uses: fuxingloh/multi-labeler@fb9bc28b2d65e406ffd208384c5095793c3fd59a # v1.8.0 with: github-token: ${{secrets.GITHUB_TOKEN}} config-path: .github/multi-labeler.yml diff --git a/HISTORY.md b/HISTORY.md index 775866a8cf..caaddcb686 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,105 @@ # Keyman Version History +## 16.0.66 alpha 2022-09-17 + +* chore: improve auto labeling (#7288) + +## 16.0.65 alpha 2022-09-16 + +* chore(linux): Remove unused IBusLookupTable (#7296) + +## 16.0.64 alpha 2022-09-15 + +* fix(android/engine): Switch keyboard if uninstalling current one (#7291) +* fix(common/models): fixes quote-adjacent pred-text suggestions (#7205) +* fix(common/models): max prediction wait check (#7290) + +## 16.0.63 alpha 2022-09-14 + +* chore(linux): Update debian changelog (#7281) +* fix(linux): Fix ignored error (#7284) + +## 16.0.62 alpha 2022-09-13 + +* fix(developer): hide key-sizes when in desktop layout in touch layout editor (#7225) +* fix(developer): show more useful error if out of space during Setup (#7267) + +## 16.0.61 alpha 2022-09-12 + +* docs(windows): add steps for using testhost debugging (#7263) +* fix(developer): compiler mismatch on currentLine (#7190) +* fix(developer): suppress repeated warnings about unreachable code (#7219) +* chore: try disabling concurrency for browserstack tests (#7258) +* chore(web): disable browserstack on non-web-specific builds (#7260) + +## 16.0.60 alpha 2022-09-10 + +* fix(web): enhanced timer for prediction algorithm (#7037) +* chore(core): fixup km_kbp_event docs (#7253) +* fix(windows) Update unit tests for tsf bkspace (#7254) +* fix(android): Standardize language ID in language picker menu (#7239) + +## 16.0.59 alpha 2022-09-09 + +* chore: use keyman.com instead of keyman-staging.com (#7233) +* feat(core): add `km_kbp_event` API endpoint (#7223) +* fix(windows): Delete both code units when deleting surrogate pairs in TSF-aware apps (#7243) + +## 16.0.58 alpha 2022-09-08 + +* fix(common/models): reconnects unit tests for worker-internal submodules (#7215) +* feat(common/models): extra Unicode-based wordbreaker unit tests (#7217) + +## 16.0.57 alpha 2022-09-07 + +* fix(android/engine): Fix Backspace key to delete without errant subkeys (#7156) + +## 16.0.56 alpha 2022-09-05 + +* chore(linux): Fix ibus-keyman.postinst script (#7192) + +## 16.0.55 alpha 2022-09-03 + +* chore(android): Reduce Toast notification noise (#7178) + +## 16.0.54 alpha 2022-08-30 + +* chore(core): rename json.hpp to jsonpp.hpp (#6993) +* chore(developer): remove unused dependencies from KeymanWeb compiler (#7000) +* chore: update core/ label (#7138) +* chore(linux): Update debian changelog (#7145) +* fix(linux): Allow downgrades for installing build deps (#7147) +* chore(core): emcc off path for linux (#7149) + +## 16.0.53 alpha 2022-08-29 + +* feat(linux): Dockerfile for linux builder (#7133) + +## 16.0.52 alpha 2022-08-26 + +* fix(android/engine): Lower the max height for landscape orientation (#7119) +* fix(linux): Remove wrong `ok_for_single_backspace` method (#7123) + +## 16.0.51 alpha 2022-08-24 + +* fix(android): verify browser before starting activity (#7001) +* chore: Change platform advocates per discussion (#7096) +* fix(windows): Add invalidate context action to non-updatable parse (#7089) +* feat(common): add parameter variable support to `builder_` functions (#7103) + +## 16.0.50 alpha 2022-08-23 + +* chore(core): Remove obsolete python keyboardprocessor (#7094) + +## 16.0.49 alpha 2022-08-22 + +* fix: remove saving and restoring context kbd options (#7077) +* chore(deps): bump @actions/core from 1.8.2 to 1.9.1 (#7087) + +## 16.0.48 alpha 2022-08-16 + +* fix(web): button, float init timer cleanup (#7036) + ## 16.0.47 alpha 2022-08-15 * chore(core): refactor kmx_file.h to common (#7062) diff --git a/VERSION.md b/VERSION.md index 84f71677bf..b90f7346b3 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -16.0.48 \ No newline at end of file +16.0.67 \ No newline at end of file diff --git a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/KMPBrowserActivity.java b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/KMPBrowserActivity.java index 7cff5b5e80..fe217c57e4 100644 --- a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/KMPBrowserActivity.java +++ b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/KMPBrowserActivity.java @@ -18,6 +18,7 @@ import android.webkit.WebChromeClient; import android.webkit.WebSettings; import android.webkit.WebView; import android.webkit.WebViewClient; +import android.widget.Toast; import androidx.appcompat.app.AppCompatActivity; import com.tavultesoft.kmea.BaseActivity; @@ -113,7 +114,11 @@ public class KMPBrowserActivity extends BaseActivity { // All links that aren't internal Keyman keyboard links open in user's browser Intent intent = new Intent(Intent.ACTION_VIEW, uri); - startActivity(intent); + if (intent.resolveActivity(getPackageManager()) != null) { + startActivity(intent); + } else { + Toast.makeText(context, getString(R.string.unable_to_open_browser), Toast.LENGTH_SHORT).show(); + } return true; } if (lowerURL.startsWith("keyman:")) { diff --git a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/KeyboardSettingsActivity.java b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/KeyboardSettingsActivity.java index 8aee2a6acb..dc5519095a 100644 --- a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/KeyboardSettingsActivity.java +++ b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/KeyboardSettingsActivity.java @@ -26,6 +26,7 @@ import android.widget.ListAdapter; import android.widget.ListView; import android.widget.SimpleAdapter; import android.widget.TextView; +import android.widget.Toast; import com.tavultesoft.kmea.ConfirmDialogFragment; import com.tavultesoft.kmea.KMHelpFileActivity; @@ -160,7 +161,11 @@ public final class KeyboardSettingsActivity extends AppCompatActivity { Bundle args = kbd.buildDownloadBundle(); Intent i = new Intent(getApplicationContext(), KMKeyboardDownloaderActivity.class); i.putExtras(args); - startActivity(i); + if (i.resolveActivity(getPackageManager()) != null) { + startActivity(i); + } else { + Toast.makeText(getApplicationContext(), getString(R.string.unable_to_open_browser), Toast.LENGTH_SHORT).show(); + } finish(); // "Help" link clicked diff --git a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/MainActivity.java b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/MainActivity.java index 6f3affe4fb..2d2710142a 100644 --- a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/MainActivity.java +++ b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/MainActivity.java @@ -762,10 +762,24 @@ public class MainActivity extends BaseActivity implements OnKeyboardEventListene // Launch PlayStore to update Chrome try { Intent intent = new Intent(Intent.ACTION_VIEW, Uri.parse("market://details?id=com.android.chrome")); - startActivity(intent); + if (intent.resolveActivity(getPackageManager()) != null) { + startActivity(intent); + } else { + intent = new Intent(Intent.ACTION_VIEW, Uri.parse("https://play.google.com/store/apps/details?id=com.android.chrome")); + if (intent.resolveActivity(getPackageManager()) != null) { + startActivity(intent); + } else { + Toast.makeText(getApplicationContext(), getString(R.string.unable_to_open_browser), Toast.LENGTH_SHORT).show(); + } + } } catch (android.content.ActivityNotFoundException e) { // Link to Chrome if user is not signed in to Play Store - startActivity(new Intent(Intent.ACTION_VIEW, Uri.parse("https://play.google.com/store/apps/details?id=com.android.chrome"))); + Intent intent = new Intent(Intent.ACTION_VIEW, Uri.parse("https://play.google.com/store/apps/details?id=com.android.chrome")); + if (intent.resolveActivity(getPackageManager()) != null) { + startActivity(intent); + } else { + Toast.makeText(getApplicationContext(), getString(R.string.unable_to_open_browser), Toast.LENGTH_SHORT).show(); + } } } }); @@ -979,10 +993,6 @@ public class MainActivity extends BaseActivity implements OnKeyboardEventListene String _downloadid = CloudLexicalModelMetaDataDownloadCallback.createDownloadId(languageID); CloudLexicalModelMetaDataDownloadCallback _callback = new CloudLexicalModelMetaDataDownloadCallback(); - Toast.makeText(context, - context.getString(R.string.query_associated_model), - Toast.LENGTH_SHORT).show(); - ArrayList aPreparedCloudApiParams = new ArrayList<>(); String url = CloudRepository.prepareLexicalModelQuery(languageID); aPreparedCloudApiParams.add(new CloudApiTypes.CloudApiParam( diff --git a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/SelectLanguageFragment.java b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/SelectLanguageFragment.java index c1128f9966..da53c2000f 100644 --- a/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/SelectLanguageFragment.java +++ b/android/KMAPro/kMAPro/src/main/java/com/tavultesoft/kmapro/SelectLanguageFragment.java @@ -32,6 +32,7 @@ import com.tavultesoft.kmea.util.KMLog; import org.json.JSONObject; import java.io.File; +import java.security.InvalidParameterException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -190,6 +191,33 @@ public final class SelectLanguageFragment extends Fragment implements BlockingSt listView.setOnItemClickListener(new AdapterView.OnItemClickListener() { + /** + * Utility to modify keyboardList. Based on BCP47.toggleLanguage + * If keyboard's languageID exists in the list, remove the keyboard. + * Otherwise, add the keyboard to the list. + * @param keyboardList + * @param k + */ + private void toggleLanguage(ArrayList keyboardList, Keyboard k) { + if (keyboardList == null) { + throw new InvalidParameterException("keyboardList must not be null"); + } + if (k == null) { + throw new InvalidParameterException("keyboard must not be null"); + } + + // See if languageID already exists in the keyboardList + for (Keyboard l: keyboardList) { + if (BCP47.languageEquals(l.getLanguageID(), k.getLanguageID())) { + keyboardList.remove(l); + return; + } + } + + k.setLanguage(k.getLanguageID().toLowerCase(), k.getLanguageName()); + keyboardList.add(k); + } + @Override public void onItemClick(AdapterView parent, View view, int position, long id) { Keyboard k = availableKeyboardsList.get(position); @@ -199,21 +227,13 @@ public final class SelectLanguageFragment extends Fragment implements BlockingSt languageList = new ArrayList(); } String selectedLanguageID = k.getLanguageID(); - if (languageList.contains(selectedLanguageID)) { - languageList.remove(selectedLanguageID); - } else { - languageList.add(selectedLanguageID); - } + BCP47.toggleLanguage(languageList, selectedLanguageID); } else { // Otherwise, add the language association if (addKeyboardsList == null) { addKeyboardsList = new ArrayList(); } - if (addKeyboardsList.contains(k)) { - addKeyboardsList.remove(k); - } else { - addKeyboardsList.add(k); - } + toggleLanguage(addKeyboardsList, k); } // Disable install button if no languages selected or all languages already installed diff --git a/android/KMAPro/kMAPro/src/main/res/values/strings.xml b/android/KMAPro/kMAPro/src/main/res/values/strings.xml index 09a91b5f60..ed4e5b85a1 100644 --- a/android/KMAPro/kMAPro/src/main/res/values/strings.xml +++ b/android/KMAPro/kMAPro/src/main/res/values/strings.xml @@ -247,4 +247,7 @@ Keyboard requires a newer version of Keyman + + + Unable to launch web browser diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KMKeyboard.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KMKeyboard.java index 1c1dfb5a6a..b67e28eef1 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KMKeyboard.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KMKeyboard.java @@ -372,8 +372,10 @@ final class KMKeyboard extends WebView { public void dismissSubKeysWindow() { try { - if (subKeysWindow != null && subKeysWindow.isShowing()) + if (subKeysWindow != null && subKeysWindow.isShowing()) { subKeysWindow.dismiss(); + } + subKeysList = null; } catch (Exception e) { KMLog.LogException(TAG, "", e); } diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardInfoActivity.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardInfoActivity.java index d9b15f0ea8..4b818acf7c 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardInfoActivity.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardInfoActivity.java @@ -136,7 +136,11 @@ public final class KeyboardInfoActivity extends BaseActivity { } else { Intent i = new Intent(Intent.ACTION_VIEW); i.setData(Uri.parse(customHelpLink)); - startActivity(i); + if (i.resolveActivity(getPackageManager()) != null) { + startActivity(i); + } else { + Toast.makeText(context, getString(R.string.unable_to_open_browser), Toast.LENGTH_SHORT).show(); + } } } } diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardPickerActivity.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardPickerActivity.java index 50a478dd30..ec32425899 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardPickerActivity.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/KeyboardPickerActivity.java @@ -433,7 +433,7 @@ public final class KeyboardPickerActivity extends BaseActivity { if(adapter != null) { adapter.notifyDataSetChanged(); } - if (position == curKbPos && listView != null) { + if (position == curKbPos) { switchKeyboard(0,false); } else if(listView != null) { // A bit of a hack, since LanguageSettingsActivity calls this method too. curKbPos = KeyboardController.getInstance().getKeyboardIndex(KMKeyboard.currentKeyboard()); diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/cloud/impl/CloudLexicalModelMetaDataDownloadCallback.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/cloud/impl/CloudLexicalModelMetaDataDownloadCallback.java index fba9a8e1ad..6791a3097d 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/cloud/impl/CloudLexicalModelMetaDataDownloadCallback.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/cloud/impl/CloudLexicalModelMetaDataDownloadCallback.java @@ -125,31 +125,12 @@ public class CloudLexicalModelMetaDataDownloadCallback implements ICloudDownload * @param aMetaDataResult the meta data results */ private void processCloudResults(Context aContext, List aMetaDataResult) { - for(MetaDataResult _r:aMetaDataResult) { - if (_r.returnjson.target== CloudApiTypes.ApiTarget.Keyboard) { + for (MetaDataResult _r : aMetaDataResult) { + if (_r.returnjson.target == CloudApiTypes.ApiTarget.Keyboard) { //handleKeyboardMetaData(_r); } - if(_r.returnjson.target== CloudApiTypes.ApiTarget.KeyboardLexicalModels) { - JSONArray lmData = _r.returnjson.jsonArray; - if (lmData != null && lmData.length() > 0) { - try { - JSONObject modelInfo = lmData.getJSONObject(0); - - if (modelInfo.has("packageFilename") && modelInfo.has("id")) { - String _modelID = modelInfo.getString("id"); - ArrayList urls = new ArrayList<>(); - urls.add(new CloudApiTypes.CloudApiParam( - CloudApiTypes.ApiTarget.LexicalModelPackage, - modelInfo.getString("packageFilename"))); - _r.additionalDownloadid = CloudLexicalPackageDownloadCallback.createDownloadId(_modelID); - _r.additionalDownloads= urls; - } - } catch (JSONException e) { - KMLog.LogException(TAG, "Error parsing lexical model from api.keyman.com. ", e); - } - } else { - BaseActivity.makeToast(aContext, R.string.no_associated_model, Toast.LENGTH_SHORT); - } + if (_r.returnjson.target == CloudApiTypes.ApiTarget.KeyboardLexicalModels) { + processCloudResultForModel(aContext, _r); } if (_r.returnjson.target == CloudApiTypes.ApiTarget.PackageVersion) { @@ -164,6 +145,32 @@ public class CloudLexicalModelMetaDataDownloadCallback implements ICloudDownload } } + private void processCloudResultForModel(Context aContext, MetaDataResult _r) { + JSONArray lmData = _r.returnjson.jsonArray; + if (lmData == null || lmData.length() == 0) { + KMLog.LogError(TAG, "Error in lexical model metadata from api.keyman.com - zero or null"); + return; + } + + try { + JSONObject modelInfo = lmData.getJSONObject(0); + if (!modelInfo.has("packageFilename") || !modelInfo.has("id")) { + KMLog.LogError(TAG, "Error in lexical model metadata from api.keyman.com - missing metadata"); + return; + } + + String _modelID = modelInfo.getString("id"); + ArrayList urls = new ArrayList<>(); + urls.add(new CloudApiTypes.CloudApiParam( + CloudApiTypes.ApiTarget.LexicalModelPackage, + modelInfo.getString("packageFilename"))); + _r.additionalDownloadid = CloudLexicalPackageDownloadCallback.createDownloadId(_modelID); + _r.additionalDownloads= urls; + } catch (JSONException e) { + KMLog.LogException(TAG, "Error in lexical model metadata from api.keyman.com. ", e); + } + } + /** * create a download id for the lexical model metadata. * @param aLanguageId the language id diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/data/CloudRepository.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/data/CloudRepository.java index 5bafb4a2df..a7f5974718 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/data/CloudRepository.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/data/CloudRepository.java @@ -39,7 +39,7 @@ public class CloudRepository { public static final String DOWNLOAD_IDENTIFIER_CATALOGUE = "catalogue"; public static final String API_PRODUCTION_HOST = "api.keyman.com"; - public static final String API_STAGING_HOST = "api.keyman-staging.com"; + public static final String API_STAGING_HOST = "api.keyman.com"; // #7227 disabling: "api.keyman-staging.com"; public static final String API_MODEL_LANGUAGE_FORMATSTR = "https://%s/model?q=bcp47:%s"; public static final String API_PACKAGE_VERSION_FORMATSTR = "https://%s/package-version?platform=android%s%s"; @@ -400,7 +400,6 @@ public class CloudRepository { BaseActivity.makeToast(context, R.string.catalog_download_is_running_in_background, Toast.LENGTH_SHORT); } else { updateIsRunning = true; - BaseActivity.makeToast(context, R.string.catalog_download_start_in_background, Toast.LENGTH_SHORT); CloudDownloadMgr.getInstance().executeAsDownload( context, DOWNLOAD_IDENTIFIER_CATALOGUE, memCachedDataset, _download_callback, params); } diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/logic/ResourcesUpdateTool.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/logic/ResourcesUpdateTool.java index 9495ce30bd..0b057cfdb8 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/logic/ResourcesUpdateTool.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/logic/ResourcesUpdateTool.java @@ -106,7 +106,6 @@ public class ResourcesUpdateTool implements KeyboardEventHandler.OnKeyboardDownl return; } - BaseActivity.makeToast(currentContext, R.string.update_check_current, Toast.LENGTH_SHORT); lastUpdateCheck = Calendar.getInstance(); SharedPreferences prefs = currentContext.getSharedPreferences(currentContext.getString(R.string.kma_prefs_name), Context.MODE_PRIVATE); SharedPreferences.Editor editor = prefs.edit(); diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/BCP47.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/BCP47.java index 7e396dc551..81542ca86d 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/BCP47.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/BCP47.java @@ -4,6 +4,9 @@ package com.tavultesoft.kmea.util; +import java.util.ArrayList; +import java.security.InvalidParameterException; + public final class BCP47 { /** * Utility to compare two language ID strings @@ -18,4 +21,29 @@ public final class BCP47 { return id1.equalsIgnoreCase(id2); } -} \ No newline at end of file + + /** + * Utility to modify languageList. + * If languageID exists in the list, remove it. Otherwise, add languageID to the list. + * @param languageList + * @param languageID + */ + public static void toggleLanguage(ArrayList languageList, String languageID) { + if (languageList == null) { + throw new InvalidParameterException("languageList must not be null"); + } + if (languageID == null) { + throw new InvalidParameterException("languageID must not be null"); + } + + // See if languageID already exists in the languageList + for (String l: languageList) { + if (languageEquals(l, languageID)) { + languageList.remove(l); + return; + } + } + + languageList.add(languageID.toLowerCase()); + } +} diff --git a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/KMPLink.java b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/KMPLink.java index ff234e85c7..19ad4aaa72 100644 --- a/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/KMPLink.java +++ b/android/KMEA/app/src/main/java/com/tavultesoft/kmea/util/KMPLink.java @@ -21,7 +21,7 @@ import java.util.regex.Pattern; public final class KMPLink { public static final String KMP_PRODUCTION_HOST = "keyman.com"; - public static final String KMP_STAGING_HOST = "keyman-staging.com"; + public static final String KMP_STAGING_HOST = "keyman.com"; // #7227 disabling: "keyman-staging.com"; private static final String KMP_INSTALL_KEYBOARDS_PATTERN_FORMATSTR = "^http(s)?://(%s|%s)/keyboards/install/([^\\?/]+)(\\?(.+))?$"; private static final String installPatternFormatStr = KMString.format(KMP_INSTALL_KEYBOARDS_PATTERN_FORMATSTR, diff --git a/android/KMEA/app/src/main/res/values-land/dimens.xml b/android/KMEA/app/src/main/res/values-land/dimens.xml index 1e0bfcf55f..00cb4ae358 100644 --- a/android/KMEA/app/src/main/res/values-land/dimens.xml +++ b/android/KMEA/app/src/main/res/values-land/dimens.xml @@ -2,7 +2,7 @@ 40dp - 150dp + 100dp 49.25dp 49.25dp 21dp diff --git a/android/KMEA/app/src/main/res/values-sw600dp-land/dimens.xml b/android/KMEA/app/src/main/res/values-sw600dp-land/dimens.xml index 020aab6a3b..613a8b8526 100644 --- a/android/KMEA/app/src/main/res/values-sw600dp-land/dimens.xml +++ b/android/KMEA/app/src/main/res/values-sw600dp-land/dimens.xml @@ -2,7 +2,7 @@ 50dp - 200dp + 140dp 73.5dp 58.5dp 32dp diff --git a/android/KMEA/app/src/main/res/values-sw720dp-land/dimens.xml b/android/KMEA/app/src/main/res/values-sw720dp-land/dimens.xml index 88a93389bb..3944826815 100644 --- a/android/KMEA/app/src/main/res/values-sw720dp-land/dimens.xml +++ b/android/KMEA/app/src/main/res/values-sw720dp-land/dimens.xml @@ -2,7 +2,7 @@ 60dp - 300dp + 200dp 98.5dp 98.5dp 42dp diff --git a/android/KMEA/app/src/main/res/values/strings.xml b/android/KMEA/app/src/main/res/values/strings.xml index 8f7d20150d..50f52c5ef3 100644 --- a/android/KMEA/app/src/main/res/values/strings.xml +++ b/android/KMEA/app/src/main/res/values/strings.xml @@ -3,10 +3,10 @@ AndroidMobile - + KMEA - + Keyboard @@ -267,4 +267,6 @@ Tap here to change keyboard + + Unable to launch web browser diff --git a/android/KMEA/app/src/test/java/com/tavultesoft/kmea/util/KMPLinkTest.java b/android/KMEA/app/src/test/java/com/tavultesoft/kmea/util/KMPLinkTest.java index 013da4132a..56743502ed 100644 --- a/android/KMEA/app/src/test/java/com/tavultesoft/kmea/util/KMPLinkTest.java +++ b/android/KMEA/app/src/test/java/com/tavultesoft/kmea/util/KMPLinkTest.java @@ -18,9 +18,9 @@ public class KMPLinkTest { Assert.assertFalse(KMPLink.isKeymanInstallLink("")); // Valid Keyman keyboard install links - Assert.assertTrue(KMPLink.isKeymanInstallLink("https://keyman-staging.com/keyboards/install/malar_malayalam")); - Assert.assertTrue(KMPLink.isKeymanInstallLink("https://keyman-staging.com/keyboards/install/malar_malayalam?bcp47=ml")); - Assert.assertTrue(KMPLink.isKeymanInstallLink("https://keyman-staging.com/keyboards/install/fv_sencoten?bcp47=str-Latn")); + // Assert.assertTrue(KMPLink.isKeymanInstallLink("https://keyman-staging.com/keyboards/install/malar_malayalam")); + // Assert.assertTrue(KMPLink.isKeymanInstallLink("https://keyman-staging.com/keyboards/install/malar_malayalam?bcp47=ml")); + // Assert.assertTrue(KMPLink.isKeymanInstallLink("https://keyman-staging.com/keyboards/install/fv_sencoten?bcp47=str-Latn")); Assert.assertTrue(KMPLink.isKeymanInstallLink("https://keyman.com/keyboards/install/lao%20unicode")); // legacy keyboard within download dialog Assert.assertTrue(KMPLink.isKeymanInstallLink("https://keyman.com/keyboards/install/khmer_angkor?_gid=112233&bcp47=lo")); // other query params appearing @@ -28,11 +28,11 @@ public class KMPLinkTest { Assert.assertTrue(KMPLink.isKeymanInstallLink("https://keyman.com/keyboards/install/malar_malayalam?bcp47=ml")); // Keyboard link is wrong - Assert.assertFalse(KMPLink.isKeymanInstallLink("https://keyman-staging.com/keyboard/install/malar_malayalam")); + // Assert.assertFalse(KMPLink.isKeymanInstallLink("https://keyman-staging.com/keyboard/install/malar_malayalam")); // link missing packageID - Assert.assertFalse(KMPLink.isKeymanInstallLink("https://keyman-staging.com/keyboards/install")); - Assert.assertFalse(KMPLink.isKeymanInstallLink("https://keyman-staging.com/keyboards/install/")); + // Assert.assertFalse(KMPLink.isKeymanInstallLink("https://keyman-staging.com/keyboards/install")); + // Assert.assertFalse(KMPLink.isKeymanInstallLink("https://keyman-staging.com/keyboards/install/")); Assert.assertFalse(KMPLink.isKeymanInstallLink("https://keyman.com/keyboards/install")); Assert.assertFalse(KMPLink.isKeymanInstallLink("https://keyman.com/keyboards/install/")); } @@ -43,14 +43,14 @@ public class KMPLinkTest { Assert.assertFalse(KMPLink.isKeymanDownloadLink("")); // Valid Keyman keyboard download links - Assert.assertTrue(KMPLink.isKeymanDownloadLink("https://keyman-staging.com/go/package/download/malar_malayalam?platform=android&tier=alpha")); - Assert.assertTrue(KMPLink.isKeymanDownloadLink("https://keyman-staging.com/go/package/download/malar_malayalam?platform=android&tier=alpha&bcp47=ml")); + // Assert.assertTrue(KMPLink.isKeymanDownloadLink("https://keyman-staging.com/go/package/download/malar_malayalam?platform=android&tier=alpha")); + // Assert.assertTrue(KMPLink.isKeymanDownloadLink("https://keyman-staging.com/go/package/download/malar_malayalam?platform=android&tier=alpha&bcp47=ml")); Assert.assertTrue(KMPLink.isKeymanDownloadLink("https://keyman.com/go/package/download/malar_malayalam?platform=android&tier=alpha")); Assert.assertTrue(KMPLink.isKeymanDownloadLink("https://keyman.com/go/package/download/malar_malayalam?platform=android&tier=alpha&bcp47=ml")); // link missing packageID - Assert.assertFalse(KMPLink.isKeymanDownloadLink("https://keyman-staging.com/go/package/download")); - Assert.assertFalse(KMPLink.isKeymanDownloadLink("https://keyman-staging.com/go/package/download/")); + // Assert.assertFalse(KMPLink.isKeymanDownloadLink("https://keyman-staging.com/go/package/download")); + // Assert.assertFalse(KMPLink.isKeymanDownloadLink("https://keyman-staging.com/go/package/download/")); Assert.assertFalse(KMPLink.isKeymanDownloadLink("https://keyman.com/go/package/download")); Assert.assertFalse(KMPLink.isKeymanDownloadLink("https://keyman.com/go/package/download/")); } diff --git a/android/Tests/KeyboardHarness/app/src/main/assets/final.kmp b/android/Tests/KeyboardHarness/app/src/main/assets/final.kmp new file mode 100644 index 0000000000..80d024623b Binary files /dev/null and b/android/Tests/KeyboardHarness/app/src/main/assets/final.kmp differ diff --git a/android/Tests/KeyboardHarness/app/src/main/java/com/keyman/android/tests/keyboardHarness/MainActivity.java b/android/Tests/KeyboardHarness/app/src/main/java/com/keyman/android/tests/keyboardHarness/MainActivity.java index 9454e940b7..c09a70c0fe 100644 --- a/android/Tests/KeyboardHarness/app/src/main/java/com/keyman/android/tests/keyboardHarness/MainActivity.java +++ b/android/Tests/KeyboardHarness/app/src/main/java/com/keyman/android/tests/keyboardHarness/MainActivity.java @@ -79,6 +79,22 @@ public class MainActivity extends AppCompatActivity implements OnKeyboardEventLi KMManager.KMDefault_KeyboardFont, KMManager.KMDefault_KeyboardFont); KMManager.addKeyboard(this, platformtestKBbInfo); + + // Final K_ENTER test keyboard + Keyboard finalKBInfo = new Keyboard( + "final", + "final", + "final Keyboard", + "en", + "English", + "1.0", + "", + "", + true, + KMManager.KMDefault_KeyboardFont, + KMManager.KMDefault_KeyboardFont); + KMManager.addKeyboard(this, finalKBInfo); + } @Override diff --git a/common/models/templates/package.json b/common/models/templates/package.json index d4eafa6f0c..b9b84c6492 100644 --- a/common/models/templates/package.json +++ b/common/models/templates/package.json @@ -33,7 +33,7 @@ }, "scripts": { "build": "tsc", - "pretest": "npm run build", + "pretest": "tsc -b tsconfig.bundled.json", "test": "mocha -r test/helpers.js" }, "bugs": { @@ -46,8 +46,8 @@ "@types/mocha": "^7.0.2", "@types/node": "^14.0.4", "chai": "^4.3.4", - "mocha": "^8.4.0", - "typescript": "^3.8.3" + "mocha": "^10.0.0", + "typescript": "^4.5.4" }, "dependencies": { "@keymanapp/models-wordbreakers": "*" diff --git a/common/models/templates/test/helpers.js b/common/models/templates/test/helpers.js index e3931d648a..268e24b7dd 100644 --- a/common/models/templates/test/helpers.js +++ b/common/models/templates/test/helpers.js @@ -7,8 +7,10 @@ var _ = global; // TODO: then mocha invocation is as follows: // TODO: mocha -r @keymanapp/models-test-helpers test/ -// KMW string must be included, so do it here: -require('@keymanapp/web-utils'); +// Ensure that we can successfully load the module & apply kmwLength, as it's +// needed for some of the unit tests. +require('../build/index.bundled.js'); + assert.ok('๐Ÿ’ฉ'.kmwLength); /** @@ -21,7 +23,7 @@ _.jsonFixture = function (name) { /** * Returns the Context of an empty buffer; no text, at both the start and * end of the buffer. - * + * * @returns {Context} */ _.emptyContext = function emptyContext() { diff --git a/common/models/templates/test/test-common.js b/common/models/templates/test/test-common.js index 299cb25443..7dfc565369 100644 --- a/common/models/templates/test/test-common.js +++ b/common/models/templates/test/test-common.js @@ -3,7 +3,7 @@ */ var assert = require('chai').assert; -var models = require('../').models; +var models = require('../build/index.bundled.js').models; describe('Common utility functions', function() { // TODO: unit tests for other common utility functions @@ -97,7 +97,7 @@ describe('Common utility functions', function() { let apple = { insert: 'apple', deleteLeft: 0, - deleteRight: 2 + deleteRight: 2 }; let banana = { @@ -143,7 +143,7 @@ describe('Common utility functions', function() { let apple = { insert: 'apple', deleteLeft: 0, - deleteRight: 2 + deleteRight: 2 }; let banana = { @@ -167,7 +167,7 @@ describe('Common utility functions', function() { let apple = { insert: 'apple', deleteLeft: 0, - deleteRight: 2 + deleteRight: 2 }; let banana = { diff --git a/common/models/templates/test/test-priority-queue.js b/common/models/templates/test/test-priority-queue.js index 9f2e48e1d2..b004027d84 100644 --- a/common/models/templates/test/test-priority-queue.js +++ b/common/models/templates/test/test-priority-queue.js @@ -3,12 +3,12 @@ */ var assert = require('chai').assert; -var PriorityQueue = require('../').models.PriorityQueue; +var PriorityQueue = require('../build/index.bundled.js').models.PriorityQueue; describe('Priority queue', function() { it('can act as a min-heap', function () { let input = [1, 10, 2, 9, 3, 8, 4, 7, 5, 6]; - + let queue = new PriorityQueue((a, b) => a - b); input.forEach((input) => queue.enqueue(input)); @@ -41,7 +41,7 @@ describe('Priority queue', function() { it('can act as a max-heap', function () { let input = [1, 10, 2, 9, 3, 8, 4, 7, 5, 6]; - + let queue = new PriorityQueue((a, b) => b - a); input.forEach((input) => queue.enqueue(input)); diff --git a/common/models/templates/test/test-quote-behavior.js b/common/models/templates/test/test-quote-behavior.js index eb169424be..dd5408f6a9 100644 --- a/common/models/templates/test/test-quote-behavior.js +++ b/common/models/templates/test/test-quote-behavior.js @@ -3,7 +3,7 @@ */ var assert = require('chai').assert; -var QuoteBehavior = require('../').models.QuoteBehavior; +var QuoteBehavior = require('../build/index.bundled.js').models.QuoteBehavior; describe('Quote behaviors', function() { describe('Script directionality', function() { @@ -144,15 +144,15 @@ describe('Quote behaviors', function() { quotesForKeepSuggestion: { open: `โ€œ`, close: `โ€`}, insertAfterWord: " " } - + assert.throws(function() { QuoteBehavior.apply(QuoteBehavior.default, "hello", englishPunctuation, QuoteBehavior.default); }); - + assert.throws(function() { QuoteBehavior.apply(QuoteBehavior.useQuotes, "hello", englishPunctuation, QuoteBehavior.default); }); - + assert.throws(function() { QuoteBehavior.apply(QuoteBehavior.noQuotes, "hello", englishPunctuation, QuoteBehavior.default); }); diff --git a/common/models/templates/test/test-tokenization.js b/common/models/templates/test/test-tokenization.js index 22b975a69c..a7ae0835e0 100644 --- a/common/models/templates/test/test-tokenization.js +++ b/common/models/templates/test/test-tokenization.js @@ -3,7 +3,7 @@ */ var assert = require('chai').assert; -var models = require('../').models; +var models = require('../build/index.bundled.js').models; var wordBreakers = require('@keymanapp/models-wordbreakers').wordBreakers; describe('Tokenization functions', function() { @@ -132,7 +132,7 @@ describe('Tokenization functions', function() { left: '', startOfBuffer: true, right: '', endOfBuffer: true }; - + let tokenization = models.tokenize(wordBreakers.default, context); let expectedResult = { @@ -140,7 +140,7 @@ describe('Tokenization functions', function() { right: [], caretSplitsToken: false }; - + assert.deepEqual(tokenization, expectedResult); }); @@ -153,7 +153,7 @@ describe('Tokenization functions', function() { right: [], caretSplitsToken: false }; - + assert.deepEqual(tokenization, expectedResult); }); @@ -163,7 +163,7 @@ describe('Tokenization functions', function() { left: ' ', startOfBuffer: true, right: '', endOfBuffer: true }; - + let tokenization = models.tokenize(wordBreakers.default, context); let expectedResult = { @@ -171,7 +171,7 @@ describe('Tokenization functions', function() { right: [], caretSplitsToken: false }; - + assert.deepEqual(tokenization, expectedResult); }); @@ -233,7 +233,7 @@ describe('Tokenization functions', function() { case 'แžŸแŸ’แžšแžปแž€แžแŸ’แž˜แŸ‚แžš': return [srok, shiftSpan(khmer, srok.length)]; // array of the two. case 'แž€แžแŸ’แž˜แŸ‚แžš': - // I'd admittedly be at least somewhat surprised if a real wordbreaker got this + // I'd admittedly be at least somewhat surprised if a real wordbreaker got this // and similar situations perfectly right... but at least it gives us what // we need for a test. return [k, shiftSpan(khmer, k.length)]; @@ -289,6 +289,82 @@ describe('Tokenization functions', function() { assert.deepEqual(tokenization, expectedResult); }); + + let midLetterNonbreaker = (text) => { + let customization = { + rules: [{ + match: (context) => { + if(context.propertyMatch(null, ["ALetter"], ["MidLetter"], ["eot"])) { + return true; + } else { + return false; + } + }, + breakIfMatch: false + }], + propertyMapping: (char) => { + let hyphens = ['\u002d', '\u2010', '\u058a', '\u30a0']; + if(hyphens.includes(char)) { + return "MidLetter"; + } else { + return null; + } + } + }; + + return wordBreakers.default(text, customization); + } + + it('treats caret as `eot` for pre-caret text', function() { + let context = { + left: "don-", // We use a hyphen here b/c single-quote is hardcoded. + right: " worry", + endOfBuffer: true, + startOfBuffer: true + }; + + let tokenization = models.tokenize(wordBreakers.default, context); + + assert.deepEqual(tokenization, { + left: ["don", "-"], + right: ["worry"], + caretSplitsToken: false + }); + + tokenization = models.tokenize(midLetterNonbreaker, context); + + assert.deepEqual(tokenization, { + left: ["don-"], + right: ["worry"], + caretSplitsToken: false + }); + }); + + it('handles mid-contraction tokenization', function() { + let context = { + left: "don:", + right: "t worry", + endOfBuffer: true, + startOfBuffer: true + }; + + let tokenization = models.tokenize(wordBreakers.default, context); + + assert.deepEqual(tokenization, { + left: ["don", ":"], // This particular case feels like a possible issue. + right: ["t", "worry"], // It'd be a three-way split token, as "don:t" would + // be a single token were it not for the caret in the middle. + caretSplitsToken: false + }) + + tokenization = models.tokenize(midLetterNonbreaker, context); + + assert.deepEqual(tokenization, { + left: ["don:"], + right: ["t", "worry"], + caretSplitsToken: true + }); + }); }); describe('getLastPreCaretToken', function() { diff --git a/common/models/templates/test/test-trie-model.js b/common/models/templates/test/test-trie-model.js index 36a5944b84..b48dc254ec 100644 --- a/common/models/templates/test/test-trie-model.js +++ b/common/models/templates/test/test-trie-model.js @@ -3,7 +3,7 @@ */ var assert = require('chai').assert; -var TrieModel = require('../').models.TrieModel; +var TrieModel = require('../build/index.bundled.js').models.TrieModel; describe('LMLayerWorker trie model for word lists', function() { describe('instantiation', function () { @@ -20,7 +20,7 @@ describe('LMLayerWorker trie model for word lists', function() { } } }) - + assert.equal(model.punctuation.insertAfterWord, spaceMark); assert.equal(model.punctuation.quotesForKeepSuggestion.open, openQuote); assert.equal(model.punctuation.quotesForKeepSuggestion.close, closeQuote); @@ -176,7 +176,7 @@ describe('LMLayerWorker trie model for word lists', function() { ); }); }); - + describe('The default key function', function () { it('uses the default key function', function () { var model = new TrieModel(jsonFixture('tries/accented')); diff --git a/common/models/templates/test/test-trie-traversal.js b/common/models/templates/test/test-trie-traversal.js index 09d1361525..31f52f854a 100644 --- a/common/models/templates/test/test-trie-traversal.js +++ b/common/models/templates/test/test-trie-traversal.js @@ -3,7 +3,7 @@ */ var assert = require('chai').assert; -var TrieModel = require('../').models.TrieModel; +var TrieModel = require('../build/index.bundled.js').models.TrieModel; // Useful for tests related to strings with supplementary pairs. var smpForUnicode = function(code){ @@ -52,7 +52,7 @@ describe('Trie traversal abstractions', function() { assert.isEmpty(child.traversal().entries); for(tChild of traversalInner1.children()) { - if(tChild.char == 'h') { + if(tChild.char == 'h') { hSuccess = true; let traversalInner2 = tChild.traversal(); assert.isDefined(traversalInner2); @@ -64,7 +64,7 @@ describe('Trie traversal abstractions', function() { eSuccess = true; let traversalInner3 = hChild.traversal(); assert.isDefined(traversalInner3); - + assert.isDefined(traversalInner3.entries); assert.equal(traversalInner3.entries[0], "the"); @@ -104,7 +104,7 @@ describe('Trie traversal abstractions', function() { assert.isEmpty(child.traversal().entries); for(tChild of traversalInner1.children()) { - if(tChild.char == 'r') { + if(tChild.char == 'r') { let traversalInner2 = tChild.traversal(); assert.isDefined(traversalInner2); assert.isArray(tChild.traversal().entries); @@ -185,7 +185,7 @@ describe('Trie traversal abstractions', function() { assert.isEmpty(child.traversal().entries); for(aChild of traversalInner1.children()) { - if(aChild.char == smpP) { + if(aChild.char == smpP) { pSuccess = true; let traversalInner2 = aChild.traversal(); assert.isDefined(traversalInner2); diff --git a/common/models/templates/tsconfig.bundled.json b/common/models/templates/tsconfig.bundled.json new file mode 100644 index 0000000000..fd8dc5f65a --- /dev/null +++ b/common/models/templates/tsconfig.bundled.json @@ -0,0 +1,21 @@ +{ + // This variant of the tsconfig.json exists to create a 'leaf', 'bundled' + // version of the models/templates build product. The same reference + // cannot be prepended twice in a composite tsc build, posing problems + // for certain down-line builds if the two tsconfigs are not differentiated. + "extends": "./tsconfig.json", + "compilerOptions": { + "outFile": "build/index.bundled.js", + }, + "references": [ + { "path": "../../web/keyman-version", "prepend": true}, + { "path": "../../web/utils", "prepend": true }, + { "path": "../types" } + ], + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "test" + ] +} diff --git a/common/models/wordbreakers/package.json b/common/models/wordbreakers/package.json index 38cf3a0175..a4b97fc973 100644 --- a/common/models/wordbreakers/package.json +++ b/common/models/wordbreakers/package.json @@ -14,7 +14,7 @@ ], "homepage": "https://github.com/keymanapp/keyman", "license": "MIT", - "main": "index.js", + "main": "build/index.js", "directories": { "lib": "lib", "test": "test" @@ -43,7 +43,7 @@ "@types/chai": "^4.2.11", "@types/mocha": "^7.0.2", "chai": "^4.3.4", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "ts-node": "^9.1.1", "typescript": "^4.5.4" }, diff --git a/common/models/wordbreakers/src/default/data.ts b/common/models/wordbreakers/src/default/data.ts index ffb6b8e79a..16a4d9beb3 100644 --- a/common/models/wordbreakers/src/default/data.ts +++ b/common/models/wordbreakers/src/default/data.ts @@ -4,7 +4,7 @@ export namespace data { /** * Valid values for a word break property. */ -export const enum WordBreakProperty { +export const enum WordBreakProperty { // Scary bit: this does not exist as an object at run-time! Other, LF, Newline, @@ -28,6 +28,32 @@ export const enum WordBreakProperty { eot }; +// Not currently built by the auto-generator tool, but it easily could be. +// If and when we import the data.ts rebuilder, we can add this in. +export const propertyMap = [ + "Other", + "LF", + "Newline", + "CR", + "WSegSpace", + "Double_Quote", + "Single_Quote", + "MidNum", + "MidNumLet", + "Numeric", + "MidLetter", + "ALetter", + "ExtendNumLet", + "Format", + "Extend", + "Hebrew_Letter", + "ZWJ", + "Katakana", + "Regional_Indicator", + "sot", + "eot" +]; + /** * Constants for indexing values in WORD_BREAK_PROPERTY. */ diff --git a/common/models/wordbreakers/src/default/index.ts b/common/models/wordbreakers/src/default/index.ts index c587ca08de..715b722932 100644 --- a/common/models/wordbreakers/src/default/index.ts +++ b/common/models/wordbreakers/src/default/index.ts @@ -1,20 +1,48 @@ // Include the word-breaking data here: /// namespace wordBreakers { + /** + * A set of options used to customize and extend the behavior of the default + * Unicode wordbreaker. + */ + export interface DefaultWordBreakerOptions { + /** + * Allows addition of custom wordbreaking rules, which will be applied + * after WB1-WB4 and before all other default wordbreaking rules. + * + * @see `WordbreakerRule` + */ + rules?: WordbreakerRule[]; + + /** + * Allows assignment of characters to different word-breaking properties than + * their standard word-breaking assignment, including to custom properties + * specified within `customProperties`. + * @param char + */ + propertyMapping?(char: string): string; + + /** + * Allows definition of extra word-breaking properties for use with custom + * rules. + */ + customProperties?: string[]; + } + /** * Word breaker based on Unicode Standard Annex #29, Section 4.1: - * Default Word Boundary Specification. - * + * Default Word Boundary Specification. + * * @see http://unicode.org/reports/tr29/#Word_Boundaries * @see https://github.com/eddieantonio/unicode-default-word-boundary/tree/v12.0.0 */ - export function default_(text: string): Span[] { - let boundaries = findBoundaries(text); + export function default_(text: string, options?: DefaultWordBreakerOptions): Span[] { + let boundaries = findBoundaries(text, options); if (boundaries.length == 0) { return []; } - // All non-empty strings have at least TWO boundaries at the start and end of + // All non-empty strings have at least TWO boundaries: at the start and at the end of // the string. let spans = []; for (let i = 0; i < boundaries.length - 1; i++) { @@ -22,7 +50,7 @@ namespace wordBreakers { let end = boundaries[i + 1]; let span = new LazySpan(text, start, end); - if (isNonSpace(span.text)) { + if (isNonSpace(span.text, options)) { spans.push(span); // Preserve a sequence-final space if it exists. Needed to signal "end of word". } else if (i == boundaries.length - 2) { // if "we just checked the final boundary"... @@ -61,13 +89,198 @@ namespace wordBreakers { } } + /** + * An abstraction supporting custom wordbreaker boundary rules. While this doesn't provide + * support for more complex rules like WB4, WB15, or WB16, this is sufficient for all other + * default word-breaking rules and can be used to define custom rules of similar structure. + * + * @see https://unicode.org/reports/tr29/#WB_Rule_Macros + */ + export interface WordbreakerRule { + /** + * Indicates whether or not the rule applies in the specified context. + * @param context + */ + match(context: BreakerContext): boolean; + + /** + * Indicates whether or not the rule indicates a word boundary at the context's site when it matches. + */ + breakIfMatch: boolean; + } + + /** + * Provides a useful presentation for wordbreaker's context for use in word-breaking rules. + * + * @see https://unicode.org/reports/tr29/#Word_Boundary_Rules + */ + export class BreakerContext { + // Referenced by this object in order to facilitate `lookahead` maintenance. + private readonly text: string; + readonly options?: DefaultWordBreakerOptions; + + /** + * Represents the property of character immediately preceding `left`'s character. + */ + readonly lookbehind: WordBreakProperty = WordBreakProperty.sot; + + /** + * Represents the property of the character immediately preceding the potential word boundary. + */ + readonly left: WordBreakProperty = WordBreakProperty.sot; + + /** + * Represents the property of the character immediately following the potential word boundary. + */ + readonly right: WordBreakProperty = WordBreakProperty.sot; + + /** + * Represents the property of the character immediately following `right`'s character. + */ + readonly lookahead: WordBreakProperty; // Always initialized by constructor. + + /** + * Initializes the word-breaking context at the start of the word-breaker's boundary-detection + * algorithm. + * @param text The text to be word-broken + * @param lookaheadPos The position corresponding to `lookahead`. + */ + constructor(text: string, options: DefaultWordBreakerOptions | undefined, lookaheadPos: number); + /** + * Used internally by the boundary-detection algorithm during context-shifting operations. + * @param text + * @param lookbehind + * @param left + * @param right + * @param lookahead + */ + constructor(text: string, + options: DefaultWordBreakerOptions | undefined, + lookbehind: WordBreakProperty, + left: WordBreakProperty, + right: WordBreakProperty, + lookahead: WordBreakProperty); + constructor(text: string, + options: DefaultWordBreakerOptions | undefined, + prop1: WordBreakProperty | number, + prop2?: WordBreakProperty, + prop3?: WordBreakProperty, + prop4?: WordBreakProperty) { + this.text = text; + this.options = options; + + if(arguments.length == 3) { + this.lookahead = this.wordbreakPropertyAt(prop1);// prop1; + } else /*if(arguments.length == 6)*/ { + this.lookbehind = prop1 as WordBreakProperty; + this.left = prop2 as WordBreakProperty; + this.right = prop3 as WordBreakProperty; + this.lookahead = prop4 as WordBreakProperty; + } + } + + /** + * The general use-case when shifting boundary-check position if WB4 is not active. + * @param lookahead The WordBreakProperty for the character to become `lookahead`. + * @returns + */ + public next(lookaheadPos: number): BreakerContext { + let newLookahead = this.wordbreakPropertyAt(lookaheadPos); + return new BreakerContext(this.text, this.options, this.left, this.right, this.lookahead, newLookahead); + } + + /** + * Used for WB4: when ignoring characters before an intervening linebreak, we + * replace `right` with the current `lookahead`, without affecting `lookbehind` + * or `left`. A new `lookahead` is then needed. + * @param lookahead + * @returns + */ + public ignoringRight(lookaheadPos: number) { + let newLookahead = this.wordbreakPropertyAt(lookaheadPos); + return new BreakerContext(this.text, this.options, this.lookbehind, this.left, this.lookahead, newLookahead); + } + + /** + * Used for WB4: when ignoring characters after an intervening linebreak, it's + * `lookahead` that gets replaced without shifting the other tracked properties. + * @param lookahead + * @returns + */ + public ignoringLookahead(lookaheadPos: number) { + let newLookahead = this.wordbreakPropertyAt(lookaheadPos); + return new BreakerContext(this.text, this.options, this.lookbehind, this.left, this.right, newLookahead); + } + + /** + * Return the value of the Word_Break property at the given string index. + * @param pos position in the text. + */ + private wordbreakPropertyAt(pos: number) { + if (pos < 0) { + return WordBreakProperty.sot; // Always "start of string" before the string starts! + } else if (pos >= this.text.length) { + return WordBreakProperty.eot; // Always "end of string" after the string ends! + } else if (isStartOfSurrogatePair(this.text[pos])) { + // Surrogate pairs the next TWO items from the string! + return property(this.text[pos] + this.text[pos + 1]); + } + return property(this.text[pos], this.options); + } + + /** + * Returns `true` if and only if each member of the context has a property included within + * its corresponding set (when specified). Any set may be replaced with null to disable + * a check against its corresponding property. + * @param lookbehindSet + * @param leftSet + * @param rightSet + * @param lookaheadSet + */ + public match(lookbehindSet: WordBreakProperty[] | null, + leftSet: WordBreakProperty[] | null, + rightSet: WordBreakProperty[] | null, + lookaheadSet: WordBreakProperty[] | null) : boolean { + let result: boolean = lookbehindSet?.includes(this.lookbehind) ?? true; + result = result && (leftSet?.includes(this.left) ?? true); + result = result && (rightSet?.includes(this.right) ?? true); + return result && (lookaheadSet?.includes(this.lookahead) ?? true); + } + + /** + * Returns `true` if and only if each member of the context has a property included within + * its corresponding set (when specified). Any set may be replaced with null to disable + * a check against its corresponding property. + * + * Names should match those found at https://unicode.org/reports/tr29/#Word_Boundary_Rules + * or defined in the word-breaker customization options; matching is case-insensitive. + * Also includes two extra properties: + * - `sot` - start of text + * - `eot` - end of text + * @param lookbehindSet + * @param leftSet + * @param rightSet + * @param lookaheadSet + */ + public propertyMatch(lookbehindSet: string[] | null, + leftSet: string[] | null, + rightSet: string[] | null, + lookaheadSet: string[] | null) : boolean { + const propMapper = (name: string) => propertyVal(name, this.options); + return this.match(lookbehindSet?.map(propMapper) as WordBreakProperty[] | null, + leftSet?.map(propMapper) as WordBreakProperty[] | null, + rightSet?.map(propMapper) as WordBreakProperty[] | null, + lookaheadSet?.map(propMapper) as WordBreakProperty[] | null); + } + } + /** * Returns true when the chunk does not solely consist of whitespace. * * @param chunk a chunk of text. Starts and ends at word boundaries. */ - function isNonSpace(chunk: string): boolean { - return !Array.from(chunk).map(property).every(wb => ( + function isNonSpace(chunk: string, options?: DefaultWordBreakerOptions): boolean { + return !Array.from(chunk).map((char) => property(char, options)).every(wb => ( wb === WordBreakProperty.CR || wb === WordBreakProperty.LF || wb === WordBreakProperty.Newline || @@ -82,13 +295,17 @@ namespace wordBreakers { * * @param text Text to find word boundaries in. */ - function findBoundaries(text: string): number[] { + function findBoundaries(text: string, options?: DefaultWordBreakerOptions): number[] { // WB1 and WB2: no boundaries if given an empty string. if (text.length === 0) { // There are no boundaries in an empty string! return []; } + if(options && !options.rules) { + options.rules = []; + } + // This algorithm works by maintaining a sliding window of four SCALAR VALUES. // // - Scalar values? JavaScript strings are NOT actually a string of @@ -112,10 +329,7 @@ namespace wordBreakers { let rightPos: number; let lookaheadPos = 0; // lookahead, one scalar value to the right of right. // Before the start of the string is also the start of the string. - let lookbehind: WordBreakProperty; - let left = WordBreakProperty.sot; - let right = WordBreakProperty.sot; - let lookahead = wordbreakPropertyAt(0); + let state = new BreakerContext(text, options, lookaheadPos); // Count RIs to make sure we're not splitting emoji flags: let nConsecutiveRegionalIndicators = 0; @@ -124,34 +338,32 @@ namespace wordBreakers { rightPos = lookaheadPos; lookaheadPos = positionAfter(lookaheadPos); // Shift all properties, one scalar value to the right. - [lookbehind, left, right, lookahead] = - [left, right, lookahead, wordbreakPropertyAt(lookaheadPos)]; + state = state.next(lookaheadPos); // Break at the start and end of text, unless the text is empty. // WB1: Break at start of text... - if (left === WordBreakProperty.sot) { + if (state.match(null, [WordBreakProperty.sot], null, null)) { boundaries.push(rightPos); continue; } // WB2: Break at the end of text... - if (right === WordBreakProperty.eot) { + if (state.match(null, null, [WordBreakProperty.eot], null)) { boundaries.push(rightPos); break; // Reached the end of the string. We're done! } // WB3: Do not break within CRLF: - if (left === WordBreakProperty.CR && right === WordBreakProperty.LF) + if (state.match(null, [WordBreakProperty.CR], [WordBreakProperty.LF], null)) { continue; + } + // WB3b: Otherwise, break after... - if (left === WordBreakProperty.Newline || - left === WordBreakProperty.CR || - left === WordBreakProperty.LF) { + const NEWLINE_SET = [WordBreakProperty.Newline, WordBreakProperty.CR, WordBreakProperty.LF]; + if(state.match(null, NEWLINE_SET, null, null)) { boundaries.push(rightPos); continue; } // WB3a: ...and before newlines - if (right === WordBreakProperty.Newline || - right === WordBreakProperty.CR || - right === WordBreakProperty.LF) { + if (state.match(null, null, NEWLINE_SET, null)) { boundaries.push(rightPos); continue; } @@ -163,107 +375,144 @@ namespace wordBreakers { // https://www.unicode.org/Public/emoji/12.0/emoji-zwj-sequences.txt // WB3d: Keep horizontal whitespace together - if (left === WordBreakProperty.WSegSpace && right == WordBreakProperty.WSegSpace) + if (state.match(null, [WordBreakProperty.WSegSpace], [WordBreakProperty.WSegSpace], null)) { continue; + } // WB4: Ignore format and extend characters // This is to keep grapheme clusters together! // See: Section 6.2: https://unicode.org/reports/tr29/#Grapheme_Cluster_and_Format_Rules // N.B.: The rule about "except after sot, CR, LF, and // Newline" already been by WB1, WB2, WB3a, and WB3b above. - while (right === WordBreakProperty.Format || - right === WordBreakProperty.Extend || - right === WordBreakProperty.ZWJ) { + const SET_WB4_IGNORE = [WordBreakProperty.Format, WordBreakProperty.Extend, WordBreakProperty.ZWJ]; + while (state.match(null, null, SET_WB4_IGNORE, null)) { // Continue advancing in the string, as if these // characters do not exist. DO NOT update left and // lookbehind however! [rightPos, lookaheadPos] = [lookaheadPos, positionAfter(lookaheadPos)]; - [right, lookahead] = [lookahead, wordbreakPropertyAt(lookaheadPos)]; + state = state.ignoringRight(lookaheadPos); } // In ignoring the characters in the previous loop, we could // have fallen off the end of the string, so end the loop // prematurely if that happens! - if (right === WordBreakProperty.eot) { + if (state.right === WordBreakProperty.eot) { boundaries.push(rightPos); break; } // WB4 (continued): Lookahead must ALSO ignore these format, // extend, ZWJ characters! - while (lookahead === WordBreakProperty.Format || - lookahead === WordBreakProperty.Extend || - lookahead === WordBreakProperty.ZWJ) { + while (state.match(null, null, null, SET_WB4_IGNORE)) { // Continue advancing in the string, as if these // characters do not exist. DO NOT update left and right, // however! lookaheadPos = positionAfter(lookaheadPos); - lookahead = wordbreakPropertyAt(lookaheadPos); + state = state.ignoringLookahead(lookaheadPos); + } + + // See: https://unicode.org/reports/tr29/#WB_Rule_Macros + const SET_AHLETTER = [WordBreakProperty.ALetter, WordBreakProperty.Hebrew_Letter]; + const SET_MIDNUMLETQ = [WordBreakProperty.MidNumLet, WordBreakProperty.Single_Quote]; + + // Custom rules may override the base ruleset aside from the first few fundamental ones. + if(options?.rules) { + let customMatch: boolean = false; + for(const rule of options.rules) { + customMatch = rule.match(state); + if(customMatch) { + if(rule.breakIfMatch) { + boundaries.push(rightPos); + } + break; // as customMatch == true here, this will trigger the `continue` that follows. + } + } + if(customMatch) { + continue; + } } // WB5: Do not break between most letters. - if (isAHLetter(left) && isAHLetter(right)) + // if (isAHLetter(state.left) && isAHLetter(state.right)) + if(state.match(null, SET_AHLETTER, SET_AHLETTER, null)) { continue; + } // Do not break across certain punctuation // WB6: (Don't break before apostrophes in contractions) - if (isAHLetter(left) && isAHLetter(lookahead) && - (right === WordBreakProperty.MidLetter || isMidNumLetQ(right))) + const SET_ALL_MIDLETTER = [WordBreakProperty.MidLetter, ...SET_MIDNUMLETQ]; + if(state.match(null, SET_AHLETTER, SET_ALL_MIDLETTER, SET_AHLETTER)) { continue; + } // WB7: (Don't break after apostrophes in contractions) - if (isAHLetter(lookbehind) && isAHLetter(right) && - (left === WordBreakProperty.MidLetter || isMidNumLetQ(left))) + if(state.match(SET_AHLETTER, SET_ALL_MIDLETTER, SET_AHLETTER, null)) { continue; + } + // WB7a - if (left === WordBreakProperty.Hebrew_Letter && right === WordBreakProperty.Single_Quote) + if(state.match(null, [WordBreakProperty.Hebrew_Letter], [WordBreakProperty.Single_Quote], null)) { continue; + } // WB7b - if (left === WordBreakProperty.Hebrew_Letter && right === WordBreakProperty.Double_Quote && - lookahead === WordBreakProperty.Hebrew_Letter) + if(state.match(null, + [WordBreakProperty.Hebrew_Letter], + [WordBreakProperty.Double_Quote], + [WordBreakProperty.Hebrew_Letter])) { continue; + } // WB7c - if (lookbehind === WordBreakProperty.Hebrew_Letter && left === WordBreakProperty.Double_Quote && - right === WordBreakProperty.Hebrew_Letter) + if(state.match([WordBreakProperty.Hebrew_Letter], + [WordBreakProperty.Double_Quote], + [WordBreakProperty.Hebrew_Letter], + null)) { continue; + } // Do not break within sequences of digits, or digits adjacent to letters. // e.g., "3a" or "A3" // WB8 - if (left === WordBreakProperty.Numeric && right === WordBreakProperty.Numeric) + if(state.match(null, [WordBreakProperty.Numeric], [WordBreakProperty.Numeric], null)) { continue; + } // WB9 - if (isAHLetter(left) && right === WordBreakProperty.Numeric) + if(state.match(null, SET_AHLETTER, [WordBreakProperty.Numeric], null)) { continue; + } // WB10 - if (left === WordBreakProperty.Numeric && isAHLetter(right)) + if(state.match(null, [WordBreakProperty.Numeric], SET_AHLETTER, null)) { continue; + } // Do not break within sequences, such as 3.2, 3,456.789 // WB11 - if (lookbehind === WordBreakProperty.Numeric && right === WordBreakProperty.Numeric && - (left === WordBreakProperty.MidNum || isMidNumLetQ(left))) + const SET_ALL_MIDNUM = [WordBreakProperty.MidNum, ...SET_MIDNUMLETQ]; + if(state.match([WordBreakProperty.Numeric], SET_ALL_MIDNUM, [WordBreakProperty.Numeric], null)) { continue; + } // WB12 - if (left === WordBreakProperty.Numeric && lookahead === WordBreakProperty.Numeric && - (right === WordBreakProperty.MidNum || isMidNumLetQ(right))) + if(state.match(null, [WordBreakProperty.Numeric], SET_ALL_MIDNUM, [WordBreakProperty.Numeric])) { continue; + } // WB13: Do not break between Katakana - if (left === WordBreakProperty.Katakana && right === WordBreakProperty.Katakana) + if(state.match(null, [WordBreakProperty.Katakana], [WordBreakProperty.Katakana], null)) { continue; + } // Do not break from extenders (e.g., U+202F NARROW NO-BREAK SPACE) // WB13a - if ((isAHLetter(left) || - left === WordBreakProperty.Numeric || - left === WordBreakProperty.Katakana || - left === WordBreakProperty.ExtendNumLet) && - right === WordBreakProperty.ExtendNumLet) + const SET_NUM_KAT_LET = [WordBreakProperty.Katakana, + WordBreakProperty.Numeric, + ...SET_AHLETTER]; + if(state.match(null, SET_NUM_KAT_LET, [WordBreakProperty.ExtendNumLet], null)) { continue; + } + if(state.match(null, [WordBreakProperty.ExtendNumLet], [WordBreakProperty.ExtendNumLet], null)) { + continue; + } // WB13b - if ((isAHLetter(right) || - right === WordBreakProperty.Numeric || - right === WordBreakProperty.Katakana) && left === WordBreakProperty.ExtendNumLet) + if(state.match(null, [WordBreakProperty.ExtendNumLet], SET_NUM_KAT_LET, null)) { continue; + } // WB15 & WB16: // Do not break within emoji flag sequences. That is, do not break between // regional indicator (RI) symbols if there is an odd number of RI // characters before the break point. - if (right === WordBreakProperty.Regional_Indicator) { + if (state.right === WordBreakProperty.Regional_Indicator) { // Emoji flags are actually composed of TWO scalar values, each being a // "regional indicator". These indicators correspond to Latin letters. Put // two of them together, and they spell out an ISO 3166-1-alpha-2 country @@ -300,34 +549,6 @@ namespace wordBreakers { } return pos + 1; } - - /** - * Return the value of the Word_Break property at the given string index. - * @param pos position in the text. - */ - function wordbreakPropertyAt(pos: number) { - if (pos < 0) { - return WordBreakProperty.sot; // Always "start of string" before the string starts! - } else if (pos >= text.length) { - return WordBreakProperty.eot; // Always "end of string" after the string ends! - } else if (isStartOfSurrogatePair(text[pos])) { - // Surrogate pairs the next TWO items from the string! - return property(text[pos] + text[pos + 1]); - } - return property(text[pos]); - } - - // Word_Break rule macros - // See: https://unicode.org/reports/tr29/#WB_Rule_Macros - function isAHLetter(prop: WordBreakProperty): boolean { - return prop === WordBreakProperty.ALetter || - prop === WordBreakProperty.Hebrew_Letter; - } - - function isMidNumLetQ(prop: WordBreakProperty): boolean { - return prop === WordBreakProperty.MidNumLet || - prop === WordBreakProperty.Single_Quote; - } } function isStartOfSurrogatePair(character: string) { @@ -340,15 +561,37 @@ namespace wordBreakers { * Note that * @param character a scalar value */ - function property(character: string): WordBreakProperty { + function property(character: string, options?: DefaultWordBreakerOptions): WordBreakProperty { + // If there is a customized mapping for the character, prioritize that. + if(options?.propertyMapping) { + let propName = options.propertyMapping(character); + if(propName) { + return propertyVal(propName, options); + } + } + // This MUST be a scalar value. // TODO: remove dependence on character.codepointAt()? let codepoint = character.codePointAt(0) as number; + return searchForProperty(codepoint, 0, WORD_BREAK_PROPERTY.length - 1); } + function propertyVal(propName: string, options?: DefaultWordBreakerOptions) { + const matcher = (name: string) => name.toLowerCase() == propName.toLowerCase() + + const customIndex = options?.customProperties?.findIndex(matcher) ?? -1; + return customIndex != -1 ? -customIndex - 1 : data.propertyMap.findIndex(matcher); + } + /** * Binary search for the word break property of a given CODE POINT. + * + * The auto-generated data.ts master array defines a **character range** + * lookup table. If a character's codepoint is equal to or greater than + * the I.Start value for an entry and exclusively less than the next entry, + * it falls in the first entry's range bucket and is classified accordingly + * by this method. */ function searchForProperty(codePoint: number, left: number, right: number): WordBreakProperty { // All items that are not found in the array are assigned the 'Other' property. diff --git a/common/models/wordbreakers/test/test-ascii-word-breaker.js b/common/models/wordbreakers/test/test-ascii-word-breaker.js index 598780ad95..55b9c3b8ba 100644 --- a/common/models/wordbreakers/test/test-ascii-word-breaker.js +++ b/common/models/wordbreakers/test/test-ascii-word-breaker.js @@ -1,5 +1,5 @@ var assert = require('chai').assert; -var breakASCIIWords = require('..').wordBreakers['ascii']; +var breakASCIIWords = require('../build').wordBreakers['ascii']; describe('The ASCII word breaker', function () { it('should break simple English sentences', function () { diff --git a/common/models/wordbreakers/test/test-default-word-breaker.js b/common/models/wordbreakers/test/test-default-word-breaker.js index 6cf2362695..85129b836c 100644 --- a/common/models/wordbreakers/test/test-default-word-breaker.js +++ b/common/models/wordbreakers/test/test-default-word-breaker.js @@ -3,21 +3,339 @@ */ const assert = require('chai').assert; -const breakWords = require('..').wordBreakers['default']; +const breakWords = require('../build').wordBreakers['default']; -const SHY = '\u00AD'; +const SHY = '\u00AD'; // Other, Format. The "Soft HYphen" - usually invisible unless needed for word-wrapping. describe('The default word breaker', function () { - it('should break multilingual text', function () { - let breaks = breakWords( - `ะ”ะพะฑั€ั‹ะน ะดะตะฝัŒ! แ‘•แปแ™ฎ โ€” after working on ka${SHY}wen${SHY}non:${SHY}nis, - let's eat phแปŸ! ๐Ÿฅฃ` - ); - let words = breaks.map(span => span.text); - assert.deepEqual(words, [ - 'ะ”ะพะฑั€ั‹ะน', 'ะดะตะฝัŒ', '!', 'แ‘•แป', 'แ™ฎ', 'โ€”', 'after', - 'working', 'on', `ka${SHY}wen${SHY}non:${SHY}nis`, ',', - "let's", 'eat', 'phแปŸ', '!', '๐Ÿฅฃ' - ]); + describe('default configuration', function() { + it('should break multilingual text', function () { + let breaks = breakWords( + `ะ”ะพะฑั€ั‹ะน ะดะตะฝัŒ! แ‘•แปแ™ฎ โ€” after working on ka${SHY}wen${SHY}non:${SHY}nis, + let's eat phแปŸ! ๐Ÿฅฃ` + ); + let words = breaks.map(span => span.text); + assert.deepEqual(words, [ + 'ะ”ะพะฑั€ั‹ะน', 'ะดะตะฝัŒ', '!', 'แ‘•แป', 'แ™ฎ', 'โ€”', 'after', + 'working', 'on', `ka${SHY}wen${SHY}non:${SHY}nis`, ',', + "let's", 'eat', 'phแปŸ', '!', '๐Ÿฅฃ' + ]); + }); + + it('handles heavily-punctuated English text', function() { + // This test case brought to you by http://unicode.org/reports/tr29/#Word_Boundaries, Figure 1. + let breaks = breakWords( + `The quick ("brown") fox can't jump 32.3 feet, right?` + ); + let words = breaks.map(span => span.text); + assert.deepEqual(words, [ + 'The', 'quick', '(', '"', 'brown', '"', ')', 'fox', "can't", + 'jump', '32.3', 'feet', ',', 'right', '?' + ]); + }); + + // The way these two tests are written is a bit much on the "white-box" style, + // but they do decently cover the boundary rules mentioned. + it('Does not split empty contexts (WB1 + WB2)', function() { + let breaks = breakWords(''); + let words = breaks.map(span => span.text); + assert.deepEqual(words, []); + }); + + it('Does split at context boundaries (WB1 + WB2)', function() { + let breaks = breakWords('a'); + let words = breaks.map(span => span.text); + assert.deepEqual(words, ['a']); + }); + + // WB3, WB3a, WB3b are all handled internally, within the top-level function. + + // iff, as in "if and only if" + it('ignores the zero-width joiner iff appropriate (WB4)', function() { + const zwj = '\u200d'; + + let breaks = breakWords(`a${zwj}b\n${zwj}c${zwj}\nd`); + let words = breaks.map(span => span.text); + + // Does NOT ignore the zwj immediately after a newline - the notable exception + // (the reason for "iff", not "if"). + assert.deepEqual(words, [`a${zwj}b`, `${zwj}`, `c${zwj}`, `d`]); + }) + + it('ignores extend characters iff appropriate (WB4)', function() { + const comboGrave = '\u0300'; // The 'combining grave accent', as used in NFD. + + let breaks = breakWords(`a${comboGrave}e\n${comboGrave}i${comboGrave}\no`); + let words = breaks.map(span => span.text); + + // Does NOT ignore the zwj immediately after a newline - the notable exception + // (the reason for "iff", not "if"). + assert.deepEqual(words, [`a${comboGrave}e`, `${comboGrave}`, `i${comboGrave}`, `o`]); + }); + + it('ignores format characters iff appropriate (WB4)', function() { + // Re-uses `const SHY` from above. + let breaks = breakWords(`a${SHY}e\n${SHY}i${SHY}\no`); + let words = breaks.map(span => span.text); + + // Does NOT ignore the zwj immediately after a newline - the notable exception + // (the reason for "iff", not "if"). + assert.deepEqual(words, [`a${SHY}e`, `${SHY}`, `i${SHY}`, `o`]); + }); + + it('does not break between most alphabetic characters (WB5)', function() { + let breaks = breakWords(`aษ™ฯƒืะ‘ ะปืสˆฮณX`); // a mix of latin, hebrew, greek, cyrillic, and IPA chars + // for both "words". + let words = breaks.map(span => span.text); + + assert.deepEqual(words, [`aษ™ฯƒืะ‘`, `ะปืสˆฮณX`]); + }); + + it('does not break letters across specific punctuation patterns (WB6, WB7)', function() { + // `'`: MidNumLetQ (from Single_Quote) + // '.': MidNumLet + // ':': MidLetter + let breaks = breakWords(`don't b.r.e.a.k t:h:e:s:e`); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, [`don't`, `b.r.e.a.k`, `t:h:e:s:e`]); + + let breaks2 = breakWords(`.drop: :the' 'extras.`); + let words2 = breaks2.map(span => span.text); + + assert.deepEqual(words2, [`.`, `drop`, `:`, `:`, `the`, `'`, `'`, `extras`, `.`]); + + // ',': MidNum (is NOT included by rule!) + let breaks3 = breakWords('do br,eak that'); + let words3 = breaks3.map(span => span.text); + + assert.deepEqual(words3, ['do', 'br', ',', 'eak', 'that']); + }); + + it('treats Hebrew properly (WB7a-c)', function() { + const aleph = 'ื'; + const bet = 'ื‘'; + + // As Hebrew is RTL... this is probably the clearest way for us LTR people to + // clearly see what's going on without ordering mechanics messing up the render. + let breaks = breakWords(`${aleph}' ${aleph}" ${aleph}"${bet}`); + let words = breaks.map(span => span.text); + + // A lingering double-quote isn't cool, but one in the middle's fine. + // Lingering single-quote is fine regardless. + assert.deepEqual(words, [`${aleph}'`, `${aleph}`, `"`, `${aleph}"${bet}`]); + }); + + it(`doesn't break within digit + digit/letter sequences (WB8-10)`, function() { + let breaks = breakWords('a1b2c3 hunter2 ab12cd34 1234567890'); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, ['a1b2c3', 'hunter2', 'ab12cd34', '1234567890']); + }); + + it('does not break within formatted number sequences (WB11-12)', function() { + // Note: `'` fits "MidNumLetQ", part of the two rules! + let breaks = breakWords(`1.2.3 3,458.01 3.45'8,01`); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, [`1.2.3`, `3,458.01`, `3.45'8,01`]); + + let breaks2 = breakWords(`.1' ,3.`); + let words2 = breaks2.map(span => span.text); + + assert.deepEqual(words2, [`.`, `1`, `'`, `,`, `3`, `.`]); + }); + + it('does not break between Katakana (WB13)', function() { + const kataSmA = '\u30a2'; //ใ‚ก + const kataA = '\u30a2'; //ใ‚ข + const kataSound = '\u309b'; // ใ‚› + + let breaks = breakWords(`${kataSound}${kataA} ${kataSmA}${kataSound}b ${kataA}${kataSound}${kataSmA}`); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, [ + `${kataSound}${kataA}`, + `${kataSmA}${kataSound}`, + 'b', + `${kataA}${kataSound}${kataSmA}` + ]); + }); + + it('does not break form extenders (WB13a-b)', function() { + // The `_` (underscore) fits the ExtendNumLet class this rule focuses on. + const kataA = '\u30a2'; //ใ‚ข + + let breaks = breakWords(`${kataA}_a__0_b_${kataA} _${kataA} 1_ _c_ ____`); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, [ + `${kataA}_a__0_b_${kataA}`, + `_${kataA}`, + `1_`, + `_c_`, + `____` + ]); + }); + + it('handles emoji flag sequences properly (WB15-16)', function() { + // For clarity on what's being tested... + let CA_FLAG = '\u{1f1e8}\u{1f1e6}' // '๐Ÿ‡จ๐Ÿ‡ฆ' (canadian flag emoji); should not be broken. + let KH_FLAG = '\u{1f1f0}\u{1f1ed}' // '๐Ÿ‡ฐ๐Ÿ‡ญ' (khmer flag emoji); same + let X_FLAG_PIECE = '\u{1f1fd}' // '๐Ÿ‡ฝ' (half of a flag emoji; '๐Ÿ‡ฝ๐Ÿ‡ฝ' doesn't match a flag) + let breaks = breakWords(`${CA_FLAG}${KH_FLAG}${X_FLAG_PIECE}${X_FLAG_PIECE}`); + let words = breaks.map(span => span.text); + + // Note that the emoji may not render well within VSCode, but they show up nicely on GitHub. + assert.deepEqual(words, ['๐Ÿ‡จ๐Ÿ‡ฆ', '๐Ÿ‡ฐ๐Ÿ‡ญ', '๐Ÿ‡ฝ๐Ÿ‡ฝ']); + }); + + it('breaks hyphenated words by default', function() { + let breaks = breakWords('Smith-Jones'); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, ['Smith', '-', 'Jones']); + }); + }); + + describe('customization', function() { + // Refer to https://unicode.org/reports/tr29/#Word_Boundary_Rules, third bullet point. + it('custom prop, rule: do not break on letter-adjacent hyphens', function() { + let customization = { + rules: [{ + match: (context) => { + if(context.propertyMatch(null, ["ALetter"], ["Hyphen"], ["ALetter"])) { + return true; + } else if(context.propertyMatch(["ALetter"], ["Hyphen"], ["ALetter"], null)) { + return true; + } else { + return false; + } + }, + breakIfMatch: false + }], + propertyMapping: (char) => { + const validHyphenCodes = [ + '\u002d', '\u2010', '\u058a', '\u30a0' + ]; + if(validHyphenCodes.includes(char)) { + return "Hyphen"; + } + + return null; + }, + customProperties: ["Hyphen"] + } + + let breaks = breakWords('Smith-Jones', customization); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, ['Smith-Jones']); + }); + + it('mid-word hyphen via reassignment to MidLetter', function() { + let customization = { + propertyMapping: (char) => { + const validHyphenCodes = [ + '\u002d', '\u2010', '\u058a', '\u30a0' + ]; + if(validHyphenCodes.includes(char)) { + return "MidLetter"; + } + + return null; + } + } + + let breaks = breakWords('Smith-Jones', customization); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, ['Smith-Jones']); + }); + + // Useful for some regional minority languages that prefer word-breaking spaces. + it('character reassignment: Khmer letters as ALetter', function() { + let customization = { + propertyMapping: (char) => { + if(char >= '\u1780' && char <= '\u17b3') { + return "ALetter"; + } else { + // The other Khmer characters already have useful word-breaking + // property assignments. + return null; + } + } + } + + let breaks = breakWords('แžŸแŸ’แžšแžปแž€ แžแŸ’แž˜แŸ‚แžš', customization); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, ['แžŸแŸ’แžšแžปแž€', 'แžแŸ’แž˜แŸ‚แžš']); + }); + + // See: suggested language-specific WB5a from the spec's notes. + it("french/italian apostrophe / vowel boundaries", function() { + let customization = { + rules: [ + // WB5, but with differentiated consonants (ALetter) and vowels (AVowel) + { + match: (context) => { + if(context.propertyMatch(null, ["ALetter", "AVowel"], ["ALetter", "AVowel"], null)) { + return true; + } else { + return false; + } + }, + breakIfMatch: false + }, + // Proposed WB5a + { + match: (context) => { + if(context.propertyMatch(null, ["Single_Quote"], ["AVowel"], null)) { + return true; + } else { + return false; + } + }, + breakIfMatch: true + }, + // WB6, 7 + { + match: (context) => { + if(context.propertyMatch(null, + ["ALetter", "AVowel"], + ["MidLetter", "MidNumLet", "Single_Quote"], + ["ALetter", "AVowel"])) { + return true; + } else if(context.propertyMatch(["ALetter", "AVowel"], + ["MidLetter", "MidNumLet", "Single_Quote"], + ["ALetter", "AVowel"], + null)) { + return true; + } else { + return false; + } + }, + breakIfMatch: false + } + // Similar extensions to WB9, 10, 13a, and 13b would also be needed for robustness. + // And I kind of left the Hebrew_Letter out of the WB5, 6, and 7 rewrites. + ], + propertyMapping: (char) => { + const vowels = ['a', 'e', 'i', 'o', 'u']; + if(vowels.includes(char)) { + return "AVowel"; + } + + return null; + }, + customProperties: ["AVowel"] + } + + let breaks = breakWords("l'objectif aujourd'hui", customization); + let words = breaks.map(span => span.text); + + assert.deepEqual(words, ["l'", "objectif", "aujourd'hui"]); + }); }); }); diff --git a/common/models/wordbreakers/test/test-placeholder-word-breaker.js b/common/models/wordbreakers/test/test-placeholder-word-breaker.js index b23fd5d6bb..dbc1aef34e 100644 --- a/common/models/wordbreakers/test/test-placeholder-word-breaker.js +++ b/common/models/wordbreakers/test/test-placeholder-word-breaker.js @@ -1,5 +1,5 @@ const assert = require('chai').assert; -const breakWords = require('..').wordBreakers['placeholder']; +const breakWords = require('../build').wordBreakers['placeholder']; describe('The placeholder word breaker', function () { diff --git a/common/predictive-text/package.json b/common/predictive-text/package.json index d7c12767ad..aedbc982b9 100644 --- a/common/predictive-text/package.json +++ b/common/predictive-text/package.json @@ -37,7 +37,7 @@ "karma-mocha-reporter": "^2.2.5", "karma-safari-launcher": "^1.0.0", "karma-teamcity-reporter": "^1.1.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "sinon": "^14.0.0", "ts-node": "^9.1.1", diff --git a/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js b/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js index 6a4148b669..fa09ca20de 100644 --- a/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js +++ b/common/predictive-text/unit_tests/headless/edit-distance/context-tracker.js @@ -45,7 +45,7 @@ describe('ContextTracker', function() { assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); }); - it("properly matches and aligns when a 'wordbreak' is added'", function() { + it("properly matches and aligns when a 'wordbreak' is added", function() { let existingContext = ["an", "apple", "a", "day", "keeps", "the", "doctor"]; let transform = { insert: ' ', @@ -56,7 +56,7 @@ describe('ContextTracker', function() { let rawTokens = ["an", null, "apple", null, "a", null, "day", null, "keeps", null, "the", null, "doctor", null, ""]; let existingState = ContextTracker.modelContextState(existingContext); - let state = ContextTracker.attemptMatchContext(newContext, existingState, null, toWrapperDistribution(transform)); + let state = ContextTracker.attemptMatchContext(newContext, existingState, toWrapperDistribution(transform)); assert.isNotNull(state); assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); @@ -65,6 +65,26 @@ describe('ContextTracker', function() { assert.isEmpty(state.tokens[state.tokens.length - 1].transformDistributions); }); + it("properly matches and aligns when an implied 'wordbreak' occurs (as when following \"'\")", function() { + let existingContext = ["'"]; + let transform = { + insert: 'a', + deleteLeft: 0 + } + let newContext = Array.from(existingContext); + newContext.push('a'); // The incoming transform should produce a new token WITH TEXT. + let rawTokens = ["'", null, "a"]; + + let existingState = ContextTracker.modelContextState(existingContext); + let state = ContextTracker.attemptMatchContext(newContext, existingState, toWrapperDistribution(transform)); + assert.isNotNull(state); + assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); + + // The 'wordbreak' transform + assert.isEmpty(state.tokens[state.tokens.length - 2].transformDistributions); + assert.isNotEmpty(state.tokens[state.tokens.length - 1].transformDistributions); + }); + it("properly matches and aligns when lead token is removed AND a 'wordbreak' is added'", function() { let existingContext = ["an", "apple", "a", "day", "keeps", "the", "doctor"]; let transform = { @@ -77,7 +97,7 @@ describe('ContextTracker', function() { let rawTokens = ["apple", null, "a", null, "day", null, "keeps", null, "the", null, "doctor", null, ""]; let existingState = ContextTracker.modelContextState(existingContext); - let state = ContextTracker.attemptMatchContext(newContext, existingState, null, toWrapperDistribution(transform)); + let state = ContextTracker.attemptMatchContext(newContext, existingState, toWrapperDistribution(transform)); assert.isNotNull(state); assert.deepEqual(state.tokens.map(token => token.raw), rawTokens); diff --git a/common/predictive-text/unit_tests/headless/transform-utils.js b/common/predictive-text/unit_tests/headless/transform-utils.js new file mode 100644 index 0000000000..b00a24a123 --- /dev/null +++ b/common/predictive-text/unit_tests/headless/transform-utils.js @@ -0,0 +1,58 @@ +var assert = require('chai').assert; + +let TransformUtils = require('../../../web/lm-worker/build/intermediate.js').TransformUtils; + +describe('TransformUtils', function () { + describe('isWhitespace', function () { + it("should not match a string containing standard alphabetic characters", function () { + let testTransforms = [{ + insert: "a ", + deleteLeft: 0 + }, { + insert: " a", + deleteLeft: 0 + }, { + insert: "ab", + deleteLeft: 0 + }]; + + testTransforms.forEach((transform) => assert.isFalse(TransformUtils.isWhitespace(transform), `failed with: '${transform.insert}'`)); + }); + + it("should match a simple ' ' transform", function() { + transform = { + insert: " ", + deleteLeft: 0 + }; + + assert.isTrue(TransformUtils.isWhitespace(transform)); + }); + + it("should match a simple ' ' transform with delete-left", function() { + transform = { + insert: " ", + deleteLeft: 1 + }; + + assert.isTrue(TransformUtils.isWhitespace(transform)); + }); + + it("should match a transform consisting of multiple characters of only whitespace", function() { + transform = { + insert: " \n\r\u00a0\t\u2000 ", + deleteLeft: 0 + }; + + assert.isTrue(TransformUtils.isWhitespace(transform)); + }); + + it("stress tests", function() { + transform = { + insert: " \n\r\u00a0\ta\u2000 ", // the 'a' should cause failure. + deleteLeft: 0 + }; + + assert.isFalse(TransformUtils.isWhitespace(transform)); + }); + }); +}); diff --git a/common/predictive-text/unit_tests/headless/worker-model-compositor.js b/common/predictive-text/unit_tests/headless/worker-model-compositor.js index f7e9173cee..8080b48f47 100644 --- a/common/predictive-text/unit_tests/headless/worker-model-compositor.js +++ b/common/predictive-text/unit_tests/headless/worker-model-compositor.js @@ -50,6 +50,45 @@ describe('ModelCompositor', function() { }); }); + it('strongly avoids corrections for single-character roots', function() { + let compositor = new ModelCompositor(plainModel); + let context = { + left: '', startOfBuffer: true, endOfBuffer: true, + }; + + // The 'weights' involved imply that we have an edge-case fat finger on the bottom of + // the 'q' key, slightly in its favor. + let inputDistribution = [ + {sample: {insert: 'q', deleteLeft: 0}, p: 0.5}, // 'quite' (679) and 'question' (644) are included! + {sample: {insert: 'a', deleteLeft: 0}, p: 0.4} // but at lower weight than 'and' (998). + ]; + + compositor.predict({insert: '', deleteLeft: 0}, context); // Initialize context tracking first! + let suggestions = compositor.predict(inputDistribution, context); + + // remove the keep suggestion; we're not testing that here. + suggestions = suggestions.filter((suggestion) => suggestion.tag != 'keep'); + + suggestions.sort((a, b) => b.p - a.p); + + // There are only 4 suggestions in this limited test model that begin with 'q'. + // We expect more than that, since 'a' is indicated to be very close by. + assert.isAbove(suggestions.length, 4, "fat-finger style corrections needed for test comparisons are missing"); + + // Note: 'and' is (currently) modeled by the text-fixture model to have 9.3x the base probability + // that the worst 'q'-rooted suggestion ('quality') does. Without single-character correction + // avoidance logic, this test _will_ fail. + // + // In case a tweak to test parameters is desired, note that 'and' beats rank #3 - 'questions' - + // at 3.36x base. At the time of writing this test, upping 'a's probability to 0.45 will block + // 'quality' while the top three 'q's (ending with 'questions') remain in place. + let qRange = suggestions.slice(0, 4); + assert.isUndefined(qRange.find((suggestion) => suggestion.transform.insert.charAt(0) != 'q')); + + let aRange = suggestions.slice(4); + assert.isUndefined(aRange.find((suggestion) => suggestion.transform.insert.charAt(0) == 'q')); + }); + it('properly handles suggestions after a backspace', function() { let compositor = new ModelCompositor(plainModel); let context = { @@ -66,12 +105,53 @@ describe('ModelCompositor', function() { // Suggestions always delete the full root of the suggestion. // // After a backspace, that means the text 'the' - 3 chars. - // Char 4 is for the original backspace, as suggstions are built + // Char 4 is for the original backspace, as suggestions are built // based on the context state BEFORE the triggering input - // here, a backspace. assert.equal(suggestion.transform.deleteLeft, 4); }); }); + + it('properly handles suggestions for the first letter after a ` `', function() { + let compositor = new ModelCompositor(plainModel); + let context = { + left: 'the', startOfBuffer: true, endOfBuffer: true, + }; + + let inputTransform = { + insert: ' ', + deleteLeft: 0 + }; + + let suggestions = compositor.predict(inputTransform, context); + suggestions.forEach(function(suggestion) { + // After a space, predictions are based on a new, zero-length root. + // With nothing to replace, .deleteLeft should be zero. + assert.equal(suggestion.transform.deleteLeft, 0); + }); + }); + + it('properly handles suggestions for the first letter after a `\'`', function() { + let compositor = new ModelCompositor(plainModel); + let context = { + left: "the '", startOfBuffer: true, endOfBuffer: true, + }; + + // This results in a new word boundary (between the `'` and the `a`). + // Basically, an implied (but nonexistent) ` `. + let inputTransform = { + insert: "a", + deleteLeft: 0 + }; + + let suggestions = compositor.predict(inputTransform, context); + suggestions.forEach(function(suggestion) { + // Suggestions always delete the full root of the suggestion. + // Which, here, didn't exist before the input. Nothing to + // replace => nothing for the suggestion to delete. + assert.equal(suggestion.transform.deleteLeft, 0); + }); + }); }); describe('applySuggestionCasing', function() { diff --git a/common/predictive-text/unit_tests/test.sh b/common/predictive-text/unit_tests/test.sh index ed8c787c3e..8329e8c417 100755 --- a/common/predictive-text/unit_tests/test.sh +++ b/common/predictive-text/unit_tests/test.sh @@ -5,6 +5,7 @@ THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BA . "$(dirname "$THIS_SCRIPT")/../../../resources/build/build-utils.sh" ## END STANDARD BUILD SCRIPT INCLUDE +. "$KEYMAN_ROOT/resources/build/build-utils-ci.inc.sh" . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" SCRIPT_ROOT="$(dirname "$THIS_SCRIPT")" @@ -33,6 +34,18 @@ test-headless ( ) { _FLAGS="$_FLAGS --reporter mocha-teamcity-reporter" fi + pushd "$KEYMAN_ROOT/common/models/wordbreakers" + npm run test || fail "models/wordbreakers tests failed" + popd + + pushd "$KEYMAN_ROOT/common/models/templates" + npm run test || fail "models/templates tests failed" + popd + + pushd "$KEYMAN_ROOT/common/models/types" + npm run test || fail "models/types tests failed" + popd + npm run mocha -- --recursive $_FLAGS ./unit_tests/headless/*.js ./unit_tests/headless/**/*.js } @@ -85,6 +98,24 @@ if (( RUN_HEADLESS )); then test-headless || fail "DOMless tests failed!" fi +if (( RUN_BROWSERS )); then + if [[ $VERSION_ENVIRONMENT == test ]]; then + # If we are running a TeamCity test build, for now, only run BrowserStack + # tests when on a PR branch with a title including "(web)" or with the label + # test-browserstack. This is because the BrowserStack tests are currently + # unreliable, and the false positive failures are masking actual failures. + # + # We do not run BrowserStack tests on master, beta, or stable-x.y test + # builds. + RUN_BROWSERS=0 + if builder_pull_get_details; then + if [[ $builder_pull_title =~ \(web\) ]] || builder_pull_has_label test-browserstack; then + RUN_BROWSERS=1 + fi + fi + fi +fi + # Run browser-based tests. if (( RUN_BROWSERS )); then test-browsers || fail "Browser-based tests failed!" diff --git a/common/test/resources/karma-browserstack-config.js b/common/test/resources/karma-browserstack-config.js index b65b46d104..fd61c40e28 100644 --- a/common/test/resources/karma-browserstack-config.js +++ b/common/test/resources/karma-browserstack-config.js @@ -141,7 +141,7 @@ module.exports = function(baseConfigParams /* the project's base configuration // Concurrency level // We are alloted 5 total at once from BrowserStack. That said, note that we have multiple build configs that may // need BrowserStack-based testing simultaneously. It may be best to avoid bottlenecking on that limitation. - concurrency: 2, + concurrency: 1, customLaunchers: FINAL_LAUNCHER_DEFS, diff --git a/common/web/gesture-recognizer/package.json b/common/web/gesture-recognizer/package.json index e00cbd7dc3..83502ddbb2 100644 --- a/common/web/gesture-recognizer/package.json +++ b/common/web/gesture-recognizer/package.json @@ -19,7 +19,7 @@ "karma-mocha-reporter": "^2.2.5", "karma-safari-launcher": "^1.0.0", "karma-teamcity-reporter": "^1.1.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "sinon": "^14.0.0", "ts-node": "^9.1.1", diff --git a/common/web/input-processor/package.json b/common/web/input-processor/package.json index 9c9fc2d3ab..bf9e13cd41 100644 --- a/common/web/input-processor/package.json +++ b/common/web/input-processor/package.json @@ -21,7 +21,7 @@ "@keymanapp/resources-gosh": "*", "@types/node": "^11.9.4", "chai": "^4.3.4", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "ts-node": "^9.1.1", "typescript": "^4.5.4" diff --git a/common/web/input-processor/src/text/inputProcessor.ts b/common/web/input-processor/src/text/inputProcessor.ts index 9471c244fe..b45098e538 100644 --- a/common/web/input-processor/src/text/inputProcessor.ts +++ b/common/web/input-processor/src/text/inputProcessor.ts @@ -12,7 +12,7 @@ namespace com.keyman.text { /** * Indicates the device (platform) to be used for non-keystroke events, - * such as those sent to `begin postkeystroke` and `begin newcontext` + * such as those sent to `begin postkeystroke` and `begin newcontext` * entry points. */ private contextDevice: utils.DeviceSpec; @@ -273,6 +273,7 @@ namespace com.keyman.text { let totalMass = 0; // Tracks sum of non-error probabilities. for(let pair of keyDistribution) { if(pair.p < KEYSTROKE_EPSILON) { + totalMass += pair.p; break; } else if(timer && timer() >= TIMEOUT_THRESHOLD) { // Note: it's always possible that the thread _executing_ our JS diff --git a/common/web/keyboard-processor/package.json b/common/web/keyboard-processor/package.json index 95f7992a20..253715b3b0 100644 --- a/common/web/keyboard-processor/package.json +++ b/common/web/keyboard-processor/package.json @@ -19,7 +19,7 @@ "devDependencies": { "@keymanapp/resources-gosh": "*", "chai": "^4.3.4", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "ts-node": "^9.1.1", "typescript": "^4.5.4" diff --git a/common/web/keyboard-processor/src/keyboards/activeLayout.ts b/common/web/keyboard-processor/src/keyboards/activeLayout.ts index b1be473bc6..7f0f2f935f 100644 --- a/common/web/keyboard-processor/src/keyboards/activeLayout.ts +++ b/common/web/keyboard-processor/src/keyboards/activeLayout.ts @@ -37,6 +37,7 @@ namespace com.keyman.keyboards { layer: string; displayLayer: string; nextlayer: string; + sp?: ButtonClass; private baseKeyEvent: text.KeyEvent; isMnemonic: boolean = false; @@ -73,6 +74,13 @@ namespace com.keyman.keyboards { return this.id; } + @Enumerable + public get isPadding(): boolean { + // Does not include 9 (class: blank) as that may be an intentional 'catch' for misplaced + // keystrokes. + return this['sp'] == 10; // Button class: hidden. + } + /** * A unique identifier based on both the key ID & the 'desktop layer' to be used for the key. * @@ -337,14 +345,12 @@ namespace com.keyman.keyboards { // Allow for right OSK margin (15 layout units) let rightMargin = ActiveKey.DEFAULT_RIGHT_MARGIN/totalWidth; - totalPercent += rightMargin; // If a single key, and padding is negative, add padding to right align the key if(keys.length == 1 && parseInt(keys[0]['pad'],10) < 0) { keyPercent=parseInt(keys[0]['width'],10)/totalWidth; keys[0]['widthpc']=keyPercent; - totalPercent += keyPercent; - keys[0]['padpc']=1-totalPercent; + keys[0]['padpc']=1-(totalPercent + keyPercent + rightMargin); // compute center's default x-coord (used in headless modes) setProportions(keys[0] as ActiveKey, padPercent, keyPercent, totalPercent); @@ -352,8 +358,7 @@ namespace com.keyman.keyboards { let j=keys.length-1; padPercent=parseInt(keys[j]['pad'],10)/totalWidth; keys[j]['padpc']=padPercent; - totalPercent += padPercent; - keys[j]['widthpc'] = keyPercent = 1-totalPercent; + keys[j]['widthpc'] = keyPercent = 1-(totalPercent + padPercent + rightMargin); // compute center's default x-coord (used in headless modes) setProportions(keys[j] as ActiveKey, padPercent, keyPercent, totalPercent); @@ -515,7 +520,7 @@ namespace com.keyman.keyboards { // Should we wish to allow multiple different transforms for distance -> probability, use a function parameter in place // of the formula in the loop below. for(let key in keyDists) { - totalMass += keyProbs[key] = 1 / (keyDists[key] + 1e-6); // Prevent div-by-0 errors. + totalMass += keyProbs[key] = 1 / (Math.pow(keyDists[key], 2) + 1e-6); // Prevent div-by-0 errors. } for(let key in keyProbs) { @@ -550,6 +555,8 @@ namespace com.keyman.keyboards { // Results in a more optimized distribution. if(text.Codes.isKnownOSKModifierKey(key.baseKeyID)) { return; + } else if(key.isPadding) { // to the user, blank / padding keys do not exist. + return; } } // These represent the within-key distance of the touch from the key's center. diff --git a/common/web/keyboard-processor/src/text/codes.ts b/common/web/keyboard-processor/src/text/codes.ts index 915ae7feed..48fde0d728 100644 --- a/common/web/keyboard-processor/src/text/codes.ts +++ b/common/web/keyboard-processor/src/text/codes.ts @@ -81,10 +81,13 @@ namespace com.keyman.text { case 'K_SHIFT': case 'K_LOPT': case 'K_ROPT': - case 'K_NUMLOCK': // Often used for numeric layers. + case 'K_NUMLOCK': // Often used for numeric layers. case 'K_CAPS': return true; default: + if(Codes.keyCodes[keyID] >= 50000) { // A few are used by `sil_euro_latin`. + return true; // is a 'K_' key defined for layer shifting or 'control' use. + } // Refer to text/codes.ts - these are Keyman-custom "keycodes" used for // layer shifting keys. To be safe, we currently let K_TABBACK and // K_TABFWD through, though we might be able to drop them too. diff --git a/common/web/lm-worker/package.json b/common/web/lm-worker/package.json index 753948495a..13f55cbc57 100644 --- a/common/web/lm-worker/package.json +++ b/common/web/lm-worker/package.json @@ -29,7 +29,7 @@ "karma-mocha-reporter": "^2.2.5", "karma-safari-launcher": "^1.0.0", "karma-teamcity-reporter": "^1.1.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "sinon": "^14.0.0", "ts-node": "^9.1.1", diff --git a/common/web/lm-worker/src/correction/context-tracker.ts b/common/web/lm-worker/src/correction/context-tracker.ts index 52e89bedc8..e17e201d78 100644 --- a/common/web/lm-worker/src/correction/context-tracker.ts +++ b/common/web/lm-worker/src/correction/context-tracker.ts @@ -32,10 +32,6 @@ namespace correction { replacements: TrackedContextSuggestion[]; activeReplacementId: number = -1; - get isNew(): boolean { - return this.transformDistributions.length == 0; - } - get currentText(): string { if(this.replacementText === undefined || this.replacementText === null) { return this.raw; @@ -89,7 +85,7 @@ namespace correction { if(token.replacementText) { copy.replacementText = token.replacementText; } - + return copy; }); this.searchSpace = obj.searchSpace; @@ -139,8 +135,8 @@ namespace correction { // Track the Transform that resulted in the whitespace 'token'. // Will be needed for phrase-level correction/prediction. - whitespaceToken.transformDistributions = [transformDistribution]; - + whitespaceToken.transformDistributions = transformDistribution ? [transformDistribution] : []; + whitespaceToken.raw = null; this.tokens.push(whitespaceToken); } @@ -149,19 +145,19 @@ namespace correction { * Used for 14.0's backspace workaround, which flattens all previous Distribution * entries because of limitations with direct use of backspace transforms. * @param tokenText - * @param transformId + * @param transformId */ replaceTailForBackspace(tokenText: USVString, transformId: number) { this.tokens.pop(); // It's a backspace transform; time for special handling! // - // For now, with 14.0, we simply compress all remaining Transforms for the token into - // multiple single-char transforms. Probabalistically modeling BKSP is quite complex, + // For now, with 14.0, we simply compress all remaining Transforms for the token into + // multiple single-char transforms. Probabalistically modeling BKSP is quite complex, // so we simplify by assuming everything remaining after a BKSP is 'true' and 'intended' text. // // Note that we cannot just use a single, monolithic transform at this point b/c - // of our current edit-distance optimization strategy; diagonalization is currently... + // of our current edit-distance optimization strategy; diagonalization is currently... // not very compatible with that. let backspacedTokenContext: Distribution[] = textToCharTransforms(tokenText, transformId).map(function(transform) { return [{sample: transform, p: 1.0}]; @@ -175,7 +171,7 @@ namespace correction { updateTail(transformDistribution: Distribution, tokenText?: USVString) { let editedToken = this.tail; - + // Preserve existing text if new text isn't specified. tokenText = tokenText || (tokenText === '' ? '' : editedToken.raw); @@ -191,7 +187,7 @@ namespace correction { toRawTokenization() { let sequence: USVString[] = []; - + for(let token of this.tokens) { // Hide any tokens representing wordbreaks. (Thinking ahead to phrase-level possibilities) if(token.currentText !== null) { @@ -281,7 +277,7 @@ namespace correction { /** * Returns items contained within the circular array, ordered from 'oldest' to 'newest' - * the same order in which the items will be dequeued. - * @param index + * @param index */ item(index: number) { if(index >= this.count) { @@ -294,7 +290,7 @@ namespace correction { } export class ContextTracker extends CircularArray { - static attemptMatchContext(tokenizedContext: USVString[], + static attemptMatchContext(tokenizedContext: USVString[], matchState: TrackedContextState, transformDistribution?: Distribution,): TrackedContextState { // Map the previous tokenized state to an edit-distance friendly version. @@ -335,7 +331,7 @@ namespace correction { } // Can happen for the first text input after backspace deletes a wordbreaking character, - // thus the new input continues a previous word while dropping the empty word after + // thus the new input continues a previous word while dropping the empty word after // that prior wordbreaking character. // // We can't handle it reliably from this match state, but a previous entry (without the empty token) @@ -353,7 +349,7 @@ namespace correction { // If we've made it here... success! We have a context match! let state: TrackedContextState; - + if(pushedTail) { // On suggestion acceptance, we should update the previous final token. // We do it first so that the acceptance is replicated in the new TrackedContextState @@ -376,7 +372,9 @@ namespace correction { if(primaryInput && primaryInput.insert == "" && primaryInput.deleteLeft == 0 && !primaryInput.deleteRight) { primaryInput = null; } - const isBackspace = primaryInput && primaryInput.insert == "" && primaryInput.deleteLeft > 0 && !primaryInput.deleteRight; + + const isWhitespace = primaryInput && TransformUtils.isWhitespace(primaryInput); + const isBackspace = primaryInput && TransformUtils.isBackspace(primaryInput); const finalToken = tokenizedContext[tokenizedContext.length-1]; /* Assumption: This is an adequate check for its two sub-branches. @@ -388,7 +386,7 @@ namespace correction { * - Assumption: one keystroke may only cause a single token to be appended to the context * - That is, no "reasonable" keystroke would emit a Transform adding two separate word tokens * - For languages using whitespace to word-break, said keystroke would have to include said whitespace to break the assumption. - */ + */ // If there is/was more than one context token available... if(editPath.length > 1) { @@ -399,17 +397,29 @@ namespace correction { // We're adding an additional context token. if(pushedTail) { - // ASSUMPTION: any transform that triggers this case is a pure-whitespace Transform, as we - // need a word-break before beginning a new word's context. - // Worth note: when invalid, the lm-layer already has problems in other aspects too. - state.pushWhitespaceToTail(transformDistribution); + const tokenizedTail = tokenizedContext[tokenizedContext.length - 1]; + /* + * Common-case: most transforms that trigger this case are from pure-whitespace Transforms. MOST. + * + * Less-common, but noteworthy: some wordbreaks may occur without whitespace. Example: + * `"o` => ['"', 'o']. Make sure to double-check against `tokenizedContext`! + */ + let pushedToken = new TrackedContextToken(); + pushedToken.raw = tokenizedTail; - let emptyToken = new TrackedContextToken(); - emptyToken.raw = ''; - // Continuing the earlier assumption, that 'pure-whitespace Transform' does not emit any initial characters - // for the new word (token), so the input keystrokes do not correspond to the new text token. - emptyToken.transformDistributions = []; - state.pushTail(emptyToken); + if(isWhitespace || !primaryInput) { + state.pushWhitespaceToTail(transformDistribution ?? []); + // Continuing the earlier assumption, that 'pure-whitespace Transform' does not emit any initial characters + // for the new word (token), so the input keystrokes do not correspond to the new text token. + pushedToken.transformDistributions = []; + } else { + state.pushWhitespaceToTail(); + // Assumption: Since we only allow one-transform-at-a-time changes between states, we shouldn't be missing + // any metadata used to construct the new context state token. + pushedToken.transformDistributions = transformDistribution ? [transformDistribution] : []; + } + + state.pushTail(pushedToken); } else { // We're editing the final context token. // TODO: Assumption: we didn't 'miss' any inputs somehow. // As is, may be prone to fragility should the lm-layer's tracked context 'desync' from its host's. @@ -442,7 +452,9 @@ namespace correction { return state; } - static modelContextState(tokenizedContext: USVString[], lexicalModel: LexicalModel): TrackedContextState { + static modelContextState(tokenizedContext: USVString[], + transformDistribution: Distribution, + lexicalModel: LexicalModel): TrackedContextState { let baseTokens = tokenizedContext.map(function(entry) { let token = new TrackedContextToken(); token.raw = entry; @@ -483,13 +495,12 @@ namespace correction { * Compares the current, post-input context against the most recently-seen contexts from previous prediction calls, returning * the most information-rich `TrackedContextState` possible. If a match is found, the state will be annotated with the * input information provided to previous prediction calls and persisted correction-search calculations for re-use. - * - * @param model - * @param context - * @param mainTransform - * @param transformDistribution + * + * @param model + * @param context + * @param transformDistribution */ - analyzeState(model: LexicalModel, + analyzeState(model: LexicalModel, context: Context, transformDistribution?: Distribution): TrackedContextState { if(!model.traverseFromRoot) { @@ -519,7 +530,7 @@ namespace correction { // // Assumption: as a caret needs to move to context before any actual transform distributions occur, // this state is only reached on caret moves; thus, transformDistribution is actually just a single null transform. - let state = ContextTracker.modelContextState(tokenizedContext.left, model); + let state = ContextTracker.modelContextState(tokenizedContext.left, transformDistribution, model); state.taggedContext = context; this.enqueue(state); return state; diff --git a/common/web/lm-worker/src/correction/distance-modeler.ts b/common/web/lm-worker/src/correction/distance-modeler.ts index 7b4456e4d8..723cbc93ca 100644 --- a/common/web/lm-worker/src/correction/distance-modeler.ts +++ b/common/web/lm-worker/src/correction/distance-modeler.ts @@ -204,6 +204,15 @@ namespace correction { // TODO: might should also track diagonalWidth. return inputString + models.SENTINEL_CODE_UNIT + matchString; } + + get isFullReplacement(): boolean { + // If the known edit-distance cost is equal to the input length, this means + // that literally every input has been full-on replaced. Thus, this is + // likely not a good 'root' to use for predictions. + // + // Logic exception: 0 cost, 0 length != a "replacement". + return this.knownCost && this.knownCost == this.priorInput.length; + } } class SearchSpaceTier { @@ -526,8 +535,6 @@ namespace correction { let searchSpace = this; let currentReturns: {[mapKey: string]: SearchNode} = {}; - // JS measures time by the number of milliseconds since Jan 1, 1970. - let timeStart = Date.now(); let maxTime: number; if(waitMillis == 0) { maxTime = Infinity; @@ -537,6 +544,136 @@ namespace correction { maxTime = waitMillis; } + /** + * This inner class is designed to help the algorithm detect its active execution time. + * While there's no official JS way to do this, we can approximate it by polling the + * current system time (in ms) after each iteration of a short-duration loop. Unusual + * spikes in system time for a single iteration is likely to indicate that an OS + * context switch occurred at some point during the iteration's execution. + */ + class ExecutionTimer { + /** + * The system time when this instance was created. + */ + private start: number; + + /** + * Marks the system time at the start of the currently-running loop, as noted + * by a call to the `startLoop` function. + */ + private loopStart: number; + + private maxExecutionTime: number; + private maxTrueTime: number; + + private executionTime: number; + + /** + * Used to track intervals in which potential context swaps by the OS may + * have occurred. Context switches generally seem to pause threads for + * at least 16 ms, while we expect each loop iteration to complete + * within just 1 ms. So, any possible context switch should have the + * longest observed change in system time. + * + * See `updateOutliers` for more details. + */ + private largestIntervals: number[] = [0]; + + constructor(maxExecutionTime: number, maxTrueTime: number) { + // JS measures time by the number of milliseconds since Jan 1, 1970. + this.loopStart = this.start = Date.now(); + this.maxExecutionTime = maxExecutionTime; + this.maxTrueTime = maxTrueTime; + } + + startLoop() { + this.loopStart = Date.now(); + } + + markIteration() { + const now = Date.now(); + const delta = now - this.loopStart; + this.executionTime += delta; + + /** + * Update the list of the three longest system-time intervals observed + * for execution of a single loop iteration. + * + * Ignore any zero-ms length intervals; they'd make the logic much + * messier than necessary otherwise. + */ + if(delta) { + // If the currently-observed interval is longer than the shortest of the 3 + // previously-observed longest intervals, replace it. + if(this.largestIntervals.length > 2 && delta > this.largestIntervals[0]) { + this.largestIntervals[0] = delta; + } else { + this.largestIntervals.push(delta); + } + // Puts the list in ascending order. Shortest of the list becomes the head, + // longest one the tail. + this.largestIntervals.sort(); + + // Then, determine if we need to update our outlier-based tweaks. + this.updateOutliers(); + } + } + + updateOutliers() { + /* Base assumption: since each loop of the search should evaluate within ~1ms, + * notably longer execution times are probably context switches. + * + * Base assumption: OS context switches generally last at least 16ms. (Based on + * a window.setTimeout() usually not evaluating for at least + * that long, even if set to 1ms.) + * + * To mitigate these assumptions: we'll track the execution time of every loop + * iteration. If the longest observation somehow matches or exceeds the length of + * the next two almost-longest observations twice over... we have a very strong + * 'context switch' candidate. + * + * Or, in near-formal math/stats: we expect a very low variance in execution + * time among the iterations of the search's loops. With a very low variance, + * ANY significant proportional spikes in execution time are outliers - outliers + * likely caused by an OS context switch. + * + * Rather than do intensive math, we use a somewhat lazy approach below that + * achieves the same net results given our assumptions, even when relaxed somewhat. + * + * The logic below relaxes the base assumptions a bit to be safe: + * - [2ms, 2ms, 8ms] will cause 8ms to be seen as an outlier. + * - [2ms, 3ms, 10ms] will cause 10ms to be seen as an outlier. + * + * Ideally: + * - [1ms, 1ms, 4ms] will view 4ms as an outlier. + * + * So we can safely handle slightly longer average intervals and slightly shorter + * OS context-switch time intervals. + */ + if(this.largestIntervals.length > 2) { + // Precondition: the `largestIntervals` array is sorted in ascending order. + // Shortest entry is at the head, longest at the tail. + if(this.largestIntervals[2] >= 2 * (this.largestIntervals[0] + this.largestIntervals[1])) { + this.executionTime -= this.largestIntervals[2]; + this.largestIntervals.pop(); + } + } + } + + shouldTimeout(): boolean { + const now = Date.now(); + if(now - this.start > this.maxTrueTime) { + return true; + } + + return this.executionTime > this.maxExecutionTime; + } + + resetOutlierCheck() { + this.largestIntervals = []; + } + } + class BatchingAssistant { currentCost = Number.MIN_SAFE_INTEGER; entries: SearchResult[] = []; @@ -582,17 +719,30 @@ namespace correction { let batcher = new BatchingAssistant(); + const timer = new ExecutionTimer(maxTime*1.5, maxTime); + // Stage 1 - if we already have extracted results, build a queue just for them and iterate over it first. let returnedValues = Object.values(this.returnedValues); if(returnedValues.length > 0) { let preprocessedQueue = new models.PriorityQueue(QUEUE_NODE_COMPARATOR, returnedValues); // Build batches of same-cost entries. + timer.startLoop(); while(preprocessedQueue.count > 0) { let entry = preprocessedQueue.dequeue(); + + // Is the entry a reasonable result? + if(entry.isFullReplacement) { + // If the entry's 'match' fully replaces the input string, we consider it + // unreasonable and ignore it. + continue; + } + let batch = batcher.checkAndAdd(entry); + timer.markIteration(); if(batch) { + // Do not track yielded time. yield batch; } } @@ -601,11 +751,14 @@ namespace correction { // finalize the last preprocessed group without issue. let batch = batcher.tryFinalize(); if(batch) { + // Do not track yielded time. yield batch; } } // Stage 2: the fun part; actually searching! + timer.resetOutlierCheck(); + timer.startLoop(); let timedOut = false; do { let newResult: PathResult; @@ -613,10 +766,9 @@ namespace correction { // Search for a 'complete' path, skipping all partial paths as long as time remains. do { newResult = this.handleNextNode(); + timer.markIteration(); - // (Naive) timeout check! - let now = Date.now(); - if(now - timeStart > maxTime) { + if(timer.shouldTimeout()) { timedOut = true; } } while(!timedOut && newResult.type == 'intermediate') @@ -626,6 +778,13 @@ namespace correction { if(newResult.type == 'none') { break; } else if(newResult.type == 'complete') { + // Is the entry a reasonable result? + if(newResult.finalNode.isFullReplacement) { + // If the entry's 'match' fully replaces the input string, we consider it + // unreasonable and ignore it. Also, if we've reached this point... + // we can(?) assume that everything thereafter is as well. + break; + } batch = batcher.checkAndAdd(newResult.finalNode); } diff --git a/common/web/lm-worker/src/index.ts b/common/web/lm-worker/src/index.ts index 7cc3bcbde1..e3c16fa2c8 100644 --- a/common/web/lm-worker/src/index.ts +++ b/common/web/lm-worker/src/index.ts @@ -32,6 +32,7 @@ /// /// /// +/// /** * Encapsulates all the state required for the LMLayer's worker thread. @@ -407,6 +408,7 @@ if (typeof module !== 'undefined' && typeof module.exports !== 'undefined') { module.exports['wordBreakers'] = wordBreakers; /// XXX: export the ModelCompositor for testing. module.exports['ModelCompositor'] = ModelCompositor; + module.exports['TransformUtils'] = TransformUtils; } else if (typeof self !== 'undefined' && 'postMessage' in self && 'importScripts' in self) { // Automatically install if we're in a Web Worker. LMLayerWorker.install(self as any); // really, 'as typeof globalThis', but we're currently getting TS errors from use of that. diff --git a/common/web/lm-worker/src/model-compositor.ts b/common/web/lm-worker/src/model-compositor.ts index 329460776d..95c2e390fd 100644 --- a/common/web/lm-worker/src/model-compositor.ts +++ b/common/web/lm-worker/src/model-compositor.ts @@ -6,6 +6,23 @@ class ModelCompositor { private static readonly MAX_SUGGESTIONS = 12; readonly punctuation: LexicalModelPunctuation; + /** + * Controls the strength of anti-corrective measures for single-character scenarios. + * The base key probability will be raised to this power for this specific case. + * + * Current selection's motivation: (0.5 / 0.4) ^ 16 ~= 35.5. + * - if the most likely has p=0.5 and second-most has p=0.4 - a highly-inaccurate key + * stroke - the net effect will apply a factor of 35.5 to the lexical probability of + * the best key's prediction roots, favoring it in this manner. + * - less extreme edge cases will have a significantly stronger factor, acting as a + * "soft threshold". + * - truly ambiguous, "coin flip" cases will have a lower factor and thus favor the + * more likely words from the pair. + * - Our OSK key-element borders aren't visible to the user, so the 'spot' where + * behavior changes might feel arbitrary to users if we used a hard threshold instead. + */ + private static readonly SINGLE_CHAR_KEY_PROB_EXPONENT = 16; + private SUGGESTION_ID_SEED = 0; constructor(lexicalModel: LexicalModel) { @@ -16,30 +33,6 @@ class ModelCompositor { this.punctuation = ModelCompositor.determinePunctuationFromModel(lexicalModel); } - protected isWhitespace(transform: Transform): boolean { - // Matches prefixed text + any instance of a character with Unicode general property Z* or the following: CR, LF, and Tab. - let whitespaceRemover = /.*[\u0009\u000A\u000D\u0020\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b\u2028\u2029\u202f\u205f\u3000]/i; - - // Filter out null-inserts; their high probability can cause issues. - if(transform.insert == '') { // Can actually register as 'whitespace'. - return false; - } - - let insert = transform.insert; - - insert = insert.replace(whitespaceRemover, ''); - - return insert == ''; - } - - protected isBackspace(transform: Transform): boolean { - return transform.insert == "" && transform.deleteLeft > 0; - } - - protected isEmpty(transform: Transform): boolean { - return transform.insert == '' && transform.deleteLeft == 0; - } - private predictFromCorrections(corrections: ProbabilityMass[], context: Context): Distribution { let returnedPredictions: Distribution = []; @@ -98,8 +91,8 @@ class ModelCompositor { })[0].sample; // Only allow new-word suggestions if space was the most likely keypress. - let allowSpace = this.isWhitespace(inputTransform); - let allowBksp = this.isBackspace(inputTransform); + let allowSpace = TransformUtils.isWhitespace(inputTransform); + let allowBksp = TransformUtils.isBackspace(inputTransform); let postContext = models.applyTransform(inputTransform, context); let keepOptionText = this.wordbreak(postContext); @@ -109,7 +102,7 @@ class ModelCompositor { // Used to restore whitespaces if operations would remove them. let prefixTransform: Transform; - let contextState: correction.TrackedContextState = null; + let postContextState: correction.TrackedContextState = null; // Section 1: determining 'prediction roots'. if(!this.contextTracker) { @@ -124,18 +117,18 @@ class ModelCompositor { predictionRoots = [{sample: inputTransform, p: 1.0}]; prefixTransform = inputTransform; } else { - predictionRoots = transformDistribution.map(function(alt) { + predictionRoots = transformDistribution.map((alt) => { let transform = alt.sample; // Filter out special keys unless they're expected. - if(this.isWhitespace(transform) && !allowSpace) { + if(TransformUtils.isWhitespace(transform) && !allowSpace) { return null; - } else if(this.isBackspace(transform) && !allowBksp) { + } else if(TransformUtils.isBackspace(transform) && !allowBksp) { return null; } return alt; - }, this); + }); } // Remove `null` entries. @@ -144,12 +137,15 @@ class ModelCompositor { // Running in bulk over all suggestions, duplicate entries may be possible. rawPredictions = this.predictFromCorrections(predictionRoots, context); } else { - contextState = this.contextTracker.analyzeState(this.lexicalModel, - postContext, - !this.isEmpty(inputTransform) ? - transformDistribution: - null - ); + // Token replacement benefits greatly from knowledge of the prior context state. + let contextState = this.contextTracker.analyzeState(this.lexicalModel, context, null); + // Corrections and predictions are based upon the post-context state, though. + postContextState = this.contextTracker.analyzeState(this.lexicalModel, + postContext, + !TransformUtils.isEmpty(inputTransform) ? + transformDistribution: + null + ); // TODO: Should we filter backspaces & whitespaces out of the transform distribution? // Ideally, the answer (in the future) will be no, but leaving it in right now may pose an issue. @@ -158,19 +154,81 @@ class ModelCompositor { // let's just note that right now, there will only ever be one. // // The 'eventual' logic will be significantly more complex, though still manageable. - let searchSpace = contextState.searchSpace[0]; + let searchSpace = postContextState.searchSpace[0]; - let newEmptyToken = false; - // Detect if we're starting a new context state. - let contextTokens = contextState.tokens; - if(contextTokens.length == 0 || contextTokens[contextTokens.length - 1].isNew) { - if(this.isEmpty(inputTransform) || this.isWhitespace(inputTransform)) { - newEmptyToken = true; + // No matter the prediction, once we know the root of the prediction, we'll always 'replace' the + // same amount of text. We can handle this before the big 'prediction root' loop. + let deleteLeft = 0; + + // The amount of text to 'replace' depends upon whatever sort of context change occurs + // from the received input. + const postContextTokens = postContextState.tokens; + let postContextLength = postContextTokens.length; + let contextLengthDelta = postContextTokens.length - contextState.tokens.length; + // If the context now has more tokens, the token we'll be 'predicting' didn't originally exist. + if(postContextLength == 0 || contextLengthDelta > 0) { + // As the word/token being corrected/predicted didn't originally exist, there's no + // part of it to 'replace'. + deleteLeft = 0; + + // If the new token is due to whitespace or due to a different input type that would + // likely imply a tokenization boundary... + if(TransformUtils.isWhitespace(inputTransform)) { + /* TODO: consider/implement: the second half of the comment above. + * For example: on input of a `'`, predict new words instead of replacing the `'`. + * (since after a letter, the `'` will be ignored, anyway) + * + * Idea: if the model's most likely prediction (with no root) would make a new + * token if appended to the current token, that's probably a good case. + * Keeps the check simple & quick. + * + * Might need a mixed mode, though: ';' is close enough that `l` is a reasonable + * fat-finger guess. So yeah, we're not addressing this idea right now. + * - so... consider multiple context behavior angles when building prediction roots? + * + * May need something similar to help handle contractions during their construction, + * but that'd be within `ContextTracker`. + * can' => [`can`, `'`] + * can't => [`can't`] (WB6, 7 of https://unicode.org/reports/tr29/#Word_Boundary_Rules) + * + * (Would also helps WB7b+c for Hebrew text) + */ + + // Infer 'new word' mode, even if we received new text when reaching + // this position. That new text didn't exist before, so still - nothing + // to 'replace'. prefixTransform = inputTransform; - context = postContext; // Ensure the whitespace token is preapplied! + context = postContext; // As far as predictions are concerned, the post-context state + // should not be replaced. Predictions are to be rooted on + // text "up for correction" - so we want a null root for this + // branch. + contextState = postContextState; } + // If the tokenized context length is shorter... sounds like a backspace (or similar). + } else if (contextLengthDelta < 0) { + /* Ooh, we've dropped context here. Almost certainly from a backspace. + * Even if we drop multiple tokens... well, we know exactly how many chars + * were actually deleted - `inputTransform.deleteLeft`. + * Since we replace a word being corrected/predicted, we take length of the remaining + * context's tail token in addition to however far was deleted to reach that state. + */ + deleteLeft = this.wordbreak(postContext).kmwLength() + inputTransform.deleteLeft; + } else { + // Suggestions are applied to the pre-input context, so get the token's original length. + // We're on the same token, so just delete its text for the replacement op. + deleteLeft = this.wordbreak(context).kmwLength(); } + // Is the token under construction newly-constructed / is there no pre-existing root? + // If so, we want to strongly avoid overcorrection, even for 'nearby' keys. + // (Strong lexical frequency differences can easily cause overcorrection when only + // one key's available.) + // + // NOTE: we only want this applied word-initially, when any corrections 'correct' + // 100% of the word. Things are generally fine once it's not "all or nothing." + let tailToken = postContextTokens[postContextTokens.length - 1]; + const isTokenStart = tailToken.transformDistributions.length <= 1; + // TODO: whitespace, backspace filtering. Do it here. // Whitespace is probably fine, actually. Less sure about backspace. @@ -192,19 +250,6 @@ class ModelCompositor { finalInput = inputTransform; // A fallback measure. Greatly matters for empty contexts. } - let deleteLeft = 0; - // remove actual token string. If new token, there should be nothing to delete. - if(!newEmptyToken) { - // If this is triggered from a backspace, make sure to use its results - // and also include its left-deletions! It's the one post-input context case. - if(allowBksp) { - deleteLeft = this.wordbreak(postContext).kmwLength() + inputTransform.deleteLeft; - } else { - // Normal case - use the pre-input context. - deleteLeft = this.wordbreak(context).kmwLength(); - } - } - // Replace the existing context with the correction. let correctionTransform: Transform = { insert: correction, // insert correction string @@ -212,9 +257,39 @@ class ModelCompositor { id: inputTransform.id // The correction should always be based on the most recent external transform/transcription ID. } + let rootCost = match.totalCost; + + /* If we're dealing with the FIRST keystroke of a new sequence, we'll **dramatically** boost + * the exponent to ensure only VERY nearby corrections have a chance of winning, and only if + * there are significantly more likely words. We only need this to allow very minor fat-finger + * adjustments for 100% keystroke-sequence corrections in order to prevent finickiness on + * key borders. + * + * Technically, the probabilities this produces won't be normalized as-is... but there's no + * true NEED to do so for it, even if it'd be 'nice to have'. Consistently tracking when + * to apply it could become tricky, so it's simpler to leave out. + * + * Worst-case, it's possible to temporarily add normalization if a code deep-dive + * is needed in the future. + */ + if(isTokenStart) { + /* Suppose a key distribution: most likely with p=0.5, second-most with 0.4 - a pretty + * ambiguous case that would only arise very near the center of the boundary between two keys. + * Raising (0.5/0.4)^16 ~= 35.53. (At time of writing, SINGLE_CHAR_KEY_PROB_EXPONENT = 16.) + * That seems 'within reason' for correction very near boundaries. + * + * So, with the second-most-likely key being that close in probability, its best suggestion + * must be ~ 35.5x more likely than that of the truly-most-likely key to "win". So, it's not + * a HARD cutoff, but more of a 'soft' one. Keeping the principles in mind documented above, + * it's possible to tweak this to a more harsh or lenient setting if desired, rather than + * being totally "all or nothing" on which key is taken for highly-ambiguous keypresses. + */ + rootCost *= ModelCompositor.SINGLE_CHAR_KEY_PROB_EXPONENT; // note the `Math.exp` below. + } + return { sample: correctionTransform, - p: Math.exp(-match.totalCost) + p: Math.exp(-rootCost) }; }, this); @@ -411,8 +486,8 @@ class ModelCompositor { // Store the suggestions on the final token of the current context state (if it exists). // Or, once phrase-level suggestions are possible, on whichever token serves as each prediction's root. - if(contextState) { - contextState.tail.replacements = suggestions.map(function(suggestion) { + if(postContextState) { + postContextState.tail.replacements = suggestions.map(function(suggestion) { return { suggestion: suggestion, tokenWidth: 1 @@ -659,7 +734,7 @@ class ModelCompositor { // than before. if(this.contextTracker) { let tokenizedContext = models.tokenize(this.lexicalModel.wordbreaker || wordBreakers.default, context); - let contextState = correction.ContextTracker.modelContextState(tokenizedContext.left, this.lexicalModel); + let contextState = correction.ContextTracker.modelContextState(tokenizedContext.left, null, this.lexicalModel); this.contextTracker.enqueue(contextState); } } diff --git a/common/web/lm-worker/src/transformUtils.ts b/common/web/lm-worker/src/transformUtils.ts new file mode 100644 index 0000000000..cbb2c151fe --- /dev/null +++ b/common/web/lm-worker/src/transformUtils.ts @@ -0,0 +1,15 @@ +class TransformUtils { + static isWhitespace(transform: Transform): boolean { + // Matches a string that is entirely one or more characters with Unicode general property Z* or the following: CR, LF, and Tab. + const whitespaceRemover = /^[\u0009\u000A\u000D\u0020\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u200b\u2028\u2029\u202f\u205f\u3000]+$/i; + return transform.insert.match(whitespaceRemover) != null; + } + + static isBackspace(transform: Transform): boolean { + return transform.insert == "" && transform.deleteLeft > 0 && !transform.deleteRight; + } + + static isEmpty(transform: Transform): boolean { + return transform.insert == '' && transform.deleteLeft == 0 && !transform.deleteRight; + } +} \ No newline at end of file diff --git a/common/web/utils/package.json b/common/web/utils/package.json index 680231d3b3..eee6e45040 100644 --- a/common/web/utils/package.json +++ b/common/web/utils/package.json @@ -1,8 +1,8 @@ { "name": "@keymanapp/web-utils", "description": "Common utility functions used throughout other Keyman packages", - "main": "./dist/index.js", - "types": "./dist/index.d.ts", + "main": "./build/index.js", + "types": "./build/index.d.ts", "scripts": { "build": "gosh ./build.sh", "tsc": "tsc" diff --git a/common/windows/cef-checkout.sh b/common/windows/cef-checkout.sh old mode 100644 new mode 100755 diff --git a/common/windows/delphi/general/Upload_Settings.pas b/common/windows/delphi/general/Upload_Settings.pas index 41355e9c0f..f38cc07890 100644 --- a/common/windows/delphi/general/Upload_Settings.pas +++ b/common/windows/delphi/general/Upload_Settings.pas @@ -108,8 +108,8 @@ const // Alpha versions will work against the staging server so that they // can access new APIs etc that will only be available there. The staging // servers have resource constraints but should be okay for limited use. - S_KeymanCom_Staging = 'https://keyman-staging.com'; - S_APIServer_Staging = 'api.keyman-staging.com'; + S_KeymanCom_Staging = 'https://keyman.com'; // #7227 disabling: 'https://keyman-staging.com'; + S_APIServer_Staging = 'api.keyman.com'; // #7227 disabling: 'api.keyman-staging.com'; const URLPath_PackageDownload_Format = '/go/package/download/%0:s?platform=windows&tier=%1:s&bcp47=%2:s&update=%3:d'; diff --git a/common/windows/delphi/setup/SFX.pas b/common/windows/delphi/setup/SFX.pas index 13491802cb..5af3972261 100644 --- a/common/windows/delphi/setup/SFX.pas +++ b/common/windows/delphi/setup/SFX.pas @@ -82,27 +82,38 @@ var fs: TFileStream; ms: TMemoryStream; begin - fs := TFileStream.Create(ParamStr(0), fmOpenRead or fmShareDenyWrite); - ms := TMemoryStream.Create; try - if not FindFirstHeader(fs) then - Exit(False); - - fs.Seek(StartOfFile, TSeekOrigin.soBeginning); - - ms.CopyFrom(fs, fs.Size - StartOfFile); - ms.Position := 0; - - with TZipFile.Create do + fs := TFileStream.Create(ParamStr(0), fmOpenRead or fmShareDenyWrite); + ms := TMemoryStream.Create; try - Open(ms, zmRead); - ExtractAll(ExtPath); + if not FindFirstHeader(fs) then + Exit(False); + + fs.Seek(StartOfFile, TSeekOrigin.soBeginning); + + ms.CopyFrom(fs, fs.Size - StartOfFile); + ms.Position := 0; + + with TZipFile.Create do + try + Open(ms, zmRead); + ExtractAll(ExtPath); + finally + Free; + end; finally - Free; + fs.Free; + ms.Free; + end; + except + on E:Exception do + begin + raise Exception.Create( + 'Failed to extract setup archive. '+ + 'You may have run out of disk space or there may be a '+ + 'problem with the source files.'#13#10#13#10+ + 'The error received was: '+E.Message); end; - finally - fs.Free; - ms.Free; end; Result := True; end; diff --git a/common/windows/mkver.sh b/common/windows/mkver.sh old mode 100644 new mode 100755 diff --git a/core/doc/meson.build b/core/doc/meson.build index cae6b4be60..2772819ce7 100644 --- a/core/doc/meson.build +++ b/core/doc/meson.build @@ -17,7 +17,7 @@ if hotdoc.found() configuration: cfg) deps = files( '../include/keyman/keyboardprocessor.h.in', - '../src/json.hpp', + '../src/jsonpp.hpp', '../src/utfcodec.hpp' ) diff --git a/core/include/keyman/keyboardprocessor.h b/core/include/keyman/keyboardprocessor.h index 3a6700c833..efac59d472 100644 --- a/core/include/keyman/keyboardprocessor.h +++ b/core/include/keyman/keyboardprocessor.h @@ -1118,6 +1118,49 @@ KMN_API km_kbp_status km_kbp_process_queued_actions(km_kbp_state *state); +/* +``` +### `km_kbp_event` +##### Description: +Tell the keyboard processor that an external event has occurred, such as a keyboard +being activated through the language switching UI. +##### Return status: +- `KM_KBP_STATUS_OK`: On success. +- `KM_KBP_STATUS_NO_MEM`: +In the event memory is unavailable to allocate internal buffers. +- `KM_KBP_STATUS_INVALID_ARGUMENT`: +In the event the `state` pointer is null or an invalid event or data is passed. + +The keyboard processor may generate actions which should be processed by the +consumer of the API. + +The action list will be cleared at the start of this call; options and context in +the state may also be modified. + +##### Parameters: +- __state__: A pointer to the opaque state object. +- __event__: The event to be processed, from km_kbp_event_code enumeration +- __data__: Additional event-specific data. Currently unused, must be nullptr. + +```c +*/ +KMN_API +km_kbp_status +km_kbp_event( + km_kbp_state *state, + uint32_t event, + void* data +); + +enum km_kbp_event_code { + /** + * A keyboard has been activated by the user. The processor may use this + * event, for example, to switch caps lock state or provide other UX. + */ + KM_KBP_EVENT_KEYBOARD_ACTIVATED = 1, + //future: KM_KBP_EVENT_KEYBOARD_DEACTIVATED = 2, +}; + #if defined(__cplusplus) } // extern "C" #endif diff --git a/core/python/keyman/keyboardprocessor/_api.py b/core/python/keyman/keyboardprocessor/_api.py deleted file mode 100644 index 0f2f59bd2c..0000000000 --- a/core/python/keyman/keyboardprocessor/_api.py +++ /dev/null @@ -1,573 +0,0 @@ -# Copyright: ยฉ 2018 SIL International. -# Description: Lowlevel C style python API. Intended to be wrapped by a more -# Pythonic higher level API. -# Create Date: 18 Oct 2018 -# Authors: Tim Eves (TSE) -# - -import ctypes -import ctypes.util -import operator -import os -from enum import auto, IntEnum, IntFlag -from ctypes import (c_uint8, - c_uint16, c_uint32, - c_size_t, - c_void_p, c_char_p, - Structure, Union, POINTER, CFUNCTYPE) -from typing import Any, Tuple - -CP = c_uint16 -USV = c_uint32 -VirtualKey = c_uint16 - -libpath = os.environ.get('PYKMNKBD_LIBRARY_PATH', - ctypes.util.find_library("kmnkbp0")) - -libkbp = ctypes.cdll.LoadLibrary(libpath) - - -# Error handling -# ============== -class StatusCode(IntEnum): - OK = 0 - NO_MEM = auto() - IO_ERROR = auto() - INVALID_ARGUMENT = auto() - KEY_ERROR = auto() - OS_ERROR = 0x80000000 - - -Status = c_uint32 - - -def __map_oserror(code: Status) -> str: - code = StatusCode.OS_ERROR ^ code - msg = '{0!s}: OS error code (' + code + '): ' + os.strerror(code) - return msg, lambda m: OSError(code, m) - - -__exceptions_map = [ - (None, '{0!s}: Success'), - (MemoryError, '{0!s}: memory allocation failed.'), - (RuntimeError, '{0!s}: IO Error: {1!s}'), - (ValueError, '{0!s}: Invalid argument passed.'), - (LookupError, '{0!s}: Item does not exist in: {1!s}'), - (OSError, __map_oserror)] - - -def status_code(code: Status, func, args): - if code == StatusCode.OK: return args - exc, msg = __exceptions_map[code] - if callable(msg): msg = msg(code) - raise exc(msg.format(libkbp._name, *args)) - - -def null_check(code, func, args): - if code is not None: return args - raise KeyError(args[1] + ': Not found in collection') - - -class Dir(IntFlag): - IN = auto() - OUT = auto() - OPT = auto() - - -def __method(iface: str, method: str, result, - *args: Tuple[Any, Dir, str], **kwds): - proto = CFUNCTYPE(result, *map(operator.itemgetter(0), args)) - params = tuple(a[1:] for a in args) - c_name = iface+'_'+method if iface else method - f = proto(('km_kbp_' + c_name, libkbp), params) - if 'errcheck' in kwds: f.errcheck = kwds.get('errcheck') - globals()[c_name] = f - - -# Context processing -# ================== -Context_p = c_void_p - - -class ContextType(IntEnum): - END = 0 - CHAR = auto() - MARKER = auto() - - -class ContextItem(Structure): - class __ContextValue(Union): - _fields_ = (('character', USV), - ('marker', c_uint32)) - _anonymous_ = ('value',) - _fields_ = (('type', c_uint8), - ('value', __ContextValue)) - - -__method('context_items', 'from_utf16', Status, - (c_void_p, Dir.IN, 'text'), - (POINTER(POINTER(ContextItem)), Dir.OUT, 'context_items'), - errcheck=status_code) - -__method('context_items', 'to_utf16', c_size_t, - (POINTER(ContextItem), Dir.IN, 'context_items'), - (c_void_p, Dir.IN | Dir.OPT, 'buffer'), - (c_size_t, Dir.IN | Dir.OPT, 'buffer_size')) - -__method('context_items', 'dispose', None, - (POINTER(ContextItem), Dir.IN, 'context_items')) - -__method('context', 'set', Status, - (Context_p, Dir.IN, 'context'), - (POINTER(ContextItem), Dir.IN, 'context_items'), - errcheck=status_code) - -__method('context', 'get', POINTER(ContextItem), - (Context_p, Dir.IN, 'context')) - -__method('context', 'clear', None, (Context_p, Dir.IN, 'context')) - -__method('context', 'length', c_size_t, (Context_p, Dir.IN, 'context')) - -__method('context', 'append', Status, - (Context_p, Dir.IN, 'context'), - (POINTER(ContextItem), Dir.IN, 'context_items'), - errcheck=status_code) - -__method('context', 'shrink', Status, - (Context_p, Dir.IN, 'context'), - (c_size_t, Dir.IN, 'num'), - (POINTER(ContextItem), Dir.IN, 'prefix'), - errcheck=status_code) - - -class ActionItem(Structure): - class __ActionItem(Union): - _fields_ = (('marker', c_size_t), - ('option', c_char_p), - ('character', USV), - ('vkey', VirtualKey)) - _anonymous_ = ('data',) - _fields_ = (('type', c_uint8), - ('reserved', c_uint8*3), - ('data', __ActionItem)) - - -class ActionType(IntEnum): - END = 0 # Marks end of action items list. - CHAR = 1 - MARKER = 2 # correlates to kmn's "deadkey" markers - ALERT = 3 - BACK = 4 - PERSIST_OPT = 5 - RESET_OPT = 6 - VKEYDOWN = 7 - VKEYUP = 8 - VSHIFTDOWN = 9 - VSHIFTUP = 10 - MAX_TYPE_ID = auto() - - -# Option processing -# ================= -OptionSet_p = c_void_p - - -class OptionScope(IntEnum): - UNKNOWN = auto() - KEYBOARD = auto() - ENVIRONMENT = auto() - - -class Option(Structure): - _fields_ = (('key', c_char_p), - ('value', c_char_p), - ('scope', c_uint8)) - - -Option.END = Option(None, None) - -__method('options_set', 'size', c_size_t, (OptionSet_p, Dir.IN, 'opts')) - -__method('options_set', 'lookup', POINTER(Option), - (OptionSet_p, Dir.IN, 'opts'), - (c_char_p, Dir.IN, 'key'), - errcheck=null_check) - -__method('options_set', 'update', Status, - (OptionSet_p, Dir.IN, 'opts'), - (POINTER(Option), Dir.IN, 'new_opts'), - errcheck=status_code) - -__method('options_set', 'to_json', Status, - (OptionSet_p, Dir.IN, 'opts'), - (c_char_p, Dir.IN | Dir.OPT, 'buffer'), - (c_size_t, Dir.IN | Dir.OUT, 'space'), - errcheck=status_code) - - -# Keyboards -# ========= -Keyboard_p = c_void_p - - -class KeyboardAttrs(Structure): - _fields_ = (('version_string', c_char_p), - ('id', c_char_p), - ('folder_path', c_char_p), - ('default_options', OptionSet_p)) - - -__method('keyboard', 'load', Status, - (c_char_p, Dir.IN, 'path'), - (POINTER(Keyboard_p), Dir.OUT, 'kb'), - errcheck=status_code) - -__method('keyboard', 'dispose', None, (Keyboard_p, Dir.IN, 'kb')) - -__method('keyboard', 'get_attrs', POINTER(KeyboardAttrs), - (Keyboard_p, Dir.IN, 'keyboard')) - - -# State processing -# ================ -State_p = c_void_p - -__method('state', 'create', Status, - (Keyboard_p, Dir.IN, 'keyboard'), - (POINTER(Option), Dir.IN, 'env',), - (POINTER(State_p), Dir.OUT, 'out'), - errcheck=status_code) - -__method('state', 'clone', Status, - (State_p, Dir.IN, 'state'), - (POINTER(State_p), Dir.OUT, 'out'), - errcheck=status_code) - -__method('state', 'dispose', None, (State_p, Dir.IN, 'state')) - -__method('state', 'context', Context_p, (State_p, Dir.IN, 'state')) - -__method('state', 'options', OptionSet_p, (State_p, Dir.IN, 'state')) - -__method('state', 'action_items', POINTER(ActionItem), - (State_p, Dir.IN, 'state'), - (POINTER(c_size_t), Dir.OUT, 'num_items')) - -__method('state', 'to_json', Status, - (State_p, Dir.IN, 'state'), - (c_char_p, Dir.IN | Dir.OPT, 'buffer'), - (c_size_t, Dir.IN | Dir.OUT, 'space'), - errcheck=status_code) - - -# Processor -# ========= -class Attributes(Structure): - _fields_ = (('max_context', c_size_t), - ('current', c_uint16), - ('revision', c_uint16), - ('age', c_uint16), - ('technology', c_uint16), - ('vendor', c_char_p)) - - -class Tech(IntFlag): - UNSPECIFIED = 0 - KMN = 1 - LDML = 2 - - -__method(None, 'get_engine_attrs', POINTER(Attributes)) - -__method(None, 'process_event', Status, - (State_p, Dir.IN, 'state'), - (VirtualKey, Dir.IN, 'vkey'), - (c_uint16, Dir.IN, 'modifier_state')) - - -class Modifier(IntFlag): - LCTRL = 1 << 0 - RCTRL = 1 << 1 - LALT = 1 << 2 - RALT = 1 << 3 - SHIFT = 1 << 4 - CTRL = 1 << 5 - ALT = 1 << 6 - CAPS = 1 << 7 - NOCAPS = 1 << 8 - NUMLOCK = 1 << 9 - NONUMLOCK = 1 << 10 - SCROLLOCK = 1 << 11 - NOSCROLLOCK = 1 << 12 - VIRTUALKEY = 1 << 13 - - -class ModifierMask(IntFlag): - ALL = 0x7f - ALT_GR_SIM = Modifier.LCTRL | Modifier.LALT - CHIRAL = 0x1f - IS_CHIRAL = 0x0f - NON_CHIRAL = 0x7f - CAPS = 0x0300 - NUMLOCK = 0x0C00 - SCROLLLOCK = 0x3000 - - -class VKey(IntEnum): - _00 = auto() - LBUTTON = auto() - RBUTTON = auto() - CANCEL = auto() - MBUTTON = auto() - _05 = auto() - _06 = auto() - _07 = auto() - BKSP = auto() - KTAB = auto() - _0A = auto() - _0B = auto() - KP5 = auto() - ENTER = auto() - _0E = auto() - _0F = auto() - SHIFT = auto() - CONTROL = auto() - ALT = auto() - PAUSE = auto() - CAPS = auto() - _15 = auto() - _16 = auto() - _17 = auto() - _18 = auto() - _19 = auto() - _1A = auto() - ESC = auto() - _1C = auto() - _1D = auto() - _1E = auto() - _1F = auto() - SPACE = auto() - PGUP = auto() - PGDN = auto() - END = auto() - HOME = auto() - LEFT = auto() - UP = auto() - RIGHT = auto() - DOWN = auto() - SEL = auto() - PRINT = auto() - EXEC = auto() - PRTSCN = auto() - INS = auto() - DEL = auto() - HELP = auto() - K0 = auto() - K1 = auto() - K2 = auto() - K3 = auto() - K4 = auto() - K5 = auto() - K6 = auto() - K7 = auto() - K8 = auto() - K9 = auto() - _3A = auto() - _3B = auto() - _3C = auto() - _3D = auto() - _3E = auto() - _3F = auto() - _40 = auto() - KA = auto() - KB = auto() - KC = auto() - KD = auto() - KE = auto() - KF = auto() - KG = auto() - KH = auto() - KI = auto() - KJ = auto() - KK = auto() - KL = auto() - KM = auto() - KN = auto() - KO = auto() - KP = auto() - KQ = auto() - KR = auto() - KS = auto() - KT = auto() - KU = auto() - KV = auto() - KW = auto() - KX = auto() - KY = auto() - KZ = auto() - _5B = auto() - _5C = auto() - _5D = auto() - _5E = auto() - _5F = auto() - NP0 = auto() - NP1 = auto() - NP2 = auto() - NP3 = auto() - NP4 = auto() - NP5 = auto() - NP6 = auto() - NP7 = auto() - NP8 = auto() - NP9 = auto() - NPSTAR = auto() - NPPLUS = auto() - SEPARATOR = auto() - NPMINUS = auto() - NPDOT = auto() - NPSLASH = auto() - F1 = auto() - F2 = auto() - F3 = auto() - F4 = auto() - F5 = auto() - F6 = auto() - F7 = auto() - F8 = auto() - F9 = auto() - F10 = auto() - F11 = auto() - F12 = auto() - F13 = auto() - F14 = auto() - F15 = auto() - F16 = auto() - F17 = auto() - F18 = auto() - F19 = auto() - F20 = auto() - F21 = auto() - F22 = auto() - F23 = auto() - F24 = auto() - _88 = auto() - _89 = auto() - _8A = auto() - _8B = auto() - _8C = auto() - _8D = auto() - _8E = auto() - _8F = auto() - NUMLOCK = auto() - SCROLL = auto() - _92 = auto() - _93 = auto() - _94 = auto() - _95 = auto() - _96 = auto() - _97 = auto() - _98 = auto() - _99 = auto() - _9A = auto() - _9B = auto() - _9C = auto() - _9D = auto() - _9E = auto() - _9F = auto() - _A0 = auto() - _A1 = auto() - _A2 = auto() - _A3 = auto() - _A4 = auto() - _A5 = auto() - _A6 = auto() - _A7 = auto() - _A8 = auto() - _A9 = auto() - _AA = auto() - _AB = auto() - _AC = auto() - _AD = auto() - _AE = auto() - _AF = auto() - _B0 = auto() - _B1 = auto() - _B2 = auto() - _B3 = auto() - _B4 = auto() - _B5 = auto() - _B6 = auto() - _B7 = auto() - _B8 = auto() - _B9 = auto() - COLON = auto() - EQUAL = auto() - COMMA = auto() - HYPHEN = auto() - PERIOD = auto() - SLASH = auto() - BKQUOTE = auto() - _C1 = auto() - _C2 = auto() - _C3 = auto() - _C4 = auto() - _C5 = auto() - _C6 = auto() - _C7 = auto() - _C8 = auto() - _C9 = auto() - _CA = auto() - _CB = auto() - _CC = auto() - _CD = auto() - _CE = auto() - _CF = auto() - _D0 = auto() - _D1 = auto() - _D2 = auto() - _D3 = auto() - _D4 = auto() - _D5 = auto() - _D6 = auto() - _D7 = auto() - _D8 = auto() - _D9 = auto() - _DA = auto() - LBRKT = auto() - BKSLASH = auto() - RBRKT = auto() - QUOTE = auto() - oDF = auto() - oE0 = auto() - oE1 = auto() - oE2 = auto() - oE3 = auto() - oE4 = auto() - _E5 = auto() - oE6 = auto() - _E7 = auto() - _E8 = auto() - oE9 = auto() - oEA = auto() - oEB = auto() - oEC = auto() - oED = auto() - oEE = auto() - oEF = auto() - oF0 = auto() - oF1 = auto() - oF2 = auto() - oF3 = auto() - oF4 = auto() - oF5 = auto() - _F6 = auto() - _F7 = auto() - _F8 = auto() - _F9 = auto() - _FA = auto() - _FB = auto() - _FC = auto() - _FD = auto() - _FE = auto() - _FF = auto() diff --git a/core/python/keyman/keyboardprocessor/api.py b/core/python/keyman/keyboardprocessor/api.py deleted file mode 100644 index 6fcedf18e5..0000000000 --- a/core/python/keyman/keyboardprocessor/api.py +++ /dev/null @@ -1,118 +0,0 @@ -import pathlib -from typing import NamedTuple, NewType, Tuple, List, Union -from enum import Enum -from collections.abc import Sequence, Mapping - -USV = int -Marker = int -VirtualKey = int - - -class Context(Sequence): - Item = Union[USV, Marker] - - def __init__(initial: str): - pass - - def __del__(self): - pass - - def __str__(self): - pass - - def set(self, ctxt: List[Item]): - self.clear() - self.apped(ctxt) - - def clear(self): - pass - - def __getitem__(self, key: int) -> Item: - pass - - def __len__(self): - pass - - def append(self, ctxt: List[Item]): - pass - - def delete(self, remove_n: int, prefix: List[Item]): - pass - - -Option = Tuple[str, str] - - -class OptionSet(Mapping): - def __getitem__(self, key: str) -> Option: - pass - - def __iter__(self): - pass - - def __len__(self): - pass - - def __str__(self): - pass - - -class Keyboard(NamedTuple('__kb_attrs', version=str, id=str, folder_path=pathlib.Path, default_options=OptionSet)): - def __new__(cls, _): - return super(Keyboard, cls).__new__(cls, *[None]*4) - - def __init__(self, kb_path: pathlib.Path): - pass - - def __del__(self): - pass - - -class Action: - VKeyDown = NewType('Action.VirtualKey', VirtualKey) - # VKeyUp = VirtualKey - # VShiftDown = VirtualKey - # VShiftUp = VirtualKey - # Char = int - # Marker = int - # Bell = NewType('Bell', None) - # Back = NewType('Back', None) - # PersistOpt = str - # ResetOpt = str - - -ActionList = List[Action] - - -class State: - def __init__(self, kb: Keyboard, env: OptionSet): - pass - - def __del__(self): - pass - - @property - def flags(self) -> int: - pass - - @property - def context(self) -> Context: - pass - - @property - def environment(self) -> OptionSet: - pass - - @property - def options(self) -> OptionSet: - pass - - def indentify_option_src(opt: Option): - pass - - -def process_event(vk: VirtualKey, - modifier_state, - state: State, - acts: ActionList): - pass diff --git a/core/src/json.cpp b/core/src/jsonpp.cpp similarity index 99% rename from core/src/json.cpp rename to core/src/jsonpp.cpp index 40cb86130c..69c791c2cc 100644 --- a/core/src/json.cpp +++ b/core/src/jsonpp.cpp @@ -12,7 +12,7 @@ #include #include -#include "json.hpp" +#include "jsonpp.hpp" #if defined(_MSC_VER) diff --git a/core/src/json.hpp b/core/src/jsonpp.hpp similarity index 100% rename from core/src/json.hpp rename to core/src/jsonpp.hpp diff --git a/core/src/keyboard.cpp b/core/src/keyboard.cpp index ac5e978a5d..977da597d3 100644 --- a/core/src/keyboard.cpp +++ b/core/src/keyboard.cpp @@ -6,7 +6,7 @@ History: 7 Oct 2018 - TSE - Refactored out of km_kbp_keyboard_api.cpp */ #include "keyboard.hpp" -#include "json.hpp" +#include "jsonpp.hpp" using namespace km::kbp; diff --git a/core/src/km_kbp_context_api.cpp b/core/src/km_kbp_context_api.cpp index 97b439f9e3..a9111f66c4 100644 --- a/core/src/km_kbp_context_api.cpp +++ b/core/src/km_kbp_context_api.cpp @@ -15,7 +15,7 @@ #include #include "context.hpp" -#include "json.hpp" +#include "jsonpp.hpp" #include "utfcodec.hpp" namespace { diff --git a/core/src/km_kbp_options_api.cpp b/core/src/km_kbp_options_api.cpp index f95159a8ab..7dc4879098 100644 --- a/core/src/km_kbp_options_api.cpp +++ b/core/src/km_kbp_options_api.cpp @@ -14,7 +14,7 @@ #include #include "processor.hpp" -#include "json.hpp" +#include "jsonpp.hpp" #include "state.hpp" diff --git a/core/src/km_kbp_processevent_api.cpp b/core/src/km_kbp_processevent_api.cpp index 0a3df41f31..57b9310754 100644 --- a/core/src/km_kbp_processevent_api.cpp +++ b/core/src/km_kbp_processevent_api.cpp @@ -12,11 +12,42 @@ #include "processor.hpp" #include "state.hpp" +km_kbp_status +km_kbp_event( + km_kbp_state *state, + uint32_t event, + void* data +) { + assert(state != nullptr); + if(state == nullptr) { + return KM_KBP_STATUS_INVALID_ARGUMENT; + } + + // event: KM_KBP_EVENT_KEYBOARD_ACTIVATED; data should be nullptr + // future event: KM_KBP_EVENT_KEYBOARD_DEACTIVATED + switch(event) { + case KM_KBP_EVENT_KEYBOARD_ACTIVATED: + assert(data == nullptr); + if(data == nullptr) { + return KM_KBP_STATUS_INVALID_ARGUMENT; + } + break; + default: + return KM_KBP_STATUS_INVALID_ARGUMENT; + } + + return state->processor().external_event(state, event, data); +} + km_kbp_status km_kbp_process_event(km_kbp_state *state, km_kbp_virtual_key vk, uint16_t modifier_state, uint8_t is_key_down) { + assert(state != nullptr); + if(state == nullptr) { + return KM_KBP_STATUS_INVALID_ARGUMENT; + } return state->processor().process_event(state, vk, modifier_state, is_key_down); } @@ -24,11 +55,19 @@ km_kbp_status km_kbp_process_queued_actions( km_kbp_state *state ) { + assert(state != nullptr); + if(state == nullptr) { + return KM_KBP_STATUS_INVALID_ARGUMENT; + } return state->processor().process_queued_actions(state); } km_kbp_attr const * km_kbp_get_engine_attrs(km_kbp_state const *state) { + assert(state != nullptr); + if(state == nullptr) { + return nullptr; + } return &state->processor().attributes(); } diff --git a/core/src/km_kbp_state_api.cpp b/core/src/km_kbp_state_api.cpp index 1d750536d7..b966688950 100644 --- a/core/src/km_kbp_state_api.cpp +++ b/core/src/km_kbp_state_api.cpp @@ -13,7 +13,7 @@ #include #include -#include "json.hpp" +#include "jsonpp.hpp" #include "processor.hpp" #include "state.hpp" diff --git a/core/src/meson.build b/core/src/meson.build index ebca69b599..0db9b1dde1 100644 --- a/core/src/meson.build +++ b/core/src/meson.build @@ -68,7 +68,7 @@ lib = library('kmnkbp0', 'km_kbp_state_api.cpp', 'km_kbp_debug_api.cpp', 'km_kbp_processevent_api.cpp', - 'json.cpp', + 'jsonpp.cpp', 'mock/mock_processor.cpp', 'kmx/kmx_consts.cpp', 'kmx/kmx_processevent.cpp', diff --git a/core/src/path.hpp b/core/src/path.hpp index f7600f3841..e13360248e 100644 --- a/core/src/path.hpp +++ b/core/src/path.hpp @@ -13,7 +13,7 @@ #include #include -#include "json.hpp" +#include "jsonpp.hpp" #include "utfcodec.hpp" // Forward declarations diff --git a/core/src/processor.hpp b/core/src/processor.hpp index d68343ce96..7041a72100 100644 --- a/core/src/processor.hpp +++ b/core/src/processor.hpp @@ -44,6 +44,15 @@ namespace kbp uint8_t is_key_down ) = 0; + virtual km_kbp_status + external_event( + km_kbp_state* _kmn_unused(state), + uint32_t _kmn_unused(event), + void* _kmn_unused(data) + ) { + return KM_KBP_STATUS_OK; + } + virtual km_kbp_attr const & attributes() const = 0; virtual km_kbp_status validate() const = 0; diff --git a/core/tests/unit/json/jsontest.cpp b/core/tests/unit/json/jsontest.cpp index fabd24eca6..3ff64d4cd5 100644 --- a/core/tests/unit/json/jsontest.cpp +++ b/core/tests/unit/json/jsontest.cpp @@ -14,7 +14,7 @@ #include #include -#include +#include #ifdef __EMSCRIPTEN__ #include diff --git a/core/tests/unit/json/meson.build b/core/tests/unit/json/meson.build index 5ad7b1ffd8..42106976cd 100644 --- a/core/tests/unit/json/meson.build +++ b/core/tests/unit/json/meson.build @@ -13,7 +13,7 @@ endif e = executable('jsontest', 'jsontest.cpp', include_directories: [libsrc], link_args: links + tests_flags, - objects: lib.extract_objects('json.cpp')) + objects: lib.extract_objects('jsonpp.cpp')) test('jsontest', e, args: 'jsontest.json') test('jsontestOutput', python, is_parallel: false, args: cmpfiles + ['jsontest.json', join_paths(stnds, 'jsontest.json')]) diff --git a/core/wasm.build.linux.in b/core/wasm.build.linux.in index 126f856341..95dd08f12b 100644 --- a/core/wasm.build.linux.in +++ b/core/wasm.build.linux.in @@ -1,2 +1,7 @@ +[binaries] +c = ['$EMSCRIPTEN_BASE/emcc.py', '-s', 'WASM=1', '-O2'] +cpp = ['$EMSCRIPTEN_BASE/em++.py', '-s', 'WASM=1','-O2'] +ar = ['$EMSCRIPTEN_BASE/emar.py'] + [properties] root = '$EMSCRIPTEN_BASE/system' diff --git a/developer/src/common/delphi/compiler/CompileErrorCodes.pas b/developer/src/common/delphi/compiler/CompileErrorCodes.pas index b61cdca9f6..be16128fc3 100644 --- a/developer/src/common/delphi/compiler/CompileErrorCodes.pas +++ b/developer/src/common/delphi/compiler/CompileErrorCodes.pas @@ -194,7 +194,7 @@ const CWARN_VisualKeyboardFileMissing = $2097; CWARN_ExtendedShiftFlagsNotSupportedInKeymanWeb = $2098; // I4118 CWARN_TouchLayoutUnidentifiedKey = $2099; // I4142 - CWARN_UnreachableKeyCode = $209A; // I4141 + CHINT_UnreachableKeyCode = $109A; // I4141 CWARN_CouldNotCopyJsonFile = $209B; // I4688 CWARN_PlatformNotInTargets = $209C; diff --git a/developer/src/kmcmpdll/DeprecationChecks.cpp b/developer/src/kmcmpdll/DeprecationChecks.cpp index 0e13eaa0ec..a1e19cb27b 100644 --- a/developer/src/kmcmpdll/DeprecationChecks.cpp +++ b/developer/src/kmcmpdll/DeprecationChecks.cpp @@ -25,7 +25,7 @@ BOOL CheckForDeprecatedFeatures(PFILE_KEYBOARD fk) { // Keyman 7 #define TSS_WINDOWSLANGUAGES 29 */ - int currentLineBackup = currentLine; + int oldCurrentLine = currentLine; DWORD i; PFILE_STORE sp; @@ -46,7 +46,7 @@ BOOL CheckForDeprecatedFeatures(PFILE_KEYBOARD fk) { } } - currentLine = currentLineBackup; + currentLine = oldCurrentLine; return TRUE; } diff --git a/developer/src/kmcmpdll/UnreachableRules.cpp b/developer/src/kmcmpdll/UnreachableRules.cpp index 70f4331966..a5c6519ef3 100644 --- a/developer/src/kmcmpdll/UnreachableRules.cpp +++ b/developer/src/kmcmpdll/UnreachableRules.cpp @@ -24,6 +24,8 @@ DWORD VerifyUnreachableRules(PFILE_GROUP gp) { PFILE_KEY kp = gp->dpKeyArray; DWORD i; + int oldCurrentLine = currentLine; + std::unordered_map map; std::unordered_set reportedLines; @@ -43,5 +45,7 @@ DWORD VerifyUnreachableRules(PFILE_GROUP gp) { } } + currentLine = oldCurrentLine; + return CERR_None; } diff --git a/developer/src/kmcmpdll/comperr.h b/developer/src/kmcmpdll/comperr.h index 17f72bce88..98235c7018 100644 --- a/developer/src/kmcmpdll/comperr.h +++ b/developer/src/kmcmpdll/comperr.h @@ -191,7 +191,7 @@ #define CWARN_VisualKeyboardFileMissing 0x00002097 #define CWARN_ExtendedShiftFlagsNotSupportedInKeymanWeb 0x00002098 // I4118 #define CWARN_TouchLayoutUnidentifiedKey 0x00002099 -#define CWARN_UnreachableKeyCode 0x0000209A +#define CHINT_UnreachableKeyCode 0x0000109A #define CWARN_CouldNotCopyJsonFile 0x0000209B #define CWARN_PlatformNotInTargets 0x0000209C diff --git a/developer/src/kmcomp/kmcomp.dpr b/developer/src/kmcomp/kmcomp.dpr index 1fcf977155..8ac6e124cf 100644 --- a/developer/src/kmcomp/kmcomp.dpr +++ b/developer/src/kmcomp/kmcomp.dpr @@ -53,7 +53,6 @@ uses CompileErrorCodes in '..\common\delphi\compiler\CompileErrorCodes.pas', TouchLayout in '..\tike\oskbuilder\TouchLayout.pas', TouchLayoutDefinitions in '..\tike\oskbuilder\TouchLayoutDefinitions.pas', - TouchLayoutUtils in '..\tike\oskbuilder\TouchLayoutUtils.pas', KeyboardFonts in '..\common\delphi\general\KeyboardFonts.pas', KeyboardParser in '..\tike\main\KeyboardParser.pas', WindowsLanguages in '..\common\delphi\general\WindowsLanguages.pas', diff --git a/developer/src/kmcomp/kmcomp.dproj b/developer/src/kmcomp/kmcomp.dproj index 8453e92639..f6f183330f 100644 --- a/developer/src/kmcomp/kmcomp.dproj +++ b/developer/src/kmcomp/kmcomp.dproj @@ -202,7 +202,6 @@ - diff --git a/developer/src/kmconvert/kmconvert.dpr b/developer/src/kmconvert/kmconvert.dpr index 1b0b9edda2..233c07d43c 100644 --- a/developer/src/kmconvert/kmconvert.dpr +++ b/developer/src/kmconvert/kmconvert.dpr @@ -73,7 +73,6 @@ uses OnScreenKeyboardData in '..\..\..\common\windows\delphi\visualkeyboard\OnScreenKeyboardData.pas', TouchLayout in '..\TIKE\oskbuilder\TouchLayout.pas', TouchLayoutDefinitions in '..\TIKE\oskbuilder\TouchLayoutDefinitions.pas', - TouchLayoutUtils in '..\TIKE\oskbuilder\TouchLayoutUtils.pas', KeyboardFonts in '..\common\delphi\general\KeyboardFonts.pas', Keyman.System.Util.RenderLanguageIcon in '..\..\..\common\windows\delphi\ui\Keyman.System.Util.RenderLanguageIcon.pas', utilicon in '..\..\..\common\windows\delphi\general\utilicon.pas', diff --git a/developer/src/kmconvert/kmconvert.dproj b/developer/src/kmconvert/kmconvert.dproj index 148585112e..48626bc67b 100644 --- a/developer/src/kmconvert/kmconvert.dproj +++ b/developer/src/kmconvert/kmconvert.dproj @@ -179,7 +179,6 @@ - diff --git a/developer/src/kmlmc/package.json b/developer/src/kmlmc/package.json index 91b2583b72..843b1ea82a 100644 --- a/developer/src/kmlmc/package.json +++ b/developer/src/kmlmc/package.json @@ -52,7 +52,7 @@ "chai": "^4.3.4", "chalk": "^2.4.2", "jszip": "^3.7.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "ts-node": "^9.1.1" }, "mocha": { diff --git a/developer/src/server/package.json b/developer/src/server/package.json index ee9efbaa58..d6db75b7d6 100644 --- a/developer/src/server/package.json +++ b/developer/src/server/package.json @@ -34,7 +34,7 @@ "@types/ws": "^8.2.2", "chai": "^4.3.4", "copyfiles": "^2.4.1", - "mocha": "^9.1.4", + "mocha": "^10.0.0", "ts-node": "^10.4.0", "tsc-watch": "^4.5.0", "typescript": "^4.5.4" diff --git a/developer/src/test/auto/keyboard-package-versions/KeyboardPackageVersionsTestSuite.dpr b/developer/src/test/auto/keyboard-package-versions/KeyboardPackageVersionsTestSuite.dpr index 44640edc55..bfe61aa975 100644 --- a/developer/src/test/auto/keyboard-package-versions/KeyboardPackageVersionsTestSuite.dpr +++ b/developer/src/test/auto/keyboard-package-versions/KeyboardPackageVersionsTestSuite.dpr @@ -72,14 +72,8 @@ uses WindowsLanguages in '..\..\..\common\delphi\general\WindowsLanguages.pas', KeymanWebKeyCodes in '..\..\..\tike\compile\KeymanWebKeyCodes.pas', kmxfileutils in '..\..\..\..\..\common\windows\delphi\keyboards\kmxfileutils.pas', - TikeUnicodeData in '..\..\..\tike\main\TikeUnicodeData.pas', - UnicodeData in '..\..\..\..\..\common\windows\delphi\charmap\UnicodeData.pas', - ADODB_TLB in '..\..\..\..\..\common\windows\delphi\tlb\ADODB_TLB.pas', - ADOX_TLB in '..\..\..\..\..\common\windows\delphi\tlb\ADOX_TLB.pas', - ttinfo in '..\..\..\..\..\common\windows\delphi\general\ttinfo.pas', TouchLayoutDefinitions in '..\..\..\tike\oskbuilder\TouchLayoutDefinitions.pas', TouchLayout in '..\..\..\tike\oskbuilder\TouchLayout.pas', - TouchLayoutUtils in '..\..\..\tike\oskbuilder\TouchLayoutUtils.pas', KeyboardFonts in '..\..\..\common\delphi\general\KeyboardFonts.pas', Keyman.Developer.System.Project.kpsProjectFile in '..\..\..\tike\project\Keyman.Developer.System.Project.kpsProjectFile.pas', Keyman.Developer.System.Project.kpsProjectFileAction in '..\..\..\tike\project\Keyman.Developer.System.Project.kpsProjectFileAction.pas', diff --git a/developer/src/test/auto/keyboard-package-versions/KeyboardPackageVersionsTestSuite.dproj b/developer/src/test/auto/keyboard-package-versions/KeyboardPackageVersionsTestSuite.dproj index 0cb4eb7ad9..c1d5645e40 100644 --- a/developer/src/test/auto/keyboard-package-versions/KeyboardPackageVersionsTestSuite.dproj +++ b/developer/src/test/auto/keyboard-package-versions/KeyboardPackageVersionsTestSuite.dproj @@ -153,14 +153,8 @@ - - - - - - diff --git a/developer/src/test/auto/keyboard-package-versions/Keyman.Test.System.CompilePackageVersioningTest.pas b/developer/src/test/auto/keyboard-package-versions/Keyman.Test.System.CompilePackageVersioningTest.pas index 9d4222195b..fae07d5eed 100644 --- a/developer/src/test/auto/keyboard-package-versions/Keyman.Test.System.CompilePackageVersioningTest.pas +++ b/developer/src/test/auto/keyboard-package-versions/Keyman.Test.System.CompilePackageVersioningTest.pas @@ -36,6 +36,8 @@ implementation uses System.SysUtils, + Winapi.ActiveX, + compile, CompilePackage, Keyman.Developer.System.Project.kmnProjectFileAction, @@ -154,5 +156,8 @@ begin end; initialization + CoInitializeEx(nil, COINIT_APARTMENTTHREADED); TDUnitX.RegisterTestFixture(TCompilePackageVersioningTest); +finalization + CoUninitialize; end. diff --git a/developer/src/tike/compile/CompileKeymanWeb.pas b/developer/src/tike/compile/CompileKeymanWeb.pas index af27c4f2fb..c15a0032a3 100644 --- a/developer/src/tike/compile/CompileKeymanWeb.pas +++ b/developer/src/tike/compile/CompileKeymanWeb.pas @@ -102,6 +102,7 @@ uses Winapi.Windows, System.Character, System.Classes, + System.Generics.Collections, System.UITypes, compile, @@ -214,6 +215,7 @@ type FTouchLayoutFont: string; FFix183_LadderLength: Integer; FCloseBrace: Boolean; // I4872 + FUnreachableKeys: TList; function JavaScript_String(ch: DWord): string; // I2242 @@ -254,6 +256,8 @@ type function IsKeyboardVersion15OrLater: Boolean; function WriteBeginStatement(const name: string; groupIndex: Integer): string; + function FormatKeyForErrorMessage(fkp: PFILE_KEY; + FMnemonic: Boolean): string; public function Compile(AOwnerProject: TProject; const InFile: string; const OutFile: string; Debug: Boolean; Callback: TCompilerCallbackW): Boolean; // I3681 // I4140 // I4688 // I4866 constructor Create; @@ -273,14 +277,11 @@ uses CompileErrorCodes, JsonUtil, KeymanDeveloperOptions, - KeyboardParser, Keyman.System.KeyboardUtils, KeymanWebKeyCodes, kmxfileutils, TouchLayout, - TouchLayoutUtils, Unicode, - UnicodeData, utilstr, VisualKeyboard, VKeys; @@ -303,6 +304,8 @@ var WarnDeprecatedCode: Boolean; Data: string; begin + FUnreachableKeys.Clear; + FCallback := Callback; FInFile := InFile; FOutFile := OutFile; // I4140 // I4155 // I4154 @@ -375,6 +378,7 @@ end; constructor TCompileKeymanWeb.Create; begin + FUnreachableKeys := TList.Create; FillChar(fk, sizeof(fk), 0); FFix183_LadderLength := FKeymanDeveloperOptions.Fix183_LadderLength; // How frequently to break ladders end; @@ -382,6 +386,7 @@ end; destructor TCompileKeymanWeb.Destroy; begin // TODO: Free FK values + FUnreachableKeys.Free; inherited; end; @@ -800,6 +805,63 @@ const // I1585 - add space to conversion VK_NUMLOCK, // &H90 VK_SCROLL); // &H91 +function TCompileKeymanWeb.FormatKeyForErrorMessage(fkp: PFILE_KEY; FMnemonic: Boolean): string; + function FormatShift(ShiftFlags: DWord): string; + const + mask: array[0..13] of string = ( + 'LCTRL', // 0X0001 + 'RCTRL', // 0X0002 + 'LALT', // 0X0004 + 'RALT', // 0X0008 + + 'SHIFT', // 0X0010 + 'CTRL', // 0X0020 + 'ALT', // 0X0040 + + '???', // Reserved + + 'CAPS', // 0X0100 + 'NCAPS', // 0X0200 + + 'NUMLOCK', // 0X0400 + 'NNUMLOCK', // 0X0800 + + 'SCROLLLOCK', // 0X1000 + 'NSCROLLLOCK' // 0X2000 + ); + var + i: Integer; + begin + Result := ''; + for i := 0 to High(mask) do + begin + if ShiftFlags and (1 shl i) <> 0 then + begin + Result := Result + mask[i] + ' '; + end; + end; + end; +begin + if not FMnemonic then + begin + if (fkp.ShiftFlags and KMX_ISVIRTUALKEY) = KMX_ISVIRTUALKEY then + begin + if Ord(fkp.Key) < 256 + then Result := Format('[%s%s]', [FormatShift(fkp.ShiftFlags), VKeyNames[Ord(fkp.Key)]]) + else Result := Format('[%sK_%x]', [FormatShift(fkp.ShiftFlags), Ord(fkp.Key)]); + end + else + begin + Result := Format('''%s''', [fkp.Key]); + end; + end + else + begin + if (fkp.ShiftFlags and KMX_VIRTUALCHARKEY) = KMX_VIRTUALCHARKEY + then Result := Format('[%s''%s'']', [FormatShift(fkp.ShiftFlags), fkp.Key]) + else Result := Format('''%s''', [fkp.Key]); + end; +end; function TCompileKeymanWeb.JavaScript_Key(fkp: PFILE_KEY; FMnemonic: Boolean): Integer; var @@ -835,7 +897,13 @@ begin if (Result = 0) or (Result >= Ord(Low(TKeymanWebTouchStandardKey))) then // I4141 begin - ReportError(fkp.Line, CWARN_UnreachableKeyCode, 'The rule will never be matched because its key code is never fired.'); + if not FUnreachableKeys.Contains(fkp) then + begin + ReportError(fkp.Line, CHINT_UnreachableKeyCode, + 'The rule will never be matched for key '+ + FormatKeyForErrorMessage(fkp,FMnemonic)+' because its key code is never fired.'); + FUnreachableKeys.Add(fkp); + end; end; end; diff --git a/developer/src/tike/xml/layoutbuilder/builder.css b/developer/src/tike/xml/layoutbuilder/builder.css index 60daff8f3c..6e40f5b5e1 100644 --- a/developer/src/tike/xml/layoutbuilder/builder.css +++ b/developer/src/tike/xml/layoutbuilder/builder.css @@ -613,6 +613,11 @@ body:not(.text-controls-in-toolbar) input#inpSubKeyCap { color: white; } +#kbd.desktop .key-size { + /* the layout is fixed on desktop so key size is not useful */ + display: none; +} + /* Position flick keys relative to the flick grid, by hand */ #flick .key { diff --git a/developer/src/tike/xml/layoutbuilder/builder.js b/developer/src/tike/xml/layoutbuilder/builder.js index 561c4d6d0a..a432f8cd97 100644 --- a/developer/src/tike/xml/layoutbuilder/builder.js +++ b/developer/src/tike/xml/layoutbuilder/builder.js @@ -12,10 +12,7 @@ $(function() { this.getPresentation = function () { - var platform = $('#selPlatformPresentation').val(); - //if(platform == 'tablet') return 'tablet-ipad'; - //if(platform == 'phone') return 'phone-iphone5'; - return platform; + return $('#selPlatformPresentation').val(); } this.saveSelection = function() { diff --git a/developer/src/tike/xml/layoutbuilder/constants.js b/developer/src/tike/xml/layoutbuilder/constants.js index 593c7e4ca0..807a8bd30a 100644 --- a/developer/src/tike/xml/layoutbuilder/constants.js +++ b/developer/src/tike/xml/layoutbuilder/constants.js @@ -604,7 +604,8 @@ $(function() { "tablet-ipad-landscape": { "x": 829, "y": 299, "name": "iPad (landscape)" }, // 829x622 = iPad tablet box size; (97,101)-(926,723) "tablet-ipad-portrait": { "x": 605, "y": 300, "name": "iPad (portrait)" }, // 605x806 = iPad tablet box size; (98,94)-(703,900) "phone-iphone5-landscape": { "x": 731, "y": 196, "name": "iPhone 5 (landscape)" }, // 731x412 = iPhone box size; (144,39)-(875,451) - "phone-iphone5-portrait": { "x": 526, "y": 266, "name": "iPhone 5 (portrait)"} // 528x936 = iPhone box size; (90,204)-(618,1040) + "phone-iphone5-portrait": { "x": 526, "y": 266, "name": "iPhone 5 (portrait)"}, // 528x936 = iPhone box size; (90,204)-(618,1040) + "desktop": { "x": 640, "y": 300, "name": "Desktop" }, }; this.keyMargin = 15; diff --git a/docs/CODEOWNERS b/docs/CODEOWNERS index eff7811f5e..021a6fc8bc 100644 --- a/docs/CODEOWNERS +++ b/docs/CODEOWNERS @@ -3,32 +3,31 @@ # # @darcywong00 @mcdurdin @ermshiperete @rc-swag @SabineSIL @sgschantz -/android/ @darcywong00 @rc-swag +/android/ @darcywong00 @mcdurdin -/common/ @mcdurdin @jahorton -/core/ @mcdurdin @jahorton +/common/ @mcdurdin @rc-swag +/core/ @mcdurdin @rc-swag /common/lexical-model-types/ @jahorton @mcdurdin /common/models/ @jahorton @mcdurdin /common/predictive-text/ @jahorton @mcdurdin /common/schemas/ @mcdurdin @jahorton /common/test/ @mcdurdin @ermshiperete -/common/web/ @jahorton @ermshiperete @mcdurdin +/common/web/ @jahorton @sgschantz @mcdurdin /developer/ @mcdurdin @darcywong00 /docs/ @mcdurdin @jahorton -/ios/ @sgschantz @mcdurdin +/ios/ @sgschantz @jahorton /linux/ @ermshiperete @darcywong00 /mac/ @sgschantz @SabineSIL -/oem/firstvoices/android/ @darcywong00 @rc-swag -/oem/firstvoices/common/ @mcdurdin @jahorton -/oem/firstvoices/ios/ @sgschantz @mcdurdin -/oem/firstvoices/windows/ @rc-swag @sgschantz +/oem/firstvoices/android/ @darcywong00 @mcdurdin +/oem/firstvoices/common/ @mcdurdin @rc-swag +/oem/firstvoices/ios/ @sgschantz @jahorton +/oem/firstvoices/windows/ @rc-swag @ermshiperete /resources/ @mcdurdin @jahorton # Web is currently shared between Marc and Joshua: -/web/ @jahorton @ermshiperete @mcdurdin +/web/ @jahorton @sgschantz @mcdurdin + +/windows/ @rc-swag @ermshiperete -/windows/ @rc-swag @sgschantz -# Override for windows/src: -/windows/src/developer/ @mcdurdin @darcywong00 diff --git a/docs/build/linux-ubuntu.md b/docs/build/linux-ubuntu.md index 1a4be89885..570f233298 100644 --- a/docs/build/linux-ubuntu.md +++ b/docs/build/linux-ubuntu.md @@ -64,3 +64,23 @@ All dependencies are already installed if you followed the instructions under [P Building: * [Building Keyman Core](../../core/doc/BUILDING.md) + +## Docker Builder + +The Docker builder allows you to perform a linux build from anywhere Docker is supported. +To build the docker image: + +```shell +cd ../../linux +docker pull ubuntu:latest +docker build . -t keymanapp/keyman-linux-builder:latest +``` + +Once the image is built, it may be used to build parts of Keyman. + +```shell +cd ../core +# keep linux build artifacts separate +mkdir -p build/linux +docker run -it --rm -v $(pwd)/..:/home/build -v $(pwd)/build/linux:/home/build/core/build keyman-linux-builder:latest bash -c 'cd core; bash build.sh -d' +``` diff --git a/ios/engine/KMEI/KeymanEngine/Classes/KeymanHosts.swift b/ios/engine/KMEI/KeymanEngine/Classes/KeymanHosts.swift index cf312a8e0d..897b162a5e 100644 --- a/ios/engine/KMEI/KeymanEngine/Classes/KeymanHosts.swift +++ b/ios/engine/KMEI/KeymanEngine/Classes/KeymanHosts.swift @@ -30,7 +30,7 @@ public enum KeymanHosts { case .alpha: fallthrough case .beta: - return URL.init(string: "https://api.keyman-staging.com")! + return URL.init(string: "https://api.keyman.com")! // #7227 disabling: "https://api.keyman-staging.com")! case .stable: return URL.init(string: "https://api.keyman.com")! } @@ -54,7 +54,7 @@ public enum KeymanHosts { case .alpha: fallthrough case .beta: - return URL.init(string: "https://help.keyman-staging.com")! + return URL.init(string: "https://help.keyman.com")! // #7227 disabling: "https://help.keyman-staging.com")! case .stable: return URL.init(string: "https://help.keyman.com")! } @@ -78,7 +78,7 @@ public enum KeymanHosts { case .alpha: fallthrough case .beta: - return URL.init(string: "https://keyman-staging.com")! + return URL.init(string: "https://keyman.com")! // #7227 disabling: "https://keyman-staging.com")! case .stable: return URL.init(string: "https://keyman.com")! } diff --git a/linux/Dockerfile b/linux/Dockerfile new file mode 100644 index 0000000000..203d33aef1 --- /dev/null +++ b/linux/Dockerfile @@ -0,0 +1,24 @@ +# Copyright (c) 2022 SIL International. All rights reserved. +# +# builder image for a linux build +# see ../docs/build/linux-ubuntu.md + +FROM --platform=amd64 ubuntu:latest +LABEL org.opencontainers.image.authors="SIL International." +LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git" +LABEL org.opencontainers.image.title="Keyman Linux Build Image" +# We will switch to a build user after some installation +USER root +RUN useradd -c "Build user" -d $HOME -m build +ENV HOME /home/build +VOLUME /home/build +WORKDIR /home/build +ENV DEBIAN_FRONTEND noninteractive +# Update to the latest +RUN apt-get -q -y update && apt-get -q -y upgrade +RUN apt-get -q -y install devscripts equivs meson python3 python3-setuptools +ADD debian/control /tmp/control +# Answer 'yes' to install questions +RUN (yes | mk-build-deps --install /tmp/control) || true +# now, switch to build user +USER build diff --git a/linux/debian/changelog b/linux/debian/changelog index 6990b7dbec..ab682e3d73 100644 --- a/linux/debian/changelog +++ b/linux/debian/changelog @@ -1,3 +1,17 @@ +keyman (15.0.270-1) unstable; urgency=medium + + * New upstream release. + * Re-release to Debian + + -- Eberhard Beilharz Tue, 13 Sep 2022 11:20:25 +0200 + +keyman (15.0.269-1) unstable; urgency=medium + + * New upstream release. + * Re-release to Debian + + -- Eberhard Beilharz Mon, 29 Aug 2022 15:14:16 +0200 + keyman (15.0.268-1) unstable; urgency=medium * New upstream release. diff --git a/linux/debian/ibus-keyman.postinst b/linux/debian/ibus-keyman.postinst index 5b66b64c2a..57c22b8d6d 100644 --- a/linux/debian/ibus-keyman.postinst +++ b/linux/debian/ibus-keyman.postinst @@ -37,8 +37,8 @@ case "$1" in fi # Verify that it's running now - if [ ! -z $SUDO_USER ]; then - ! ibusdaemon=`ps --user $SUDO_USER -o s= -o cmd | grep --regexp="^[^ZT] ibus-daemon .*--xim.*"` + if [ ! -z $SUDO_USER ] && id $SUDO_USER > /dev/null 2>/dev/null; then + ! ibusdaemon=$(ps --user $SUDO_USER -o s= -o cmd | grep --regexp="^[^ZT] ibus-daemon .*--xim.*") if [ "x$ibusdaemon" = "x" ]; then # otherwise try to start it for the user installing the package if [ "x$is_gnome_shell" = "x1" ]; then diff --git a/linux/ibus-keyman/src/engine.c b/linux/ibus-keyman/src/engine.c index 81625cc672..b0c03bb439 100644 --- a/linux/ibus-keyman/src/engine.c +++ b/linux/ibus-keyman/src/engine.c @@ -70,7 +70,6 @@ struct _IBusKeymanEngine { gboolean lalt_pressed; gboolean ralt_pressed; gboolean emitting_keystroke; - IBusLookupTable *table; IBusProperty *status_prop; IBusPropList *prop_list; #ifdef GDK_WINDOWING_X11 @@ -262,8 +261,6 @@ ibus_keyman_engine_init(IBusKeymanEngine *keyman) { g_object_ref_sink(keyman->prop_list); ibus_prop_list_append(keyman->prop_list, keyman->status_prop); - keyman->table = ibus_lookup_table_new(9, 0, TRUE, TRUE); - g_object_ref_sink(keyman->table); keyman->state = NULL; #ifdef GDK_WINDOWING_X11 keyman->xdisplay = NULL; @@ -469,11 +466,6 @@ ibus_keyman_engine_destroy (IBusKeymanEngine *keyman) keyman->status_prop = NULL; } - if (keyman->table) { - g_debug("DAR: unref keyman->table"); - g_object_unref (keyman->table); - keyman->table = NULL; - } if (keyman->state) { km_kbp_state_dispose(keyman->state); keyman->state = NULL; @@ -508,16 +500,6 @@ static void forward_backspace(IBusKeymanEngine *keyman, unsigned int state) ibus_engine_forward_key_event((IBusEngine *)keyman, KEYMAN_BACKSPACE_KEYSYM, KEYMAN_BACKSPACE, state); } -static gboolean ok_for_single_backspace(const km_kbp_action_item *action_items, int i, size_t num_actions) -{ - for (int j=i+1; j < num_actions; j++) { - if (action_items[i].type == KM_KBP_IT_BACK || action_items[i].type == KM_KBP_IT_CHAR || action_items[i].type == KM_KBP_IT_EMIT_KEYSTROKE) { - return FALSE; - } - } - return TRUE; -} - static gboolean process_unicode_char_action( IBusKeymanEngine *keyman, @@ -601,10 +583,6 @@ process_backspace_action( } g_free(keyman->char_buffer); keyman->char_buffer = new_buffer; - } else if (ok_for_single_backspace(action_items, i, num_action_items)) { - // single backspace can be handled by ibus as normal - g_message("no char actions, just single back"); - return FALSE; } else { g_message( "DAR: process_backspace_action - client_capabilities=%x, %x", engine->client_capabilities, IBUS_CAP_SURROUNDING_TEXT); diff --git a/linux/ibus-keyman/tests/Makefile.am b/linux/ibus-keyman/tests/Makefile.am index 041acc0fa9..eeeb89317b 100644 --- a/linux/ibus-keyman/tests/Makefile.am +++ b/linux/ibus-keyman/tests/Makefile.am @@ -54,7 +54,7 @@ ibus_keyman_tests_SOURCES = \ testfixture.cpp \ testmodule.c \ testmodule.h \ - ../../../core/src/json.cpp \ + ../../../core/src/jsonpp.cpp \ ../../../core/src/keyboard.cpp \ ../../../core/src/kmx/kmx_actions.cpp \ ../../../core/src/kmx/kmx_actions.h \ diff --git a/linux/keyman-config/Makefile b/linux/keyman-config/Makefile index ee01e2bfc9..cb0fe8ab51 100644 --- a/linux/keyman-config/Makefile +++ b/linux/keyman-config/Makefile @@ -55,8 +55,10 @@ deb: dist man: ./build-help.sh --man --no-reconf -version: +version_reconf: cd .. && ./scripts/reconf.sh keyman-config + +version: version_reconf $(eval VERSION := $(shell python3 -c "from keyman_config import __releaseversion__; print(__releaseversion__)")) # i18n diff --git a/linux/keyman-config/keyman_config/__init__.py b/linux/keyman-config/keyman_config/__init__.py index 7c3972c247..ee2326a86f 100644 --- a/linux/keyman-config/keyman_config/__init__.py +++ b/linux/keyman-config/keyman_config/__init__.py @@ -40,15 +40,15 @@ def secure_lookup(data, key1, key2 = None): gettext.bindtextdomain('keyman-config', '/usr/share/locale') gettext.textdomain('keyman-config') -if __tier__ == 'alpha' or __tier__ == 'beta': +#if __tier__ == 'alpha' or __tier__ == 'beta': // #7227 disabling: # Alpha and beta versions will work against the staging server so that they # can access new APIs etc that will only be available there. The staging # servers have resource constraints but should be okay for limited use. - KeymanComUrl = 'https://keyman-staging.com' - KeymanApiUrl = 'https://api.keyman-staging.com' -else: - KeymanComUrl = 'https://keyman.com' - KeymanApiUrl = 'https://api.keyman.com' +# KeymanComUrl = 'https://keyman-staging.com' +# KeymanApiUrl = 'https://api.keyman-staging.com' +#else: +KeymanComUrl = 'https://keyman.com' +KeymanApiUrl = 'https://api.keyman.com' # There's no staging site for downloads KeymanDownloadsUrl = 'https://downloads.keyman.com' diff --git a/linux/scripts/jenkins.sh b/linux/scripts/jenkins.sh index c7406d8831..bc40800fa2 100755 --- a/linux/scripts/jenkins.sh +++ b/linux/scripts/jenkins.sh @@ -55,7 +55,7 @@ checkAndInstallRequirements() fi sudo mk-build-deps debian/control - sudo apt-get -qy install ./keyman-build-deps_*.deb + sudo apt-get -qy --allow-downgrades install ./keyman-build-deps_*.deb sudo rm -f keyman-buid-deps_* } diff --git a/mac/Keyman4MacIM/Keyman4MacIM/KMInputMethodAppDelegate.m b/mac/Keyman4MacIM/Keyman4MacIM/KMInputMethodAppDelegate.m index 61558b7ecf..8a4cc0734b 100644 --- a/mac/Keyman4MacIM/Keyman4MacIM/KMInputMethodAppDelegate.m +++ b/mac/Keyman4MacIM/Keyman4MacIM/KMInputMethodAppDelegate.m @@ -130,16 +130,16 @@ NSString* _keymanDataPath = nil; result.tier = [keymanInfo objectForKey:@"Tier"]; result.versionRelease = [keymanInfo objectForKey:@"VersionRelease"]; result.versionWithTag = [keymanInfo objectForKey:@"VersionWithTag"]; - if([result.tier isEqualToString:@"stable"]) { - result.keymanCom = @"keyman.com"; - result.helpKeymanCom = @"help.keyman.com"; - result.apiKeymanCom = @"api.keyman.com"; - } + // if([result.tier isEqualToString:@"stable"]) { // #7227 disabling: + result.keymanCom = @"keyman.com"; + result.helpKeymanCom = @"help.keyman.com"; + result.apiKeymanCom = @"api.keyman.com"; + /*} else { result.keymanCom = @"keyman-staging.com"; result.helpKeymanCom = @"help.keyman-staging.com"; result.apiKeymanCom = @"api.keyman-staging.com"; - } + }*/ return result; } @@ -632,11 +632,11 @@ CGEventRef eventTapFunction(CGEventTapProxy proxy, CGEventType type, CGEventRef NSString *path = [[self keymanDataPath] stringByAppendingPathComponent:packageFolder]; KMPackageInfo *packageInfo = [self.packageReader loadPackageInfo:path]; - + if (packageInfo) { packageName = packageInfo.packageName; } - + return packageName; } @@ -1149,7 +1149,7 @@ CGEventRef eventTapFunction(CGEventTapProxy proxy, CGEventType type, CGEventRef [self.receivedData writeToFile:filePath atomically:YES]; [self unzipFile:filePath]; [[NSFileManager defaultManager] removeItemAtPath:filePath error:nil]; - + [_downloadInfoView setMessageText:NSLocalizedString(@"message-keyboard-download-complete", nil)]; NSButton *button = (NSButton *)[_downloadInfoView.buttons objectAtIndex:0]; [button setTitle:NSLocalizedString(@"button-download-complete", nil)]; diff --git a/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/FVShared.java b/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/FVShared.java index 854175e4cc..d308049417 100644 --- a/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/FVShared.java +++ b/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/FVShared.java @@ -6,6 +6,7 @@ import android.content.SharedPreferences; import android.content.res.AssetManager; import android.net.Uri; import android.util.Log; +import android.widget.Toast; import com.tavultesoft.kmea.KMManager; import com.tavultesoft.kmea.data.Keyboard; import com.tavultesoft.kmea.packages.PackageProcessor; @@ -252,7 +253,11 @@ final class FVShared { String helpUrl = String.format("%s%s", FVKeyboardHelpLink, id); Intent i = new Intent(Intent.ACTION_VIEW); i.setData(Uri.parse(helpUrl)); - localContext.startActivity(i); + if (i.resolveActivity(localContext.getPackageManager()) != null) { + localContext.startActivity(i); + } else { + Toast.makeText(localContext, localContext.getString(R.string.unable_to_open_browser), Toast.LENGTH_SHORT).show(); + } } private void updateActiveKeyboardsList() { diff --git a/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/MainActivity.java b/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/MainActivity.java index 5d3e5c5939..95309efe48 100644 --- a/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/MainActivity.java +++ b/oem/firstvoices/android/app/src/main/java/com/firstvoices/keyboards/MainActivity.java @@ -105,7 +105,11 @@ public class MainActivity extends AppCompatActivity implements OnKeyboardDownloa } else { Intent i = new Intent(Intent.ACTION_VIEW, Uri.parse(url)); - startActivity(i); + if (i.resolveActivity(getPackageManager()) != null) { + startActivity(i); + } else { + Toast.makeText(context, getString(R.string.unable_to_open_browser), Toast.LENGTH_SHORT).show(); + } } return true; diff --git a/oem/firstvoices/android/app/src/main/res/values/strings.xml b/oem/firstvoices/android/app/src/main/res/values/strings.xml index 83edf8f86b..e443e0f717 100644 --- a/oem/firstvoices/android/app/src/main/res/values/strings.xml +++ b/oem/firstvoices/android/app/src/main/res/values/strings.xml @@ -42,4 +42,6 @@ Invalid/Missing metadata in package + + Unable to launch web browser diff --git a/package-lock.json b/package-lock.json index 0149000800..985a430af4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ ], "devDependencies": { "chai": "^4.3.4", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "ts-node": "^9.1.1", "typescript": "^4.5.4" @@ -116,8 +116,8 @@ "@types/mocha": "^7.0.2", "@types/node": "^14.0.4", "chai": "^4.3.4", - "mocha": "^8.4.0", - "typescript": "^3.8.3" + "mocha": "^10.0.0", + "typescript": "^4.5.4" } }, "common/models/templates/node_modules/@types/node": { @@ -126,19 +126,6 @@ "integrity": "sha512-q4jlIR71hUpWTnGhXWcakgkZeHa3CCjcQcnuzU8M891BAWA2jHiziiWEPEkdS5pFsz7H9HJiy8BrK7tBRNrY7A==", "dev": true }, - "common/models/templates/node_modules/typescript": { - "version": "3.9.10", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.10.tgz", - "integrity": "sha512-w6fIxVE/H1PkLKcCPsFqKE7Kv7QUwhU8qQY2MueZXWx5cPZdwFupLgKK3vntcK98BtNHZtAF4LA/yl2a7k8R6Q==", - "dev": true, - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=4.2.0" - } - }, "common/models/types": { "name": "@keymanapp/models-types", "license": "MIT", @@ -157,7 +144,7 @@ "@types/chai": "^4.2.11", "@types/mocha": "^7.0.2", "chai": "^4.3.4", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "ts-node": "^9.1.1", "typescript": "^4.5.4" } @@ -197,7 +184,7 @@ "karma-mocha-reporter": "^2.2.5", "karma-safari-launcher": "^1.0.0", "karma-teamcity-reporter": "^1.1.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "sinon": "^14.0.0", "ts-node": "^9.1.1", @@ -242,7 +229,7 @@ "karma-mocha-reporter": "^2.2.5", "karma-safari-launcher": "^1.0.0", "karma-teamcity-reporter": "^1.1.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "sinon": "^14.0.0", "ts-node": "^9.1.1", @@ -265,7 +252,7 @@ "@keymanapp/resources-gosh": "*", "@types/node": "^11.9.4", "chai": "^4.3.4", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "ts-node": "^9.1.1", "typescript": "^4.5.4" @@ -283,7 +270,7 @@ "devDependencies": { "@keymanapp/resources-gosh": "*", "chai": "^4.3.4", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "ts-node": "^9.1.1", "typescript": "^4.5.4" @@ -351,7 +338,7 @@ "karma-mocha-reporter": "^2.2.5", "karma-safari-launcher": "^1.0.0", "karma-teamcity-reporter": "^1.1.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "sinon": "^14.0.0", "ts-node": "^9.1.1", @@ -497,7 +484,7 @@ "chai": "^4.3.4", "chalk": "^2.4.2", "jszip": "^3.7.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "ts-node": "^9.1.1" }, "engines": { @@ -601,7 +588,7 @@ "@types/ws": "^8.2.2", "chai": "^4.3.4", "copyfiles": "^2.4.1", - "mocha": "^9.1.4", + "mocha": "^10.0.0", "ts-node": "^10.4.0", "tsc-watch": "^4.5.0", "typescript": "^4.5.4" @@ -611,18 +598,6 @@ "node-windows-trayicon": "keymanapp/node-windows-trayicon#keyman-15.0" } }, - "developer/src/server/node_modules/@cspotcode/source-map-support": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", - "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", - "dev": true, - "dependencies": { - "@jridgewell/trace-mapping": "0.3.9" - }, - "engines": { - "node": ">=12" - } - }, "developer/src/server/node_modules/@types/mocha": { "version": "9.1.1", "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-9.1.1.tgz", @@ -635,207 +610,6 @@ "integrity": "sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==", "dev": true }, - "developer/src/server/node_modules/chokidar": { - "version": "3.5.3", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", - "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", - "dev": true, - "funding": [ - { - "type": "individual", - "url": "https://paulmillr.com/funding/" - } - ], - "dependencies": { - "anymatch": "~3.1.2", - "braces": "~3.0.2", - "glob-parent": "~5.1.2", - "is-binary-path": "~2.1.0", - "is-glob": "~4.0.1", - "normalize-path": "~3.0.0", - "readdirp": "~3.6.0" - }, - "engines": { - "node": ">= 8.10.0" - }, - "optionalDependencies": { - "fsevents": "~2.3.2" - } - }, - "developer/src/server/node_modules/debug": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", - "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==", - "dev": true, - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "developer/src/server/node_modules/escape-string-regexp": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", - "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "developer/src/server/node_modules/glob": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", - "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", - "dev": true, - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "developer/src/server/node_modules/log-symbols": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", - "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", - "dev": true, - "dependencies": { - "chalk": "^4.1.0", - "is-unicode-supported": "^0.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "developer/src/server/node_modules/mocha": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-9.2.2.tgz", - "integrity": "sha512-L6XC3EdwT6YrIk0yXpavvLkn8h+EU+Y5UcCHKECyMbdUIxyMuZj4bX4U9e1nvnvUUvQVsV2VHQr5zLdcUkhW/g==", - "dev": true, - "dependencies": { - "@ungap/promise-all-settled": "1.1.2", - "ansi-colors": "4.1.1", - "browser-stdout": "1.3.1", - "chokidar": "3.5.3", - "debug": "4.3.3", - "diff": "5.0.0", - "escape-string-regexp": "4.0.0", - "find-up": "5.0.0", - "glob": "7.2.0", - "growl": "1.10.5", - "he": "1.2.0", - "js-yaml": "4.1.0", - "log-symbols": "4.1.0", - "minimatch": "4.2.1", - "ms": "2.1.3", - "nanoid": "3.3.1", - "serialize-javascript": "6.0.0", - "strip-json-comments": "3.1.1", - "supports-color": "8.1.1", - "which": "2.0.2", - "workerpool": "6.2.0", - "yargs": "16.2.0", - "yargs-parser": "20.2.4", - "yargs-unparser": "2.0.0" - }, - "bin": { - "_mocha": "bin/_mocha", - "mocha": "bin/mocha" - }, - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/mochajs" - } - }, - "developer/src/server/node_modules/mocha/node_modules/minimatch": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-4.2.1.tgz", - "integrity": "sha512-9Uq1ChtSZO+Mxa/CL1eGizn2vRn3MlLgzhT0Iz8zaY8NdvxvB0d5QdPFmCKf7JKA9Lerx5vRrnwO03jsSfGG9g==", - "dev": true, - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": ">=10" - } - }, - "developer/src/server/node_modules/mocha/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true - }, - "developer/src/server/node_modules/nanoid": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.1.tgz", - "integrity": "sha512-n6Vs/3KGyxPQd6uO0eH4Bv0ojGSUvuLlIHtC3Y0kEO23YRge8H9x1GCzLn28YX0H66pMkxuaeESFq4tKISKwdw==", - "dev": true, - "bin": { - "nanoid": "bin/nanoid.cjs" - }, - "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" - } - }, - "developer/src/server/node_modules/readdirp": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", - "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "dev": true, - "dependencies": { - "picomatch": "^2.2.1" - }, - "engines": { - "node": ">=8.10.0" - } - }, - "developer/src/server/node_modules/serialize-javascript": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz", - "integrity": "sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==", - "dev": true, - "dependencies": { - "randombytes": "^2.1.0" - } - }, - "developer/src/server/node_modules/supports-color": { - "version": "8.1.1", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", - "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", - "dev": true, - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/supports-color?sponsor=1" - } - }, "developer/src/server/node_modules/ts-node": { "version": "10.8.1", "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.8.1.tgz", @@ -888,18 +662,13 @@ "node": ">=0.3.1" } }, - "developer/src/server/node_modules/workerpool": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.2.0.tgz", - "integrity": "sha512-Rsk5qQHJ9eowMH28Jwhe8HEbmdYDX4lwoMWshiCXugjtHqMD9ZbiqSDLxcsfdqsETPzVUtX5s1Z5kStiIM6l4A==", - "dev": true - }, "node_modules/@actions/core": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.8.2.tgz", - "integrity": "sha512-FXcBL7nyik8K5ODeCKlxi+vts7torOkoDAKfeh61EAkAy1HAvwn9uVzZBY0f15YcQTcZZ2/iSGBFHEuioZWfDA==", + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.9.1.tgz", + "integrity": "sha512-5ad+U2YGrmmiw6du20AQW5XuWo7UKN2052FjSV7MX+Wfjf8sCqcsZe62NfgHys4QI4/Y+vQvLKYL8jWtA1ZBTA==", "dependencies": { - "@actions/http-client": "^2.0.1" + "@actions/http-client": "^2.0.1", + "uuid": "^8.3.2" } }, "node_modules/@actions/github": { @@ -937,6 +706,18 @@ "node": ">=0.1.90" } }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/@jridgewell/resolve-uri": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.0.7.tgz", @@ -1783,16 +1564,6 @@ "integrity": "sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q==", "dev": true }, - "node_modules/@zeit/ncc": { - "version": "0.21.1", - "resolved": "https://registry.npmjs.org/@zeit/ncc/-/ncc-0.21.1.tgz", - "integrity": "sha512-M9WzgquSOt2nsjRkYM9LRylBLmmlwNCwYbm3Up3PDEshfvdmIfqpFNSK8EJvR18NwZjGHE5z2avlDtYQx2JQnw==", - "deprecated": "@zeit/ncc is no longer maintained. Please use @vercel/ncc instead.", - "dev": true, - "bin": { - "ncc": "dist/ncc/cli.js" - } - }, "node_modules/abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -1872,7 +1643,7 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", - "devOptional": true, + "optional": true, "engines": { "node": ">=0.10.0" } @@ -2416,7 +2187,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", - "devOptional": true, + "optional": true, "engines": { "node": ">=0.10.0" } @@ -3581,15 +3352,6 @@ "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", "devOptional": true }, - "node_modules/growl": { - "version": "1.10.5", - "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", - "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", - "dev": true, - "engines": { - "node": ">=4.x" - } - }, "node_modules/har-schema": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", @@ -3805,7 +3567,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", - "devOptional": true, + "optional": true, "dependencies": { "number-is-nan": "^1.0.0" }, @@ -4368,18 +4130,6 @@ "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==" }, - "node_modules/log-symbols": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.0.0.tgz", - "integrity": "sha512-FN8JBzLx6CzeMrB0tg6pqlGU1wCrXW+ZXGH481kfsBqer0hToTIiHdjH4Mq8xJUbvATujKCvaREGWpGUionraA==", - "dev": true, - "dependencies": { - "chalk": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/log4js": { "version": "6.4.4", "resolved": "https://registry.npmjs.org/log4js/-/log4js-6.4.4.tgz", @@ -4594,43 +4344,40 @@ } }, "node_modules/mocha": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-8.4.0.tgz", - "integrity": "sha512-hJaO0mwDXmZS4ghXsvPVriOhsxQ7ofcpQdm8dE+jISUOKopitvnXFQmpRR7jd2K6VBG6E26gU3IAbXXGIbu4sQ==", + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.0.0.tgz", + "integrity": "sha512-0Wl+elVUD43Y0BqPZBzZt8Tnkw9CMUdNYnUsTfOM1vuhJVZL+kiesFYsqwBkEEuEixaiPe5ZQdqDgX2jddhmoA==", "dev": true, "dependencies": { "@ungap/promise-all-settled": "1.1.2", "ansi-colors": "4.1.1", "browser-stdout": "1.3.1", - "chokidar": "3.5.1", - "debug": "4.3.1", + "chokidar": "3.5.3", + "debug": "4.3.4", "diff": "5.0.0", "escape-string-regexp": "4.0.0", "find-up": "5.0.0", - "glob": "7.1.6", - "growl": "1.10.5", + "glob": "7.2.0", "he": "1.2.0", - "js-yaml": "4.0.0", - "log-symbols": "4.0.0", - "minimatch": "3.0.4", + "js-yaml": "4.1.0", + "log-symbols": "4.1.0", + "minimatch": "5.0.1", "ms": "2.1.3", - "nanoid": "3.1.20", - "serialize-javascript": "5.0.1", + "nanoid": "3.3.3", + "serialize-javascript": "6.0.0", "strip-json-comments": "3.1.1", "supports-color": "8.1.1", - "which": "2.0.2", - "wide-align": "1.1.3", - "workerpool": "6.1.0", + "workerpool": "6.2.1", "yargs": "16.2.0", "yargs-parser": "20.2.4", "yargs-unparser": "2.0.0" }, "bin": { "_mocha": "bin/_mocha", - "mocha": "bin/mocha" + "mocha": "bin/mocha.js" }, "engines": { - "node": ">= 10.12.0" + "node": ">= 14.0.0" }, "funding": { "type": "opencollective", @@ -4649,6 +4396,56 @@ "mocha": ">=6" } }, + "node_modules/mocha/node_modules/chokidar": { + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", + "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + ], + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/mocha/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/mocha/node_modules/debug/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, "node_modules/mocha/node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -4661,16 +4458,73 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/mocha/node_modules/js-yaml": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", - "integrity": "sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==", + "node_modules/mocha/node_modules/glob": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", "dev": true, "dependencies": { - "argparse": "^2.0.1" + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" }, - "bin": { - "js-yaml": "bin/js-yaml.js" + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/mocha/node_modules/glob/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/mocha/node_modules/log-symbols": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", + "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", + "dev": true, + "dependencies": { + "chalk": "^4.1.0", + "is-unicode-supported": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mocha/node_modules/minimatch": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.0.1.tgz", + "integrity": "sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==", + "dev": true, + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/mocha/node_modules/minimatch/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0" } }, "node_modules/mocha/node_modules/ms": { @@ -4679,6 +4533,39 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true }, + "node_modules/mocha/node_modules/nanoid": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.3.tgz", + "integrity": "sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==", + "dev": true, + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/mocha/node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/mocha/node_modules/serialize-javascript": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz", + "integrity": "sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==", + "dev": true, + "dependencies": { + "randombytes": "^2.1.0" + } + }, "node_modules/mocha/node_modules/supports-color": { "version": "8.1.1", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", @@ -4694,6 +4581,12 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, + "node_modules/mocha/node_modules/workerpool": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.2.1.tgz", + "integrity": "sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw==", + "dev": true + }, "node_modules/modernizr": { "version": "3.12.0", "resolved": "https://registry.npmjs.org/modernizr/-/modernizr-3.12.0.tgz", @@ -4912,18 +4805,6 @@ "node": ">= 0.10.0" } }, - "node_modules/nanoid": { - "version": "3.1.20", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.20.tgz", - "integrity": "sha512-a1cQNyczgKbLX9jwbS/+d7W8fX/RfgYR7lVWwWOGIPNgK2m0MWvrGF6/m4kk6U3QcFMnZf3RIhL0v2Jgh/0Uxw==", - "dev": true, - "bin": { - "nanoid": "bin/nanoid.cjs" - }, - "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" - } - }, "node_modules/negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -5177,7 +5058,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", - "devOptional": true, + "optional": true, "engines": { "node": ">=0.10.0" } @@ -5778,15 +5659,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, - "node_modules/serialize-javascript": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz", - "integrity": "sha512-SaaNal9imEO737H2c05Og0/8LUXG7EnsZyMa8MzkmuHoELfT6txuj0cMqRj6zfPKnmQ1yasR4PCJc8x+M4JSPA==", - "dev": true, - "dependencies": { - "randombytes": "^2.1.0" - } - }, "node_modules/serve-static": { "version": "1.14.2", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.2.tgz", @@ -6056,7 +5928,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", - "devOptional": true, + "optional": true, "dependencies": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -6080,7 +5952,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", - "devOptional": true, + "optional": true, "dependencies": { "ansi-regex": "^2.0.0" }, @@ -6552,7 +6424,7 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", - "devOptional": true, + "optional": true, "dependencies": { "string-width": "^1.0.2 || 2" } @@ -6571,12 +6443,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/workerpool": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.1.0.tgz", - "integrity": "sha512-toV7q9rWNYha963Pl/qyeZ6wG+3nnsyvolaNUS8+R5Wtw6qJPTxIlOP1ZSvcGhEJw+l3HMMmtiNo9Gl61G4GVg==", - "dev": true - }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -6867,7 +6733,7 @@ "name": "@keymanapp/auto-history-action", "license": "MIT", "dependencies": { - "@actions/core": "^1.2.6", + "@actions/core": "^1.9.1", "@actions/github": "^2.1.0", "typescript": "^4.5.4", "yargs": "^15.1.0" @@ -6875,7 +6741,6 @@ "devDependencies": { "@types/node": "^13.7.0", "@types/semver": "^7.1.0", - "@zeit/ncc": "^0.21.0", "semver": "^7.1.2", "ts-node": "^8.6.2" }, @@ -7113,7 +6978,7 @@ "karma-mocha-reporter": "^2.2.5", "karma-safari-launcher": "^1.0.0", "karma-teamcity-reporter": "^1.1.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "modernizr": "^3.11.7", "ts-node": "^9.1.1", "typescript": "^4.5.4" @@ -7122,11 +6987,12 @@ }, "dependencies": { "@actions/core": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.8.2.tgz", - "integrity": "sha512-FXcBL7nyik8K5ODeCKlxi+vts7torOkoDAKfeh61EAkAy1HAvwn9uVzZBY0f15YcQTcZZ2/iSGBFHEuioZWfDA==", + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.9.1.tgz", + "integrity": "sha512-5ad+U2YGrmmiw6du20AQW5XuWo7UKN2052FjSV7MX+Wfjf8sCqcsZe62NfgHys4QI4/Y+vQvLKYL8jWtA1ZBTA==", "requires": { - "@actions/http-client": "^2.0.1" + "@actions/http-client": "^2.0.1", + "uuid": "^8.3.2" } }, "@actions/github": { @@ -7163,6 +7029,15 @@ "integrity": "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==", "dev": true }, + "@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "requires": { + "@jridgewell/trace-mapping": "0.3.9" + } + }, "@jridgewell/resolve-uri": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.0.7.tgz", @@ -7188,11 +7063,10 @@ "@keymanapp/auto-history-action": { "version": "file:resources/build/version", "requires": { - "@actions/core": "^1.2.6", + "@actions/core": "^1.9.1", "@actions/github": "^2.1.0", "@types/node": "^13.7.0", "@types/semver": "^7.1.0", - "@zeit/ncc": "^0.21.0", "semver": "^7.1.2", "ts-node": "^8.6.2", "typescript": "^4.5.4", @@ -7356,7 +7230,7 @@ "copyfiles": "^2.4.1", "express": "^4.17.2", "hetrodo-node-hide-console-window-napi": "keymanapp/hetrodo-node-hide-console-window-napi#keyman-15.0", - "mocha": "^9.1.4", + "mocha": "^10.0.0", "multer": "^1.4.4", "ngrok": "^4.2.2", "node-windows-trayicon": "keymanapp/node-windows-trayicon#keyman-15.0", @@ -7367,15 +7241,6 @@ "ws": "^8.3.0" }, "dependencies": { - "@cspotcode/source-map-support": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", - "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", - "dev": true, - "requires": { - "@jridgewell/trace-mapping": "0.3.9" - } - }, "@types/mocha": { "version": "9.1.1", "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-9.1.1.tgz", @@ -7388,143 +7253,6 @@ "integrity": "sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==", "dev": true }, - "chokidar": { - "version": "3.5.3", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", - "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", - "dev": true, - "requires": { - "anymatch": "~3.1.2", - "braces": "~3.0.2", - "fsevents": "~2.3.2", - "glob-parent": "~5.1.2", - "is-binary-path": "~2.1.0", - "is-glob": "~4.0.1", - "normalize-path": "~3.0.0", - "readdirp": "~3.6.0" - } - }, - "debug": { - "version": "4.3.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", - "integrity": "sha512-/zxw5+vh1Tfv+4Qn7a5nsbcJKPaSvCDhojn6FEl9vupwK2VCSDtEiEtqr8DFtzYFOdz63LBkxec7DYuc2jon6Q==", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, - "escape-string-regexp": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", - "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", - "dev": true - }, - "glob": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", - "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", - "dev": true, - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, - "log-symbols": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", - "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", - "dev": true, - "requires": { - "chalk": "^4.1.0", - "is-unicode-supported": "^0.1.0" - } - }, - "mocha": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-9.2.2.tgz", - "integrity": "sha512-L6XC3EdwT6YrIk0yXpavvLkn8h+EU+Y5UcCHKECyMbdUIxyMuZj4bX4U9e1nvnvUUvQVsV2VHQr5zLdcUkhW/g==", - "dev": true, - "requires": { - "@ungap/promise-all-settled": "1.1.2", - "ansi-colors": "4.1.1", - "browser-stdout": "1.3.1", - "chokidar": "3.5.3", - "debug": "4.3.3", - "diff": "5.0.0", - "escape-string-regexp": "4.0.0", - "find-up": "5.0.0", - "glob": "7.2.0", - "growl": "1.10.5", - "he": "1.2.0", - "js-yaml": "4.1.0", - "log-symbols": "4.1.0", - "minimatch": "4.2.1", - "ms": "2.1.3", - "nanoid": "3.3.1", - "serialize-javascript": "6.0.0", - "strip-json-comments": "3.1.1", - "supports-color": "8.1.1", - "which": "2.0.2", - "workerpool": "6.2.0", - "yargs": "16.2.0", - "yargs-parser": "20.2.4", - "yargs-unparser": "2.0.0" - }, - "dependencies": { - "minimatch": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-4.2.1.tgz", - "integrity": "sha512-9Uq1ChtSZO+Mxa/CL1eGizn2vRn3MlLgzhT0Iz8zaY8NdvxvB0d5QdPFmCKf7JKA9Lerx5vRrnwO03jsSfGG9g==", - "dev": true, - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true - } - } - }, - "nanoid": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.1.tgz", - "integrity": "sha512-n6Vs/3KGyxPQd6uO0eH4Bv0ojGSUvuLlIHtC3Y0kEO23YRge8H9x1GCzLn28YX0H66pMkxuaeESFq4tKISKwdw==", - "dev": true - }, - "readdirp": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", - "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "dev": true, - "requires": { - "picomatch": "^2.2.1" - } - }, - "serialize-javascript": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz", - "integrity": "sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==", - "dev": true, - "requires": { - "randombytes": "^2.1.0" - } - }, - "supports-color": { - "version": "8.1.1", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", - "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", - "dev": true, - "requires": { - "has-flag": "^4.0.0" - } - }, "ts-node": { "version": "10.8.1", "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.8.1.tgz", @@ -7553,12 +7281,6 @@ "dev": true } } - }, - "workerpool": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.2.0.tgz", - "integrity": "sha512-Rsk5qQHJ9eowMH28Jwhe8HEbmdYDX4lwoMWshiCXugjtHqMD9ZbiqSDLxcsfdqsETPzVUtX5s1Z5kStiIM6l4A==", - "dev": true } } }, @@ -7583,7 +7305,7 @@ "karma-mocha-reporter": "^2.2.5", "karma-safari-launcher": "^1.0.0", "karma-teamcity-reporter": "^1.1.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "sinon": "^14.0.0", "ts-node": "^9.1.1", @@ -7603,7 +7325,7 @@ "@types/node": "^11.9.4", "chai": "^4.3.4", "eventemitter3": "^4.0.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "ts-node": "^9.1.1", "typescript": "^4.5.4" @@ -7618,7 +7340,7 @@ "@keymanapp/web-utils": "*", "@types/node": "^11.9.4", "chai": "^4.3.4", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "ts-node": "^9.1.1", "typescript": "^4.5.4" @@ -7645,7 +7367,7 @@ "chalk": "^2.4.2", "commander": "^3.0.0", "jszip": "^3.7.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "ts-node": "^9.1.1", "typescript": "^4.5.4", "xml2js": "^0.4.19" @@ -7741,7 +7463,7 @@ "karma-mocha-reporter": "^2.2.5", "karma-safari-launcher": "^1.0.0", "karma-teamcity-reporter": "^1.1.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "sinon": "^14.0.0", "string.prototype.codepointat": "^0.2.1", @@ -7789,7 +7511,7 @@ "karma-mocha-reporter": "^2.2.5", "karma-safari-launcher": "^1.0.0", "karma-teamcity-reporter": "^1.1.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "sinon": "^14.0.0", "string.prototype.codepointat": "^0.2.1", @@ -7816,8 +7538,8 @@ "@types/mocha": "^7.0.2", "@types/node": "^14.0.4", "chai": "^4.3.4", - "mocha": "^8.4.0", - "typescript": "^3.8.3" + "mocha": "^10.0.0", + "typescript": "^4.5.4" }, "dependencies": { "@types/node": { @@ -7825,12 +7547,6 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.12.tgz", "integrity": "sha512-q4jlIR71hUpWTnGhXWcakgkZeHa3CCjcQcnuzU8M891BAWA2jHiziiWEPEkdS5pFsz7H9HJiy8BrK7tBRNrY7A==", "dev": true - }, - "typescript": { - "version": "3.9.10", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.10.tgz", - "integrity": "sha512-w6fIxVE/H1PkLKcCPsFqKE7Kv7QUwhU8qQY2MueZXWx5cPZdwFupLgKK3vntcK98BtNHZtAF4LA/yl2a7k8R6Q==", - "dev": true } } }, @@ -7848,7 +7564,7 @@ "@types/mocha": "^7.0.2", "@types/node": "^14.0.3", "chai": "^4.3.4", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "ts-node": "^9.1.1", "typescript": "^4.5.4" }, @@ -8581,12 +8297,6 @@ "integrity": "sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q==", "dev": true }, - "@zeit/ncc": { - "version": "0.21.1", - "resolved": "https://registry.npmjs.org/@zeit/ncc/-/ncc-0.21.1.tgz", - "integrity": "sha512-M9WzgquSOt2nsjRkYM9LRylBLmmlwNCwYbm3Up3PDEshfvdmIfqpFNSK8EJvR18NwZjGHE5z2avlDtYQx2JQnw==", - "dev": true - }, "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -8644,7 +8354,7 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", - "devOptional": true + "optional": true }, "ansi-styles": { "version": "4.3.0", @@ -9103,7 +8813,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", - "devOptional": true + "optional": true }, "color-convert": { "version": "2.0.1", @@ -10022,12 +9732,6 @@ "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", "devOptional": true }, - "growl": { - "version": "1.10.5", - "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", - "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", - "dev": true - }, "har-schema": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", @@ -10187,7 +9891,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", - "devOptional": true, + "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -10617,7 +10321,7 @@ "karma-mocha-reporter": "^2.2.5", "karma-safari-launcher": "^1.0.0", "karma-teamcity-reporter": "^1.1.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "modernizr": "^3.11.7", "ts-node": "^9.1.1", "typescript": "^4.5.4" @@ -10685,15 +10389,6 @@ "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==" }, - "log-symbols": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.0.0.tgz", - "integrity": "sha512-FN8JBzLx6CzeMrB0tg6pqlGU1wCrXW+ZXGH481kfsBqer0hToTIiHdjH4Mq8xJUbvATujKCvaREGWpGUionraA==", - "dev": true, - "requires": { - "chalk": "^4.0.0" - } - }, "log4js": { "version": "6.4.4", "resolved": "https://registry.npmjs.org/log4js/-/log4js-6.4.4.tgz", @@ -10851,51 +10546,127 @@ } }, "mocha": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-8.4.0.tgz", - "integrity": "sha512-hJaO0mwDXmZS4ghXsvPVriOhsxQ7ofcpQdm8dE+jISUOKopitvnXFQmpRR7jd2K6VBG6E26gU3IAbXXGIbu4sQ==", + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.0.0.tgz", + "integrity": "sha512-0Wl+elVUD43Y0BqPZBzZt8Tnkw9CMUdNYnUsTfOM1vuhJVZL+kiesFYsqwBkEEuEixaiPe5ZQdqDgX2jddhmoA==", "dev": true, "requires": { "@ungap/promise-all-settled": "1.1.2", "ansi-colors": "4.1.1", "browser-stdout": "1.3.1", - "chokidar": "3.5.1", - "debug": "4.3.1", + "chokidar": "3.5.3", + "debug": "4.3.4", "diff": "5.0.0", "escape-string-regexp": "4.0.0", "find-up": "5.0.0", - "glob": "7.1.6", - "growl": "1.10.5", + "glob": "7.2.0", "he": "1.2.0", - "js-yaml": "4.0.0", - "log-symbols": "4.0.0", - "minimatch": "3.0.4", + "js-yaml": "4.1.0", + "log-symbols": "4.1.0", + "minimatch": "5.0.1", "ms": "2.1.3", - "nanoid": "3.1.20", - "serialize-javascript": "5.0.1", + "nanoid": "3.3.3", + "serialize-javascript": "6.0.0", "strip-json-comments": "3.1.1", "supports-color": "8.1.1", - "which": "2.0.2", - "wide-align": "1.1.3", - "workerpool": "6.1.0", + "workerpool": "6.2.1", "yargs": "16.2.0", "yargs-parser": "20.2.4", "yargs-unparser": "2.0.0" }, "dependencies": { + "chokidar": { + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", + "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", + "dev": true, + "requires": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "fsevents": "~2.3.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + } + }, + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + }, + "dependencies": { + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + } + } + }, "escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", "dev": true }, - "js-yaml": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", - "integrity": "sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==", + "glob": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", "dev": true, "requires": { - "argparse": "^2.0.1" + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "dependencies": { + "minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "requires": { + "brace-expansion": "^1.1.7" + } + } + } + }, + "log-symbols": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", + "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", + "dev": true, + "requires": { + "chalk": "^4.1.0", + "is-unicode-supported": "^0.1.0" + } + }, + "minimatch": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.0.1.tgz", + "integrity": "sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==", + "dev": true, + "requires": { + "brace-expansion": "^2.0.1" + }, + "dependencies": { + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0" + } + } } }, "ms": { @@ -10904,6 +10675,30 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true }, + "nanoid": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.3.tgz", + "integrity": "sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==", + "dev": true + }, + "readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "requires": { + "picomatch": "^2.2.1" + } + }, + "serialize-javascript": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz", + "integrity": "sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==", + "dev": true, + "requires": { + "randombytes": "^2.1.0" + } + }, "supports-color": { "version": "8.1.1", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", @@ -10912,6 +10707,12 @@ "requires": { "has-flag": "^4.0.0" } + }, + "workerpool": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.2.1.tgz", + "integrity": "sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw==", + "dev": true } } }, @@ -11094,12 +10895,6 @@ "xtend": "^4.0.0" } }, - "nanoid": { - "version": "3.1.20", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.20.tgz", - "integrity": "sha512-a1cQNyczgKbLX9jwbS/+d7W8fX/RfgYR7lVWwWOGIPNgK2m0MWvrGF6/m4kk6U3QcFMnZf3RIhL0v2Jgh/0Uxw==", - "dev": true - }, "negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -11302,7 +11097,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", - "devOptional": true + "optional": true }, "oauth-sign": { "version": "0.9.0", @@ -11749,15 +11544,6 @@ } } }, - "serialize-javascript": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz", - "integrity": "sha512-SaaNal9imEO737H2c05Og0/8LUXG7EnsZyMa8MzkmuHoELfT6txuj0cMqRj6zfPKnmQ1yasR4PCJc8x+M4JSPA==", - "dev": true, - "requires": { - "randombytes": "^2.1.0" - } - }, "serve-static": { "version": "1.14.2", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.2.tgz", @@ -11969,7 +11755,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", - "devOptional": true, + "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -11990,7 +11776,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", - "devOptional": true, + "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -12338,7 +12124,7 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", - "devOptional": true, + "optional": true, "requires": { "string-width": "^1.0.2 || 2" } @@ -12351,12 +12137,6 @@ "execa": "^1.0.0" } }, - "workerpool": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.1.0.tgz", - "integrity": "sha512-toV7q9rWNYha963Pl/qyeZ6wG+3nnsyvolaNUS8+R5Wtw6qJPTxIlOP1ZSvcGhEJw+l3HMMmtiNo9Gl61G4GVg==", - "dev": true - }, "wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", diff --git a/package.json b/package.json index 291c69d225..df60de59f8 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "private": true, "devDependencies": { "chai": "^4.3.4", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "mocha-teamcity-reporter": "^4.0.0", "ts-node": "^9.1.1", "typescript": "^4.5.4" diff --git a/resources/build/build-utils-ci.inc.sh b/resources/build/build-utils-ci.inc.sh new file mode 100644 index 0000000000..6212f9720c --- /dev/null +++ b/resources/build/build-utils-ci.inc.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# +# This script gets CI / pull request details for builds, part of the build-utils +# builder_ suite of functions. All functions and variables in this file have the +# prefix builder_pull_. +# + +. "$KEYMAN_ROOT/resources/build/jq.inc.sh" + +# +# Returns 0 if current build is in CI and triggered from a pull request. If it +# returns 0, then a call is made to GitHub to get pull request details, and the +# PR details are added to $builder_pull_title, $builder_pull_number, and +# the $builder_pull_labels array. +# +# Note that the GitHub REST call is made with credentials if $GITHUB_TOKEN is +# available; without credentials it will be subject to IP-based rate limits. +# +builder_pull_get_details() { + builder_pull_title= + builder_pull_number= + builder_pull_labels=() + if [[ ! ${TEAMCITY_PR_NUMBER-} =~ ^[0-9]+$ ]]; then + return 1 + fi + + if [ -z "${GITHUB_TOKEN-}" ]; then + local pull_data=`curl -s https://api.github.com/repos/keymanapp/keyman/pulls/$TEAMCITY_PR_NUMBER` + else + local pull_data=`curl -H "Authorization: Bearer $GITHUB_TOKEN" -s https://api.github.com/repos/keymanapp/keyman/pulls/$TEAMCITY_PR_NUMBER` + fi + + builder_pull_title=`echo $pull_data | $JQ .title` + + # Simple poor bash test if data returned from API is valid + if [[ -z builder_pull_title ]]; then + return 1 + fi + + builder_pull_number=$TEAMCITY_PR_NUMBER + builder_pull_labels=(`echo $pull_data | $JQ -jr '.labels[].name|.," "'`) + + return 0 +} + + +# +# Returns 0 if the current PR has a particular label. Requires +# builder_pull_get_details to have successfully run first +# +# Usage: +# builder_pull_has_label "label-name" +# Parameters +# 1: $label label to test +# +function builder_pull_has_label() { + local label="$1" + if [[ " ${builder_pull_labels[*]} " =~ " $label " ]]; then + return 0 + fi + return 1 +} diff --git a/resources/build/build-utils-ci.test.sh b/resources/build/build-utils-ci.test.sh new file mode 100755 index 0000000000..b80be2d1b1 --- /dev/null +++ b/resources/build/build-utils-ci.test.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# +# This script tests build-utils-ci.sh +# + +set -eu + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")" +. "$(dirname "$THIS_SCRIPT")/build-utils.sh" +# END STANDARD BUILD SCRIPT INCLUDE + +. "$KEYMAN_ROOT/resources/build/build-utils-ci.inc.sh" +. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" + +# Tests + +echo "${COLOR_BLUE}## Testing: builder_pull_get_details with \$TEAMCITY_PR_NUMBER not set${COLOR_RESET}" +if ! builder_pull_get_details; then + echo "PASS: builder_pull_get_details should get no details if \$TEAMCITY_PR_NUMBER is not set" +else + fail "FAIL: builder_pull_get_details should get no details if \$TEAMCITY_PR_NUMBER is not set" +fi + + +echo "${COLOR_BLUE}## Testing: builder_pull_get_details with \$TEAMCITY_PR_NUMBER=master${COLOR_RESET}" +TEAMCITY_PR_NUMBER=master +if ! builder_pull_get_details; then + echo "PASS: builder_pull_get_details should get no details if \$TEAMCITY_PR_NUMBER=master" +else + fail "FAIL: builder_pull_get_details should get no details if \$TEAMCITY_PR_NUMBER=master" +fi + + +echo "${COLOR_BLUE}## Testing: builder_pull_get_details with \$TEAMCITY_PR_NUMBER=$TEAMCITY_PR_NUMBER${COLOR_RESET}" +TEAMCITY_PR_NUMBER=7248 +if ! builder_pull_get_details; then + fail "FAIL: builder_pull_get_details was unable to read and parse PR details for #$TEAMCITY_PR_NUMBER from GitHub" +fi + +# print metadata from PR +echo " pull number: $builder_pull_number" +echo " title: $builder_pull_title" +echo " labels: ${builder_pull_labels[*]}" + +echo "PASS: builder_pull_get_details with \$TEAMCITY_PR_NUMBER=$TEAMCITY_PR_NUMBER" + + +echo "${COLOR_BLUE}## Testing: builder_pull_has_label epic-ldml${COLOR_RESET}" +if ! builder_pull_has_label epic-ldml; then + fail "FAIL: builder_pull_has_label epic-ldml" +fi + +echo "PASS: builder_pull_has_label epic-ldml" + +echo "All tests passed." diff --git a/resources/build/build-utils.sh b/resources/build/build-utils.sh index ab9c2a469c..9a26531bf7 100755 --- a/resources/build/build-utils.sh +++ b/resources/build/build-utils.sh @@ -27,6 +27,11 @@ # Note: keep changes to version, tier and tag determination in sync with mkver (windows/src/buildutils/mkver) # +# +# Prevents 'clear' on exit of mingw64 bash shell +# +SHLVL=0 + # Setup variable for calling script's path and name if [ ! -z ${THIS_SCRIPT+x} ]; then THIS_SCRIPT_PATH="$(dirname "$THIS_SCRIPT")" @@ -410,12 +415,16 @@ _builder_trim() { # # There are three types of parameters that may be specified: # -# * Option, param_desc format: "--option[,-o] [One line description]" +# * Option, param_desc format: "--option[,-o][=var] [One line description]" # All options must have a longhand form with two prefix hyphens, # e.g. --option. The ",-o" shorthand form is optional. When testing if # the option is set with `builder_has_option``, always use the longhand # form. # +# if =var is specified, then the next parameter will be a variable stored +# in $var for that option. e.g. --option=opt means $opt will have the value +# 'foo' when the script is called for --option foo. +# # * Action, param_desc format: "action [One line description]" # Actions must be a single word, lower case. To specify an action # as the default, append a '+' to the action name, e.g. @@ -433,6 +442,7 @@ builder_describe() { _builder_default_action=build declare -A -g _builder_params declare -A -g _builder_options_short + declare -A -g _builder_options_var shift # describe each target, action, and option possibility while [[ $# -gt 0 ]]; do @@ -449,14 +459,30 @@ builder_describe() { elif [[ $value =~ ^-- ]]; then # Parameter is an option # Look for a shorthand version of the option + local option_var= + if [[ $value =~ = ]]; then + option_var="$(echo "$value" | cut -d= -f 2 -)" + value="$(echo "$value" | cut -d= -f 1 -)" + fi + if [[ $value =~ , ]]; then local option_long="$(echo "$value" | cut -d, -f 1 -)" local option_short="$(echo "$value" | cut -d, -f 2 -)" _builder_options+=($option_long) _builder_options_short[$option_short]="$option_long" + if [[ ! -z "$option_var" ]]; then + _builder_options_var[$option_long]="$option_var" + fi value="$option_long, $option_short" else _builder_options+=($value) + if [[ ! -z "$option_var" ]]; then + _builder_options_var[$value]="$option_var" + fi + fi + + if [[ ! -z $option_var ]]; then + value="$value $option_var" fi else # Parameter is an action @@ -574,6 +600,18 @@ builder_parse() { _builder_chosen_action_targets+=("$_builder_default_action$target") elif (( has_option )); then _builder_chosen_options+=("$key") + if [[ ! -z ${_builder_options_var[$key]+x} ]]; then + shift + if [[ $# -eq 0 ]]; then + _builder_parameter_error "$0" parameter "$key" + fi + # Set the variable associated with this option to the next parameter value + # A little bit of hoop jumping here to avoid issues with cygwin paths being + # corrupted too early in the game + local varname=${_builder_options_var[$key]} + declare -g $varname="$1" + fi + else case "$key" in --help|-h) @@ -632,7 +670,9 @@ builder_display_usage() { local width=12 for e in "${!_builder_params[@]}"; do - if (( ${#e} > $width )); then width = ${#e}; fi + if (( ${#e} > $width )); then + width=${#e} + fi done width=$((width + 6)) diff --git a/resources/build/build-utils.test.sh b/resources/build/build-utils.test.sh index 3fa7902940..efc06ae483 100755 --- a/resources/build/build-utils.test.sh +++ b/resources/build/build-utils.test.sh @@ -94,7 +94,7 @@ builder_parse_test() { shift shift local parameters="$@" - echo "Testing: builder_parse $parameters" + echo "${COLOR_BLUE}## Testing: builder_parse $parameters${COLOR_RESET}" builder_parse $parameters || fail "builder_parse died under curious circumstances" if [[ "$expected" != "${_builder_chosen_action_targets[@]}" ]]; then fail " Test: builder_parse $parameters action:target != \"$expected\"" @@ -112,7 +112,8 @@ builder_describe \ ":app" \ ":engine Thomas, y'know" \ "--power,-p Use powerful mode" \ - "--zoom,-z Use zoom mode" + "--zoom,-z Use zoom mode" \ + "--feature=FOO Enable feature foo" # Test --options @@ -132,6 +133,21 @@ else fail "FAIL: --zoom option not found" fi +# Test --feature + +echo "${COLOR_BLUE}## Testing: builder_parse --feature xyzzy${COLOR_RESET}" +builder_parse --feature xyzzy + +if builder_has_option --feature; then + if [[ $FOO == xyzzy ]]; then + echo "PASS: --feature option variable \$FOO has expected value 'xyzzy'" + else + echo "FAIL: --feature option variable \$FOO had value '$FOO' but should have had 'xyzzy'" + fi +else + echo "FAIL: --feature option not found" +fi + # Finally, run with --help so we can see what it looks like echo "${COLOR_BLUE}## Testing --help${COLOR_RESET}" diff --git a/resources/build/version/package.json b/resources/build/version/package.json index 1a4e8355c8..5902dce8cc 100644 --- a/resources/build/version/package.json +++ b/resources/build/version/package.json @@ -1,6 +1,6 @@ { "dependencies": { - "@actions/core": "^1.2.6", + "@actions/core": "^1.9.1", "@actions/github": "^2.1.0", "typescript": "^4.5.4", "yargs": "^15.1.0" @@ -9,7 +9,6 @@ "devDependencies": { "@types/node": "^13.7.0", "@types/semver": "^7.1.0", - "@zeit/ncc": "^0.21.0", "semver": "^7.1.2", "ts-node": "^8.6.2" }, @@ -17,20 +16,16 @@ "node": "16" }, "files": [ - "dist", "src" ], "license": "MIT", - "main": "dist/index.js", + "main": "lib/index.js", "name": "@keymanapp/auto-history-action", "private": true, "scripts": { - "build": "npm run clean:dist; npm run build:dist", - "build:dist": "ncc build src/index.ts --minify --source-map --v8-cache", "build:ts": "tsc --project tsconfig.production.json", "build:ts:watch": "tsc --project tsconfig.production.json --watch", "clean": "run-p clean:*", - "clean:dist": "rm -rf dist", "clean:lib": "rm -rf lib", "types": "tsc --noEmit" } diff --git a/resources/devbox/iis-https/setup-iis-https.sh b/resources/devbox/iis-https/setup-iis-https.sh old mode 100644 new mode 100755 diff --git a/resources/devbox/macos/keyman.macos.env.sh b/resources/devbox/macos/keyman.macos.env.sh old mode 100644 new mode 100755 diff --git a/resources/git-hooks/pre-commit b/resources/git-hooks/pre-commit new file mode 100644 index 0000000000..3fd01c7d02 --- /dev/null +++ b/resources/git-hooks/pre-commit @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# +# We want to make sure that .sh files are executable; however, .inc.sh need not +# be as they are always source-included in scripts. +# +SH_NON_EXECUTABLE=`git ls-files --stage | grep "\\.sh$" | grep -v "\\.inc\\.sh$" | grep -v 755 | cut -f 2 -` + +if [ ! -z "$SH_NON_EXECUTABLE" ]; then + echo "ERROR: The following scripts are not marked as executable:" + echo + echo "$SH_NON_EXECUTABLE" + echo + echo "You can mark them as executable with:" + echo " git add --chmod=+x" + exit 1 +fi + +exit 0 diff --git a/resources/standards-data/unicode-character-database/Blocks.txt b/resources/standards-data/unicode-character-database/Blocks.txt index cc5d61988b..12684594c9 100644 --- a/resources/standards-data/unicode-character-database/Blocks.txt +++ b/resources/standards-data/unicode-character-database/Blocks.txt @@ -1,10 +1,10 @@ -# Blocks-14.0.0.txt -# Date: 2021-01-22, 23:29:00 GMT [KW] -# ยฉ 2021 Unicodeยฎ, Inc. -# For terms of use, see http://www.unicode.org/terms_of_use.html +# Blocks-15.0.0.txt +# Date: 2022-01-28, 20:58:00 GMT [KW] +# ยฉ 2022 Unicodeยฎ, Inc. +# For terms of use, see https://www.unicode.org/terms_of_use.html # # Unicode Character Database -# For documentation, see http://www.unicode.org/reports/tr44/ +# For documentation, see https://www.unicode.org/reports/tr44/ # # Format: # Start Code..End Code; Block Name @@ -15,7 +15,7 @@ # and underbars are ignored. # For example, "Latin Extended-A" and "latin extended a" are equivalent. # For more information on the comparison of property values, -# see UAX #44: http://www.unicode.org/reports/tr44/ +# see UAX #44: https://www.unicode.org/reports/tr44/ # # All block ranges start with a value where (cp MOD 16) = 0, # and end with a value where (cp MOD 16) = 15. In other words, @@ -241,6 +241,7 @@ FFF0..FFFF; Specials 10D00..10D3F; Hanifi Rohingya 10E60..10E7F; Rumi Numeral Symbols 10E80..10EBF; Yezidi +10EC0..10EFF; Arabic Extended-C 10F00..10F2F; Old Sogdian 10F30..10F6F; Sogdian 10F70..10FAF; Old Uyghur @@ -272,11 +273,13 @@ FFF0..FFFF; Specials 11A50..11AAF; Soyombo 11AB0..11ABF; Unified Canadian Aboriginal Syllabics Extended-A 11AC0..11AFF; Pau Cin Hau +11B00..11B5F; Devanagari Extended-A 11C00..11C6F; Bhaiksuki 11C70..11CBF; Marchen 11D00..11D5F; Masaram Gondi 11D60..11DAF; Gunjala Gondi 11EE0..11EFF; Makasar +11F00..11F5F; Kawi 11FB0..11FBF; Lisu Supplement 11FC0..11FFF; Tamil Supplement 12000..123FF; Cuneiform @@ -284,7 +287,7 @@ FFF0..FFFF; Specials 12480..1254F; Early Dynastic Cuneiform 12F90..12FFF; Cypro-Minoan 13000..1342F; Egyptian Hieroglyphs -13430..1343F; Egyptian Hieroglyph Format Controls +13430..1345F; Egyptian Hieroglyph Format Controls 14400..1467F; Anatolian Hieroglyphs 16800..16A3F; Bamum Supplement 16A40..16A6F; Mro @@ -309,6 +312,7 @@ FFF0..FFFF; Specials 1D000..1D0FF; Byzantine Musical Symbols 1D100..1D1FF; Musical Symbols 1D200..1D24F; Ancient Greek Musical Notation +1D2C0..1D2DF; Kaktovik Numerals 1D2E0..1D2FF; Mayan Numerals 1D300..1D35F; Tai Xuan Jing Symbols 1D360..1D37F; Counting Rod Numerals @@ -316,9 +320,11 @@ FFF0..FFFF; Specials 1D800..1DAAF; Sutton SignWriting 1DF00..1DFFF; Latin Extended-G 1E000..1E02F; Glagolitic Supplement +1E030..1E08F; Cyrillic Extended-D 1E100..1E14F; Nyiakeng Puachue Hmong 1E290..1E2BF; Toto 1E2C0..1E2FF; Wancho +1E4D0..1E4FF; Nag Mundari 1E7E0..1E7FF; Ethiopic Extended-B 1E800..1E8DF; Mende Kikakui 1E900..1E95F; Adlam @@ -348,6 +354,7 @@ FFF0..FFFF; Specials 2CEB0..2EBEF; CJK Unified Ideographs Extension F 2F800..2FA1F; CJK Compatibility Ideographs Supplement 30000..3134F; CJK Unified Ideographs Extension G +31350..323AF; CJK Unified Ideographs Extension H E0000..E007F; Tags E0100..E01EF; Variation Selectors Supplement F0000..FFFFF; Supplementary Private Use Area-A diff --git a/resources/standards-data/unicode-character-database/UnicodeData.txt b/resources/standards-data/unicode-character-database/UnicodeData.txt index b5abef7ed4..ea963a7162 100644 --- a/resources/standards-data/unicode-character-database/UnicodeData.txt +++ b/resources/standards-data/unicode-character-database/UnicodeData.txt @@ -2975,6 +2975,7 @@ 0CEF;KANNADA DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 0CF1;KANNADA SIGN JIHVAMULIYA;Lo;0;L;;;;;N;;;;; 0CF2;KANNADA SIGN UPADHMANIYA;Lo;0;L;;;;;N;;;;; +0CF3;KANNADA SIGN COMBINING ANUSVARA ABOVE RIGHT;Mc;0;L;;;;;N;;;;; 0D00;MALAYALAM SIGN COMBINING ANUSVARA ABOVE;Mn;0;NSM;;;;;N;;;;; 0D01;MALAYALAM SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;; 0D02;MALAYALAM SIGN ANUSVARA;Mc;0;L;;;;;N;;;;; @@ -3339,6 +3340,7 @@ 0ECB;LAO TONE MAI CATAWA;Mn;122;NSM;;;;;N;;;;; 0ECC;LAO CANCELLATION MARK;Mn;0;NSM;;;;;N;;;;; 0ECD;LAO NIGGAHITA;Mn;0;NSM;;;;;N;;;;; +0ECE;LAO YAMAKKAN;Mn;0;NSM;;;;;N;;;;; 0ED0;LAO DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; 0ED1;LAO DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; 0ED2;LAO DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; @@ -19393,6 +19395,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 10EAD;YEZIDI HYPHENATION MARK;Pd;0;R;;;;;N;;;;; 10EB0;YEZIDI LETTER LAM WITH DOT ABOVE;Lo;0;R;;;;;N;;;;; 10EB1;YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE;Lo;0;R;;;;;N;;;;; +10EFD;ARABIC SMALL LOW WORD SAKTA;Mn;220;NSM;;;;;N;;;;; +10EFE;ARABIC SMALL LOW WORD QASR;Mn;220;NSM;;;;;N;;;;; +10EFF;ARABIC SMALL LOW WORD MADDA;Mn;220;NSM;;;;;N;;;;; 10F00;OLD SOGDIAN LETTER ALEPH;Lo;0;R;;;;;N;;;;; 10F01;OLD SOGDIAN LETTER FINAL ALEPH;Lo;0;R;;;;;N;;;;; 10F02;OLD SOGDIAN LETTER BETH;Lo;0;R;;;;;N;;;;; @@ -20058,6 +20063,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1123C;KHOJKI DOUBLE SECTION MARK;Po;0;L;;;;;N;;;;; 1123D;KHOJKI ABBREVIATION SIGN;Po;0;L;;;;;N;;;;; 1123E;KHOJKI SIGN SUKUN;Mn;0;NSM;;;;;N;;;;; +1123F;KHOJKI LETTER QA;Lo;0;L;;;;;N;;;;; +11240;KHOJKI LETTER SHORT I;Lo;0;L;;;;;N;;;;; +11241;KHOJKI VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;; 11280;MULTANI LETTER A;Lo;0;L;;;;;N;;;;; 11281;MULTANI LETTER I;Lo;0;L;;;;;N;;;;; 11282;MULTANI LETTER U;Lo;0;L;;;;;N;;;;; @@ -21256,6 +21264,16 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11AF6;PAU CIN HAU LOW-FALLING TONE LONG FINAL;Lo;0;L;;;;;N;;;;; 11AF7;PAU CIN HAU LOW-FALLING TONE FINAL;Lo;0;L;;;;;N;;;;; 11AF8;PAU CIN HAU GLOTTAL STOP FINAL;Lo;0;L;;;;;N;;;;; +11B00;DEVANAGARI HEAD MARK;Po;0;L;;;;;N;;;;; +11B01;DEVANAGARI HEAD MARK WITH HEADSTROKE;Po;0;L;;;;;N;;;;; +11B02;DEVANAGARI SIGN BHALE;Po;0;L;;;;;N;;;;; +11B03;DEVANAGARI SIGN BHALE WITH HOOK;Po;0;L;;;;;N;;;;; +11B04;DEVANAGARI SIGN EXTENDED BHALE;Po;0;L;;;;;N;;;;; +11B05;DEVANAGARI SIGN EXTENDED BHALE WITH HOOK;Po;0;L;;;;;N;;;;; +11B06;DEVANAGARI SIGN WESTERN FIVE-LIKE BHALE;Po;0;L;;;;;N;;;;; +11B07;DEVANAGARI SIGN WESTERN NINE-LIKE BHALE;Po;0;L;;;;;N;;;;; +11B08;DEVANAGARI SIGN REVERSED NINE-LIKE BHALE;Po;0;L;;;;;N;;;;; +11B09;DEVANAGARI SIGN MINDU;Po;0;L;;;;;N;;;;; 11C00;BHAIKSUKI LETTER A;Lo;0;L;;;;;N;;;;; 11C01;BHAIKSUKI LETTER AA;Lo;0;L;;;;;N;;;;; 11C02;BHAIKSUKI LETTER I;Lo;0;L;;;;;N;;;;; @@ -21584,6 +21602,92 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11EF6;MAKASAR VOWEL SIGN O;Mc;0;L;;;;;N;;;;; 11EF7;MAKASAR PASSIMBANG;Po;0;L;;;;;N;;;;; 11EF8;MAKASAR END OF SECTION;Po;0;L;;;;;N;;;;; +11F00;KAWI SIGN CANDRABINDU;Mn;0;NSM;;;;;N;;;;; +11F01;KAWI SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; +11F02;KAWI SIGN REPHA;Lo;0;L;;;;;N;;;;; +11F03;KAWI SIGN VISARGA;Mc;0;L;;;;;N;;;;; +11F04;KAWI LETTER A;Lo;0;L;;;;;N;;;;; +11F05;KAWI LETTER AA;Lo;0;L;;;;;N;;;;; +11F06;KAWI LETTER I;Lo;0;L;;;;;N;;;;; +11F07;KAWI LETTER II;Lo;0;L;;;;;N;;;;; +11F08;KAWI LETTER U;Lo;0;L;;;;;N;;;;; +11F09;KAWI LETTER UU;Lo;0;L;;;;;N;;;;; +11F0A;KAWI LETTER VOCALIC R;Lo;0;L;;;;;N;;;;; +11F0B;KAWI LETTER VOCALIC RR;Lo;0;L;;;;;N;;;;; +11F0C;KAWI LETTER VOCALIC L;Lo;0;L;;;;;N;;;;; +11F0D;KAWI LETTER VOCALIC LL;Lo;0;L;;;;;N;;;;; +11F0E;KAWI LETTER E;Lo;0;L;;;;;N;;;;; +11F0F;KAWI LETTER AI;Lo;0;L;;;;;N;;;;; +11F10;KAWI LETTER O;Lo;0;L;;;;;N;;;;; +11F12;KAWI LETTER KA;Lo;0;L;;;;;N;;;;; +11F13;KAWI LETTER KHA;Lo;0;L;;;;;N;;;;; +11F14;KAWI LETTER GA;Lo;0;L;;;;;N;;;;; +11F15;KAWI LETTER GHA;Lo;0;L;;;;;N;;;;; +11F16;KAWI LETTER NGA;Lo;0;L;;;;;N;;;;; +11F17;KAWI LETTER CA;Lo;0;L;;;;;N;;;;; +11F18;KAWI LETTER CHA;Lo;0;L;;;;;N;;;;; +11F19;KAWI LETTER JA;Lo;0;L;;;;;N;;;;; +11F1A;KAWI LETTER JHA;Lo;0;L;;;;;N;;;;; +11F1B;KAWI LETTER NYA;Lo;0;L;;;;;N;;;;; +11F1C;KAWI LETTER TTA;Lo;0;L;;;;;N;;;;; +11F1D;KAWI LETTER TTHA;Lo;0;L;;;;;N;;;;; +11F1E;KAWI LETTER DDA;Lo;0;L;;;;;N;;;;; +11F1F;KAWI LETTER DDHA;Lo;0;L;;;;;N;;;;; +11F20;KAWI LETTER NNA;Lo;0;L;;;;;N;;;;; +11F21;KAWI LETTER TA;Lo;0;L;;;;;N;;;;; +11F22;KAWI LETTER THA;Lo;0;L;;;;;N;;;;; +11F23;KAWI LETTER DA;Lo;0;L;;;;;N;;;;; +11F24;KAWI LETTER DHA;Lo;0;L;;;;;N;;;;; +11F25;KAWI LETTER NA;Lo;0;L;;;;;N;;;;; +11F26;KAWI LETTER PA;Lo;0;L;;;;;N;;;;; +11F27;KAWI LETTER PHA;Lo;0;L;;;;;N;;;;; +11F28;KAWI LETTER BA;Lo;0;L;;;;;N;;;;; +11F29;KAWI LETTER BHA;Lo;0;L;;;;;N;;;;; +11F2A;KAWI LETTER MA;Lo;0;L;;;;;N;;;;; +11F2B;KAWI LETTER YA;Lo;0;L;;;;;N;;;;; +11F2C;KAWI LETTER RA;Lo;0;L;;;;;N;;;;; +11F2D;KAWI LETTER LA;Lo;0;L;;;;;N;;;;; +11F2E;KAWI LETTER WA;Lo;0;L;;;;;N;;;;; +11F2F;KAWI LETTER SHA;Lo;0;L;;;;;N;;;;; +11F30;KAWI LETTER SSA;Lo;0;L;;;;;N;;;;; +11F31;KAWI LETTER SA;Lo;0;L;;;;;N;;;;; +11F32;KAWI LETTER HA;Lo;0;L;;;;;N;;;;; +11F33;KAWI LETTER JNYA;Lo;0;L;;;;;N;;;;; +11F34;KAWI VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +11F35;KAWI VOWEL SIGN ALTERNATE AA;Mc;0;L;;;;;N;;;;; +11F36;KAWI VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +11F37;KAWI VOWEL SIGN II;Mn;0;NSM;;;;;N;;;;; +11F38;KAWI VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +11F39;KAWI VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;; +11F3A;KAWI VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;; +11F3E;KAWI VOWEL SIGN E;Mc;0;L;;;;;N;;;;; +11F3F;KAWI VOWEL SIGN AI;Mc;0;L;;;;;N;;;;; +11F40;KAWI VOWEL SIGN EU;Mn;0;NSM;;;;;N;;;;; +11F41;KAWI SIGN KILLER;Mc;9;L;;;;;N;;;;; +11F42;KAWI CONJOINER;Mn;9;NSM;;;;;N;;;;; +11F43;KAWI DANDA;Po;0;L;;;;;N;;;;; +11F44;KAWI DOUBLE DANDA;Po;0;L;;;;;N;;;;; +11F45;KAWI PUNCTUATION SECTION MARKER;Po;0;L;;;;;N;;;;; +11F46;KAWI PUNCTUATION ALTERNATE SECTION MARKER;Po;0;L;;;;;N;;;;; +11F47;KAWI PUNCTUATION FLOWER;Po;0;L;;;;;N;;;;; +11F48;KAWI PUNCTUATION SPACE FILLER;Po;0;L;;;;;N;;;;; +11F49;KAWI PUNCTUATION DOT;Po;0;L;;;;;N;;;;; +11F4A;KAWI PUNCTUATION DOUBLE DOT;Po;0;L;;;;;N;;;;; +11F4B;KAWI PUNCTUATION TRIPLE DOT;Po;0;L;;;;;N;;;;; +11F4C;KAWI PUNCTUATION CIRCLE;Po;0;L;;;;;N;;;;; +11F4D;KAWI PUNCTUATION FILLED CIRCLE;Po;0;L;;;;;N;;;;; +11F4E;KAWI PUNCTUATION SPIRAL;Po;0;L;;;;;N;;;;; +11F4F;KAWI PUNCTUATION CLOSING SPIRAL;Po;0;L;;;;;N;;;;; +11F50;KAWI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +11F51;KAWI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +11F52;KAWI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +11F53;KAWI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +11F54;KAWI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +11F55;KAWI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +11F56;KAWI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +11F57;KAWI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +11F58;KAWI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +11F59;KAWI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 11FB0;LISU LETTER YHA;Lo;0;L;;;;;N;;;;; 11FC0;TAMIL FRACTION ONE THREE-HUNDRED-AND-TWENTIETH;No;0;L;;;;1/320;N;;;;; 11FC1;TAMIL FRACTION ONE ONE-HUNDRED-AND-SIXTIETH;No;0;L;;;;1/160;N;;;;; @@ -24040,6 +24144,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1342C;EGYPTIAN HIEROGLYPH AA030;Lo;0;L;;;;;N;;;;; 1342D;EGYPTIAN HIEROGLYPH AA031;Lo;0;L;;;;;N;;;;; 1342E;EGYPTIAN HIEROGLYPH AA032;Lo;0;L;;;;;N;;;;; +1342F;EGYPTIAN HIEROGLYPH V011D;Lo;0;L;;;;;N;;;;; 13430;EGYPTIAN HIEROGLYPH VERTICAL JOINER;Cf;0;L;;;;;N;;;;; 13431;EGYPTIAN HIEROGLYPH HORIZONTAL JOINER;Cf;0;L;;;;;N;;;;; 13432;EGYPTIAN HIEROGLYPH INSERT AT TOP START;Cf;0;L;;;;;N;;;;; @@ -24049,6 +24154,35 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 13436;EGYPTIAN HIEROGLYPH OVERLAY MIDDLE;Cf;0;L;;;;;N;;;;; 13437;EGYPTIAN HIEROGLYPH BEGIN SEGMENT;Cf;0;L;;;;;N;;;;; 13438;EGYPTIAN HIEROGLYPH END SEGMENT;Cf;0;L;;;;;N;;;;; +13439;EGYPTIAN HIEROGLYPH INSERT AT MIDDLE;Cf;0;L;;;;;N;;;;; +1343A;EGYPTIAN HIEROGLYPH INSERT AT TOP;Cf;0;L;;;;;N;;;;; +1343B;EGYPTIAN HIEROGLYPH INSERT AT BOTTOM;Cf;0;L;;;;;N;;;;; +1343C;EGYPTIAN HIEROGLYPH BEGIN ENCLOSURE;Cf;0;L;;;;;N;;;;; +1343D;EGYPTIAN HIEROGLYPH END ENCLOSURE;Cf;0;L;;;;;N;;;;; +1343E;EGYPTIAN HIEROGLYPH BEGIN WALLED ENCLOSURE;Cf;0;L;;;;;N;;;;; +1343F;EGYPTIAN HIEROGLYPH END WALLED ENCLOSURE;Cf;0;L;;;;;N;;;;; +13440;EGYPTIAN HIEROGLYPH MIRROR HORIZONTALLY;Mn;0;NSM;;;;;N;;;;; +13441;EGYPTIAN HIEROGLYPH FULL BLANK;Lo;0;L;;;;;N;;;;; +13442;EGYPTIAN HIEROGLYPH HALF BLANK;Lo;0;L;;;;;N;;;;; +13443;EGYPTIAN HIEROGLYPH LOST SIGN;Lo;0;L;;;;;N;;;;; +13444;EGYPTIAN HIEROGLYPH HALF LOST SIGN;Lo;0;L;;;;;N;;;;; +13445;EGYPTIAN HIEROGLYPH TALL LOST SIGN;Lo;0;L;;;;;N;;;;; +13446;EGYPTIAN HIEROGLYPH WIDE LOST SIGN;Lo;0;L;;;;;N;;;;; +13447;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START;Mn;0;NSM;;;;;N;;;;; +13448;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM START;Mn;0;NSM;;;;;N;;;;; +13449;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT START;Mn;0;NSM;;;;;N;;;;; +1344A;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP END;Mn;0;NSM;;;;;N;;;;; +1344B;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP;Mn;0;NSM;;;;;N;;;;; +1344C;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM START AND TOP END;Mn;0;NSM;;;;;N;;;;; +1344D;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT START AND TOP;Mn;0;NSM;;;;;N;;;;; +1344E;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM END;Mn;0;NSM;;;;;N;;;;; +1344F;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START AND BOTTOM END;Mn;0;NSM;;;;;N;;;;; +13450;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM;Mn;0;NSM;;;;;N;;;;; +13451;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT START AND BOTTOM;Mn;0;NSM;;;;;N;;;;; +13452;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT END;Mn;0;NSM;;;;;N;;;;; +13453;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP AND END;Mn;0;NSM;;;;;N;;;;; +13454;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM AND END;Mn;0;NSM;;;;;N;;;;; +13455;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED;Mn;0;NSM;;;;;N;;;;; 14400;ANATOLIAN HIEROGLYPH A001;Lo;0;L;;;;;N;;;;; 14401;ANATOLIAN HIEROGLYPH A002;Lo;0;L;;;;;N;;;;; 14402;ANATOLIAN HIEROGLYPH A003;Lo;0;L;;;;;N;;;;; @@ -27289,9 +27423,11 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1B120;KATAKANA LETTER ARCHAIC YI;Lo;0;L;;;;;N;;;;; 1B121;KATAKANA LETTER ARCHAIC YE;Lo;0;L;;;;;N;;;;; 1B122;KATAKANA LETTER ARCHAIC WU;Lo;0;L;;;;;N;;;;; +1B132;HIRAGANA LETTER SMALL KO;Lo;0;L;;;;;N;;;;; 1B150;HIRAGANA LETTER SMALL WI;Lo;0;L;;;;;N;;;;; 1B151;HIRAGANA LETTER SMALL WE;Lo;0;L;;;;;N;;;;; 1B152;HIRAGANA LETTER SMALL WO;Lo;0;L;;;;;N;;;;; +1B155;KATAKANA LETTER SMALL KO;Lo;0;L;;;;;N;;;;; 1B164;KATAKANA LETTER SMALL WI;Lo;0;L;;;;;N;;;;; 1B165;KATAKANA LETTER SMALL WE;Lo;0;L;;;;;N;;;;; 1B166;KATAKANA LETTER SMALL WO;Lo;0;L;;;;;N;;;;; @@ -28573,6 +28709,26 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1D243;COMBINING GREEK MUSICAL TETRASEME;Mn;230;NSM;;;;;N;;;;; 1D244;COMBINING GREEK MUSICAL PENTASEME;Mn;230;NSM;;;;;N;;;;; 1D245;GREEK MUSICAL LEIMMA;So;0;ON;;;;;N;;;;; +1D2C0;KAKTOVIK NUMERAL ZERO;No;0;L;;;;0;N;;;;; +1D2C1;KAKTOVIK NUMERAL ONE;No;0;L;;;;1;N;;;;; +1D2C2;KAKTOVIK NUMERAL TWO;No;0;L;;;;2;N;;;;; +1D2C3;KAKTOVIK NUMERAL THREE;No;0;L;;;;3;N;;;;; +1D2C4;KAKTOVIK NUMERAL FOUR;No;0;L;;;;4;N;;;;; +1D2C5;KAKTOVIK NUMERAL FIVE;No;0;L;;;;5;N;;;;; +1D2C6;KAKTOVIK NUMERAL SIX;No;0;L;;;;6;N;;;;; +1D2C7;KAKTOVIK NUMERAL SEVEN;No;0;L;;;;7;N;;;;; +1D2C8;KAKTOVIK NUMERAL EIGHT;No;0;L;;;;8;N;;;;; +1D2C9;KAKTOVIK NUMERAL NINE;No;0;L;;;;9;N;;;;; +1D2CA;KAKTOVIK NUMERAL TEN;No;0;L;;;;10;N;;;;; +1D2CB;KAKTOVIK NUMERAL ELEVEN;No;0;L;;;;11;N;;;;; +1D2CC;KAKTOVIK NUMERAL TWELVE;No;0;L;;;;12;N;;;;; +1D2CD;KAKTOVIK NUMERAL THIRTEEN;No;0;L;;;;13;N;;;;; +1D2CE;KAKTOVIK NUMERAL FOURTEEN;No;0;L;;;;14;N;;;;; +1D2CF;KAKTOVIK NUMERAL FIFTEEN;No;0;L;;;;15;N;;;;; +1D2D0;KAKTOVIK NUMERAL SIXTEEN;No;0;L;;;;16;N;;;;; +1D2D1;KAKTOVIK NUMERAL SEVENTEEN;No;0;L;;;;17;N;;;;; +1D2D2;KAKTOVIK NUMERAL EIGHTEEN;No;0;L;;;;18;N;;;;; +1D2D3;KAKTOVIK NUMERAL NINETEEN;No;0;L;;;;19;N;;;;; 1D2E0;MAYAN NUMERAL ZERO;No;0;L;;;;0;N;;;;; 1D2E1;MAYAN NUMERAL ONE;No;0;L;;;;1;N;;;;; 1D2E2;MAYAN NUMERAL TWO;No;0;L;;;;2;N;;;;; @@ -30404,6 +30560,12 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1DF1C;LATIN SMALL LETTER TESH DIGRAPH WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;; 1DF1D;LATIN SMALL LETTER C WITH RETROFLEX HOOK;Ll;0;L;;;;;N;;;;; 1DF1E;LATIN SMALL LETTER S WITH CURL;Ll;0;L;;;;;N;;;;; +1DF25;LATIN SMALL LETTER D WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; +1DF26;LATIN SMALL LETTER L WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; +1DF27;LATIN SMALL LETTER N WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; +1DF28;LATIN SMALL LETTER R WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; +1DF29;LATIN SMALL LETTER S WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; +1DF2A;LATIN SMALL LETTER T WITH MID-HEIGHT LEFT HOOK;Ll;0;L;;;;;N;;;;; 1E000;COMBINING GLAGOLITIC LETTER AZU;Mn;230;NSM;;;;;N;;;;; 1E001;COMBINING GLAGOLITIC LETTER BUKY;Mn;230;NSM;;;;;N;;;;; 1E002;COMBINING GLAGOLITIC LETTER VEDE;Mn;230;NSM;;;;;N;;;;; @@ -30442,6 +30604,69 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1E028;COMBINING GLAGOLITIC LETTER BIG YUS;Mn;230;NSM;;;;;N;;;;; 1E029;COMBINING GLAGOLITIC LETTER IOTATED BIG YUS;Mn;230;NSM;;;;;N;;;;; 1E02A;COMBINING GLAGOLITIC LETTER FITA;Mn;230;NSM;;;;;N;;;;; +1E030;MODIFIER LETTER CYRILLIC SMALL A;Lm;0;L; 0430;;;;N;;;;; +1E031;MODIFIER LETTER CYRILLIC SMALL BE;Lm;0;L; 0431;;;;N;;;;; +1E032;MODIFIER LETTER CYRILLIC SMALL VE;Lm;0;L; 0432;;;;N;;;;; +1E033;MODIFIER LETTER CYRILLIC SMALL GHE;Lm;0;L; 0433;;;;N;;;;; +1E034;MODIFIER LETTER CYRILLIC SMALL DE;Lm;0;L; 0434;;;;N;;;;; +1E035;MODIFIER LETTER CYRILLIC SMALL IE;Lm;0;L; 0435;;;;N;;;;; +1E036;MODIFIER LETTER CYRILLIC SMALL ZHE;Lm;0;L; 0436;;;;N;;;;; +1E037;MODIFIER LETTER CYRILLIC SMALL ZE;Lm;0;L; 0437;;;;N;;;;; +1E038;MODIFIER LETTER CYRILLIC SMALL I;Lm;0;L; 0438;;;;N;;;;; +1E039;MODIFIER LETTER CYRILLIC SMALL KA;Lm;0;L; 043A;;;;N;;;;; +1E03A;MODIFIER LETTER CYRILLIC SMALL EL;Lm;0;L; 043B;;;;N;;;;; +1E03B;MODIFIER LETTER CYRILLIC SMALL EM;Lm;0;L; 043C;;;;N;;;;; +1E03C;MODIFIER LETTER CYRILLIC SMALL O;Lm;0;L; 043E;;;;N;;;;; +1E03D;MODIFIER LETTER CYRILLIC SMALL PE;Lm;0;L; 043F;;;;N;;;;; +1E03E;MODIFIER LETTER CYRILLIC SMALL ER;Lm;0;L; 0440;;;;N;;;;; +1E03F;MODIFIER LETTER CYRILLIC SMALL ES;Lm;0;L; 0441;;;;N;;;;; +1E040;MODIFIER LETTER CYRILLIC SMALL TE;Lm;0;L; 0442;;;;N;;;;; +1E041;MODIFIER LETTER CYRILLIC SMALL U;Lm;0;L; 0443;;;;N;;;;; +1E042;MODIFIER LETTER CYRILLIC SMALL EF;Lm;0;L; 0444;;;;N;;;;; +1E043;MODIFIER LETTER CYRILLIC SMALL HA;Lm;0;L; 0445;;;;N;;;;; +1E044;MODIFIER LETTER CYRILLIC SMALL TSE;Lm;0;L; 0446;;;;N;;;;; +1E045;MODIFIER LETTER CYRILLIC SMALL CHE;Lm;0;L; 0447;;;;N;;;;; +1E046;MODIFIER LETTER CYRILLIC SMALL SHA;Lm;0;L; 0448;;;;N;;;;; +1E047;MODIFIER LETTER CYRILLIC SMALL YERU;Lm;0;L; 044B;;;;N;;;;; +1E048;MODIFIER LETTER CYRILLIC SMALL E;Lm;0;L; 044D;;;;N;;;;; +1E049;MODIFIER LETTER CYRILLIC SMALL YU;Lm;0;L; 044E;;;;N;;;;; +1E04A;MODIFIER LETTER CYRILLIC SMALL DZZE;Lm;0;L; A689;;;;N;;;;; +1E04B;MODIFIER LETTER CYRILLIC SMALL SCHWA;Lm;0;L; 04D9;;;;N;;;;; +1E04C;MODIFIER LETTER CYRILLIC SMALL BYELORUSSIAN-UKRAINIAN I;Lm;0;L; 0456;;;;N;;;;; +1E04D;MODIFIER LETTER CYRILLIC SMALL JE;Lm;0;L; 0458;;;;N;;;;; +1E04E;MODIFIER LETTER CYRILLIC SMALL BARRED O;Lm;0;L; 04E9;;;;N;;;;; +1E04F;MODIFIER LETTER CYRILLIC SMALL STRAIGHT U;Lm;0;L; 04AF;;;;N;;;;; +1E050;MODIFIER LETTER CYRILLIC SMALL PALOCHKA;Lm;0;L; 04CF;;;;N;;;;; +1E051;CYRILLIC SUBSCRIPT SMALL LETTER A;Lm;0;L; 0430;;;;N;;;;; +1E052;CYRILLIC SUBSCRIPT SMALL LETTER BE;Lm;0;L; 0431;;;;N;;;;; +1E053;CYRILLIC SUBSCRIPT SMALL LETTER VE;Lm;0;L; 0432;;;;N;;;;; +1E054;CYRILLIC SUBSCRIPT SMALL LETTER GHE;Lm;0;L; 0433;;;;N;;;;; +1E055;CYRILLIC SUBSCRIPT SMALL LETTER DE;Lm;0;L; 0434;;;;N;;;;; +1E056;CYRILLIC SUBSCRIPT SMALL LETTER IE;Lm;0;L; 0435;;;;N;;;;; +1E057;CYRILLIC SUBSCRIPT SMALL LETTER ZHE;Lm;0;L; 0436;;;;N;;;;; +1E058;CYRILLIC SUBSCRIPT SMALL LETTER ZE;Lm;0;L; 0437;;;;N;;;;; +1E059;CYRILLIC SUBSCRIPT SMALL LETTER I;Lm;0;L; 0438;;;;N;;;;; +1E05A;CYRILLIC SUBSCRIPT SMALL LETTER KA;Lm;0;L; 043A;;;;N;;;;; +1E05B;CYRILLIC SUBSCRIPT SMALL LETTER EL;Lm;0;L; 043B;;;;N;;;;; +1E05C;CYRILLIC SUBSCRIPT SMALL LETTER O;Lm;0;L; 043E;;;;N;;;;; +1E05D;CYRILLIC SUBSCRIPT SMALL LETTER PE;Lm;0;L; 043F;;;;N;;;;; +1E05E;CYRILLIC SUBSCRIPT SMALL LETTER ES;Lm;0;L; 0441;;;;N;;;;; +1E05F;CYRILLIC SUBSCRIPT SMALL LETTER U;Lm;0;L; 0443;;;;N;;;;; +1E060;CYRILLIC SUBSCRIPT SMALL LETTER EF;Lm;0;L; 0444;;;;N;;;;; +1E061;CYRILLIC SUBSCRIPT SMALL LETTER HA;Lm;0;L; 0445;;;;N;;;;; +1E062;CYRILLIC SUBSCRIPT SMALL LETTER TSE;Lm;0;L; 0446;;;;N;;;;; +1E063;CYRILLIC SUBSCRIPT SMALL LETTER CHE;Lm;0;L; 0447;;;;N;;;;; +1E064;CYRILLIC SUBSCRIPT SMALL LETTER SHA;Lm;0;L; 0448;;;;N;;;;; +1E065;CYRILLIC SUBSCRIPT SMALL LETTER HARD SIGN;Lm;0;L; 044A;;;;N;;;;; +1E066;CYRILLIC SUBSCRIPT SMALL LETTER YERU;Lm;0;L; 044B;;;;N;;;;; +1E067;CYRILLIC SUBSCRIPT SMALL LETTER GHE WITH UPTURN;Lm;0;L; 0491;;;;N;;;;; +1E068;CYRILLIC SUBSCRIPT SMALL LETTER BYELORUSSIAN-UKRAINIAN I;Lm;0;L; 0456;;;;N;;;;; +1E069;CYRILLIC SUBSCRIPT SMALL LETTER DZE;Lm;0;L; 0455;;;;N;;;;; +1E06A;CYRILLIC SUBSCRIPT SMALL LETTER DZHE;Lm;0;L; 045F;;;;N;;;;; +1E06B;MODIFIER LETTER CYRILLIC SMALL ES WITH DESCENDER;Lm;0;L; 04AB;;;;N;;;;; +1E06C;MODIFIER LETTER CYRILLIC SMALL YERU WITH BACK YER;Lm;0;L; A651;;;;N;;;;; +1E06D;MODIFIER LETTER CYRILLIC SMALL STRAIGHT U WITH STROKE;Lm;0;L; 04B1;;;;N;;;;; +1E08F;COMBINING CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I;Mn;230;NSM;;;;;N;;;;; 1E100;NYIAKENG PUACHUE HMONG LETTER MA;Lo;0;L;;;;;N;;;;; 1E101;NYIAKENG PUACHUE HMONG LETTER TSA;Lo;0;L;;;;;N;;;;; 1E102;NYIAKENG PUACHUE HMONG LETTER NTA;Lo;0;L;;;;;N;;;;; @@ -30603,6 +30828,48 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1E2F8;WANCHO DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; 1E2F9;WANCHO DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 1E2FF;WANCHO NGUN SIGN;Sc;0;ET;;;;;N;;;;; +1E4D0;NAG MUNDARI LETTER O;Lo;0;L;;;;;N;;;;; +1E4D1;NAG MUNDARI LETTER OP;Lo;0;L;;;;;N;;;;; +1E4D2;NAG MUNDARI LETTER OL;Lo;0;L;;;;;N;;;;; +1E4D3;NAG MUNDARI LETTER OY;Lo;0;L;;;;;N;;;;; +1E4D4;NAG MUNDARI LETTER ONG;Lo;0;L;;;;;N;;;;; +1E4D5;NAG MUNDARI LETTER A;Lo;0;L;;;;;N;;;;; +1E4D6;NAG MUNDARI LETTER AJ;Lo;0;L;;;;;N;;;;; +1E4D7;NAG MUNDARI LETTER AB;Lo;0;L;;;;;N;;;;; +1E4D8;NAG MUNDARI LETTER ANY;Lo;0;L;;;;;N;;;;; +1E4D9;NAG MUNDARI LETTER AH;Lo;0;L;;;;;N;;;;; +1E4DA;NAG MUNDARI LETTER I;Lo;0;L;;;;;N;;;;; +1E4DB;NAG MUNDARI LETTER IS;Lo;0;L;;;;;N;;;;; +1E4DC;NAG MUNDARI LETTER IDD;Lo;0;L;;;;;N;;;;; +1E4DD;NAG MUNDARI LETTER IT;Lo;0;L;;;;;N;;;;; +1E4DE;NAG MUNDARI LETTER IH;Lo;0;L;;;;;N;;;;; +1E4DF;NAG MUNDARI LETTER U;Lo;0;L;;;;;N;;;;; +1E4E0;NAG MUNDARI LETTER UC;Lo;0;L;;;;;N;;;;; +1E4E1;NAG MUNDARI LETTER UD;Lo;0;L;;;;;N;;;;; +1E4E2;NAG MUNDARI LETTER UK;Lo;0;L;;;;;N;;;;; +1E4E3;NAG MUNDARI LETTER UR;Lo;0;L;;;;;N;;;;; +1E4E4;NAG MUNDARI LETTER E;Lo;0;L;;;;;N;;;;; +1E4E5;NAG MUNDARI LETTER ENN;Lo;0;L;;;;;N;;;;; +1E4E6;NAG MUNDARI LETTER EG;Lo;0;L;;;;;N;;;;; +1E4E7;NAG MUNDARI LETTER EM;Lo;0;L;;;;;N;;;;; +1E4E8;NAG MUNDARI LETTER EN;Lo;0;L;;;;;N;;;;; +1E4E9;NAG MUNDARI LETTER ETT;Lo;0;L;;;;;N;;;;; +1E4EA;NAG MUNDARI LETTER ELL;Lo;0;L;;;;;N;;;;; +1E4EB;NAG MUNDARI SIGN OJOD;Lm;0;L;;;;;N;;;;; +1E4EC;NAG MUNDARI SIGN MUHOR;Mn;232;NSM;;;;;N;;;;; +1E4ED;NAG MUNDARI SIGN TOYOR;Mn;232;NSM;;;;;N;;;;; +1E4EE;NAG MUNDARI SIGN IKIR;Mn;220;NSM;;;;;N;;;;; +1E4EF;NAG MUNDARI SIGN SUTUH;Mn;230;NSM;;;;;N;;;;; +1E4F0;NAG MUNDARI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +1E4F1;NAG MUNDARI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +1E4F2;NAG MUNDARI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +1E4F3;NAG MUNDARI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +1E4F4;NAG MUNDARI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +1E4F5;NAG MUNDARI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +1E4F6;NAG MUNDARI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +1E4F7;NAG MUNDARI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +1E4F8;NAG MUNDARI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +1E4F9;NAG MUNDARI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 1E7E0;ETHIOPIC SYLLABLE HHYA;Lo;0;L;;;;;N;;;;; 1E7E1;ETHIOPIC SYLLABLE HHYU;Lo;0;L;;;;;N;;;;; 1E7E2;ETHIOPIC SYLLABLE HHYI;Lo;0;L;;;;;N;;;;; @@ -32678,6 +32945,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F6D5;HINDU TEMPLE;So;0;ON;;;;;N;;;;; 1F6D6;HUT;So;0;ON;;;;;N;;;;; 1F6D7;ELEVATOR;So;0;ON;;;;;N;;;;; +1F6DC;WIRELESS;So;0;ON;;;;;N;;;;; 1F6DD;PLAYGROUND SLIDE;So;0;ON;;;;;N;;;;; 1F6DE;WHEEL;So;0;ON;;;;;N;;;;; 1F6DF;RING BUOY;So;0;ON;;;;;N;;;;; @@ -32823,6 +33091,14 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F771;ALCHEMICAL SYMBOL FOR MONTH;So;0;ON;;;;;N;;;;; 1F772;ALCHEMICAL SYMBOL FOR HALF DRAM;So;0;ON;;;;;N;;;;; 1F773;ALCHEMICAL SYMBOL FOR HALF OUNCE;So;0;ON;;;;;N;;;;; +1F774;LOT OF FORTUNE;So;0;ON;;;;;N;;;;; +1F775;OCCULTATION;So;0;ON;;;;;N;;;;; +1F776;LUNAR ECLIPSE;So;0;ON;;;;;N;;;;; +1F77B;HAUMEA;So;0;ON;;;;;N;;;;; +1F77C;MAKEMAKE;So;0;ON;;;;;N;;;;; +1F77D;GONGGONG;So;0;ON;;;;;N;;;;; +1F77E;QUAOAR;So;0;ON;;;;;N;;;;; +1F77F;ORCUS;So;0;ON;;;;;N;;;;; 1F780;BLACK LEFT-POINTING ISOSCELES RIGHT TRIANGLE;So;0;ON;;;;;N;;;;; 1F781;BLACK UP-POINTING ISOSCELES RIGHT TRIANGLE;So;0;ON;;;;;N;;;;; 1F782;BLACK RIGHT-POINTING ISOSCELES RIGHT TRIANGLE;So;0;ON;;;;;N;;;;; @@ -32912,6 +33188,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F7D6;NEGATIVE CIRCLED TRIANGLE;So;0;ON;;;;;N;;;;; 1F7D7;CIRCLED SQUARE;So;0;ON;;;;;N;;;;; 1F7D8;NEGATIVE CIRCLED SQUARE;So;0;ON;;;;;N;;;;; +1F7D9;NINE POINTED WHITE STAR;So;0;ON;;;;;N;;;;; 1F7E0;LARGE ORANGE CIRCLE;So;0;ON;;;;;N;;;;; 1F7E1;LARGE YELLOW CIRCLE;So;0;ON;;;;;N;;;;; 1F7E2;LARGE GREEN CIRCLE;So;0;ON;;;;;N;;;;; @@ -33434,6 +33711,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FA72;BRIEFS;So;0;ON;;;;;N;;;;; 1FA73;SHORTS;So;0;ON;;;;;N;;;;; 1FA74;THONG SANDAL;So;0;ON;;;;;N;;;;; +1FA75;LIGHT BLUE HEART;So;0;ON;;;;;N;;;;; +1FA76;GREY HEART;So;0;ON;;;;;N;;;;; +1FA77;PINK HEART;So;0;ON;;;;;N;;;;; 1FA78;DROP OF BLOOD;So;0;ON;;;;;N;;;;; 1FA79;ADHESIVE BANDAGE;So;0;ON;;;;;N;;;;; 1FA7A;STETHOSCOPE;So;0;ON;;;;;N;;;;; @@ -33446,6 +33726,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FA84;MAGIC WAND;So;0;ON;;;;;N;;;;; 1FA85;PINATA;So;0;ON;;;;;N;;;;; 1FA86;NESTING DOLLS;So;0;ON;;;;;N;;;;; +1FA87;MARACAS;So;0;ON;;;;;N;;;;; +1FA88;FLUTE;So;0;ON;;;;;N;;;;; 1FA90;RINGED PLANET;So;0;ON;;;;;N;;;;; 1FA91;CHAIR;So;0;ON;;;;;N;;;;; 1FA92;RAZOR;So;0;ON;;;;;N;;;;; @@ -33475,6 +33757,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAAA;IDENTIFICATION CARD;So;0;ON;;;;;N;;;;; 1FAAB;LOW BATTERY;So;0;ON;;;;;N;;;;; 1FAAC;HAMSA;So;0;ON;;;;;N;;;;; +1FAAD;FOLDING HAND FAN;So;0;ON;;;;;N;;;;; +1FAAE;HAIR PICK;So;0;ON;;;;;N;;;;; +1FAAF;KHANDA;So;0;ON;;;;;N;;;;; 1FAB0;FLY;So;0;ON;;;;;N;;;;; 1FAB1;WORM;So;0;ON;;;;;N;;;;; 1FAB2;BEETLE;So;0;ON;;;;;N;;;;; @@ -33486,12 +33771,18 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAB8;CORAL;So;0;ON;;;;;N;;;;; 1FAB9;EMPTY NEST;So;0;ON;;;;;N;;;;; 1FABA;NEST WITH EGGS;So;0;ON;;;;;N;;;;; +1FABB;HYACINTH;So;0;ON;;;;;N;;;;; +1FABC;JELLYFISH;So;0;ON;;;;;N;;;;; +1FABD;WING;So;0;ON;;;;;N;;;;; +1FABF;GOOSE;So;0;ON;;;;;N;;;;; 1FAC0;ANATOMICAL HEART;So;0;ON;;;;;N;;;;; 1FAC1;LUNGS;So;0;ON;;;;;N;;;;; 1FAC2;PEOPLE HUGGING;So;0;ON;;;;;N;;;;; 1FAC3;PREGNANT MAN;So;0;ON;;;;;N;;;;; 1FAC4;PREGNANT PERSON;So;0;ON;;;;;N;;;;; 1FAC5;PERSON WITH CROWN;So;0;ON;;;;;N;;;;; +1FACE;MOOSE;So;0;ON;;;;;N;;;;; +1FACF;DONKEY;So;0;ON;;;;;N;;;;; 1FAD0;BLUEBERRIES;So;0;ON;;;;;N;;;;; 1FAD1;BELL PEPPER;So;0;ON;;;;;N;;;;; 1FAD2;OLIVE;So;0;ON;;;;;N;;;;; @@ -33502,6 +33793,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAD7;POURING LIQUID;So;0;ON;;;;;N;;;;; 1FAD8;BEANS;So;0;ON;;;;;N;;;;; 1FAD9;JAR;So;0;ON;;;;;N;;;;; +1FADA;GINGER ROOT;So;0;ON;;;;;N;;;;; +1FADB;PEA POD;So;0;ON;;;;;N;;;;; 1FAE0;MELTING FACE;So;0;ON;;;;;N;;;;; 1FAE1;SALUTING FACE;So;0;ON;;;;;N;;;;; 1FAE2;FACE WITH OPEN EYES AND HAND OVER MOUTH;So;0;ON;;;;;N;;;;; @@ -33510,6 +33803,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAE5;DOTTED LINE FACE;So;0;ON;;;;;N;;;;; 1FAE6;BITING LIP;So;0;ON;;;;;N;;;;; 1FAE7;BUBBLES;So;0;ON;;;;;N;;;;; +1FAE8;SHAKING FACE;So;0;ON;;;;;N;;;;; 1FAF0;HAND WITH INDEX FINGER AND THUMB CROSSED;So;0;ON;;;;;N;;;;; 1FAF1;RIGHTWARDS HAND;So;0;ON;;;;;N;;;;; 1FAF2;LEFTWARDS HAND;So;0;ON;;;;;N;;;;; @@ -33517,6 +33811,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAF4;PALM UP HAND;So;0;ON;;;;;N;;;;; 1FAF5;INDEX POINTING AT THE VIEWER;So;0;ON;;;;;N;;;;; 1FAF6;HEART HANDS;So;0;ON;;;;;N;;;;; +1FAF7;LEFTWARDS PUSHING HAND;So;0;ON;;;;;N;;;;; +1FAF8;RIGHTWARDS PUSHING HAND;So;0;ON;;;;;N;;;;; 1FB00;BLOCK SEXTANT-1;So;0;ON;;;;;N;;;;; 1FB01;BLOCK SEXTANT-2;So;0;ON;;;;;N;;;;; 1FB02;BLOCK SEXTANT-12;So;0;ON;;;;;N;;;;; @@ -33732,7 +34028,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 20000;;Lo;0;L;;;;;N;;;;; 2A6DF;;Lo;0;L;;;;;N;;;;; 2A700;;Lo;0;L;;;;;N;;;;; -2B738;;Lo;0;L;;;;;N;;;;; +2B739;;Lo;0;L;;;;;N;;;;; 2B740;;Lo;0;L;;;;;N;;;;; 2B81D;;Lo;0;L;;;;;N;;;;; 2B820;;Lo;0;L;;;;;N;;;;; @@ -34283,6 +34579,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 2FA1D;CJK COMPATIBILITY IDEOGRAPH-2FA1D;Lo;0;L;2A600;;;;N;;;;; 30000;;Lo;0;L;;;;;N;;;;; 3134A;;Lo;0;L;;;;;N;;;;; +31350;;Lo;0;L;;;;;N;;;;; +323AF;;Lo;0;L;;;;;N;;;;; E0001;LANGUAGE TAG;Cf;0;BN;;;;;N;;;;; E0020;TAG SPACE;Cf;0;BN;;;;;N;;;;; E0021;TAG EXCLAMATION MARK;Cf;0;BN;;;;;N;;;;; diff --git a/web/package.json b/web/package.json index 13622917c3..8cd671bf35 100644 --- a/web/package.json +++ b/web/package.json @@ -36,7 +36,7 @@ "karma-mocha-reporter": "^2.2.5", "karma-safari-launcher": "^1.0.0", "karma-teamcity-reporter": "^1.1.0", - "mocha": "^8.4.0", + "mocha": "^10.0.0", "modernizr": "^3.11.7", "ts-node": "^9.1.1", "typescript": "^4.5.4" diff --git a/web/source/osk/banner.ts b/web/source/osk/banner.ts index e71a2e8f32..c4f28a6e19 100644 --- a/web/source/osk/banner.ts +++ b/web/source/osk/banner.ts @@ -560,13 +560,17 @@ namespace com.keyman.osk { private doAccept(suggestion: BannerSuggestion) { let _this = this; + // Selecting a suggestion or a reversion should both clear selection + // and clear the reversion-displaying state of the banner. + this.selected = null; + this.doRevert = false; + this.revertAcceptancePromise = suggestion.apply(); if(!this.revertAcceptancePromise) { // We get here either if suggestion acceptance fails or if it was a reversion. if(suggestion.suggestion && suggestion.suggestion.tag == 'revert') { // Reversion state management this.recentAccept = false; - this.doRevert = false; this.recentRevert = true; this.doUpdate(); @@ -581,9 +585,7 @@ namespace com.keyman.osk { } }); - this.selected = null; this.recentAccept = true; - this.doRevert = false; this.recentRevert = false; this.swallowPrediction = true; diff --git a/web/unit_tests/test.sh b/web/unit_tests/test.sh index c10ea639a8..6f5ce3e06f 100755 --- a/web/unit_tests/test.sh +++ b/web/unit_tests/test.sh @@ -9,9 +9,11 @@ WORKING_DIRECTORY=`pwd` # adjust relative paths as necessary THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BASH_SOURCE[0]}")" . "$(dirname "$THIS_SCRIPT")/../../resources/build/build-utils.sh" -. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" ## END STANDARD BUILD SCRIPT INCLUDE +. "$KEYMAN_ROOT/resources/build/build-utils-ci.inc.sh" +. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" + # A simple utility script to facilitate our different modes for unit-testing KMW. # It's rigged to be callable by NPM to facilitate testing during development when in other folders. diff --git a/windows/src/desktop/locale/crowdin-control.sh b/windows/src/desktop/locale/crowdin-control.sh old mode 100644 new mode 100755 diff --git a/windows/src/engine/keyman32/appint/aiTIP.cpp b/windows/src/engine/keyman32/appint/aiTIP.cpp index 9043c9ece1..23240b2a06 100644 --- a/windows/src/engine/keyman32/appint/aiTIP.cpp +++ b/windows/src/engine/keyman32/appint/aiTIP.cpp @@ -258,15 +258,12 @@ extern "C" __declspec(dllexport) BOOL WINAPI TIPProcessKey(WPARAM wParam, LPARAM _td->TIPGetContext = ctfunc; AppContextWithStores *savedContext = NULL; // I4370 // I4978 - AppContext *savedContextUsingCore = NULL; // used for common core - km_kbp_option_item *SavedKBDOptions = NULL; // used for common core + if (!Updateable) { - if (isUsingCoreProcessor) { - savedContextUsingCore = new AppContext(); - _td->app->CopyContext(savedContextUsingCore); - SavedKBDOptions = SaveKeyboardOptionsCore(_td->lpActiveKeyboard); - } else { // I4370 - savedContext = new AppContextWithStores(_td->lpActiveKeyboard->Keyboard->cxStoreArray); // I4978 + // The core processor km_kbp_process_event is only called once per key stroke + // therefore there is no need to preserve context and keyboard actions + if (!isUsingCoreProcessor) { + savedContext = new AppContextWithStores(_td->lpActiveKeyboard->Keyboard->cxStoreArray); // I4370 // I4978 _td->app->SaveContext(savedContext); } } @@ -274,17 +271,8 @@ extern "C" __declspec(dllexport) BOOL WINAPI TIPProcessKey(WPARAM wParam, LPARAM BOOL res = ProcessHook(); if (!Updateable) { - if (isUsingCoreProcessor) { - if (res) { - // Reset the context if match found - _td->app->RestoreContextOnly(savedContextUsingCore); - RestoreKeyboardOptionsCore(_td->lpActiveKeyboard->lpCoreKeyboardState, SavedKBDOptions); - DisposeKeyboardOptionsCore(&SavedKBDOptions); - delete savedContextUsingCore; - savedContextUsingCore = NULL; - } - } else { // I4370 - if (res) { // I4585 + if (!isUsingCoreProcessor) { + if (res) { // I4585 // I4370 // Reset the context if match found _td->app->RestoreContext(savedContext); delete savedContext; diff --git a/windows/src/engine/keyman32/keyman-engine.vcxproj b/windows/src/engine/keyman32/keyman-engine.vcxproj index 6249823191..df421a3619 100644 --- a/windows/src/engine/keyman32/keyman-engine.vcxproj +++ b/windows/src/engine/keyman32/keyman-engine.vcxproj @@ -55,7 +55,7 @@ $(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\src;$(ProjectDir)..\..\..\..\core\build\rust\x86\$(Configuration);$(LibraryPath) - $(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\include;$(ProjectDir)..\..\..\..\core\include;$(IncludePath) + $(ProjectDir)..\..\..\..\common\include;$(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\include;$(ProjectDir)..\..\..\..\core\include;$(IncludePath) $(ProjectDir)..\..\..\..\core\build\x86\$(Configuration)\src;$(ProjectDir)..\..\..\..\core\build\rust\x86\$(Configuration);$(LibraryPath) @@ -402,4 +402,4 @@ - + \ No newline at end of file diff --git a/windows/src/engine/keyman32/kmprocess.cpp b/windows/src/engine/keyman32/kmprocess.cpp index 8b7286afbc..35e30c6857 100644 --- a/windows/src/engine/keyman32/kmprocess.cpp +++ b/windows/src/engine/keyman32/kmprocess.cpp @@ -190,7 +190,7 @@ BOOL ProcessHook() } if (!_td->TIPFUpdateable) { - ProcessActionsTestParse(&fOutputKeystroke); + ProcessActionsNonUpdatableParse(&fOutputKeystroke); } else { ProcessActions(&fOutputKeystroke); } diff --git a/windows/src/engine/keyman32/kmprocessactions.cpp b/windows/src/engine/keyman32/kmprocessactions.cpp index f0aa36256d..1e07523925 100644 --- a/windows/src/engine/keyman32/kmprocessactions.cpp +++ b/windows/src/engine/keyman32/kmprocessactions.cpp @@ -30,8 +30,17 @@ static BOOL processAlert(AITIP* app) { static BOOL processBack(AITIP* app, const km_kbp_action_item* actionItem) { if (actionItem->backspace.expected_type == KM_KBP_BT_MARKER) { app->QueueAction(QIT_BACK, BK_DEADKEY); - } else /* actionItem->backspace.expected_type == KM_KBP_BT_CHAR, KM_KBP_BT_UNKNOWN */ { - app->QueueAction(QIT_BACK, 0); + } else if(actionItem->backspace.expected_type == KM_KBP_BT_CHAR) { + // If this is a TSF-aware app we need to set the BK_SURROGATE flag to delete + // both parts of the surrogate pair. Legacy apps receive a BKSP WM_KEYDOWN event + // which results in deleting both parts in one action. + if (!app->IsLegacy() && Uni_IsSMP(actionItem->backspace.expected_value)) { + app->QueueAction(QIT_BACK, BK_DEFAULT | BK_SURROGATE); + } else { + app->QueueAction(QIT_BACK, BK_DEFAULT); + } + } else { // KM_KBP_BT_UNKNOWN + app->QueueAction(QIT_BACK, BK_DEFAULT); } return TRUE; } @@ -167,7 +176,7 @@ BOOL ProcessActions(BOOL* emitKeyStroke) } BOOL -ProcessActionsTestParse(BOOL* emitKeyStroke) { +ProcessActionsNonUpdatableParse(BOOL* emitKeyStroke) { PKEYMAN64THREADDATA _td = ThreadGlobals(); if (!_td) { return FALSE; @@ -184,13 +193,16 @@ ProcessActionsTestParse(BOOL* emitKeyStroke) { switch (act->type) { case KM_KBP_IT_EMIT_KEYSTROKE: *emitKeyStroke = TRUE; - SendDebugMessageFormat(0, sdmGlobal, 0, "ProcessActionsTestParse EMIT_KEYSTROKE: act->type=%d", act->type); + SendDebugMessageFormat(0, sdmGlobal, 0, "ProcessActionsNonUpdatableParse EMIT_KEYSTROKE: act->type=[%d]", act->type); continueProcessingActions = TRUE; _td->CoreProcessEventRun = FALSE; // If we emit the key stroke on this parse we don't need the second parse break; case KM_KBP_IT_CAPSLOCK: continueProcessingActions = processCapsLock(act, !_td->state.isDown, _td->TIPFUpdateable); break; + case KM_KBP_IT_INVALIDATE_CONTEXT: + continueProcessingActions = processInvalidateContext(_td->app, _td->lpActiveKeyboard->lpCoreKeyboardState); + break; } if (!continueProcessingActions) { return FALSE; diff --git a/windows/src/engine/keyman32/kmprocessactions.h b/windows/src/engine/keyman32/kmprocessactions.h index e96f2226d1..db4a95f758 100644 --- a/windows/src/engine/keyman32/kmprocessactions.h +++ b/windows/src/engine/keyman32/kmprocessactions.h @@ -6,7 +6,7 @@ * Keyman for Windows engine. */ #ifndef _KMPROCESSACTIONS_H -#define _KMPROCESSACTIONS_h +#define _KMPROCESSACTIONS_H /** * Process the actions queued in the core processor @@ -18,12 +18,12 @@ BOOL ProcessActions(BOOL* emitKeyStroke); /** * This function process the actions queued in the core processor in - * the non updateable parse of a keystroke. - * Emit keystroke and capslock are required to be processed in this phase. + * the non-updateable parse of a keystroke. + * Emit keystroke, capslock and possibly invalidate key stroke are required to be processed in this parse. * * @param [in, out] emitKeyStroke is set to true if requested by the core action queue * @return BOOL True if actions were successfully processed */ -BOOL ProcessActionsTestParse(BOOL* emitKeyStroke); +BOOL ProcessActionsNonUpdatableParse(BOOL* emitKeyStroke); #endif diff --git a/windows/src/engine/keyman32/tests/keyman-engine-tests/keyman-engine-tests.vcxproj b/windows/src/engine/keyman32/tests/keyman-engine-tests/keyman-engine-tests.vcxproj index 4980f99332..94ec1574c2 100644 --- a/windows/src/engine/keyman32/tests/keyman-engine-tests/keyman-engine-tests.vcxproj +++ b/windows/src/engine/keyman32/tests/keyman-engine-tests/keyman-engine-tests.vcxproj @@ -34,10 +34,10 @@ - $(ProjectDir)..\..\..\..\..\..\core\include;$(ProjectDir)..\..\..\..\..\..\core\build\x86\$(Configuration)\include;$(ProjectDir)..\..\..\..\global\inc;$(ProjectDir)..\..;$(IncludePath) + $(ProjectDir)..\..\..\..\..\..\common\include;$(ProjectDir)..\..\..\..\..\..\core\include;$(ProjectDir)..\..\..\..\..\..\core\build\x86\$(Configuration)\include;$(ProjectDir)..\..\..\..\global\inc;$(ProjectDir)..\..;$(IncludePath) $(ProjectDir)bin\$(Platform)\$(Configuration)\ $(ProjectDir)obj\$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\..\..\..\..\core\build\x86\$(Configuration)\src;$(ProjectDir)..\..\..\..\..\..\core\build\rust\x86\$(Configuration);$(LibraryPath) + $(ProjectDir)..\..\..\..\..\..\core\build\x86\$(Configuration)\src;$(ProjectDir)..\..\..\..\..\..\core\build\rust\x86\$(Configuration);$(ProjectDir)..\..\..\kmtip\bin\Win32\$(Configuration);$(LibraryPath) true @@ -46,7 +46,7 @@ $(ProjectDir)obj\$(Platform)\$(Configuration)\ - $(ProjectDir)..\..\..\..\global\inc;$(ProjectDir)..\..;$(IncludePath) + $(ProjectDir)..\..\..\..\..\..\common\include;$(ProjectDir)..\..\..\..\..\..\core\build\x86\$(Configuration)\include;$(ProjectDir)..\..\..\..\..\..\core\include;$(ProjectDir)..\..\..\..\global\inc;$(ProjectDir)..\..;$(IncludePath) $(ProjectDir)bin\$(Platform)\$(Configuration)\ $(ProjectDir)obj\$(Platform)\$(Configuration)\ @@ -59,6 +59,7 @@ + @@ -99,6 +100,9 @@ Console version.lib;psapi.lib;imm32.lib;libkmnkbp0.a;%(AdditionalDependencies) + + false + @@ -153,4 +157,4 @@ - + \ No newline at end of file diff --git a/windows/src/engine/keyman32/tests/keyman-engine-tests/kmprocessactionstests.cpp b/windows/src/engine/keyman32/tests/keyman-engine-tests/kmprocessactionstests.cpp index 3df1d70be3..5b4e2bfff8 100644 --- a/windows/src/engine/keyman32/tests/keyman-engine-tests/kmprocessactionstests.cpp +++ b/windows/src/engine/keyman32/tests/keyman-engine-tests/kmprocessactionstests.cpp @@ -106,6 +106,32 @@ TEST_F(KMPROCESSACTIONS, processBackCharactertest) { EXPECT_STREQ(contextBuf, expectedContextFinal); } + +// KM_KBP_IT_BACK - processBack +// Press Backspace for a surrogate character. +// TODO: This test doesn't test for the PostKey handling +// of TSF and two backspaces being sent to the App. +TEST_F(KMPROCESSACTIONS, processBackSurrogateTSFtest) { + WCHAR callbuf[MAXCONTEXT]; + AITIP testApp; + WCHAR expectedContext[] = { 0 }; + WCHAR expectedStringSurrogate[] = {0xD801, 0xDC37, 0}; + + km_kbp_usv testSurrogateChar = Uni_SurrogateToUTF32(0xD801, 0xDC37); //๐ท'; + km_kbp_action_item itemAddChar = {KM_KBP_IT_CHAR, {0,}, {testSurrogateChar}}; + processUnicodeChar(&testApp, &itemAddChar); + + km_kbp_action_item itemBackSpace = {KM_KBP_IT_BACK}; + itemBackSpace.backspace.expected_type = KM_KBP_IT_CHAR; + itemBackSpace.backspace.expected_value = testSurrogateChar; + WCHAR *contextBuf = testApp.ContextBufMax(MAXCONTEXT); + EXPECT_STREQ(contextBuf, expectedStringSurrogate); + // backspace + processBack(&testApp, &itemBackSpace); + contextBuf = testApp.ContextBufMax(MAXCONTEXT); + EXPECT_STREQ(contextBuf, expectedContext); +} + // KM_KBP_IT_BACK - processBack // Press Backspace for a character doesn't match expected character // Note currently we don't check for a character match this should be updated diff --git a/windows/src/engine/testhost/README.md b/windows/src/engine/testhost/README.md index d0c10e7ade..0f7c559010 100644 --- a/windows/src/engine/testhost/README.md +++ b/windows/src/engine/testhost/README.md @@ -5,3 +5,13 @@ This project is a debug host for keyman32 (and keyman64). It is setup to run a s The primary use of testhost is in interactive debugging of Keyman Core, and intermediate Engine code around Core. The final stage of characters reaching the document will not be identical to Keyman Engine in normal use, and there may be other interactive limitations as well. Before trying to run, make sure you have built keyman32 in Debug configuration. Testhost will attempt to load keyman32.dll from ../keyman32/bin/Win32/Debug/keyman32.dll, and if that is not present, from ../keyman32/bin/Win32/Release/keyman32.dll, and finally, from a hardcoded "C:\\Program Files (x86)\\Common Files\\Keyman\\Keyman Engine\\keyman32.dll" (which is hacky but probably suffices for now). + +The work flow that works well using testhost. +With Visual Studio(VS) +1. Build Keyman32 with Debug Configuration. +2. Build testhost with Debug Configuration. +3. Ensure Keyman is not running and that language associated with keyboard you want to test is not selected in the language bar. +4. Press F5 to start debugging. +5. Now use the language bar to select the language associated with a keyman keyboard. +6. You should now be able to debug both the platform and keyman core code. +7. Once you stop the debug session you will need to go back to step 3 to start a new session. (or step 1 if you are making code changes; you should not need to reboot windows.) diff --git a/windows/src/test/manual-tests/i18n-check-unused-strings/i18n-check-unused-strings.sh b/windows/src/test/manual-tests/i18n-check-unused-strings/i18n-check-unused-strings.sh old mode 100644 new mode 100755