diff --git a/.github/workflows/crowdin.yml b/.github/workflows/crowdin.yml index a7d1c1df19..27b4d705ba 100644 --- a/.github/workflows/crowdin.yml +++ b/.github/workflows/crowdin.yml @@ -2,8 +2,8 @@ name: Upload translation sources to Crowdin translate.keyman.com on: schedule: - # At 06:00 every two weeks - - cron: '0 6 1,15 * *' + # At 06:00 every day. https://crontab.cronhub.io/ + - cron: '0 6 * * *' jobs: upload-sources-to-crowdin: diff --git a/.github/workflows/deb-packaging.yml b/.github/workflows/deb-packaging.yml index ee0b8a5bd8..7106ed49b3 100644 --- a/.github/workflows/deb-packaging.yml +++ b/.github/workflows/deb-packaging.yml @@ -1,5 +1,5 @@ name: "Ubuntu packaging" -run-name: "Ubuntu packaging - ${{ github.ref_name }} by @${{ github.actor }}" +run-name: "Ubuntu packaging - ${{ github.event.client_payload.branch }} (branch ${{ github.head_ref }}), by @${{ github.actor }}" on: repository_dispatch: types: ['deb-release-packaging:*', 'deb-pr-packaging:*'] @@ -18,6 +18,8 @@ jobs: VERSION: ${{ steps.version_step.outputs.VERSION }} PRERELEASE_TAG: ${{ steps.prerelease_tag.outputs.PRERELEASE_TAG }} GIT_SHA: ${{ steps.set_status.outputs.GIT_SHA }} + GHA_TEST_BUILD: ${{ github.event.client_payload.isTestBuild }} + GHA_BRANCH: ${{ github.event.client_payload.branch }} steps: - name: Checkout uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c #v3.3.0 @@ -103,7 +105,10 @@ jobs: strategy: fail-fast: true matrix: - dist: [focal, jammy, kinetic, lunar] + # Currently not building mantic until ibus version on mantic stabilizied + # and we can provide a patched version + # dist: [focal, jammy, lunar, mantic] + dist: [focal, jammy, lunar] arch: [amd64] runs-on: ubuntu-latest diff --git a/HISTORY.md b/HISTORY.md index 335f2ccc79..3da35e3d94 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,60 @@ # Keyman Version History +## 17.0.153 alpha 2023-08-03 + +* docs(windows): Update OS requirement to Windows 10 (#9381) +* fix(web): maintenance of focus when changing keyboard via Toolbar UI (#9397) +* chore(linux): Remove Kinetic from GHA (#9399) +* chore(linux): Properly treat test builds with packaging GHA (#9400) + +## 17.0.152 alpha 2023-08-02 + +* fix(developer): more wasm uset fixes (#9382) +* docs(windows): corrected nmake cmd for certificates (#9376) +* chore: add run-name to deb-packaging (#9386) +* chore: try another variable for reporting (#9388) +* chore(linux): Remove package build on Jenkins for Keyman 17 (#9380) +* docs(linux): Add build doc for Keyman Web and Android (#9383) + +## 17.0.151 alpha 2023-08-01 + +* feat(developer) marker steps (#9364) +* feat(common): marker processing (#9365) +* chore(linux): Don't fail on parallel builds (#9368) +* fix(developer): fix breakage from emscripten 3.1.44 (#9375) +* docs(core): Document how to build Core on Linux (#9328) + +## 17.0.150 alpha 2023-07-31 + +* chore(linux): Update debian changelog (#9358) +* chore(linux): Fix creation of PRs after uploading to Debian (#9360) + +## 17.0.149 alpha 2023-07-30 + +* fix(core): Better range check for Uni_IsValid() (#9346) +* chore(core): update documentation in transform logic and processor (#9352) + +## 17.0.148 alpha 2023-07-27 + +* feat(core): merge transform/reorder processing w/ u32 (#9293) +* chore(developer): make unknown vkey a hint, not error (#9344) +* chore(linux): Update supported Ubuntu versions (#9341) + +## 17.0.147 alpha 2023-07-25 + +* chore(linux): Update debian changelog (#9327) + +## 17.0.146 alpha 2023-07-24 + +* chore(deps-dev): bump word-wrap from 1.2.3 to 1.2.4 (#9314) + +## 17.0.145 alpha 2023-07-21 + +* fix(linux): Fix logging (#9310) +* fix(windows): open pdf in an external browser (#9295) +* fix(linux): Fix installation of keyboards with lang tag `mul` (#9027) +* fix(web): allows registering precached keyboards (#9304) + ## 17.0.144 alpha 2023-07-20 * refactor(linux): Use better way to get username (#9313) diff --git a/VERSION.md b/VERSION.md index 56d15fc7ec..0f0cc2cceb 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -17.0.145 \ No newline at end of file +17.0.154 \ No newline at end of file diff --git a/common/web/lm-worker/build-wrap-and-minify.js b/common/web/lm-worker/build-wrap-and-minify.js index 009a8f496c..1572238b7a 100644 --- a/common/web/lm-worker/build-wrap-and-minify.js +++ b/common/web/lm-worker/build-wrap-and-minify.js @@ -35,6 +35,7 @@ if(MINIFY) { sourcesContent: DEBUG, minify: true, keepNames: true, + target: 'es5', outfile: `build/lib/worker-main.polyfilled.min.js` }); } diff --git a/common/web/types/src/kmx/kmx-plus-builder/build-list.ts b/common/web/types/src/kmx/kmx-plus-builder/build-list.ts index e770e7b278..26659f8eb6 100644 --- a/common/web/types/src/kmx/kmx-plus-builder/build-list.ts +++ b/common/web/types/src/kmx/kmx-plus-builder/build-list.ts @@ -86,6 +86,9 @@ export function build_list(source_list: List, sect_strs: BUILDER_STRS): BUILDER_ * @returns */ export function build_list_index(sect_list: BUILDER_LIST, value: ListItem) : BUILDER_LIST_REF { + if (!value) { + return 0; // empty list + } if(!(value instanceof ListItem)) { throw new Error('unexpected value '+ value); } diff --git a/common/web/types/src/kmx/kmx-plus-builder/build-vars.ts b/common/web/types/src/kmx/kmx-plus-builder/build-vars.ts index 5f70689e96..daa0e4a46c 100644 --- a/common/web/types/src/kmx/kmx-plus-builder/build-vars.ts +++ b/common/web/types/src/kmx/kmx-plus-builder/build-vars.ts @@ -2,7 +2,7 @@ import { constants } from "@keymanapp/ldml-keyboard-constants"; import { KMXPlusData } from "../kmx-plus.js"; import { build_strs_index, BUILDER_STR_REF, BUILDER_STRS } from "./build-strs.js"; import { BUILDER_SECTION } from "./builder-section.js"; -import { BUILDER_LIST_REF } from "./build-list.js"; +import { build_list_index, BUILDER_LIST, BUILDER_LIST_REF } from "./build-list.js"; import { build_elem_index, BUILDER_ELEM, BUILDER_ELEM_REF } from "./build-elem.js"; @@ -22,7 +22,7 @@ export interface BUILDER_VARS extends BUILDER_SECTION { /** * Builder for the 'vars' section */ -export function build_vars(kmxplus: KMXPlusData, sect_strs: BUILDER_STRS, sect_elem: BUILDER_ELEM) : BUILDER_VARS { +export function build_vars(kmxplus: KMXPlusData, sect_strs: BUILDER_STRS, sect_elem: BUILDER_ELEM, sect_list: BUILDER_LIST) : BUILDER_VARS { if(!kmxplus.vars) { return null; } @@ -49,7 +49,7 @@ export function build_vars(kmxplus: KMXPlusData, sect_strs: BUILDER_STRS, sect_e size: constants.length_vars + (constants.length_vars_item * kmxplus.vars.totalCount()), _offset: 0, - markers: 0, + markers: build_list_index(sect_list, kmxplus.vars.markers), varCount: kmxplus.vars.totalCount(), varEntries: [ ...stringVars, diff --git a/common/web/types/src/kmx/kmx-plus-builder/kmx-plus-builder.ts b/common/web/types/src/kmx/kmx-plus-builder/kmx-plus-builder.ts index 082e5495eb..3cc58c9fe7 100644 --- a/common/web/types/src/kmx/kmx-plus-builder/kmx-plus-builder.ts +++ b/common/web/types/src/kmx/kmx-plus-builder/kmx-plus-builder.ts @@ -99,7 +99,7 @@ export default class KMXPlusBuilder { this.sect.name = build_name(this.file.kmxplus, this.sect.strs); this.sect.tran = build_tran(this.file.kmxplus.tran, this.sect.strs, this.sect.elem); this.sect.uset = build_uset(this.file.kmxplus, this.sect.strs); - this.sect.vars = build_vars(this.file.kmxplus, this.sect.strs, this.sect.elem); + this.sect.vars = build_vars(this.file.kmxplus, this.sect.strs, this.sect.elem, this.sect.list); this.sect.vkey = build_vkey(this.file.kmxplus); // Finalize the sect (index) section diff --git a/common/web/types/src/kmx/kmx-plus.ts b/common/web/types/src/kmx/kmx-plus.ts index 9ca0a13ad7..795cebc696 100644 --- a/common/web/types/src/kmx/kmx-plus.ts +++ b/common/web/types/src/kmx/kmx-plus.ts @@ -6,6 +6,7 @@ import { isOneChar, toOneChar, unescapeString } from '../util/util.js'; import { KMXFile } from './kmx.js'; import { UnicodeSetParser, UnicodeSet } from '@keymanapp/common-types'; import { VariableParser } from '../ldml-keyboard/pattern-parser.js'; +import { MarkerParser } from '../ldml-keyboard/pattern-parser.js'; // Implementation of file structures from /core/src/ldml/C7043_ldml.md // Writer in kmx-builder.ts @@ -292,6 +293,9 @@ export class Vars extends Section { return v[0]; } } + substituteMarkerString(s : string) : string { + return MarkerParser.toSentinelString(s, this.markers); + } }; /** diff --git a/common/web/types/src/kmx/string-list.ts b/common/web/types/src/kmx/string-list.ts index 4b5716b85d..f9f4b46e91 100644 --- a/common/web/types/src/kmx/string-list.ts +++ b/common/web/types/src/kmx/string-list.ts @@ -1,3 +1,4 @@ +import { OrderedStringList } from 'src/ldml-keyboard/pattern-parser.js'; import { Strs, StrsItem } from './kmx-plus.js'; /** @@ -22,7 +23,7 @@ export class ListIndex { * A string list in memory. This will be replaced with an index * into the string table at finalization. */ -export class ListItem extends Array { +export class ListItem extends Array implements OrderedStringList { /** * Construct a new list from an array of strings. * Use List. This is meant to be called by the List.allocString*() functions. @@ -41,6 +42,9 @@ export class ListItem extends Array { this.push(index); } } + getItemOrder(item: string): number { + return this.findIndex(({value}) => value.value === item); + } isEqual(a: ListItem | string[]): boolean { if (a.length != this.length) { return false; @@ -68,7 +72,12 @@ export class ListItem extends Array { return 0; } } + /** for debugging, print as single string */ toString(): string { - return this.map(v => v.value.value).join(' '); + return this.toStringArray().join(' '); + } + /** for debugging, map to string array */ + toStringArray(): string[] { + return this.map(v => v.value.value); } }; diff --git a/common/web/types/src/ldml-keyboard/pattern-parser.ts b/common/web/types/src/ldml-keyboard/pattern-parser.ts index 1b41b9416c..faee3b1594 100644 --- a/common/web/types/src/ldml-keyboard/pattern-parser.ts +++ b/common/web/types/src/ldml-keyboard/pattern-parser.ts @@ -2,6 +2,7 @@ * Utilities for transform and marker processing */ +import { constants } from "@keymanapp/ldml-keyboard-constants"; import { MATCH_QUAD_ESCAPE, isOneChar, unescapeOneQuadString, unescapeString } from "../util/util.js"; @@ -21,6 +22,12 @@ function matchArray(str: string, match: RegExp) : string[] { */ const COMMON_ID = /^[0-9A-Za-z_]{1,32}$/; +/** for use with markers, means an ordering can be determined */ +export interface OrderedStringList { + /** @returns the ordering of an item (0..), or -1 if not found */ + getItemOrder(item : string) : number; +} + /** * Class for helping with markers */ @@ -40,6 +47,25 @@ export class MarkerParser { */ public static readonly ANY_MARKER_ID = '.'; + /** + * Marker sentinel as a string - U+FFFF + */ + public static readonly SENTINEL = String.fromCodePoint(constants.marker_sentinel); + + /** + * Matches all markers. + */ + public static readonly SENTINEL_ALL_MARKERS = this.SENTINEL + this.SENTINEL; + + /** Minimum ID (trailing code unit) */ + public static readonly MIN_MARKER_INDEX = constants.marker_min_index; + /** Index meaning 'any marker' == `\m{.}` */ + public static readonly ANY_MARKER_INDEX = constants.marker_any_index; + /** Maximum usable marker index */ + public static readonly MAX_MARKER_INDEX = constants.marker_max_index; + /** Max count of markers */ + public static readonly MAX_MARKER_COUNT = constants.marker_max_count; + /** * Pattern for matching a marker reference, OR the special marker \m{.} */ @@ -51,8 +77,40 @@ export class MarkerParser { * @returns `[]` or an array of all markers referenced */ public static allReferences(str: string): string[] { + if (!str) { + return []; + } return matchArray(str, this.REFERENCE); } + + /** @returns string for marker #n */ + public static markerOutput(n: number): string { + if (n < MarkerParser.MIN_MARKER_INDEX || n > MarkerParser.ANY_MARKER_INDEX) { + throw RangeError(`Internal Error: marker index out of range ${n}`); + } + return this.SENTINEL + String.fromCharCode(n); + } + + /** @returns all marker strings as sentinel values */ + public static toSentinelString(s: string, markers?: OrderedStringList) : string { + if (!s) return s; + return s.replaceAll(this.REFERENCE, (sub, arg) => { + if (arg === MarkerParser.ANY_MARKER_ID) { + return MarkerParser.SENTINEL_ALL_MARKERS; + } + if (!markers) { + throw RangeError(`Internal Error: Could not find marker \\m{${arg}} (no markers defined)`); + } + const order = markers.getItemOrder(arg); + if (order === -1) { + throw RangeError(`Internal Error: Could not find marker \\m{${arg}}`); + } else if(order >= MarkerParser.MAX_MARKER_INDEX) { + throw RangeError(`Internal Error: marker \\m{${arg}} has out of range index ${order}`); + } else { + return MarkerParser.markerOutput(order+1); + } + }); + } } /** diff --git a/common/web/types/test/ldml-keyboard/test-pattern-parser.ts b/common/web/types/test/ldml-keyboard/test-pattern-parser.ts index f8b881b644..9d00ab82bf 100644 --- a/common/web/types/test/ldml-keyboard/test-pattern-parser.ts +++ b/common/web/types/test/ldml-keyboard/test-pattern-parser.ts @@ -1,6 +1,6 @@ import 'mocha'; import { assert } from 'chai'; -import { ElementParser, ElementSegment, ElementType, MarkerParser, VariableParser } from '../../src/ldml-keyboard/pattern-parser.js'; +import { ElementParser, ElementSegment, ElementType, MarkerParser, OrderedStringList, VariableParser } from '../../src/ldml-keyboard/pattern-parser.js'; describe('Test of Pattern Parsers', () => { describe('should test MarkerParser', () => { @@ -51,6 +51,57 @@ describe('Test of Pattern Parsers', () => { assert.deepEqual(MarkerParser.allReferences(str), [], `expected no markers: ${str}`); } }); + it('should be able to emit sentinel values', () => { + assert.equal(MarkerParser.markerOutput(295), '\uFFFF\u0127', 'Wrong sentinel value emitted'); + assert.equal(MarkerParser.markerOutput(MarkerParser.ANY_MARKER_INDEX), MarkerParser.SENTINEL_ALL_MARKERS, 'Wrong sentinel value emitted for ffff'); + assert.throws(() => MarkerParser.markerOutput(0)); // below MIN + assert.throws(() => MarkerParser.markerOutput(0x10000)); // above MAX + }); + it('should be able to output sentinel strings', () => { + // with nothing (no markers) + assert.equal( + MarkerParser.toSentinelString(`No markers here!`), + `No markers here!` + ); + assert.throws(() => + MarkerParser.toSentinelString(`Marker \\m{sorryNoMarkers}`) + ); + // with a custom class + class MyMarkers implements OrderedStringList { + getItemOrder(item: string): number { + const m : any = { + 'a': 0, + 'b': 1, + 'c': 2, + 'zzz': 0x2FFFFF, + }; + const o = m[item]; + if (o === undefined) return -1; + return o; + } + }; + const markers = new MyMarkers(); + assert.equal(MarkerParser.toSentinelString( + `No markers here!`, markers), + `No markers here!` + ); + assert.equal(MarkerParser.toSentinelString( + `Give me \\m{a} and \\m{c}, or \\m{.}.`, markers), + `Give me \uFFFF\u0001 and \uFFFF\u0003, or \uFFFF\uFFFF.` + ); + assert.throws(() => + MarkerParser.toSentinelString( + `Want to see something funny? \\m{zzz}`, // out of range + markers + ) + ); + assert.throws(() => + MarkerParser.toSentinelString( + `Want to see something sad? \\m{nothing}`, // non existent + markers + ) + ); + }); }); describe('should test VariableParser', () => { // same test as for markers diff --git a/core/include/ldml/keyboardprocessor_ldml.h b/core/include/ldml/keyboardprocessor_ldml.h index a92731bfa3..262a18e6f0 100644 --- a/core/include/ldml/keyboardprocessor_ldml.h +++ b/core/include/ldml/keyboardprocessor_ldml.h @@ -93,6 +93,11 @@ #define LDML_LENGTH_VARS_ITEM 0x10 #define LDML_LENGTH_VKEY 0xC #define LDML_LENGTH_VKEY_ITEM 0x8 +#define LDML_MARKER_ANY_INDEX 0xFFFF +#define LDML_MARKER_MAX_COUNT 0xFFFD +#define LDML_MARKER_MAX_INDEX 0xFFFE +#define LDML_MARKER_MIN_INDEX 0x1 +#define LDML_MARKER_SENTINEL 0xFFFF #define LDML_META_SETTINGS_FALLBACK_OMIT 0x1 #define LDML_META_SETTINGS_TRANSFORMFAILURE_OMIT 0x2 #define LDML_META_SETTINGS_TRANSFORMPARTIAL_HIDE 0x4 diff --git a/core/include/ldml/keyboardprocessor_ldml.ts b/core/include/ldml/keyboardprocessor_ldml.ts index 4d59700895..fdfc63da18 100644 --- a/core/include/ldml/keyboardprocessor_ldml.ts +++ b/core/include/ldml/keyboardprocessor_ldml.ts @@ -613,6 +613,19 @@ class Constants { } return chars.join(''); } + + // ---- marker stuff ---- + /** sentinel value indicating a marker follows */ + readonly marker_sentinel = 0xFFFF; + /** minimum usable marker index */ + readonly marker_min_index = 0x0001; + /** index value referring to the 'any' marker match */ + readonly marker_any_index = 0xFFFF; + /** maximum marker index prior to the 'any' value */ + readonly marker_max_index = this.marker_any_index - 1; + /** maximum count of markers (not including 'any') */ + readonly marker_max_count = this.marker_max_index - this.marker_min_index; + }; export const constants = new Constants(); diff --git a/core/src/kmx/kmx_plus.cpp b/core/src/kmx/kmx_plus.cpp index e5132193f5..b02fec77d8 100644 --- a/core/src/kmx/kmx_plus.cpp +++ b/core/src/kmx/kmx_plus.cpp @@ -18,6 +18,20 @@ namespace km { namespace kbp { namespace kmx { +/** + * \def KMXPLUS_DEBUG_LOAD set to 1 to print messages on KMXPLUS loading. + * Off by default. +*/ +#ifndef KMXPLUS_DEBUG_LOAD +#define KMXPLUS_DEBUG_LOAD 0 +#endif + +#if KMXPLUS_DEBUG_LOAD +#define DebugLoad(msg,...) DebugLog(msg, __VA_ARGS__) +#else +#define DebugLoad(msg,...) +#endif + // double check these modifier mappings static_assert(LCTRLFLAG == LDML_KEYS_MOD_CTRLL, "LDML modifier bitfield vs. kmx_file.h #define mismatch"); static_assert(RCTRLFLAG == LDML_KEYS_MOD_CTRLR, "LDML modifier bitfield vs. kmx_file.h #define mismatch"); @@ -231,7 +245,7 @@ COMP_KMXPLUS_DISP::valid(KMX_DWORD _kmn_unused(length)) const { DebugLog("disp: baseCharacter str#0x%X", baseCharacter); } for (KMX_DWORD i=0; i str0x%X", i, entries[i].to, entries[i].display); + DebugLoad("disp#%d: to: str0x%X -> str0x%X", i, entries[i].to, entries[i].display); if (entries[i].to == 0 || entries[i].display == 0) { DebugLog("disp to: or display: has a zero string"); assert(false); @@ -265,7 +279,7 @@ COMP_KMXPLUS_STRS::valid(KMX_DWORD _kmn_unused(length)) const { return false; } // TODO-LDML: validate valid UTF-16LE? - DebugLog("strs #0x%X: '%s'", i, Debug_UnicodeString(start)); + DebugLoad("strs #0x%X: '%s'", i, Debug_UnicodeString(start)); } return true; } @@ -740,7 +754,7 @@ COMP_KMXPLUS_KEYS_Helper::setKeys(const COMP_KMXPLUS_KEYS *newKeys) { for(KMX_DWORD i = 0; is_valid && i < key2->keyCount; i++) { const auto &key = keys[i]; // is the count off the end? - DebugLog( " id=0x%X, to=0x%X, flicks=%d", i, key.id, key.to, key.flicks); // TODO-LDML: could dump more fields here + DebugLoad( " id=0x%X, to=0x%X, flicks=%d", i, key.id, key.to, key.flicks); // TODO-LDML: could dump more fields here if (key.flicks >0 && key.flicks >= key2->flicksCount) { DebugLog("key[%d] has invalid flicks index %d", i, key.flicks); is_valid = false; @@ -750,7 +764,7 @@ COMP_KMXPLUS_KEYS_Helper::setKeys(const COMP_KMXPLUS_KEYS *newKeys) { for(KMX_DWORD i = 0; is_valid && i < key2->flicksCount; i++) { const auto &e = flickLists[i]; // is the count off the end? - DebugLog(" %d: index %d, count %d", i, e.flick, e.count); + DebugLoad(" %d: index %d, count %d", i, e.flick, e.count); if (i == 0) { if (e.flick != 0 || e.count != 0) { DebugLog("Error: Invalid Flick #0"); @@ -765,18 +779,22 @@ COMP_KMXPLUS_KEYS_Helper::setKeys(const COMP_KMXPLUS_KEYS *newKeys) { } for(KMX_DWORD i = 0; is_valid && i < key2->flickCount; i++) { const auto &e = flickElements[i]; - // is the count off the end? - DebugLog(" %d: to=0x%X, directions=0x%X, flags=0x%X", i, e.to, e.directions, e.flags); + // validate to is present + if (e.to == 0 || e.directions == 0) { + DebugLog("flickElement[%d] has empty to=%0x%X or directions=%0x%X", i, e.to, e.directions); + is_valid = false; + assert(is_valid); + } + DebugLoad(" %d: to=0x%X, directions=0x%X, flags=0x%X", i, e.to, e.directions, e.flags); } // now the kmap - DebugLog(" kmap count: #0x%X", key2->kmapCount); + DebugLoad(" kmap count: #0x%X", key2->kmapCount); for (KMX_DWORD i = 0; i < key2->kmapCount; i++) { - // These are pretty noisy, drop them from the log - // DebugLog(" #0x%d\n", i); + DebugLoad(" #0x%d\n", i); auto &entry = kmap[i]; - // DebugLog(" vkey\t0x%X", entry.vkey); - // DebugLog(" mod\t0x%X", entry.mod); - // DebugLog(" key\t#0x%X", entry.key); + DebugLoad(" vkey\t0x%X", entry.vkey); + DebugLoad(" mod\t0x%X", entry.mod); + DebugLoad(" key\t#0x%X", entry.key); if (!LDML_IS_VALID_MODIFIER_BITS(entry.mod)) { DebugLog("Invalid modifier value"); assert(false); @@ -928,10 +946,12 @@ COMP_KMXPLUS_LIST_Helper::setList(const COMP_KMXPLUS_LIST *newList) { assert(is_valid); } } +#if KMXPLUS_DEBUG_LOAD for (KMX_DWORD i = 0; is_valid && i < list->indexCount; i++) { const auto &e = indices[i]; - DebugLog(" index %d: str 0x%X", i, e); + DebugLoad(" index %d: str 0x%X", i, e); } +#endif } // Return results DebugLog("COMP_KMXPLUS_LIST_Helper.setList(): %s", is_valid ? "valid" : "invalid"); @@ -969,7 +989,13 @@ COMP_KMXPLUS_USET::valid(KMX_DWORD _kmn_unused(length)) const { assert(false); return false; } - return true; + return true; // see helper +} + +COMP_KMXPLUS_USET_RANGE::COMP_KMXPLUS_USET_RANGE(KMX_DWORD s, KMX_DWORD e) : start(s), end(e) { +} + +COMP_KMXPLUS_USET_RANGE::COMP_KMXPLUS_USET_RANGE(const COMP_KMXPLUS_USET_RANGE &other) : start(other.start), end(other.end) { } COMP_KMXPLUS_USET_Helper::COMP_KMXPLUS_USET_Helper() : uset(nullptr), is_valid(false), usets(nullptr), ranges(nullptr) { @@ -977,7 +1003,7 @@ COMP_KMXPLUS_USET_Helper::COMP_KMXPLUS_USET_Helper() : uset(nullptr), is_valid(f bool COMP_KMXPLUS_USET_Helper::setUset(const COMP_KMXPLUS_USET *newUset) { - DebugLog("validating newUset=%p", newUset); + DebugLoad("validating newUset=%p", newUset); is_valid = true; if (newUset == nullptr) { // Note: kmx_plus::kmx_plus has already called section_from_bytes() @@ -1017,9 +1043,13 @@ COMP_KMXPLUS_USET_Helper::setUset(const COMP_KMXPLUS_USET *newUset) { } else { /** last lastEnd value */ KMX_DWORD lastEnd = 0x0; - for (KMX_DWORD r = 0; r < e.count; r++) { + for (KMX_DWORD r = 0; is_valid && r < e.count; r++) { const auto &range = ranges[e.range + r]; // already range-checked 'r' above - if (range.end < range.start) { + if (!Uni_IsValid(range.start, range.end)) { + DebugLog("uset[%d][%d] not valid: [U+%04X-U+%04X]", i, r, range.start, range.end); + is_valid = false; + assert(is_valid); + } else if (range.end < range.start) { // range swapped DebugLog("uset[%d]: range[%d+%d] end 0x%X= ch) { return true; } @@ -1058,6 +1090,30 @@ bool USet::contains(km_kbp_usv ch) const { return false; } +bool +USet::valid() const { + // double check + for (const auto &range : ranges) { + if (!Uni_IsValid(range.start, range.end)) { + DebugLog("Invalid UnicodeSet (contains noncharacters): [U+%04X,U+%04X]", (int)range.start, (int)range.end); + return false; + } + } + return true; +} + +void +USet::dump() const { + DebugLog(" - USet size=%d", ranges.size()); + for (const auto &range : ranges) { + if (range.start == range.end) { + DebugLog(" - [U+%04X]", (uint32_t)range.start); + } else { + DebugLog(" - [U+%04X-U+%04X]", (uint32_t)range.start, (uint32_t)range.end); + } + } +} + USet COMP_KMXPLUS_USET_Helper::getUset(KMXPLUS_USET i) const { if (!valid() || i >= uset->usetCount) { @@ -1083,6 +1139,9 @@ kmx_plus::kmx_plus(const COMP_KEYBOARD *keyboard, size_t length) : bksp(nullptr), disp(nullptr), elem(nullptr), key2(nullptr), layr(nullptr), list(nullptr), loca(nullptr), meta(nullptr), sect(nullptr), strs(nullptr), tran(nullptr), vars(nullptr), vkey(nullptr), valid(false) { DebugLog("kmx_plus: Got a COMP_KEYBOARD at %p\n", keyboard); +#if !KMXPLUS_DEBUG_LOAD + DebugLog("Note: define KMXPLUS_DEBUG_LOAD=1 at compile time for more verbosity in loading"); +#endif if (!(keyboard->dwFlags & KF_KMXPLUS)) { DebugLog("Err: flags COMP_KEYBOARD.dwFlags did not have KF_KMXPLUS set"); valid = false; diff --git a/core/src/kmx/kmx_plus.h b/core/src/kmx/kmx_plus.h index 3e8fe92285..27fc0c1b93 100644 --- a/core/src/kmx/kmx_plus.h +++ b/core/src/kmx/kmx_plus.h @@ -11,6 +11,7 @@ #include #include #include +#include namespace km { namespace kbp { @@ -683,24 +684,27 @@ struct COMP_KMXPLUS_USET_USET { struct COMP_KMXPLUS_USET_RANGE { km_kbp_usv start; km_kbp_usv end; + public: + COMP_KMXPLUS_USET_RANGE(const COMP_KMXPLUS_USET_RANGE& other); + COMP_KMXPLUS_USET_RANGE(KMX_DWORD start, KMX_DWORD end); }; /** * represents one of the uset elements - * Aliases, does not copy memory. - * The original KMX+ memory must stay around while this object is held. */ class USet { public: - /** construct a set over the specified range. */ + /** construct a set over the specified range. Data is copied. */ USet(const COMP_KMXPLUS_USET_RANGE* newStart, size_t newCount); /** empty set */ USet(); /** true if the uset contains this char */ bool contains(km_kbp_usv ch) const; + /** debugging */ + void dump() const; + bool valid() const; private: - const COMP_KMXPLUS_USET_RANGE *ranges; - size_t count; + std::list ranges; }; class COMP_KMXPLUS_USET_Helper { diff --git a/core/src/kmx/kmx_xstring.h b/core/src/kmx/kmx_xstring.h index fafa277ed7..9b479dac7a 100644 --- a/core/src/kmx/kmx_xstring.h +++ b/core/src/kmx/kmx_xstring.h @@ -6,22 +6,43 @@ namespace km { namespace kbp { namespace kmx { +const char16_t Uni_LEAD_SURROGATE_START = 0xD800; +const char16_t Uni_LEAD_SURROGATE_END = 0xDBFF; +const char16_t Uni_TRAIL_SURROGATE_START = 0xDC00; +const char16_t Uni_TRAIL_SURROGATE_END = 0xDFFF; +const char16_t Uni_SURROGATE_START = Uni_LEAD_SURROGATE_START; +const char16_t Uni_SURROGATE_END = Uni_TRAIL_SURROGATE_END; +const char16_t Uni_FD_NONCHARACTER_START = 0xFDD0; +const char16_t Uni_FD_NONCHARACTER_END = 0xFDEF; +const char16_t Uni_FFFE_NONCHARACTER = 0xFFFE; +const char16_t Uni_FFFF_NONCHARACTER = 0xFFFF; +const char16_t Uni_BMP_END = 0xFFFF; +const km_kbp_usv Uni_SMP_START = 0x010000; +const km_kbp_usv Uni_PLANE_MASK = 0x1F0000; +const km_kbp_usv Uni_MAX_CODEPOINT = 0x10FFFF; + /** * @brief True if a lead surrogate * \def Uni_IsSurrogate1 */ -#define Uni_IsSurrogate1(ch) ((ch) >= 0xD800 && (ch) <= 0xDBFF) +#define Uni_IsSurrogate1(ch) ((ch) >= km::kbp::kmx::Uni_LEAD_SURROGATE_START && (ch) <= km::kbp::kmx::Uni_LEAD_SURROGATE_END) /** * @brief True if a trail surrogate * \def Uni_IsSurrogate2 */ -#define Uni_IsSurrogate2(ch) ((ch) >= 0xDC00 && (ch) <= 0xDFFF) +#define Uni_IsSurrogate2(ch) ((ch) >= km::kbp::kmx::Uni_TRAIL_SURROGATE_START && (ch) <= km::kbp::kmx::Uni_TRAIL_SURROGATE_END) + +/** + * @brief True if any surrogate + * \def UniIsSurrogate +*/ +#define Uni_IsSurrogate(ch) (Uni_IsSurrogate1(ch) || Uni_IsSurrogate2(ch)) /** * @brief Returns true if BMP (Plane 0) * \def Uni_IsBMP */ -#define Uni_IsBMP(ch) ((ch) < 0x10000) +#define Uni_IsBMP(ch) ((ch) <= km::kbp::kmx::Uni_BMP_END) /** * @brief Convert two UTF-16 surrogates into one UTF-32 codepoint @@ -29,17 +50,35 @@ namespace kmx { * @param cl trail surrogate - Uni_IsSurrogate2(cl) must == true * \def Uni_SurrogateToUTF */ -#define Uni_SurrogateToUTF32(ch, cl) (((ch) - 0xD800) * 0x400 + ((cl) - 0xDC00) + 0x10000) +#define Uni_SurrogateToUTF32(ch, cl) (((ch) - km::kbp::kmx::Uni_LEAD_SURROGATE_START) * 0x400 + ((cl) - km::kbp::kmx::Uni_TRAIL_SURROGATE_START) + km::kbp::kmx::Uni_SMP_START) /** * @brief Convert UTF-32 BMP to UTF-16 BMP * @param ch codepoint - Uni_IsBMP(ch) must == true * \def Uni_UTF32BMPToUTF16 */ -#define Uni_UTF32BMPToUTF16(ch) (ch & 0xFFFF) +#define Uni_UTF32BMPToUTF16(ch) ((ch) & Uni_FFFF_NONCHARACTER) -#define Uni_UTF32ToSurrogate1(ch) (char16_t)(((ch) - 0x10000) / 0x400 + 0xD800) -#define Uni_UTF32ToSurrogate2(ch) (char16_t)(((ch) - 0x10000) % 0x400 + 0xDC00) +#define Uni_UTF32ToSurrogate1(ch) (char16_t)(((ch) - km::kbp::kmx::Uni_SMP_START) / 0x400 + km::kbp::kmx::Uni_LEAD_SURROGATE_START) +#define Uni_UTF32ToSurrogate2(ch) (char16_t)(((ch) - km::kbp::kmx::Uni_SMP_START) % 0x400 + km::kbp::kmx::Uni_TRAIL_SURROGATE_START) + +/** + * @returns true if the character is a noncharacter +*/ +bool Uni_IsNonCharacter(km_kbp_usv ch); + +/** + * @returns true if the character is a valid Unicode code point. + * Surrogates belong to UTF-16 and are invalid. +*/ +bool Uni_IsValid(km_kbp_usv ch); + +/** + * @returns true if the character is a valid Unicode code point range, that is, [start-end] are all + * valid. + * Surrogates belong to UTF-16 and are invalid. +*/ +bool Uni_IsValid(km_kbp_usv start, km_kbp_usv range); /** * char16_t array big enough to hold a single Unicode codepoint, @@ -133,6 +172,44 @@ u16string_to_u32string(const std::u16string &source) { return out; } +inline bool Uni_IsEndOfPlaneNonCharacter(km_kbp_usv ch) { + return (((ch) & Uni_FFFE_NONCHARACTER) == Uni_FFFE_NONCHARACTER); // matches FFFF or FFFE +} + +inline bool Uni_IsNoncharacter(km_kbp_usv ch) { + return (((ch) >= Uni_FD_NONCHARACTER_START && (ch) <= Uni_FD_NONCHARACTER_END) || Uni_IsEndOfPlaneNonCharacter(ch)); +} + +inline bool Uni_InCodespace(km_kbp_usv ch) { + return ((ch) <= Uni_MAX_CODEPOINT); +}; + +inline bool Uni_IsValid(km_kbp_usv ch) { + return (Uni_InCodespace(ch) && !Uni_IsSurrogate(ch) && !Uni_IsNoncharacter(ch)); +} + +inline bool Uni_IsValid(km_kbp_usv start, km_kbp_usv end) { + if (!Uni_IsValid(end) || !Uni_IsValid(start) || (end < start)) { + // start or end out of range, or inverted range + return false; + } else if ((start <= Uni_SURROGATE_END) && (end >= Uni_SURROGATE_START)) { + // contains some of the surrogate range + return false; + } else if ((start <= Uni_FD_NONCHARACTER_END) && (end >= Uni_FD_NONCHARACTER_START)) { + // contains some of the noncharacter range + return false; + } else if ((start & Uni_PLANE_MASK) != (end & Uni_PLANE_MASK)) { + // start and end are on different planes, meaning that the U+__FFFE/U+__FFFF noncharacters + // are contained. + // As a reminder, we already checked that start/end are themselves valid, + // so we know that 'end' is not on a noncharacter at end of plane. + return false; + } else { + return true; + } +} + + } // namespace kmx } // namespace kbp } // namespace km diff --git a/core/src/ldml/C9134_ldml_markers.md b/core/src/ldml/C9134_ldml_markers.md index 7a0f5d6959..0b2436b97f 100644 --- a/core/src/ldml/C9134_ldml_markers.md +++ b/core/src/ldml/C9134_ldml_markers.md @@ -27,7 +27,6 @@ Markers can appear in both 'emitting' and 'matching-only' areas: #### Match only - `transform from=` to match markers -- `transform after=` to match markers - `display to=` for matching keys which contain markers ## Theory / Encoding diff --git a/core/src/ldml/ldml_processor.cpp b/core/src/ldml/ldml_processor.cpp index 06257c8b11..41ed5d419f 100644 --- a/core/src/ldml/ldml_processor.cpp +++ b/core/src/ldml/ldml_processor.cpp @@ -240,51 +240,50 @@ ldml_processor::process_event( // Construct a context buffer of all the KM_KBP_BT_CHAR items // Extract the context into 'ctxt' for transforms to process if (!!transforms) { - // if no transforms, no reason to do this extraction + // if no transforms, no reason to do this extraction (ctxt will remain empty) auto &cp = state->context(); // We're only interested in as much of the context as is a KM_KBP_BT_CHAR. uint8_t last_type = KM_KBP_BT_UNKNOWN; for (auto c = cp.rbegin(); c != cp.rend(); c++) { last_type = c->type; if (last_type != KM_KBP_BT_CHAR) { - // not a char, get out + // not a char, stop here + // TODO-LDML: markers? break; } + ctxt.emplace_front(1, c->character); // extract UTF-32 to 1 or 2 UTF-16 chars in a string - km::kbp::kmx::char16_single buf; - const int len = km::kbp::kmx::Utf32CharToUtf16(c->character, buf); - const std::u16string str(buf.ch, len); - ctxt.push_front(str); // prepend to string } } // Look up the key const std::u16string str = keys.lookup(vk, modifier_state); if (str.empty()) { - // not found + // not found, so pass the keystroke on to the Engine state->actions().push_invalidate_context(); state->actions().push_emit_keystroke(); break; // ----- commit and exit } // found the correct string - push it into the context and actions const std::u32string str32 = kmx::u16string_to_u32string(str); - for(size_t i=0; icontext().push_character(str32[i]); - state->actions().push_character(str32[i]); + for (const auto &ch : str32) { + state->context().push_character(ch); + state->actions().push_character(ch); } // Now process transforms // Process the transforms if (!!transforms) { // add the newly added char to ctxt - ctxt.push_back(str); + ctxt.push_back(str32); + + std::u32string outputString; - std::u16string outputString; - // TODO-LDML: unroll ctxt into a str. Would be better to have transforms be able to process a vector - std::u16string ctxtstr; - for (size_t i = 0; i < ctxt.size(); i++) { - ctxtstr.append(ctxt[i]); + std::u32string ctxtstr; + for (const auto &ch : ctxt) { + ctxtstr.append(ch); } + // check if the context matched, and if so how much (at the end) const size_t matchedContext = transforms->apply(ctxtstr, outputString); if (matchedContext > 0) { @@ -296,10 +295,9 @@ ldml_processor::process_event( state->actions().push_backspace(KM_KBP_BT_CHAR, deletedChar); // Cause prior char to be removed } // Now, add in the updated text - const std::u32string outstr32 = kmx::u16string_to_u32string(outputString); - for (size_t i = 0; i < outstr32.length(); i++) { - state->context().push_character(outstr32[i]); - state->actions().push_character(outstr32[i]); + for (const auto &ch : outputString) { + state->context().push_character(ch); + state->actions().push_character(ch); } } } diff --git a/core/src/ldml/ldml_transforms.cpp b/core/src/ldml/ldml_transforms.cpp index e2ebce56ea..821f48e0dd 100644 --- a/core/src/ldml/ldml_transforms.cpp +++ b/core/src/ldml/ldml_transforms.cpp @@ -9,6 +9,8 @@ #include "debuglog.h" #include #include +#include "kmx/kmx_xstring.h" + #ifndef assert #define assert(x) // TODO-LDML @@ -18,6 +20,22 @@ namespace km { namespace kbp { namespace ldml { +/** + * \def KMXPLUS_DEBUG_TRANSFORM + * define KMXPLUS_DEBUG_TRANSFORM=1 to enable verbose processing of transforms/reorders + * The default is 0, which only notes initialization and exceptional cases +*/ + +#ifndef KMXPLUS_DEBUG_TRANSFORM +#define KMXPLUS_DEBUG_TRANSFORM 0 +#endif + +#if KMXPLUS_DEBUG_TRANSFORM +#define DebugTran(msg, ...) DebugLog(msg, ##__VA_ARGS__) +#else +#define DebugTran(msg, ...) +#endif + element::element(const USet &new_u, KMX_DWORD new_flags) : chr(), uset(new_u), flags((new_flags & ~LDML_ELEM_FLAGS_TYPE) | LDML_ELEM_FLAGS_TYPE_USET) { } @@ -67,6 +85,16 @@ element::matches(km_kbp_usv ch) const { } } +void +element::dump() const { + if (is_uset()) { + DebugLog("element order=%d USET", (int)get_order()); + uset.dump(); + } else { + DebugLog("element order=%d U+%04X", (int)get_order(), (int)chr); + } +} + int reorder_sort_key::compare(const reorder_sort_key &other) const { int primaryResult = (int)primary - (int)other.primary; @@ -83,7 +111,10 @@ reorder_sort_key::compare(const reorder_sort_key &other) const { } else if (quaternaryResult) { return quaternaryResult; } else { - assert(quaternaryResult); // quaternary is a string index, should always be != + // We don't expect to get here. quaternaryResult is the string index, which + // should be unequal. + assert(quaternaryResult); + // We have the underlying character, so use the binary order as a tiebreaker. int identityResult = (int)ch - (int)other.ch; // tie breaker return identityResult; } @@ -94,6 +125,11 @@ reorder_sort_key::operator<(const reorder_sort_key &other) const { return (compare(other) < 0); } +bool +reorder_sort_key::operator>(const reorder_sort_key &other) const { + return (compare(other) > 0); +} + std::deque reorder_sort_key::from(const std::u32string &str) { // construct a 'baseline' sort key, that is, in the absence of @@ -102,6 +138,10 @@ reorder_sort_key::from(const std::u32string &str) { auto s = str.begin(); // str iterator size_t c = 0; // str index for (auto e = str.begin(); e < str.end(); e++, s++, c++) { + // primary weight: 0 + // seconary weight: c (the string index) + // tertiary weight: 0 + // quaternary weight: c (the index again) keylist.emplace_back(reorder_sort_key{*s, 0, c, 0, c}); } return keylist; @@ -116,9 +156,13 @@ reorder_sort_key::dump() const { size_t element_list::match_end(const std::u32string &str) const { if (str.size() < size()) { - return 0; // input string too short, can't possibly match + // input string too short, can't possibly match. + // This assumes each element is a single char, no string elements. + return 0; } // s: iterate from end to front of string + // For example, if str = 'abcd', we try to match 'd', then 'c', then 'b', then 'a' + // starting with the end of the element list. auto s = str.rbegin(); // e: end to front on elements. // we know the # of elements is <= length of string, @@ -137,8 +181,8 @@ element_list::match_end(const std::u32string &str) const { bool element_list::load(const kmx::kmx_plus &kplus, kmx::KMXPLUS_ELEM id) { KMX_DWORD elementsLength; - auto elements = kplus.elem->getElementList(id, elementsLength); - assert((elementsLength == 0) || (elements != nullptr)); + auto elements = kplus.elem->getElementList(id, elementsLength); // pointer to beginning of element list + assert((elementsLength == 0) || (elements != nullptr)); // it could be a 0-length list for (size_t i = 0; i & element_list::update_sort_key(size_t offset, std::deque &key) const { + /** string index */ size_t c = 0; - for (auto e = begin(); e < end(); e++) { + for (auto e = begin(); e < end(); e++, c++) { + /** update this key */ auto &k = key.at(offset + c); + // we double check that the character matches. otherwise something + // has really gone awry, because we shouldn't be here if this element list doesn't apply. if (!e->matches(k.ch)) { - DebugLog("!! updateSortKey(%d+%d): element did not re-match the sortkey", offset, c); + DebugLog("!! Internal Error: updateSortKey(%d+%d): element did not re-match the sortkey", offset, c); k.dump(); + // TODO-LDML: assertion follows + assert(e->matches(k.ch)); // double check that this element matches } - assert(e->matches(k.ch)); // double check that this element matches + // we only update primary and tertiary weights k.primary = e->get_order(); - k.tertiary = e->get_tertiary(); // TODO-LDML: need more detailed tertiary work - c++; + // TODO-LDML: need more detailed tertiary work + k.tertiary = e->get_tertiary(); +#if KMXPLUS_DEBUG_TRANSFORM + DebugTran("Updating at +%d", c); + k.dump(); +#endif } return key; } +void +element_list::dump() const { + DebugLog("element_list[%d]", size()); + for (const auto &e : *this) { + e.dump(); + } +} + reorder_entry::reorder_entry(const element_list &new_elements) : elements(new_elements), before() { } reorder_entry::reorder_entry(const element_list &new_elements, const element_list &new_before) : elements(new_elements), before(new_before) { @@ -183,11 +256,14 @@ reorder_entry::reorder_entry(const element_list &new_elements, const element_lis size_t reorder_entry::match_end(std::u32string &str, size_t offset, size_t len) const { auto substr = str.substr(offset, len); + // first, see if the elements match. If not, this entry doesn't apply size_t match_len = elements.match_end(substr); if (match_len == 0) { return 0; } + // Now we need to check if there is a "before=" element string that + // is also a precondition. if (!before.empty()) { // does not match before offset std::u32string prefix = substr.substr(0, substr.size() - match_len); @@ -209,95 +285,138 @@ reorder_group::apply(std::u32string &str) const { // get a baseline sort key auto sort_keys = reorder_sort_key::from(str); - // DebugLog("Baseline Keys:"); - // for (auto e = sort_keys.begin(); e < sort_keys.end(); e++) { - // e->dump(); - // } - // apply ALL reorders in the group. - // size_t c = 0; - for (auto r = list.begin(); r < list.end(); r++) { + for (const auto &r : list) { // work backward from end of string forward + // That is, see if "abc" matches "abc" or "ab" or "a" for (size_t s = str.size(); s > 0; s--) { - size_t submatch = r->match_end(str, 0, s); + size_t submatch = r.match_end(str, 0, s); if (submatch != 0) { +#if KMXPLUS_DEBUG_TRANSFORM + DebugTran("Matched: %S (off=%d, len=%d)", str.c_str(), 0, s); + r.elements.dump(); +#endif // update the sort key size_t sub_match_start = s - submatch; - r->elements.update_sort_key(sub_match_start, sort_keys); - some_match = true; + r.elements.update_sort_key(sub_match_start, sort_keys); + some_match = true; // record that there was a match } } - // c++; } if (!some_match) { - // DebugLog("Skip: No reorder elements matched."); + // get out if nothing matched. + // the sortkey won't be "interesting", and the sort + // will be a no-op. + DebugTran("Skip: No reorder elements matched."); return false; // nothing matched, so no work. } - size_t match_len = str.size(); // TODO-LDML: for now, assume entire match - - // DebugLog("Updated Keys:"); - // for (auto e = sort_keys.begin(); e < sort_keys.end(); e++) { - // e->dump(); - // } - - std::u32string prefix = str; - prefix.resize(str.size() - match_len); // just the part before the matched part. - // just the suffix (the matched part) - std::u32string suffix = str.substr(prefix.size(), match_len); - // sort it! Here's where the reorder happens - // TODO: need to sort only between primary bases… - std::sort(sort_keys.begin(), sort_keys.end()); -#if 0 - // TODO-LDML :need to sort sub-runs - for(auto e = sort_keys.end(); !applied && e > sort_keys.begin(); e--) { - if (e->primary == 0) { - // Got it. - std::sort(e, sort_keys.end()); - // DebugLog("… sorting at q=%d", (int)e->quaternary); - } +#if KMXPLUS_DEBUG_TRANSFORM + DebugTran("Updated sortkey"); + for (const auto &r : sort_keys) { + r.dump(); } #endif - // recombine into a str + + // TODO-LDML: for now, assume matches entire string. + // A needed optimization here would be to detect a common substring + // at the end of the old and new strings, and keep the match_len + // minimal. This reduces thrash in core's context. + size_t match_len = str.size(); + + // 'prefix' is the unmatched string before the match + // TODO-LDML: right now, this is empty. + std::u32string prefix = str; + prefix.resize(str.size() - match_len); // just the part before the matched part. + + // Now, we need to actually do the sorting, but we must only sort + // 'runs' beginning with 0-weight keys. + + // Consider the 'roast' example in the spec, you might end up with the following: + // codepoint (pri, sec, ter, quat) + // U+1A21 (0, 0, 0, 0) + // U+1A60 (127, 1, 0, 1) + // U+1A45 (0, 2, 0, 2) + // U+1A6B (42, 3, 0, 3) + // U+1A76 (55, 4, 0, 4) + // This example happens to be in order, but must be sorted in two diferent ranges, + // with secondary (index) values of [0,1] and [2,4] + // + // Another example might look like the following: + // U+1A21 (0, 0, 0, 0) + // U+1A6B (42, 1, 0, 1) + // U+1A76 (55, 2, 0, 2) + // U+1A60 (10, 3, 0, 3) + // U+1A45 (10, 4, 0, 3) + // Here there is only a single range to sort [0,4] + + /** pointer to the beginning of the current run. */ + std::deque::iterator run_start = sort_keys.begin(); + for(auto e = run_start; e != sort_keys.end(); e++) { + if ((e->primary == 0) && (e != run_start)) { // it's a base + auto run_end = e - 1; + DebugTran("Sorting subrange quaternary=[%d..]", run_start->quaternary); + std::sort(run_start, run_end); // reversed because it's a reverse iterator…? + // move the start + run_start = e; // next run starts here + } + } + // sort the last run in the string as well. + if (run_start != sort_keys.end()) { // TODO-LDML: skip if a single-char run + DebugTran("Sorting final subrange quaternary=[%d..]", run_start->quaternary); + std::sort(run_start, sort_keys.end()); // reversed because it's a reverse iterator…? + } + // recombine into a string by pulling out the 'ch' value + // that's in each sortkey element. std::u32string newSuffix; - size_t q = sort_keys.begin()->quaternary; // + size_t q = sort_keys.begin()->quaternary; // start with the first quaternary for (auto e = sort_keys.begin(); e < sort_keys.end(); e++, q++) { - if (q != e->quaternary) { // something rearranged in this subrange + if (q != e->quaternary) { + // something rearranged in this subrange, because the quaternary values are out of order. applied = true; } + // collect the characters newSuffix.append(1, e->ch); } if (applied) { - // DebugLog("Final Sort"); - // for (auto e = sort_keys.begin(); e < sort_keys.end(); e++) { - // e->dump(); - // } str.resize(prefix.size()); str.append(newSuffix); } else { - // DebugLog("Skip: no reordering change detected"); + DebugTran("Skip: sorting caused no reordering"); } +#if KMXPLUS_DEBUG_TRANSFORM + DebugTran("Sorted sortkey"); + for (const auto &r : sort_keys) { + r.dump(); + } +#endif return applied; } -transform_entry::transform_entry(const std::u16string &from, const std::u16string &to) : fFrom(from), fTo(to) { +transform_entry::transform_entry(const std::u32string &from, const std::u32string &to) : fFrom(from), fTo(to) { } size_t -transform_entry::match(const std::u16string &input) const { +transform_entry::match(const std::u32string &input) const { if (input.length() < fFrom.length()) { + // TODO-LDML: regex + // Too small, can't match. return 0; } // string at end auto substr = input.substr(input.length() - fFrom.length(), fFrom.length()); if (substr != fFrom) { + // end of string doesn't match return 0; } + // match length == fFrom.length return substr.length(); } -std::u16string -transform_entry::apply(const std::u16string & /*input*/, size_t /*matchLen*/) const { +std::u32string +transform_entry::apply(const std::u32string & /*input*/, size_t /*matchLen*/) const { + // TODO-LDML: regex + // For now, we just return the 'to' string literally. return fTo; } @@ -326,7 +445,7 @@ transform_group::transform_group() { * return the first transform match in this group */ const transform_entry * -transform_group::match(const std::u16string &input, size_t &subMatched) const { +transform_group::match(const std::u32string &input, size_t &subMatched) const { for (auto transform = begin(); (subMatched == 0) && (transform < end()); transform++) { // TODO-LDML: non regex implementation // is the match area too short? @@ -346,7 +465,7 @@ transform_group::match(const std::u16string &input, size_t &subMatched) const { * @return match length: number of chars at end of input string to modify. 0 if no match. */ size_t -transforms::apply(const std::u16string &input, std::u16string &output) { +transforms::apply(const std::u32string &input, std::u32string &output) { /** * Example: * Group0: za -> c, a -> bb @@ -381,7 +500,7 @@ transforms::apply(const std::u16string &input, std::u16string &output) { */ size_t matched = 0; /** modified copy of input */ - std::u16string updatedInput = input; + std::u32string updatedInput = input; for (auto group = transform_groups.begin(); group < transform_groups.end(); group++) { // for each transform group // break out once there's a match @@ -393,13 +512,14 @@ transforms::apply(const std::u16string &input, std::u16string &output) { // find the first match in this group (if present) // TODO-LDML: check if reorder if (group->type == any_group_type::transform) { - auto transform = group->transform.match(updatedInput, subMatched); + auto entry = group->transform.match(updatedInput, subMatched); - if (transform != nullptr) { + if (entry != nullptr) { // now apply the found transform // update subOutput (string) and subMatched - std::u16string subOutput = transform->apply(updatedInput, subMatched); + // the returned string must replace the last "subMatched" chars of the string. + std::u32string subOutput = entry->apply(updatedInput, subMatched); // remove the matched part of the updatedInput updatedInput.resize(updatedInput.length() - subMatched); // chop of the subMatched part at end @@ -421,7 +541,17 @@ transforms::apply(const std::u16string &input, std::u16string &output) { } } } else if (group->type == any_group_type::reorder) { - // TODO-LDML reorder + // TODO-LDML: cheesy solution. We should be finding a smaller + // common match here. + std::u32string str2 = updatedInput; + if (group->reorder.apply(str2)) { + // pretend the whole thing matched + output.resize(0); + output.append(str2); + updatedInput.resize(0); + updatedInput.append(str2); + matched = output.length(); + } } // else: continue to next group } @@ -441,10 +571,10 @@ transforms::apply(const std::u16string &input, std::u16string &output) { return matched; } -// simple impl bool -transforms::apply(std::u16string &str) { - std::u16string output; +transforms::apply(std::u32string &str) { + // simple implementation for tests + std::u32string output; size_t matchLength = apply(str, output); if (matchLength == 0) { return false; @@ -454,23 +584,6 @@ transforms::apply(std::u16string &str) { return true; } -bool -transforms::apply(std::u32string &str) { - bool rc = false; - // TODO-LDML: PoC implementation for now, need to refactor into fcns - // ONLY reorder - for (auto group = transform_groups.begin(); group < transform_groups.end(); group++) { - assert(group->type == reorder); // TODO-LDML - auto rgroup = group->reorder; - if (rgroup.apply(str)) { - rc = true; - } - } - return rc; -} - -// Loader - transforms * transforms::load( const kmx::kmx_plus &kplus, @@ -516,17 +629,18 @@ transforms::load( for (KMX_DWORD itemNumber = 0; itemNumber < group->count; itemNumber++) { const kmx::COMP_KMXPLUS_TRAN_TRANSFORM *element = tranHelper.getTransform(group->index + itemNumber); - const std::u16string fromStr = kplus.strs->get(element->from); - const std::u16string toStr = kplus.strs->get(element->to); + const std::u32string fromStr = kmx::u16string_to_u32string(kplus.strs->get(element->from)); + const std::u32string toStr = kmx::u16string_to_u32string(kplus.strs->get(element->to)); std::u16string mapFrom, mapTo; if (element->mapFrom && element->mapTo) { - // strings: variable name + // strings: variable name of from/to + // TODO-LDML: not implemented mapFrom = kplus.strs->get(element->mapFrom); mapTo = kplus.strs->get(element->mapTo); } - newGroup.emplace_back(fromStr, toStr); // creating a transform_entry + newGroup.emplace_back(fromStr, toStr /* ,mapFrom, mapTo */); // creating a transform_entry } transforms->addGroup(newGroup); } else if (group->type == LDML_TRAN_GROUP_TYPE_REORDER) { @@ -544,6 +658,7 @@ transforms::load( if (load_ok) { newGroup.list.emplace_back(elements, before); } else { + DebugLog("reorder elements(%d+%d) failed to load", group->index, itemNumber); return nullptr; } } diff --git a/core/src/ldml/ldml_transforms.hpp b/core/src/ldml/ldml_transforms.hpp index 7904eeffc8..6296399f2b 100644 --- a/core/src/ldml/ldml_transforms.hpp +++ b/core/src/ldml/ldml_transforms.hpp @@ -24,7 +24,7 @@ using km::kbp::kmx::USet; * Type of a group */ enum any_group_type { - transform = LDML_TRAN_GROUP_TYPE_REORDER, + transform = LDML_TRAN_GROUP_TYPE_TRANSFORM, reorder = LDML_TRAN_GROUP_TYPE_REORDER, }; @@ -33,24 +33,30 @@ enum any_group_type { */ class element { public: - /** from a USet */ + /** construct from a USet */ element(const USet &u, KMX_DWORD flags); - /** from a single char */ + /** construct from a single char */ element(km_kbp_usv ch, KMX_DWORD flags); /** @returns true if a USet type */ bool is_uset() const; + /** @returns true if prebase bit set*/ bool is_prebase() const; + /** @returns true if tertiary base bit set */ bool is_tertiary_base() const; - signed char get_tertiary() const; + /** @returns the primary order */ signed char get_order() const; + /** @returns the tertiary order */ + signed char get_tertiary() const; /** @returns raw elem flags */ KMX_DWORD get_flags() const; /** @returns true if matches this character*/ bool matches(km_kbp_usv ch) const; + /** debugging: dump this element via DebugLog() */ + void dump() const; private: - // TODO-LDML: support multi-char strings + // TODO-LDML: support multi-char strings? const km_kbp_usv chr; const USet uset; const KMX_DWORD flags; @@ -62,30 +68,30 @@ private: class transform_entry { public: transform_entry( - const std::u16string &from, - const std::u16string &to + const std::u32string &from, + const std::u32string &to /*TODO-LDML: mapFrom, mapTo*/ ); /** * @returns length if it's a match */ - size_t match(const std::u16string &input) const; + size_t match(const std::u32string &input) const; /** * @returns output string */ - std::u16string apply(const std::u16string &input, size_t matchLen) const; + std::u32string apply(const std::u32string &input, size_t matchLen) const; private: - const std::u16string fFrom; // TODO-LDML: regex - const std::u16string fTo; + const std::u32string fFrom; // TODO-LDML: regex + const std::u32string fTo; }; /** * An ordered list of strings. */ -typedef std::deque string_list; +typedef std::deque string_list; /** * a group of entries - a @@ -100,7 +106,7 @@ public: * @param subMatched on output, the matched length * @returns alias to transform_entry or nullptr */ - const transform_entry *match(const std::u16string &input, size_t &subMatched) const; + const transform_entry *match(const std::u32string &input, size_t &subMatched) const; }; /** a single char, categorized according to reorder rules*/ @@ -111,13 +117,12 @@ struct reorder_sort_key { signed char tertiary; // tertiary value, defaults to 0 size_t quaternary; // index again - /** - * Return -1, 0, 1 depending on order - */ + /** @returns -1, 0, 1 depending on ordering */ int compare(const reorder_sort_key &other) const; bool operator<(const reorder_sort_key &other) const; + bool operator>(const reorder_sort_key &other) const; - /** create a 'baseline' sort key, all 0 primary weights */ + /** create a 'baseline' sort key, with each character having primary weight 0 */ static std::deque from(const std::u32string &str); /** TODO-LDML: for debugging. */ @@ -136,7 +141,7 @@ public: * Update the deque (see reorder_sort_key::from()) with the weights from this element list * starting at the beginning of this element list * @param offset start at this offset in the deque. Still starts at the first element - * @param the key deque to update + * @param key key deque to update * @returns the key parameter */ std::deque &update_sort_key(size_t offset, std::deque &key) const; @@ -144,6 +149,9 @@ public: /** construct from KMX+ elem id*/ bool load(const kmx::kmx_plus& kplus, kmx::KMXPLUS_ELEM id); + + /** TODO-LDML: for debugging */ + void dump() const; }; class reorder_entry { @@ -217,13 +225,7 @@ public: * @param output if matched, contains the replacement output text * @return length in chars of the input (counting from the end) which matched context */ - size_t apply(const std::u16string &input, std::u16string &output); - - /** - * For tests - * @return true if str was altered - */ - bool apply(std::u16string &str); + size_t apply(const std::u32string &input, std::u32string &output); /** * For tests - TODO-LDML only supports reorder @@ -232,12 +234,13 @@ public: bool apply(std::u32string &str); public: + /** load from a kmx_plus data section, either tran or bksp */ static transforms * - load(const kmx::kmx_plus &kplus, const kbp::kmx::COMP_KMXPLUS_TRAN *tran, const kbp::kmx::COMP_KMXPLUS_TRAN_Helper &tranHelper); + load(const kmx::kmx_plus &kplus, + const kbp::kmx::COMP_KMXPLUS_TRAN *tran, + const kbp::kmx::COMP_KMXPLUS_TRAN_Helper &tranHelper); }; -/** - * Loader for transform groups (from tran or bksp) - */ + } // namespace ldml } // namespace kbp } // namespace km diff --git a/core/tests/unit/kmnkbd/test_kmx_xstring.cpp b/core/tests/unit/kmnkbd/test_kmx_xstring.cpp index d578fa7e38..e3c4077622 100644 --- a/core/tests/unit/kmnkbd/test_kmx_xstring.cpp +++ b/core/tests/unit/kmnkbd/test_kmx_xstring.cpp @@ -1238,6 +1238,7 @@ test_xstrlen_ignoreifopt() { void test_utf32() { + std::cout << "== " << __FUNCTION__ << std::endl; const KMX_DWORD u295 = 0x0127; // ħ assert(Uni_IsBMP(u295)); @@ -1270,6 +1271,7 @@ test_utf32() { void test_u16string_to_u32string() { + std::cout << "== " << __FUNCTION__ << std::endl; // normal cases { const std::u32string str = u16string_to_u32string(u""); @@ -1327,6 +1329,56 @@ test_u16string_to_u32string() { } } +void test_is_valid() { + std::cout << "== " << __FUNCTION__ << std::endl; + // valid + assert_equal(Uni_IsValid(0x0000), true); + assert_equal(Uni_IsValid(0x0127), true); + assert_equal(Uni_IsValid(U'🙀'), true); + + // invalid + assert_equal(Uni_IsValid(0xDECAFBAD), false); // out of range + assert_equal(Uni_IsValid(0x566D4128), false); + assert_equal(Uni_IsValid(0xFFFF), false); // nonchar + assert_equal(Uni_IsValid(0xFFFE), false); // nonchar + assert_equal(Uni_IsValid(0x10FFFF), false); // nonchar + assert_equal(Uni_IsValid(0x10FFFE), false); // nonchar + assert_equal(Uni_IsValid(0x01FFFF), false); // nonchar + assert_equal(Uni_IsValid(0x01FFFE), false); // nonchar + assert_equal(Uni_IsValid(0x02FFFF), false); // nonchar + assert_equal(Uni_IsValid(0x02FFFE), false); // nonchar + assert_equal(Uni_IsValid(0xFDD1), false); // nonchar + assert_equal(Uni_IsValid(0xFDD0), false); // nonchar + + + // positive range test + assert_equal(Uni_IsValid(0x100000, 0x10FFFD), true); + assert_equal(Uni_IsValid(0x10, 0x20), true); + assert_equal(Uni_IsValid(0x100000, 0x10FFFD), true); + + // all valid ranges in BMP + assert_equal(Uni_IsValid(0x0000, 0xD7FF), true); + assert_equal(Uni_IsValid(0xD800, 0xDFFF), false); + assert_equal(Uni_IsValid(0xE000, 0xFDCF), true); + assert_equal(Uni_IsValid(0xFDD0, 0xFDEF), false); + assert_equal(Uni_IsValid(0xFDF0, 0xFDFF), true); + assert_equal(Uni_IsValid(0xFDF0, 0xFFFD), true); + + // negative range test + assert_equal(Uni_IsValid(0, 0x10FFFF), false); // ends with nonchar + assert_equal(Uni_IsValid(0, 0x10FFFD), false); // contains lots o' nonchars + assert_equal(Uni_IsValid(0x20, 0x10), false); // swapped + assert_equal(Uni_IsValid(0xFDEF, 0xFDF0), false); // just outside range + assert_equal(Uni_IsValid(0x0000, 0x010000), false); // crosses noncharacter plane boundary and other stuff + assert_equal(Uni_IsValid(0x010000, 0x020000), false); // crosses noncharacter plane boundary + assert_equal(Uni_IsValid(0x0000, 0xFFFF), false); // crosses other BMP prohibited and plane boundary + assert_equal(Uni_IsValid(0x0000, 0xFFFD), false); // crosses other BMP prohibited + assert_equal(Uni_IsValid(0x0000, 0xE000), false); // crosses surrogate space + assert_equal(Uni_IsValid(0x0000, 0x20FFFF), false); // out of bounds + assert_equal(Uni_IsValid(0x10FFFD, 0x20FFFF), false); // out of bounds + + } + constexpr const auto help_str = u"\ test_kmx_xstring [--color]\n\ \n\ @@ -1349,6 +1401,7 @@ int main(int argc, char *argv []) { test_xstrlen_ignoreifopt(); test_utf32(); test_u16string_to_u32string(); + test_is_valid(); return 0; } diff --git a/core/tests/unit/ldml/keyboards/fr-t-k0-azerty-test.xml b/core/tests/unit/ldml/keyboards/k_020_fr-test.xml similarity index 85% rename from core/tests/unit/ldml/keyboards/fr-t-k0-azerty-test.xml rename to core/tests/unit/ldml/keyboards/k_020_fr-test.xml index 21f9d64cc4..04066ae220 100644 --- a/core/tests/unit/ldml/keyboards/fr-t-k0-azerty-test.xml +++ b/core/tests/unit/ldml/keyboards/k_020_fr-test.xml @@ -1,10 +1,11 @@ + - + - + diff --git a/core/tests/unit/ldml/keyboards/fr-t-k0-azerty.xml b/core/tests/unit/ldml/keyboards/k_020_fr.xml similarity index 94% rename from core/tests/unit/ldml/keyboards/fr-t-k0-azerty.xml rename to core/tests/unit/ldml/keyboards/k_020_fr.xml index 8a634bda65..1f6fc2c32f 100644 --- a/core/tests/unit/ldml/keyboards/fr-t-k0-azerty.xml +++ b/core/tests/unit/ldml/keyboards/k_020_fr.xml @@ -1,4 +1,5 @@ + @@ -7,11 +8,11 @@ - + - - + + @@ -22,15 +23,15 @@ - + - + - + diff --git a/core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana-test.xml b/core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana-test.xml index 5fcfd321e2..d23862ca2b 100644 --- a/core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana-test.xml +++ b/core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana-test.xml @@ -3,16 +3,30 @@ - + + + + + + + + + + - + diff --git a/core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana.xml b/core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana.xml index da16c18717..8df0d5ffd7 100644 --- a/core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana.xml +++ b/core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana.xml @@ -13,16 +13,20 @@ + + + + - + diff --git a/core/tests/unit/ldml/keyboards/meson.build b/core/tests/unit/ldml/keyboards/meson.build index b4ba719548..3d1bfdddcc 100644 --- a/core/tests/unit/ldml/keyboards/meson.build +++ b/core/tests/unit/ldml/keyboards/meson.build @@ -9,8 +9,8 @@ # tests in resources/standards-data/ldml-keyboards/techpreview/test/ tests_from_cldr = [ 'ja-Latn', - 'pt-k0-abnt2', - # 'fr-t-k0-azerty', # vkey issues + # 'pt-k0-abnt2', #TODO-LDML: marker syntax fail! + 'fr-t-k0-azerty', ] tests_without_testdata = [ @@ -32,7 +32,7 @@ tests_without_testdata = [ # These tests have a k_001_tiny-test.xml file as well. tests_with_testdata = [ 'k_001_tiny', - 'fr-t-k0-azerty', # TODO-LDML: move to cldr above (fix vkey) + 'k_020_fr', # TODO-LDML: move to cldr above (fix vkey) 'k_200_reorder_nod_Lana', ] diff --git a/core/tests/unit/ldml/test_transforms.cpp b/core/tests/unit/ldml/test_transforms.cpp index c72fd6bcf8..7097bcb64b 100644 --- a/core/tests/unit/ldml/test_transforms.cpp +++ b/core/tests/unit/ldml/test_transforms.cpp @@ -40,7 +40,7 @@ test_transforms() { std::cout << __FILE__ << ":" << __LINE__ << " - basic " << std::endl; { // start with one - transform_entry te(std::u16string(u"e^"), std::u16string(u"E")); // keep it simple + transform_entry te(std::u32string(U"e^"), std::u32string(U"E")); // keep it simple // OK now make a group do it transforms tr; transform_group st; @@ -51,17 +51,17 @@ test_transforms() { // see if we can match the same { - std::u16string src(u"barQ^"); + std::u32string src(U"barQ^"); bool res = tr.apply(src); zassert_equal(res, false); - zassert_string_equal(src, std::u16string(u"barQ^")); // no change + zassert_string_equal(src, std::u32string(U"barQ^")); // no change } { - std::u16string src(u"fooe^"); + std::u32string src(U"fooe^"); bool res = tr.apply(src); zassert_equal(res, true); - zassert_string_equal(src, std::u16string(u"fooE")); + zassert_string_equal(src, std::u32string(U"fooE")); } } @@ -73,23 +73,23 @@ test_transforms() { // setup { transform_group st; - st.emplace_back(std::u16string(u"za"), std::u16string(u"c")); - st.emplace_back(std::u16string(u"a"), std::u16string(u"bb")); + st.emplace_back(std::u32string(U"za"), std::u32string(U"c")); + st.emplace_back(std::u32string(U"a"), std::u32string(U"bb")); tr.addGroup(st); } { transform_group st; - st.emplace_back(std::u16string(u"bb"), std::u16string(u"ccc")); + st.emplace_back(std::u32string(U"bb"), std::u32string(U"ccc")); tr.addGroup(st); } { transform_group st; - st.emplace_back(std::u16string(u"cc"), std::u16string(u"d")); + st.emplace_back(std::u32string(U"cc"), std::u32string(U"d")); tr.addGroup(st); } { transform_group st; - st.emplace_back(std::u16string(u"tcd"), std::u16string(u"e")); + st.emplace_back(std::u32string(U"tcd"), std::u32string(U"e")); tr.addGroup(st); } @@ -97,31 +97,31 @@ test_transforms() { // see if we can match the same { - std::u16string src(u"ta"); + std::u32string src(U"ta"); bool res = tr.apply(src); // pipe (|) symbol shows where the 'output' is delineated // t|a --> t|bb --> t|ccc --> t|cd --> |e - zassert_string_equal(src, std::u16string(u"e")); + zassert_string_equal(src, std::u32string(U"e")); zassert_equal(res, true); } { - std::u16string src(u"qza"); + std::u32string src(U"qza"); bool res = tr.apply(src); // pipe (|) symbol shows where the 'output' is delineated // q|za -> q|c - zassert_string_equal(src, std::u16string(u"qc")); + zassert_string_equal(src, std::u32string(U"qc")); zassert_equal(res, true); } { - std::u16string src(u"qa"); + std::u32string src(U"qa"); bool res = tr.apply(src); - zassert_string_equal(src, std::u16string(u"qcd")); + zassert_string_equal(src, std::u32string(U"qcd")); zassert_equal(res, true); } { - std::u16string src(u"tb"); + std::u32string src(U"tb"); bool res = tr.apply(src); - zassert_string_equal(src, std::u16string(u"tb")); + zassert_string_equal(src, std::u32string(U"tb")); zassert_equal(res, false); } } @@ -132,13 +132,13 @@ test_transforms() { transforms tr; { transform_group st; - st.emplace_back(std::u16string(u"िह"), std::u16string(u"हि")); + st.emplace_back(std::u32string(U"िह"), std::u32string(U"हि")); tr.addGroup(st); } { - std::u16string src(u"िह"); + std::u32string src(U"िह"); bool res = tr.apply(src); - zassert_string_equal(src, std::u16string(u"हि")); + zassert_string_equal(src, std::u32string(U"हि")); zassert_equal(res, true); } } @@ -163,7 +163,7 @@ test_reorder_standalone() { const std::u32string expect = roasts[0]; // now setup the rules const COMP_KMXPLUS_USET_RANGE ranges[] = {// 0 - {0x1A75, 0x1A79}}; + COMP_KMXPLUS_USET_RANGE(0x1A75, 0x1A79)}; const COMP_KMXPLUS_USET_USET usets[] = {{0, 1, 0xFFFFFFFF}}; const COMP_KMXPLUS_USET_USET &toneMarksUset = usets[0]; const USet toneMarks(&ranges[toneMarksUset.range], toneMarksUset.count); @@ -288,7 +288,7 @@ test_reorder_standalone() { // element_list e0; - e0.emplace_back(U'\u1A6B', 127 << LDML_ELEM_FLAGS_ORDER_BITSHIFT); + e0.emplace_back(U'\u1A60', 127 << LDML_ELEM_FLAGS_ORDER_BITSHIFT); rg.list.emplace_back(e0); // @@ -334,19 +334,41 @@ test_reorder_standalone() { std::cout << __FILE__ << ":" << __LINE__ << " - back to nod-Lana " << std::endl; // TODO-LDML: move this into test code perhaps for (size_t r = 0; r < sizeof(roasts) / sizeof(roasts[0]); r++) { - std::cout << __FILE__ << ":" << __LINE__ << " - trying roast #" << r << std::endl; const auto &roast = roasts[r]; + std::cout << __FILE__ << ":" << __LINE__ << " - trying roast #" << r << "=" << roast << std::endl; + // try apply with string + { + std::cout << "- try apply(text, output)" << std::endl; + std::u32string text = roast; + std::u32string output; + size_t len = tr.apply(text, output); + if (len == 0) { + std::cout << " (did not apply)" << std::endl; + } else { + std::cout << " applied, matchLen= " << len << std::endl; + text.resize(text.size()-len); // shrink + text.append(output); + std::cout << " = " << text << std::endl; + } + zassert_string_equal(text, expect); + } // try all-at-once { + std::cout << "- try apply(text)" << std::endl; std::u32string text = roast; if (!tr.apply(text)) { std::cout << " (did not apply)" << std::endl; + } else if (text == roast) { + std::cout << " (suboptimal: apply returned true but made no change)" << std::endl; + } else { + std::cout << " changed to " << text; } zassert_string_equal(text, expect); std::cout << " matched (converting all at once)!" << std::endl; } // simulate typing this one char at a time; { + std::cout << "- try key-at-a-time" << std::endl; std::u32string text; for (auto ch = roast.begin(); ch < roast.end(); ch++) { // append the string @@ -362,6 +384,24 @@ test_reorder_standalone() { std::cout << std::endl; } } + // special test + { + std::cout << __FILE__ << ":" << __LINE__ << " - special test " << std::endl; + const std::u32string expect = U"\u1A21\u1A60\u1A45"; // this string shouldn't mutate at all. + { + std::u32string text = expect; + tr.apply(text); + zassert_string_equal(text, expect); + } + { + // try submatch + std::u32string text = expect; + std::u32string output; + size_t len = tr.apply(text, output); + zassert_string_equal(output, U""); + assert_equal(len, 0); + } + } } return EXIT_SUCCESS; } diff --git a/developer/src/kmc-kmn/src/compiler/compiler.ts b/developer/src/kmc-kmn/src/compiler/compiler.ts index 9d0da39740..33d1b545f5 100644 --- a/developer/src/kmc-kmn/src/compiler/compiler.ts +++ b/developer/src/kmc-kmn/src/compiler/compiler.ts @@ -89,12 +89,18 @@ let callbackProcIdentifier = 0; const callbackPrefix = 'kmnCompilerCallbacks_'; +interface MallocAndFree { + malloc(sz: number) : number; + free(p: number) : null; +}; + let Module: any; export class KmnCompiler implements UnicodeSetParser { callbackID: string; // a unique numeric id added to globals with prefixed names callbacks: CompilerCallbacks; + wasmExports: MallocAndFree; constructor() { this.callbackID = callbackPrefix + callbackProcIdentifier.toString(); @@ -112,7 +118,7 @@ export class KmnCompiler implements UnicodeSetParser { return false; } } - + this.wasmExports = (Module.wasmExports ?? Module.asm); return this.verifyInitialized(); } @@ -427,25 +433,23 @@ export class KmnCompiler implements UnicodeSetParser { return null; } - const buf = Module.asm.malloc(rangeCount * 2 * Module.HEAPU32.BYTES_PER_ELEMENT); // TODO-LDML: Catch OOM - /** return code, if positive: range count */ + const buf = this.wasmExports.malloc(rangeCount * 2 * Module.HEAPU32.BYTES_PER_ELEMENT); + /** If <= 0: return code. If positive: range count */ const rc = Module.kmcmp_parseUnicodeSet(pattern, buf, rangeCount * 2); if (rc >= 0) { const ranges = []; const startu = (buf / Module.HEAPU32.BYTES_PER_ELEMENT); for (let i = 0; i < rc; i++) { - const low = Module.HEAPU32[startu + (i * 2) + 0]; - const high = Module.HEAPU32[startu + (i * 2) + 1]; - ranges.push([low, high]); + const start = Module.HEAPU32[startu + (i * 2) + 0]; + const end = Module.HEAPU32[startu + (i * 2) + 1]; + ranges.push([start, end]); } - // TODO-LDML: no free?? - // Module.asm.free(buf); + this.wasmExports.free(buf); return new UnicodeSet(pattern, ranges); } else { - // translate error - // TODO-LDML: no free?? - // Module.asm.free(buf); + this.wasmExports.free(buf); + // translate error code into callback this.callbacks.reportMessage(getUnicodeSetError(rc)); return null; } @@ -455,6 +459,7 @@ export class KmnCompiler implements UnicodeSetParser { /* c8 ignore next 2 */ return null; } + // call with rangeCount = 0 to invoke in 'preflight' mode. const rc = Module.kmcmp_parseUnicodeSet(pattern, 0, 0); if (rc >= 0) { return rc; diff --git a/developer/src/kmc-kmn/test/test-wasm-uset.ts b/developer/src/kmc-kmn/test/test-wasm-uset.ts index 2a05c24f26..a924f9fe05 100644 --- a/developer/src/kmc-kmn/test/test-wasm-uset.ts +++ b/developer/src/kmc-kmn/test/test-wasm-uset.ts @@ -65,12 +65,25 @@ describe('Compiler UnicodeSet function', function() { '[[]': CompilerMessages.ERROR_UnicodeSetSyntaxError, }; for(const [pat, expected] of Object.entries(failures)) { - callbacks.clear(); - assert.notOk(compiler.parseUnicodeSet(pat, 1)); - assert.equal(callbacks.messages.length, 1); - const firstMessage = callbacks.messages[0]; - const code = firstMessage.code; - assert.equal(code, expected, `${compilerErrorFormatCode(code)}≠${compilerErrorFormatCode(expected)} got ${firstMessage.message} for ${pat}`); + { + // verify fails parse + callbacks.clear(); + assert.notOk(compiler.parseUnicodeSet(pat, 1)); + assert.equal(callbacks.messages.length, 1); + const firstMessage = callbacks.messages[0]; + const code = firstMessage.code; + assert.equal(code, expected, `${compilerErrorFormatCode(code)}≠${compilerErrorFormatCode(expected)} got ${firstMessage.message} for parsing ${pat}`); + } + // skip 'out of range' because that one won't fail during sizing. + if (expected !== CompilerMessages.FATAL_UnicodeSetOutOfRange) { + // verify fails size + callbacks.clear(); + assert.equal(compiler.sizeUnicodeSet(pat), -1, `sizing ${pat}`); + assert.equal(callbacks.messages.length, 1); + const firstMessage = callbacks.messages[0]; + const code = firstMessage.code; + assert.equal(code, expected, `${compilerErrorFormatCode(code)}≠${compilerErrorFormatCode(expected)} got ${firstMessage.message} for sizing ${pat}`); + } } }); }); diff --git a/developer/src/kmc-ldml/src/compiler/disp.ts b/developer/src/kmc-ldml/src/compiler/disp.ts index 1ed2e14b14..241c5af685 100644 --- a/developer/src/kmc-ldml/src/compiler/disp.ts +++ b/developer/src/kmc-ldml/src/compiler/disp.ts @@ -1,5 +1,5 @@ import { constants } from "@keymanapp/ldml-keyboard-constants"; -import { KMXPlus } from '@keymanapp/common-types'; +import { KMXPlus, LDMLKeyboard, MarkerParser } from '@keymanapp/common-types'; import { CompilerMessages } from "./messages.js"; import { SectionCompiler } from "./section-compiler.js"; @@ -7,8 +7,14 @@ import { SectionCompiler } from "./section-compiler.js"; import DependencySections = KMXPlus.DependencySections; import Disp = KMXPlus.Disp; import DispItem = KMXPlus.DispItem; +import { MarkerTracker, MarkerUse } from "./marker-tracker.js"; export class DispCompiler extends SectionCompiler { + static validateMarkers(keyboard: LDMLKeyboard.LKKeyboard, mt : MarkerTracker): boolean { + keyboard.displays?.display?.forEach(({ to }) => + mt.add(MarkerUse.match, MarkerParser.allReferences(to))); + return true; + } public get id() { return constants.section.disp; @@ -38,9 +44,11 @@ export class DispCompiler extends SectionCompiler { // displayOptions result.baseCharacter = sections.strs.allocAndUnescapeString(this.keyboard.displays?.displayOptions?.baseCharacter); + // TODO-LDML: substitute variables! + // displays result.disps = this.keyboard.displays?.display.map(display => ({ - to: sections.strs.allocAndUnescapeString(display.to), + to: sections.strs.allocAndUnescapeString(sections.vars.substituteMarkerString(display.to)), display: sections.strs.allocAndUnescapeString(display.display), })) || []; // TODO-LDML: need coverage for the [] diff --git a/developer/src/kmc-ldml/src/compiler/empty-compiler.ts b/developer/src/kmc-ldml/src/compiler/empty-compiler.ts index e167c784b2..93476d4497 100644 --- a/developer/src/kmc-ldml/src/compiler/empty-compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/empty-compiler.ts @@ -1,6 +1,7 @@ import { SectionIdent, constants } from '@keymanapp/ldml-keyboard-constants'; import { SectionCompiler } from "./section-compiler.js"; import { LDMLKeyboard, KMXPlus, CompilerCallbacks } from "@keymanapp/common-types"; +import { VarsCompiler } from './vars.js'; /** * Compiler for typrs that don't actually consume input XML @@ -69,6 +70,6 @@ export class UsetCompiler extends EmptyCompiler { } /** - * For test use. The top three compilers. + * For test use. The top compilers. */ -export const BASIC_DEPENDENCIES = [ StrsCompiler, ListCompiler, ElemCompiler ]; +export const BASIC_DEPENDENCIES = [ StrsCompiler, ListCompiler, ElemCompiler, VarsCompiler ]; diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index b3e13b70fc..89d93178d3 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -1,5 +1,5 @@ import { constants } from '@keymanapp/ldml-keyboard-constants'; -import { LDMLKeyboard, KMXPlus, Constants } from '@keymanapp/common-types'; +import { LDMLKeyboard, KMXPlus, Constants, MarkerParser } from '@keymanapp/common-types'; import { CompilerMessages } from './messages.js'; import { SectionCompiler } from "./section-compiler.js"; @@ -8,8 +8,18 @@ import Keys = KMXPlus.Keys; import ListItem = KMXPlus.ListItem; import KeysFlicks = KMXPlus.KeysFlicks; import { allUsedKeyIdsInLayers, calculateUniqueKeys, translateLayerAttrToModifier, validModifier } from '../util/util.js'; +import { MarkerTracker, MarkerUse } from './marker-tracker.js'; export class KeysCompiler extends SectionCompiler { + static validateMarkers( + keyboard: LDMLKeyboard.LKKeyboard, + mt: MarkerTracker + ): boolean { + keyboard.keys?.key?.forEach(({ to }) => + mt.add(MarkerUse.emit, MarkerParser.allReferences(to)) + ); + return true; + } public get id() { return constants.section.keys; @@ -20,7 +30,7 @@ export class KeysCompiler extends SectionCompiler { * @returns just the non-touch layers. */ public hardwareLayers() { - return this.keyboard.layers?.filter(({form}) => form !== 'touch'); + return this.keyboard.layers?.filter(({ form }) => form !== "touch"); } public validate() { @@ -30,7 +40,7 @@ export class KeysCompiler extends SectionCompiler { const usedKeys = allUsedKeyIdsInLayers(this.keyboard?.layers); const uniqueKeys = calculateUniqueKeys([...this.keyboard.keys?.key]); for (let key of uniqueKeys) { - const {id, flicks} = key; + const { id, flicks } = key; if (!usedKeys.has(id)) { continue; // unused key, ignore } @@ -38,10 +48,14 @@ export class KeysCompiler extends SectionCompiler { if (!flicks) { continue; // no flicks } - const flickEntry = this.keyboard.keys?.flicks?.find(x => x.id === flicks); - if (!flickEntry ) { + const flickEntry = this.keyboard.keys?.flicks?.find( + (x) => x.id === flicks + ); + if (!flickEntry) { valid = false; - this.callbacks.reportMessage(CompilerMessages.Error_MissingFlicks({flicks, id})); + this.callbacks.reportMessage( + CompilerMessages.Error_MissingFlicks({ flicks, id }) + ); } } @@ -53,8 +67,9 @@ export class KeysCompiler extends SectionCompiler { if (hardwareLayers.length >= 1) { // validate all errors for (let layers of hardwareLayers) { - for(let layer of layers.layer) { - valid = this.validateHardwareLayerForKmap(layers.form, layer) && valid; // note: always validate even if previously invalid results found + for (let layer of layers.layer) { + valid = + this.validateHardwareLayerForKmap(layers.form, layer) && valid; // note: always validate even if previously invalid results found } } // TODO-LDML: } else { touch? @@ -84,11 +99,13 @@ export class KeysCompiler extends SectionCompiler { /* c8 ignore next 3 */ if (hardwareLayers.length > 1) { // validation should have already caught this - throw Error(`Internal error: Expected 0 or 1 hardware layer, not ${hardwareLayers.length}`); + throw Error( + `Internal error: Expected 0 or 1 hardware layer, not ${hardwareLayers.length}` + ); } else if (hardwareLayers.length === 1) { const theLayers = hardwareLayers[0]; const { form } = theLayers; - for(let layer of theLayers.layer) { + for (let layer of theLayers.layer) { this.compileHardwareLayerToKmap(sections, layer, sect, form); } } // else: TODO-LDML do nothing if only touch layers @@ -98,7 +115,9 @@ export class KeysCompiler extends SectionCompiler { public loadFlicks(sections: DependencySections, sect: Keys) { for (let lkflicks of this.keyboard.keys.flicks) { - let flicks: KeysFlicks = new KeysFlicks(sections.strs.allocString(lkflicks.id)); + let flicks: KeysFlicks = new KeysFlicks( + sections.strs.allocString(lkflicks.id) + ); for (let lkflick of lkflicks.flick) { let flags = 0; @@ -106,10 +125,14 @@ export class KeysCompiler extends SectionCompiler { if (!to.isOneChar) { flags |= constants.keys_flick_flags_extend; } - let directions: ListItem = sections.list.allocListFromSpaces(sections.strs, lkflick.directions); + let directions: ListItem = sections.list.allocListFromSpaces( + sections.strs, + lkflick.directions + ); flicks.flicks.push({ directions, flags, + // TODO-LDML: markers,variables to, }); } @@ -132,19 +155,33 @@ export class KeysCompiler extends SectionCompiler { if (!!key.gap) { flags |= constants.keys_key_flags_gap; } - if (key.transform === 'no') { + if (key.transform === "no") { flags |= constants.keys_key_flags_notransform; } const id = sections.strs.allocString(key.id); - const longPress: ListItem = sections.list.allocListFromEscapedSpaces(sections.strs, key.longPress); - const longPressDefault = sections.strs.allocAndUnescapeString(key.longPressDefault); - const multiTap: ListItem = sections.list.allocListFromEscapedSpaces(sections.strs, key.multiTap); + const longPress: ListItem = sections.list.allocListFromEscapedSpaces( + sections.strs, + // TODO-LDML: markers,variables + key.longPress + ); + const longPressDefault = sections.strs.allocAndUnescapeString( + // TODO-LDML: markers,variables + key.longPressDefault + ); + const multiTap: ListItem = sections.list.allocListFromEscapedSpaces( + sections.strs, + // TODO-LDML: markers,variables + key.multiTap + ); const keySwitch = sections.strs.allocString(key.switch); // 'switch' is a reserved word - const to = sections.strs.allocAndUnescapeString(key.to, true); + const toRaw = key.to; + // TODO-LDML: variables + let toCooked = sections.vars.substituteMarkerString(toRaw); + const to = sections.strs.allocAndUnescapeString(toCooked, true); if (!to.isOneChar) { flags |= constants.keys_key_flags_extend; } - const width = Math.ceil((key.width || 1) * 10.0); // default, width=1 + const width = Math.ceil((key.width || 1) * 10.0); // default, width=1 sect.keys.push({ flags, flicks, @@ -166,12 +203,17 @@ export class KeysCompiler extends SectionCompiler { * @param layer * @returns */ - private validateHardwareLayerForKmap(hardware: string, layer: LDMLKeyboard.LKLayer) { + private validateHardwareLayerForKmap( + hardware: string, + layer: LDMLKeyboard.LKLayer + ) { let valid = true; const { modifier } = layer; if (!validModifier(modifier)) { - this.callbacks.reportMessage(CompilerMessages.Error_InvalidModifier({ modifier, layer: layer.id })); + this.callbacks.reportMessage( + CompilerMessages.Error_InvalidModifier({ modifier, layer: layer.id }) + ); valid = false; } @@ -179,21 +221,31 @@ export class KeysCompiler extends SectionCompiler { /* c8 ignore next 5 */ if (!keymap) { // not reached due to XML validation - this.callbacks.reportMessage(CompilerMessages.Error_InvalidHardware({ form: hardware })); + this.callbacks.reportMessage( + CompilerMessages.Error_InvalidHardware({ form: hardware }) + ); valid = false; } const uniqueKeys = calculateUniqueKeys([...this.keyboard.keys?.key]); if (layer.row.length > keymap.length) { - this.callbacks.reportMessage(CompilerMessages.Error_HardwareLayerHasTooManyRows()); + this.callbacks.reportMessage( + CompilerMessages.Error_HardwareLayerHasTooManyRows() + ); valid = false; } for (let y = 0; y < layer.row.length && y < keymap.length; y++) { - const keys = layer.row[y].keys.split(' '); + const keys = layer.row[y].keys.split(" "); if (keys.length > keymap[y].length) { - this.callbacks.reportMessage(CompilerMessages.Error_RowOnHardwareLayerHasTooManyKeys({ row: y + 1, hardware, modifier })); + this.callbacks.reportMessage( + CompilerMessages.Error_RowOnHardwareLayerHasTooManyKeys({ + row: y + 1, + hardware, + modifier, + }) + ); valid = false; } @@ -201,14 +253,24 @@ export class KeysCompiler extends SectionCompiler { for (let key of keys) { x++; - let keydef = uniqueKeys.find(x => x.id == key); + let keydef = uniqueKeys.find((x) => x.id == key); if (!keydef) { - this.callbacks.reportMessage(CompilerMessages.Error_KeyNotFoundInKeyBag({ keyId: key, col: x + 1, row: y + 1, layer: layer.id, form: 'hardware' })); + this.callbacks.reportMessage( + CompilerMessages.Error_KeyNotFoundInKeyBag({ + keyId: key, + col: x + 1, + row: y + 1, + layer: layer.id, + form: "hardware", + }) + ); valid = false; continue; } if (!keydef.to && !keydef.gap && !keydef.switch) { - this.callbacks.reportMessage(CompilerMessages.Error_KeyMissingToGapOrSwitch({ keyId: key })); + this.callbacks.reportMessage( + CompilerMessages.Error_KeyMissingToGapOrSwitch({ keyId: key }) + ); valid = false; continue; } @@ -222,7 +284,7 @@ export class KeysCompiler extends SectionCompiler { sections: DependencySections, layer: LDMLKeyboard.LKLayer, sect: Keys, - hardware: string, + hardware: string ): Keys { const mod = translateLayerAttrToModifier(layer); const keymap = Constants.HardwareToKeymap.get(hardware); @@ -231,7 +293,7 @@ export class KeysCompiler extends SectionCompiler { for (let row of layer.row) { y++; - const keys = row.keys.split(' '); + const keys = row.keys.split(" "); let x = -1; for (let key of keys) { x++; diff --git a/developer/src/kmc-ldml/src/compiler/marker-tracker.ts b/developer/src/kmc-ldml/src/compiler/marker-tracker.ts new file mode 100644 index 0000000000..8007f29e5b --- /dev/null +++ b/developer/src/kmc-ldml/src/compiler/marker-tracker.ts @@ -0,0 +1,72 @@ +/** + * Verb for MarkerTracker.add() + */ +export enum MarkerUse { + /** outputs this marker into context (e.g. transform to= or key to=) */ + emit, + /** consumes this marker out of the context (e.g. transform from=) */ + consume, + /** matches the marker, but doesn't consume (e.g. display to=) */ + match, + /** variable definition: might consume, emit, or match. */ + variable, +} + +type MarkerSet = Set; + +/** Tracks usage of markers */ +export class MarkerTracker { + /** markers that were emitted */ + emitted: MarkerSet; + /** markers that were consumed and removed from the context */ + consumed: MarkerSet; + /** markers that were matched, but not necessarily consumed */ + matched: MarkerSet; + /** all markers */ + all: MarkerSet; + + constructor() { + this.emitted = new Set(); + this.consumed = new Set(); + this.matched = new Set(); + this.all = new Set(); + } + + /** + * + * @param verb what kind of use we are adding + * @param markers list of markers to add + */ + add(verb: MarkerUse, markers: string[]) { + if (!markers.length) { + return; // skip if empty + } + if (verb == MarkerUse.emit) { + markers.forEach((m) => { + this.emitted.add(m); + this.all.add(m); + }); + } else if (verb == MarkerUse.consume) { + markers.forEach((m) => { + this.consumed.add(m); + this.all.add(m); + }); + } else if (verb == MarkerUse.match) { + markers.forEach((m) => { + this.matched.add(m); + this.all.add(m); + }); + } else if (verb == MarkerUse.variable) { + markers.forEach((m) => { + // we don't know, so add it to all three + this.matched.add(m); + this.emitted.add(m); + this.consumed.add(m); + this.all.add(m); + }); + /* c8 skip next 3 */ + } else { + throw Error(`Internal error: unsupported verb ${verb} for match`); + } + } +} diff --git a/developer/src/kmc-ldml/src/compiler/messages.ts b/developer/src/kmc-ldml/src/compiler/messages.ts index f0feb6f599..e3ec8bcbf2 100644 --- a/developer/src/kmc-ldml/src/compiler/messages.ts +++ b/developer/src/kmc-ldml/src/compiler/messages.ts @@ -2,7 +2,7 @@ import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler; -// const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.KeyboardCompiler; +// const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevError = CompilerErrorSeverity.Error | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevFatal = CompilerErrorSeverity.Fatal | CompilerErrorNamespace.LdmlKeyboardCompiler; @@ -35,9 +35,9 @@ export class CompilerMessages { m(this.HINT_LocaleIsNotMinimalAndClean, `Locale '${o.sourceLocale}' is not minimal or correctly formatted and should be '${o.locale}'`); static HINT_LocaleIsNotMinimalAndClean = SevHint | 0x0008; - static Error_VkeyIsNotValid = (o:{vkey: string}) => - m(this.ERROR_VkeyIsNotValid, `Virtual key '${o.vkey}' is not found in the CLDR VKey Enum table.`); - static ERROR_VkeyIsNotValid = SevError | 0x0009; + static Hint_VkeyIsNotValid = (o:{vkey: string}) => + m(this.HINT_VkeyIsNotValid, `Virtual key '${o.vkey}' is not found in the CLDR VKey Enum table.`); + static HINT_VkeyIsNotValid = SevHint | 0x0009; static Hint_VkeyIsRedundant = (o:{vkey: string}) => m(this.HINT_VkeyIsRedundant, `Virtual key '${o.vkey}' is mapped to itself, which is redundant.`); @@ -130,5 +130,9 @@ export class CompilerMessages { static Error_CantReferenceSetFromUnicodeSet = (o:{id: string}) => m(this.ERROR_CantReferenceSetFromUnicodeSet, `Illegal use of set variable from within UnicodeSet: \$[${o.id}]`); static ERROR_CantReferenceSetFromUnicodeSet = SevError | 0x0020; + + static Error_MissingMarkers = (o: { ids: string[] }) => + m(this.ERROR_MissingMarkers, `Markers used for matching but not defined: ${o.ids?.join(',')}`); + static ERROR_MissingMarkers = SevError | 0x0021; } diff --git a/developer/src/kmc-ldml/src/compiler/section-compiler.ts b/developer/src/kmc-ldml/src/compiler/section-compiler.ts index 5150108e6c..9e187fafe6 100644 --- a/developer/src/kmc-ldml/src/compiler/section-compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/section-compiler.ts @@ -32,7 +32,8 @@ export class SectionCompiler { const defaults = new Set([ constants.section.strs, constants.section.list, - constants.section.elem + constants.section.elem, + constants.section.vars, ]); return defaults; } diff --git a/developer/src/kmc-ldml/src/compiler/tran.ts b/developer/src/kmc-ldml/src/compiler/tran.ts index d0173b7bfb..fa1f828622 100644 --- a/developer/src/kmc-ldml/src/compiler/tran.ts +++ b/developer/src/kmc-ldml/src/compiler/tran.ts @@ -1,5 +1,5 @@ import { constants, SectionIdent } from "@keymanapp/ldml-keyboard-constants"; -import { KMXPlus, LDMLKeyboard, CompilerCallbacks, VariableParser } from '@keymanapp/common-types'; +import { KMXPlus, LDMLKeyboard, CompilerCallbacks, VariableParser, MarkerParser } from '@keymanapp/common-types'; import { SectionCompiler } from "./section-compiler.js"; import Bksp = KMXPlus.Bksp; @@ -15,10 +15,21 @@ import LKTransform = LDMLKeyboard.LKTransform; import LKTransforms = LDMLKeyboard.LKTransforms; import { verifyValidAndUnique } from "../util/util.js"; import { CompilerMessages } from "./messages.js"; +import { MarkerTracker, MarkerUse } from "./marker-tracker.js"; type TransformCompilerType = 'simple' | 'backspace'; -class TransformCompiler extends SectionCompiler { +export class TransformCompiler extends SectionCompiler { + + static validateMarkers(keyboard: LDMLKeyboard.LKKeyboard, mt : MarkerTracker): boolean { + keyboard?.transforms?.forEach(transforms => + transforms.transformGroup.forEach(transformGroup => { + transformGroup.transform?.forEach(({ to, from }) => { + mt.add(MarkerUse.emit, MarkerParser.allReferences(to)); + mt.add(MarkerUse.consume, MarkerParser.allReferences(from)); + })})); + return true; + } protected type: T; @@ -130,6 +141,10 @@ class TransformCompiler cookedTo = sections.vars.substituteStrings(cookedTo, sections); } + // add in markers. idempotent if no markers. + cookedFrom = sections.vars.substituteMarkerString(cookedFrom); // TODO-LDML: need to support \m{.} here, maybe other edge cases + cookedTo = sections.vars.substituteMarkerString(cookedTo); + result.from = sections.strs.allocAndUnescapeString(cookedFrom); // TODO-LDML: not unescaped here, done previously result.to = sections.strs.allocAndUnescapeString(cookedTo); // TODO-LDML: not unescaped here, done previously return result; diff --git a/developer/src/kmc-ldml/src/compiler/vars.ts b/developer/src/kmc-ldml/src/compiler/vars.ts index 19451c4db4..b6e6136750 100644 --- a/developer/src/kmc-ldml/src/compiler/vars.ts +++ b/developer/src/kmc-ldml/src/compiler/vars.ts @@ -1,5 +1,5 @@ import { SectionIdent, constants } from "@keymanapp/ldml-keyboard-constants"; -import { KMXPlus, LDMLKeyboard, CompilerCallbacks } from '@keymanapp/common-types'; +import { KMXPlus, LDMLKeyboard, CompilerCallbacks, MarkerParser } from '@keymanapp/common-types'; import { VariableParser } from '@keymanapp/common-types'; import { SectionCompiler } from "./section-compiler.js"; import Vars = KMXPlus.Vars; @@ -9,6 +9,10 @@ import UnicodeSetItem = KMXPlus.UnicodeSetItem; import DependencySections = KMXPlus.DependencySections; import LDMLKeyboardXMLSourceFile = LDMLKeyboard.LDMLKeyboardXMLSourceFile; import { CompilerMessages } from "./messages.js"; +import { KeysCompiler } from "./keys.js"; +import { TransformCompiler } from "./tran.js"; +import { DispCompiler } from "./disp.js"; +import { MarkerTracker, MarkerUse } from "./marker-tracker.js"; export class VarsCompiler extends SectionCompiler { public get id() { return constants.section.vars; @@ -17,7 +21,8 @@ export class VarsCompiler extends SectionCompiler { public get dependencies(): Set { const defaults = new Set([ constants.section.strs, - constants.section.elem + constants.section.elem, + constants.section.list, ]); defaults.delete(this.id); return defaults; @@ -29,7 +34,6 @@ export class VarsCompiler extends SectionCompiler { public validate(): boolean { let valid = true; - // TODO-LDML scan for markers? // Check for duplicate ids const allIds = new Set(); @@ -130,14 +134,61 @@ export class VarsCompiler extends SectionCompiler { })); valid = false; } + + valid = this.validateMarkers() && valid; // accumulate validity + return valid; } + private collectMarkers(mt : MarkerTracker) : boolean { + let valid = true; + + // call our friends to validate + valid = this.validateVarsMarkers(this.keyboard, mt) && valid; // accumulate validity + valid = KeysCompiler.validateMarkers(this.keyboard, mt) && valid; // accumulate validity + valid = TransformCompiler.validateMarkers(this.keyboard, mt) && valid; // accumulate validity + valid = DispCompiler.validateMarkers(this.keyboard, mt) && valid; // accumulate validity + + return valid; + } + + private validateMarkers(): boolean { + const mt = new MarkerTracker(); + let valid = this.collectMarkers(mt); + // see if there are any matched-but-not-emitted + const matchedNotEmitted : Set = new Set(); + for (const m of mt.matched.values()) { + if (m === MarkerParser.ANY_MARKER_ID) continue; // match-all marker + if (!mt.emitted.has(m)) { + matchedNotEmitted.add(m); + } + } + for (const m of mt.consumed.values()) { + if (m === MarkerParser.ANY_MARKER_ID) continue; // match-all marker + if (!mt.emitted.has(m)) { + matchedNotEmitted.add(m); + } + } + + // report once + if (matchedNotEmitted.size > 0) { + this.callbacks.reportMessage(CompilerMessages.Error_MissingMarkers({ ids: Array.from(matchedNotEmitted.values()).sort() })); + valid = false; + } + return valid; + } + + validateVarsMarkers(keyboard: LDMLKeyboard.LKKeyboard, mt : MarkerTracker) : boolean { + keyboard?.variables?.string?.forEach(({value}) => + mt.add(MarkerUse.variable, MarkerParser.allReferences(value))); + return true; + } + public compile(sections: DependencySections): Vars { const result = new Vars(); const variables = this.keyboard?.variables; - + // we always have vars, it's depended on by other sections if (!variables) return result; // Empty vars, to simplify other sections // we already know the variables do not conflict with each other @@ -153,6 +204,14 @@ export class VarsCompiler extends SectionCompiler { variables?.unicodeSet?.forEach((e) => this.addUnicodeSet(result, e, sections)); + // reload markers - TODO-LDML: double work! + const mt = new MarkerTracker(); + this.collectMarkers(mt); + + // collect all markers, excluding the match-all + const allMarkers : string[] = Array.from(mt.all).filter(m => m !== MarkerParser.ANY_MARKER_ID).sort(); + result.markers = sections.list.allocList(sections.strs, allMarkers); + return result.valid() ? result : null; } diff --git a/developer/src/kmc-ldml/src/compiler/visual-keyboard-compiler.ts b/developer/src/kmc-ldml/src/compiler/visual-keyboard-compiler.ts index 33ecf182ab..d328f8c336 100644 --- a/developer/src/kmc-ldml/src/compiler/visual-keyboard-compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/visual-keyboard-compiler.ts @@ -40,6 +40,10 @@ export class LdmlKeyboardVisualKeyboardCompiler { let keydef = source.keyboard.keys?.key?.find(x => x.id == key); + if (!keydef) { + throw Error(`Internal Error: could not find key id="${key}" in layer "${layer.id || ''}", row "${y}"`); + } + vk.keys.push({ flags: VisualKeyboard.VisualKeyboardKeyFlags.kvkkUnicode, shift: shift, diff --git a/developer/src/kmc-ldml/src/compiler/vkey.ts b/developer/src/kmc-ldml/src/compiler/vkey.ts index 1b7d71f300..2d6d44d9d0 100644 --- a/developer/src/kmc-ldml/src/compiler/vkey.ts +++ b/developer/src/kmc-ldml/src/compiler/vkey.ts @@ -19,13 +19,15 @@ export class VkeyCompiler extends SectionCompiler { this.keyboard.vkeys.vkey.forEach(vk => { if(LdmlVkeyNames[vk.from] === undefined) { - this.callbacks.reportMessage(CompilerMessages.Error_VkeyIsNotValid({vkey: vk.from})); - valid = false; + // TODO-LDML: When we do #7135 this may need to change back to an error. + this.callbacks.reportMessage(CompilerMessages.Hint_VkeyIsNotValid({vkey: vk.from})); + return; } if(LdmlVkeyNames[vk.to] === undefined) { - this.callbacks.reportMessage(CompilerMessages.Error_VkeyIsNotValid({vkey: vk.to})); - valid = false; + // TODO-LDML: When we do #7135 this may need to change back to an error. + this.callbacks.reportMessage(CompilerMessages.Hint_VkeyIsNotValid({vkey: vk.to})); + return; } if(vk.from == vk.to) { diff --git a/developer/src/kmc-ldml/test/fixtures/basic.txt b/developer/src/kmc-ldml/test/fixtures/basic.txt index 0fe6d67a30..cbbb403891 100644 --- a/developer/src/kmc-ldml/test/fixtures/basic.txt +++ b/developer/src/kmc-ldml/test/fixtures/basic.txt @@ -257,7 +257,7 @@ block(keys) # struct COMP_KMXPLUS_KEYS { index(strNull,strHmaqtugha,2) # KMXPLUS_STR 'hmaqtugha' 00 00 00 00 # KMXPLUS_STR switch 0A 00 00 00 # KMX_DWORD width*10 - 01 00 00 00 # TODO: index(listNull,indexAe,4) # LIST longPress 'a e' + 02 00 00 00 # TODO: index(listNull,indexAe,4) # LIST longPress 'a e' 00 00 00 00 # STR longPressDefault 00 00 00 00 # TODO: index(listNull,listNull,4) # LIST multiTap 00 00 00 00 # flicks 0 @@ -324,21 +324,25 @@ block(layr) # struct COMP_KMXPLUS_LAYR { block(list) # struct COMP_KMXPLUS_LAYR_LIST { 6c 69 73 74 # KMX_DWORD header.ident; // 0000 Section name - list diff(list,endList) # KMX_DWORD header.size; // 0004 Section length - 02 00 00 00 # KMX_DWORD listCount (should be 2) - 02 00 00 00 # KMX_DWORD indexCount (should be 2) + 03 00 00 00 # KMX_DWORD listCount (should be 2) + 03 00 00 00 # KMX_DWORD indexCount (should be 2) # list #0 the null list block(listNull) 00 00 00 00 #index(indexNull,indexNull,2) # KMX_DWORD list index (0) 00 00 00 00 # KMX_DWORD lists[0].count - # list #1 the ae list + block(listA) + 00 00 00 00 # first index + 01 00 00 00 #count block(listAe) - 00 00 00 00 # index(indexAe,indexNull,2) # KMX_DWORD list index (also 0) + 01 00 00 00 # index(indexAe,indexNull,2) # KMX_DWORD list index (also 0) 02 00 00 00 # KMX_DWORD count block(endLists) # indices #block(indexNull) # No null index # index(strNull,strNull,2) # KMXPLUS_STR string index + block(indexA) + index(strNull,strA,2) # a block(indexAe) index(strNull,strA,2) # KMXPLUS_STR a index(strNull,strElemBkspFrom2,2) # KMXPLUS_STR e @@ -401,6 +405,7 @@ block(strs) # struct COMP_KMXPLUS_STRS { diff(strs,strName) sizeof(strName,2) diff(strs,strFromSet) sizeof(strFromSet,2) diff(strs,strUSet) sizeof(strUSet,2) + diff(strs,strAmarker) sizeof(strAmarker,2) diff(strs,strElemTranFrom1) sizeof(strElemTranFrom1,2) diff(strs,strElemTranFrom1a) sizeof(strElemTranFrom1a,2) diff(strs,strElemTranFrom1b) sizeof(strElemTranFrom1b,2) @@ -421,6 +426,7 @@ block(strs) # struct COMP_KMXPLUS_STRS { diff(strs,strTranTo) sizeof(strTranTo,2) diff(strs,strKeys) sizeof(strKeys,2) diff(strs,strIndicator) sizeof(strIndicator,2) + diff(strs,strSentinel0001) sizeof(strSentinel0001,2) # String table -- block(x) is used to store the null u16char at end of each string @@ -433,6 +439,7 @@ block(strs) # struct COMP_KMXPLUS_STRS { block(strName) 54 00 65 00 73 00 74 00 4b 00 62 00 64 00 block(x) 00 00 # 'TestKbd' block(strFromSet) 5B 00 5C 00 75 00 31 00 41 00 37 00 35 00 2D 00 5C 00 75 00 31 00 41 00 37 00 39 00 5D 00 block(x) 00 00 # [\u1a75-\u1a79] block(strUSet) 5b 00 61 00 62 00 63 00 5d 00 block(x) 00 00 # '[abc]' + block(strAmarker) 5C 00 6D 00 7B 00 61 00 7D 00 block(x) 00 00 # '\m{a}' block(strElemTranFrom1) 5E 00 block(x) 00 00 # '^' block(strElemTranFrom1a) 5E 00 61 00 block(x) 00 00 # '^a' block(strElemTranFrom1b) 5E 00 65 00 block(x) 00 00 # '^e' @@ -458,7 +465,7 @@ block(strs) # struct COMP_KMXPLUS_STRS { block(strKeys) 90 17 b6 17 block(x) 00 00 # 'ថា' # block(strIndicator) 3d d8 40 de block(x) 00 00 # '🙀' - + block(strSentinel0001) FF FF 01 00 block(x) 00 00 # U+FFFF U+0001 @@ -478,10 +485,15 @@ block(tran) # struct COMP_KMXPLUS_TRAN { block(tranGroupStart) # COMP_KMXPLUS_TRAN_GROUP # group 0 00 00 00 00 # KMX_DWORD type = transform - 01 00 00 00 # KMX_DWORD count + 02 00 00 00 # KMX_DWORD count diff(tranTransformStart,tranTransform0,16) # KMX_DWORD index # group 1 + 00 00 00 00 # KMX_DWORD type = transform + 01 00 00 00 # KMX_DWORD count + diff(tranTransformStart,tranTransform2,16) # KMX_DWORD index + + # group 2 01 00 00 00 # KMX_DWORD type = reorder 01 00 00 00 # KMX_DWORD count diff(tranReorderStart,tranReorder0,8) # KMX_DWORD index @@ -494,6 +506,18 @@ block(tran) # struct COMP_KMXPLUS_TRAN { index(strNull,strNull,2) # mapFrom index(strNull,strNull,2) # mapTo + block(tranTransform1) + index(strNull,strA,2) # KMXPLUS_STR from; 'a' + index(strNull,strSentinel0001,2) # KMXPLUS_STR to; \m{a} + index(strNull,strNull,2) # mapFrom + index(strNull,strNull,2) # mapTo + + block(tranTransform2) # Next group + index(strNull,strSentinel0001,2) # KMXPLUS_STR from; (\m{a}) + index(strNull,strNull,2) # KMXPLUS_STR to; (none) + index(strNull,strNull,2) # mapFrom + index(strNull,strNull,2) # mapTo + # reorders block(tranReorderStart) # COMP_KMXPLUS_TRAN_REORDER block(tranReorder0) @@ -520,23 +544,29 @@ block(uset) block(vars) # struct COMP_KMXPLUS_VARS { 76 61 72 73 # KMX_DWORD header.ident; // 0000 Section name - vars diff(vars,varsEnd) # KMX_DWORD header.size; // 0004 Section length - 00 00 00 00 # KMX_DWORD markers - list + 01 00 00 00 # KMX_DWORD markers - list 1 ['a'] diff(varsBegin,varsEnd,16) # KMX_DWORD varCount - # var 0 block(varsBegin) + # var 0 + 00 00 00 00 # KMX_DWORD type = str + index(strNull,strA,2) # KMXPLUS_STR id 'a' + index(strNull,strAmarker,2) # KMXPLUS_STR value '\m{a}' + 00 00 00 00 # KMXPLUS_ELEM + + # var 1 01 00 00 00 # KMX_DWORD type = set index(strNull,strVse,2) # KMXPLUS_STR id 'vse' index(strNull,strSet,2) # KMXPLUS_STR value 'a b c' 01 00 00 00 # KMXPLUS_ELEM elem 'a b c' see 'elemSet' - # var 1 + # var 2 00 00 00 00 # KMX_DWORD type = string index(strNull,strVst,2) # KMXPLUS_STR id 'vst' index(strNull,strSet2,2) # KMXPLUS_STR value 'abc' 00 00 00 00 # KMXPLUS_ELEM elem - # var 2 + # var 3 02 00 00 00 # KMX_DWORD type = string index(strNull,strVus,2) # KMXPLUS_STR id 'vus' index(strNull,strUSet,2) # KMXPLUS_STR value '[abc]' diff --git a/developer/src/kmc-ldml/test/fixtures/basic.xml b/developer/src/kmc-ldml/test/fixtures/basic.xml index 0eb6676490..2d345324d9 100644 --- a/developer/src/kmc-ldml/test/fixtures/basic.xml +++ b/developer/src/kmc-ldml/test/fixtures/basic.xml @@ -37,6 +37,7 @@ + @@ -45,6 +46,11 @@ + + + + + diff --git a/developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml b/developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml new file mode 100644 index 0000000000..fd63c36b02 --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/fixtures/sections/vars/markers-maximal.xml b/developer/src/kmc-ldml/test/fixtures/sections/vars/markers-maximal.xml new file mode 100644 index 0000000000..cbe36d4db1 --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/sections/vars/markers-maximal.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/helpers/index.ts b/developer/src/kmc-ldml/test/helpers/index.ts index 38e30b967e..db0704bf75 100644 --- a/developer/src/kmc-ldml/test/helpers/index.ts +++ b/developer/src/kmc-ldml/test/helpers/index.ts @@ -19,6 +19,7 @@ import DependencySections = KMXPlus.DependencySections; import Section = KMXPlus.Section; import { ElemCompiler, ListCompiler, StrsCompiler } from '../../src/compiler/empty-compiler.js'; import { KmnCompiler } from '@keymanapp/kmc-kmn'; +import { VarsCompiler } from '../../src/compiler/vars.js'; // import Vars = KMXPlus.Vars; /** @@ -92,7 +93,7 @@ async function loadDepsFor(sections: DependencySections, parentCompiler: Section const parentId = parentCompiler.id; if (!dependencies) { // default dependencies - dependencies = [ StrsCompiler, ListCompiler, ElemCompiler ]; + dependencies = [ StrsCompiler, ListCompiler, ElemCompiler, VarsCompiler ]; } for (const dep of dependencies) { const compiler = new dep(source, callbacks); diff --git a/developer/src/kmc-ldml/test/test-tran.ts b/developer/src/kmc-ldml/test/test-tran.ts index 6e25cb4ac8..ec77ab02f2 100644 --- a/developer/src/kmc-ldml/test/test-tran.ts +++ b/developer/src/kmc-ldml/test/test-tran.ts @@ -1,7 +1,6 @@ import 'mocha'; import { assert } from 'chai'; import { TranCompiler, BkspCompiler } from '../src/compiler/tran.js'; -import { VarsCompiler } from '../src/compiler/vars.js'; import { BASIC_DEPENDENCIES, UsetCompiler } from '../src/compiler/empty-compiler.js'; import { CompilerMessages } from '../src/compiler/messages.js'; import { compilerTestCallbacks, testCompilationCases } from './helpers/index.js'; @@ -10,7 +9,7 @@ import { KMXPlus } from '@keymanapp/common-types'; import Tran = KMXPlus.Tran;// for tests… import Bksp = KMXPlus.Bksp;// for tests… import { constants } from '@keymanapp/ldml-keyboard-constants'; -const tranDependencies = [ ...BASIC_DEPENDENCIES, UsetCompiler, VarsCompiler ]; +const tranDependencies = [ ...BASIC_DEPENDENCIES, UsetCompiler ]; const bkspDependencies = tranDependencies; describe('tran', function () { diff --git a/developer/src/kmc-ldml/test/test-vars.ts b/developer/src/kmc-ldml/test/test-vars.ts index a3cb303b82..b18f546f27 100644 --- a/developer/src/kmc-ldml/test/test-vars.ts +++ b/developer/src/kmc-ldml/test/test-vars.ts @@ -5,6 +5,9 @@ import { CompilerMessages } from '../src/compiler/messages.js'; import { CompilerMessages as KmnCompilerMessages } from '@keymanapp/kmc-kmn'; import { testCompilationCases } from './helpers/index.js'; import { KMXPlus } from '@keymanapp/common-types'; +import { BASIC_DEPENDENCIES } from '../src/compiler/empty-compiler.js'; +// now that 'everything' depends on vars, we need an explicit dependency here +const varsDependencies = BASIC_DEPENDENCIES.filter(c => c !== VarsCompiler); import Vars = KMXPlus.Vars; @@ -182,5 +185,32 @@ describe('vars', function () { CompilerMessages.Error_MissingStringVariable({id: 'missingStringInSet'}) ], }, - ]); + ], varsDependencies); + describe('markers', function () { + this.slow(500); // 0.5 sec -- json schema validation takes a while + + testCompilationCases(VarsCompiler, [ + { + subpath: 'sections/vars/markers-maximal.xml', + callback(sect) { + const vars = sect; + assert.ok(vars.markers); + assert.sameDeepOrderedMembers(vars.markers.toStringArray(), + ['m','x']); + }, + }, + { + subpath: 'sections/vars/fail-markers-badref-0.xml', + errors: [ + CompilerMessages.Error_MissingMarkers({ + ids: [ + 'doesnt_exist_1', + 'doesnt_exist_2', + 'doesnt_exist_3', + ] + }), + ], + }, + ], varsDependencies); + }); }); diff --git a/developer/src/kmc-ldml/test/test-vkey.ts b/developer/src/kmc-ldml/test/test-vkey.ts index 03c27cbd3f..dec24125e2 100644 --- a/developer/src/kmc-ldml/test/test-vkey.ts +++ b/developer/src/kmc-ldml/test/test-vkey.ts @@ -37,19 +37,19 @@ describe('vkey compiler', function () { assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Info_MultipleVkeysHaveSameTarget({vkey: 'Q'})); }); - it('should error on invalid "from" vkey', async function() { + it('should hint on invalid "from" vkey', async function() { let vkey = await loadSectionFixture(VkeyCompiler, 'sections/vkey/invalid-from-vkey.xml', compilerTestCallbacks) as Vkey; - assert.isNull(vkey); + assert.isNotNull(vkey); assert.equal(compilerTestCallbacks.messages.length, 2); - assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Error_VkeyIsNotValid({vkey: 'q'})); - assert.deepEqual(compilerTestCallbacks.messages[1], CompilerMessages.Error_VkeyIsNotValid({vkey: 'HYFEN'})); + assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Hint_VkeyIsNotValid({vkey: 'q'})); + assert.deepEqual(compilerTestCallbacks.messages[1], CompilerMessages.Hint_VkeyIsNotValid({vkey: 'HYFEN'})); }); - it('should error on invalid "to" vkey', async function() { + it('should hint on invalid "to" vkey', async function() { let vkey = await loadSectionFixture(VkeyCompiler, 'sections/vkey/invalid-to-vkey.xml', compilerTestCallbacks) as Vkey; - assert.isNull(vkey); + assert.isNotNull(vkey); assert.equal(compilerTestCallbacks.messages.length, 1); - assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Error_VkeyIsNotValid({vkey: 'A-ACUTE'})); + assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Hint_VkeyIsNotValid({vkey: 'A-ACUTE'})); }); it('should error on repeated vkeys', async function() { diff --git a/developer/src/kmcmplib/src/meson.build b/developer/src/kmcmplib/src/meson.build index 4e6059afbd..e3e4efcd48 100644 --- a/developer/src/kmcmplib/src/meson.build +++ b/developer/src/kmcmplib/src/meson.build @@ -38,13 +38,22 @@ if cpp_compiler.get_id() == 'emscripten' links += [ # Forcing inclusion of debug symbols '-g', '-Wlimited-postlink-optimizations', - '-fwasm-exceptions', '--bind', '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\']'] + sanitize + '-fwasm-exceptions', '--bind'] + sanitize # Alternative to sanitize is adding '-sASSERTIONS', '-sSAFE_HEAP' to lib_links: # lib_links += ['-sASSERTIONS', '-sSAFE_HEAP'] # For profiling, add: # lib_links += ['--profiling-funcs', '-sDEMANGLE_SUPPORT=1'] + + if cpp_compiler.version().version_compare('>=3.1.44') + # emscripten 3.1.44 removes .asm object and so we need to export `wasmExports` + # #9375; https://github.com/emscripten-core/emscripten/blob/main/ChangeLog.md#3144---072523 + links += ['-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\',\'wasmExports\']'] + else + # emscripten < 3.1.44 does not include `wasmExports` + links += ['-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\']'] + endif endif icu = subproject('icu-for-uset', default_options: [ 'default_library=static', 'cpp_std=c++17', 'warning_level=0', 'werror=false']) diff --git a/docs/build/linux-ubuntu.md b/docs/build/linux-ubuntu.md index ee45d4eb4a..3c4f4c57c1 100644 --- a/docs/build/linux-ubuntu.md +++ b/docs/build/linux-ubuntu.md @@ -4,28 +4,30 @@ On Linux, you can build the following projects: -* [Keyman for Linux](#keyman-for-linux) -* [Keyman Core](#keyman-core) (Linux only) (aka core) -* [Keyman for Android](#keyman-for-android) - -* Keyman Core (wasm targets) -* Common/Web -* KeymanWeb +- [Keyman Core](#keyman-core) (aka core) +- [Keyman for Linux](#keyman-for-linux) +- [Keyman Web](#keyman-web) +- [Keyman for Android](#keyman-for-android) + +- Common/Web The following projects **cannot** be built on Linux: -* Keyman for Windows -* Keyman Developer -* Keyman for macOS -* Keyman for iOS +- Keyman for Windows +- Keyman Developer +- Keyman for macOS +- Keyman for iOS -## System Requirements +## Requirements -* Minimum Ubuntu version: Ubuntu 20.04 +### System Requirements + +- Minimum Ubuntu version: Ubuntu 20.04 Other Linux distributions will also work if appropriate dependencies are installed. -## Repository Paths +### Repository Paths Recommended filesystem layout: @@ -38,7 +40,7 @@ $HOME/keyman/ ... ``` -## Prerequisites +### Prerequisites The current list of dependencies can be found in the `Build-Depends` section of `linux/debian/control`. They are most easily installed with the `mk-build-deps` tool: @@ -49,101 +51,139 @@ sudo apt install devscripts equivs sudo mk-build-deps --install linux/debian/control ``` -### Node.js +#### Node.js -Node.js v18 is required for Core build, Web tests, and Developer command line tools. +Node.js v18 is required for Core builds, Web builds, and Developer command line tool builds and usage. -## Keyman for Linux +You can install it with: -All dependencies are already installed if you followed the instructions under [Prerequisites](#Prerequisites). +```shell +curl -sL https://deb.nodesource.com/setup_18.x | bash +apt-get -q -y install nodejs +``` -Building: +#### Emscripten -* [Building Keyman for Linux](../../linux/README.md) +You'll also have to install `emscripten` (version 3.1.44 is known to work): + +```shell +git clone https://github.com/emscripten-core/emsdk.git +cd emsdk +./emsdk install latest +./emsdk activate latest +export EMSCRIPTEN_BASE=$(pwd)/upstream/emscripten +``` + +**NOTE:** Don't put EMSDK on the path, i.e. don't source `emsdk_env.sh`. ## Keyman Core -All dependencies are already installed if you followed the instructions under [Prerequisites](#Prerequisites). +All dependencies are already installed if you followed the instructions under +[Prerequisites](#prerequisites). -Building: +### Building Keyman Core -* [Building Keyman Core](../../core/doc/BUILDING.md) +Keyman Core can be built with the `core/build.sh` script. -## Docker Builder +- [Building Keyman Core](../../core/doc/BUILDING.md) -The Docker builder allows you to perform a linux build from anywhere Docker is supported. -To build the docker image: +## Keyman for Linux -```shell -cd linux -docker pull ubuntu:latest -docker build . -t keymanapp/keyman-linux-builder:latest +All dependencies are already installed if you followed the instructions +under [Prerequisites](#prerequisites). + +### Building Keyman for Linux + +Keyman for Linux can be built with the `linux/build.sh` script. + +- [Building Keyman for Linux](../../linux/README.md) + +## Keyman Web + +Most dependencies are already installed if you followed the instructions under +[Prerequisites](#prerequisites). You'll still have to install Chrome: + +```bash +wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb +sudo apt install ./google-chrome-stable_current_amd64.deb ``` -Once the image is built, it may be used to build parts of Keyman. +And add the `CHROME_BIN` environment variable to `.bashrc: -- core - -```shell -# build 'core' in docker -cd ../core -# keep linux build artifacts separate -mkdir -p build/linux -docker run -it --rm -v $(pwd)/..:/home/build -v $(pwd)/build/linux:/home/build/core/build keymanapp/keyman-linux-builder:latest bash -c 'core/build.sh --debug' +```bash +export CHROME_BIN=/opt/google/chrome/chrome ``` -- linux +### Environment variables for Keyman Web -```shell -# build 'linux' installation in docker -cd keymanapp/keyman -docker run -it --rm -v $(pwd):/home/build/src/keyman -w /home/build/src/keyman keymanapp/keyman-linux-builder:latest bash -c "DESTDIR=. linux/build.sh --debug build install" -``` +`CHROME_BIN` pointing to the Google Chrome binary. + +### Building Keyman Web + +Keyman Web can be built with the `web/build.sh` script. + +- [Building Keyman Web](../../web/README.md) ## Keyman for Android **Dependencies:** -* [Base](#base-dependencies) -* [Web](./windows#web-dependencies) +Most dependencies are already installed if you followed the instructions +under [Prerequisites](#prerequisites). **Additional requirements:** -* Android SDK -* [Android Studio](https://developer.android.com/studio/install#linux) -* Gradle -* Maven -* OpenJDK 11 (for Keyman 17.0+) -* pandoc +- [Android Studio](https://developer.android.com/studio/install#linux) + or sdkmanager +- Maven +- pandoc +- Android SDK +- Gradle +- jq + +If you only use the command line you don't need Android Studio, however +to do development it's recommended to install it. Run Android Studio once after installation to install additional components such as emulator images and SDK updates. +Maven, jq and pandoc can be installed with: + +```shell +sudo apt update +sudo apt install maven pandoc jq +``` + +If necessary, Android SDK and Gradle will be installed by the build script. +In order for that to work, run the following command once. You won't need +this if you install Android SDK through Android Studio. + +```shell +sudo apt install sdkmanager +sudo sdkmanager platform-tools +sudo chown -R $USER:$USER /opt/android-sdk/ +sdkmanager --licenses +``` + +### Environment variables for Keyman for Android + **Required environment variable:** -* `ANDROID_HOME` pointing to Android SDK (`$HOME/Android/Sdk`) +- `ANDROID_HOME` pointing to Android SDK (`$HOME/Android/Sdk`) **Recommended environment variable:** -* [`JAVA_HOME`](#java_home) +- [`JAVA_HOME`](#java_home) -Building: +### Building Keyman for Android -* [Building Keyman for Android](../../android/README.md) +Keyman for Android can be built with the `android/build.sh` script. -## Prerequisites +- [Building Keyman for Android](../../android/README.md) -Many dependencies are only required for specific projects. +### Notes on Environment Variables -### Base Dependencies - -**Environment variables:** - -* -- - -## Notes on Environment Variables - -### JAVA_HOME +#### JAVA_HOME This environment variable tells Gradle what version of Java to use for building Keyman for Android. OpenJDK 11 is used for master. @@ -166,3 +206,59 @@ older versions, you can set `JAVA_HOME_11` to the OpenJDK 11 path and from command line. But note that you do need to update your `JAVA_HOME` env var to the associated version before opening Android Studio and loading any Android projects. `JAVA_HOME_11` is mostly used by CI. + +## Docker Builder + +The Docker builder allows you to perform a build from anywhere Docker is supported. + +To build the docker image: + +```shell +cd linux +docker pull ubuntu:latest +docker build . -t keymanapp/keyman-linux-builder:latest +``` + +Once the image is built, it may be used to build parts of Keyman. + +**Note** that it's not yet possible to run tests in the Docker container. + +- core + + ```shell + # build 'Keyman Core' in docker + # keep linux build artifacts separate + mkdir -p $(git rev-parse --show-toplevel)/core/build/linux + docker run -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \ + -v $(git rev-parse --show-toplevel)/core/build/linux:/home/build/build/core/build \ + keymanapp/keyman-linux-builder:latest \ + core/build.sh --debug + ``` + +- linux + + ```shell + # build 'Keyman for Linux' installation in docker + docker run -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \ + --entrypoint /bin/bash keymanapp/keyman-linux-builder:latest \ + -c 'DESTDIR=/home/build /usr/bin/bashwrapper linux/build.sh --debug build install' + ``` + +- Keyman Web + + ```shell + # build 'Keyman Web' in docker + docker run --privileged -it --rm \ + -v $(git rev-parse --show-toplevel):/home/build/build \ + keymanapp/keyman-linux-builder:latest \ + web/build.sh --debug + ``` + +- Keyman for Android + + ```shell + # build 'Keyman for Android' in docker + docker run -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \ + keymanapp/keyman-linux-builder:latest \ + android/build.sh --debug + ``` diff --git a/docs/build/windows.md b/docs/build/windows.md index fd66445b81..9faebc3d63 100644 --- a/docs/build/windows.md +++ b/docs/build/windows.md @@ -153,6 +153,7 @@ choco install git jq python ninja pandoc refreshenv # choco meson (0.55) is too old, 1.0 required: python -m pip install meson +``` **Environment variables**: * [`KEYMAN_ROOT`](#keyman_root) diff --git a/docs/linux/README.md b/docs/linux/README.md index 18e65699ad..d848e215ed 100644 --- a/docs/linux/README.md +++ b/docs/linux/README.md @@ -2,27 +2,18 @@ ## Projects -- [keyman-config](../../linux/keyman-config) - km-config and some other tools to install, uninstall - and view information about Keyman keyboard packages. +- [keyman-config](../../linux/keyman-config) - `km-config` and some other tools + to install, uninstall and view information about Keyman keyboard packages. - [ibus-keyman](../../linux/ibus-keyman) - IBUS integration to use .kmp Keyman keyboards +- [keyman-system-service](../../linux/keyman-system-service) - A DBus system service + that allows to perform keyboard related actions when running under Wayland. - [core](../../core) - common keyboardprocessor library See [license information](../../linux/LICENSE.md) about licensing. ## Linux Requirements/Setup -- It is helpful to be using the [packages.sil.org](http://packages.sil.org) repo - -- Install packages required for building and developing Keyman for Linux. - The list of required packages can be seen in `linux/debian/control`. - It is easiest to use the `mk-build-deps` tool to install the - dependencies: - - ```bash - sudo apt update - sudo apt install devscripts equivs - sudo mk-build-deps --install linux/debian/control - ``` +See [document in ../build](../build/linux-ubuntu.md). ## Compiling from Command Line @@ -61,15 +52,18 @@ for details on building Linux packages for Keyman. ## Testing -### keyman-config - -The unit tests can be run with the following command: +The tests can be run with the following command: ```bash -cd linux/keyman-config -./run-tests.sh +linux/build.sh test ``` +To just run the unit tests without integration tests, add the +`--no-integration` parameter. + +It's also possible to only run the tests for one of the subprojects. You +can use `build.sh` in the subdirectory for that. + ### ibus-keyman If you want to run the ibus-keyman tests with Wayland, you'll have to diff --git a/docs/linux/packaging.md b/docs/linux/packaging.md index bfb6a9d13d..c133575d04 100644 --- a/docs/linux/packaging.md +++ b/docs/linux/packaging.md @@ -10,7 +10,7 @@ We use different channels to build and distribute the Linux packages: [alpha](https://launchpad.net/~keymanapp/+archive/ubuntu/keyman-alpha) versions - [pso](http://packages.sil.org/) and [llso](http://linux.lsdev.sil.org/ubuntu/) for stable, beta, and alpha versions -- artifacts on [Jenkins](https://jenkins.lsdev.sil.org/view/Keyman/view/Pipeline/job/pipeline-keyman-packaging/view/change-requests/) +- artifacts on [GitHub](https://github.com/keymanapp/keyman/actions/workflows/deb-packaging.yml) for pull requests Packages on [llso](http://linux.lsdev.sil.org/ubuntu/) are uploaded automatically and are @@ -21,191 +21,63 @@ pso enabled. ## Package builds Package builds happen on [Launchpad](#package-builds-on-launchpad) and -[Jenkins](#package-builds-on-jenkins). Package builds for the official Ubuntu/Debian +[GitHub](#github-actions-package-builds). Package builds for the official Ubuntu/Debian repos happen outside of our control. However, we [upload source packages](#uploading-debian-source-packages) to the Debian community. -## Package builds on Jenkins +## GitHub Actions package builds ### Build jobs -The definition of the packaging jobs, the triggering of the jobs and the necessary build scripts -are scattered over several source repos: +The [Keyman GitHub repo](https://github.com/keymanapp/keyman) contains various +scripts that are used to trigger a build and as part of the package build, +and of course the source code for the packages: -- [ci-builder-scripts](https://github.com/sillsdev/ci-builder-scripts) contains the definition of - a meta job (multi-branch pipeline job) that gets triggered when a change gets pushed to the - [Keyman GitHub repo](https://github.com/keymanapp/keyman). The meta job creates a new build - configuration/job for each branch/pull request on GitHub. The new job gets triggered to initialize - itself, but then exits immediately. We use the Jenkins - [Job DSL plugin](https://github.com/jenkinsci/job-dsl-plugin/wiki) to define the meta job. +- [.github/workflows/deb-packaging.yml](https://github.com/keymanapp/keyman/blob/master/.github/workflows/deb-packaging.yml) + contains the definition of the packaging GHA +- [resources/build/run-required-test-builds.sh](https://github.com/keymanapp/keyman/blob/master/resources/build/run-required-test-builds.sh) + runs on [TeamCity](https://build.palaso.org/buildConfiguration/Keyman_Test?) + to trigger the builds for the various platforms, among them the GHA package build. +- [resources/build/increment-version.sh](https://github.com/keymanapp/keyman/blob/master/resources/build/increment-version.sh) + runs on [TeamCity](https://build.palaso.org/buildConfiguration/Keyman_TriggerReleaseBuildsMaster?) + and increments the version number before triggering the builds for the + various platforms. +- The [linux/scripts](https://github.com/keymanapp/keyman/tree/master/linux/scripts) + subdirectory contains `bash` scripts that are used during the package build. + Some are only needed for Launchpad builds. - ci-builder-scripts also contains several generic scripts to set up a package build environment - (using `sbuilder`) and for building source and binary packages. These scripts are shared with - other projects. + - [deb-packaging.sh](https://github.com/keymanapp/keyman/blob/master/linux/scripts/deb-packaging.sh) + gets called by the packaging GHA to install dependencies, create the source + package and to verify the API. - The Keyman GitHub repo defines a webhook that triggers the meta job on Jenkins. - - Changes to ci-builder-scripts go through [Gerrit](https://gerrit.lsdev.sil.org). See - [CONTRIBUTING.md](https://github.com/sillsdev/ci-builder-scripts/blob/master/CONTRIBUTING.md) - for details. - - File structure: - - - [groovy/KeymanPackagingJobs.groovy](https://github.com/sillsdev/ci-builder-scripts/blob/master/groovy/KeymanPackagingJobs.groovy) - contains the meta job definition - - The [bash/](https://github.com/sillsdev/ci-builder-scripts/tree/master/bash) subdirectory - contains `bash` scripts: - - - [setup.sh](https://github.com/sillsdev/ci-builder-scripts/blob/master/bash/setup.sh) - - setup sbuild chroot environment - - [update](https://github.com/sillsdev/ci-builder-scripts/blob/master/bash/update) - - update the sbuild chroot environment - - [build-package](https://github.com/sillsdev/ci-builder-scripts/blob/master/bash/build-package) - - create a binary package - -- [lsdev-pipeline-library](https://github.com/sillsdev/lsdev-pipeline-library) contains a reusable - Jenkins pipeline library. The - [vars/keymanPackaging.groovy](https://github.com/sillsdev/lsdev-pipeline-library/blob/master/vars/keymanPackaging.groovy) - file contains the bulk of the logic of the Keyman packaging job. - -- The [Keyman GitHub repo](https://github.com/keymanapp/keyman) contains various scripts that are - used to trigger a build and as part of the package build, and of course the source code for the - packages: - - - [resources/build/run-required-test-builds.sh](https://github.com/keymanapp/keyman/blob/master/resources/build/run-required-test-builds.sh) - runs on [TeamCity](https://build.palaso.org/buildConfiguration/Keyman_Test?) to trigger the - builds for the various platforms, among them the Jenkins package build. - - [resources/build/increment-version.sh](https://github.com/keymanapp/keyman/blob/master/resources/build/increment-version.sh) - runs on [TeamCity](https://build.palaso.org/buildConfiguration/Keyman_TriggerReleaseBuildsMaster?) - and increments the version number before triggering the builds for the various platforms. - - [linux/Jenkinsfile](https://github.com/keymanapp/keyman/blob/master/linux/Jenkinsfile) is a flag - for the meta job. If the meta job finds this file, it will create a new build configuration. This - file simply calls the packaging functionality defined in `lsdev-pipeline-library` and passes the - distributions and architectures to build as parameters. - - [linux/build/agent/install-deps](https://github.com/keymanapp/keyman/blob/master/linux/build/agent/install-deps) - installs dependencies on the current build agent. - - The [linux/scripts](https://github.com/keymanapp/keyman/tree/master/linux/scripts) subdirectory - contains `bash` scripts that are used during the package build. Some are only needed for - Launchpad builds. - - - [jenkins.sh](https://github.com/keymanapp/keyman/blob/master/linux/scripts/jenkins.sh) - gets called from `lsdev-pipeline-library` to create a source package. - - - [linux/debian](https://github.com/keymanapp/keyman/tree/master/linux/debian) - this is the `debian` - subdirectory for Keyman for Linux with the meta data for the Linux package. - See [Debian New Maintainers' Guide](https://www.debian.org/doc/manuals/maint-guide/) for - details to the various files. +- [linux/debian](https://github.com/keymanapp/keyman/tree/master/linux/debian) - + this is the `debian` subdirectory for Keyman for Linux with the meta data + for the Linux package. + See [Debian New Maintainers' Guide](https://www.debian.org/doc/manuals/maint-guide/) + for details to the various files in the `debian` directory. ### Flow of a Linux package build -- TeamCity jobs [Keyman_Test](https://build.palaso.org/buildConfiguration/Keyman_Test) or - [Keyman_TriggerReleaseBuilds*](https://build.palaso.org/buildConfiguration/Keyman_TriggerReleaseBuildsBeta) - trigger a build on [Jenkins](https://jenkins.lsdev.sil.org/view/Keyman/view/Pipeline/job/pipeline-keyman-packaging/) -- Jenkins verifies the build parameters and starts the matching build configuration for the PR or - branch -- The [build job](https://github.com/sillsdev/lsdev-pipeline-library/blob/master/vars/keymanPackaging.groovy) runs several checks: - - - it exits immediately if the build is not manually triggered and no parameters are passed in - (i.e. it got triggered by the GitHub webhook) - - it doesn't build if this is a PR, didn't get triggered manually and the PR is not from a trusted - user - - it doesn't build if no Linux-relevant files changed unless the parameter `force` was passed - - manually triggered builds will always build - -- build job installs - [dependencies](https://github.com/keymanapp/keyman/blob/master/linux/build/agent/install-deps) - on the current build agent -- build job creates a source package for the linux packages (keyman, kmflcomp, - libkmfl, and ibus-kmfl). This is done by calling - [scripts/jenkins.sh](https://github.com/keymanapp/keyman/blob/master/linux/scripts/jenkins.sh). -- build job creates the binary package for each linux package on each distribution (currently - bionic, focal, and groovy) and each architecture (amd64, i386 only for bionic) -- at the end of the build if it is not a build of a PR, the `.deb` file gets uploaded to llso - (alpha packages to e.g. `bionic-experimental`, beta packages to `bionic-proposed` and - packages build from the stable branch to the main section `bionic`) +- TeamCity jobs [Keyman_Test](https://build.palaso.org/buildConfiguration/Keyman_Test) + or [Keyman_TriggerReleaseBuilds*](https://build.palaso.org/buildConfiguration/Keyman_TriggerReleaseBuildsBeta) + trigger a packaging GHA build +- packaging GHA calls [deb-packaging.sh](https://github.com/keymanapp/keyman/blob/master/linux/scripts/deb-packaging.sh) + which installs dependencies and creates the source package +- packaging GHA creates the binary package for each linux package on each + distribution +- packaging GHA verifies that the API didn't change with the help of + [deb-packaging.sh](https://github.com/keymanapp/keyman/blob/master/linux/scripts/deb-packaging.sh) +- at the end of the build if it is not a build of a PR, the `.deb` files get + uploaded to llso (alpha packages to e.g. `jammy-experimental`, beta + packages to `jammy-proposed` and packages build from the stable branch + to the main section `jammy`) - if the build is successful the job archives the artifacts -The Jenkins build progress is visible in two ways: - -- [traditional view](https://jenkins.lsdev.sil.org/view/Keyman/view/Pipeline/job/pipeline-keyman-packaging/) -- [blue ocean view](https://jenkins.lsdev.sil.org/blue/organizations/jenkins/pipeline-keyman-packaging/activity) - -**Note:** TC release builds pass the git tag to build to the Jenkins job. The same tag -gets passed twice as parameters `tag` and `tag2`. The first parameter gets persisted between -builds, allowing to retrigger a tag-build. The second parameter is necessary to distinguish -if this is a retriggered build of a tag-build. - -### Local package builds - -It is possible to use the usual Debian/Ubuntu tools to create the package locally. For someone who -only occasionally deals with packaging it might be easier to use the scripts that Jenkins runs: - -#### Prerequisites for local package builds - -Install `sbuild` (and probably some other packages that I forgot). - -You’ll need a chroot image before you can use sbuild. The scripts in -[ci-builder-scripts](https://github.com/sillsdev/ci-builder-scripts) will help -with that. [`setup.sh`](https://github.com/sillsdev/ci-builder-scripts/blob/master/bash/setup.sh) -can setup such chroots: - -```bash -bash/setup.sh --dists "focal bionic" --arches "amd64 i386" -``` - -[`update`](https://github.com/sillsdev/ci-builder-scripts/blob/master/bash/update) is used to -later update those chroots: - -```bash -bash/update --dists "focal bionic" --arches "amd64 i386" -``` - -Set the `DEBSIGNKEY` environment variable to your public GPG key that will be used to sign -the packages. - -#### Building packages - -Building packages happen in the [Keyman source tree](https://github.com/keymanapp/keyman). - -The Keyman -[`linux/scripts/jenkins.sh`](https://github.com/keymanapp/keyman/blob/master/linux/scripts/jenkins.sh) -script can be used to create a source package. - -```bash -cd linux -./scripts/jenkins.sh keyman ${DEBSIGNKEY} -``` - -This creates a source package (`keyman_-1.dsc`) and some `*.tar.?z` -files in the source root directory for `keyman`. - -ci-builder-script's [`build-package`](https://github.com/sillsdev/ci-builder-scripts/blob/master/bash/build-package) -script creates the binary packages: - -```bash -cd $KEYMAN_ROOT -~/ci-builder-scripts/bash/build-package \ - --dists "focal bionic" --arches "amd64 i386" \ - --debkeyid ${DEBSIGNKEY} --build-in-place --no-upload -``` - -This will create the binary package `keyman_-1+1_.deb`. - -To speed up package building you might want to limit the build to a single dist -(e.g. `--dists "bionic"`) and arch (e.g. `--arches "amd64"`). - -After building packages it might be a good idea to clean up the source tree -before doing further work: - -```bash -git clean -dxf -``` - -### Local package builds (Docker) +### Local package builds with Docker It is possible to use the usual Debian/Ubuntu tools to create the package locally. For someone who only occasionally deals with packaging it might be easier to use -the scripts that run on GitHub actions: +Docker and the scripts that run on GitHub actions: #### Prerequisites for local package builds with Docker diff --git a/linux/.pbuilderrc b/linux/.pbuilderrc index 1384eec453..6a77a0ab08 100644 --- a/linux/.pbuilderrc +++ b/linux/.pbuilderrc @@ -14,7 +14,7 @@ DEBIAN_SUITES=($UNSTABLE_CODENAME $TESTING_CODENAME $STABLE_CODENAME $STABLE_BAC "experimental" "unstable" "testing" "stable") # List of Ubuntu suites. Update these when needed. -UBUNTU_SUITES=("lunar" "kinetic" "jammy" "focal") +UBUNTU_SUITES=("mantic" "lunar" "jammy" "focal") # Mirrors to use. Update these to your preferred mirror. DEBIAN_MIRROR="deb.debian.org" diff --git a/linux/Dockerfile b/linux/Dockerfile index d6cf7d0c63..6852b54238 100644 --- a/linux/Dockerfile +++ b/linux/Dockerfile @@ -1,4 +1,4 @@ -# Copyright (c) 2022 SIL International. All rights reserved. +# Copyright (c) 2022-2023 SIL International. All rights reserved. # # builder image for a linux build # see ../docs/build/linux-ubuntu.md @@ -7,27 +7,80 @@ FROM --platform=amd64 ubuntu:latest LABEL org.opencontainers.image.authors="SIL International." LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git" LABEL org.opencontainers.image.title="Keyman Linux Build Image" + # We will switch to a build user after some installation USER root -RUN useradd -c "Build user" -d $HOME -m build ENV HOME /home/build -VOLUME /home/build -WORKDIR /home/build +RUN useradd -c "Build user" --home-dir $HOME --create-home --shell /usr/bin/bashwrapper build +VOLUME /home/build/build +WORKDIR /home/build/build ENV DEBIAN_FRONTEND noninteractive ENV DEBIAN_PRIORITY critical ENV DEBCONF_NOWARNINGS yes # Update to the latest RUN apt-get -q -y update && \ - apt-get -q -y install devscripts equivs meson python3 python3-setuptools software-properties-common && \ + apt-get -q -y install devscripts equivs meson python3 python3-setuptools software-properties-common curl && \ add-apt-repository ppa:keymanapp/keyman && \ - add-apt-repository ppa:keymanapp/keyman-alpha && \ + add-apt-repository ppa:keymanapp/keyman-alpha +RUN apt-get -q -y update && \ apt-get -q -y upgrade +# Install dependencies ADD debian/control /tmp/control # Answer 'yes' to install questions -RUN (yes | mk-build-deps --install /tmp/control) || true -RUN curl -sL https://deb.nodesource.com/setup_18.x | bash -RUN apt-get -q -y install nodejs +RUN (yes | mk-build-deps --install /tmp/control) || true && \ + rm /tmp/control + +# Install Node +RUN curl -sL https://deb.nodesource.com/setup_18.x | bash && \ + apt-get -q -y install nodejs + +# Install emscripten +RUN cd /usr/share && \ + git clone https://github.com/emscripten-core/emsdk.git && \ + cd emsdk && \ + ./emsdk install latest && \ + ./emsdk activate latest && \ + echo "#!/bin/bash" > /usr/bin/bashwrapper && \ + echo "export EMSCRIPTEN_BASE=/usr/share/emsdk/upstream/emscripten" >> /usr/bin/bashwrapper + +# Keyman Web +RUN curl --output google-chrome-stable_current_amd64.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ + apt-get -q -y install ./google-chrome-stable_current_amd64.deb && \ + rm google-chrome-stable_current_amd64.deb && \ + echo "export CHROME_BIN=/opt/google/chrome/chrome" >> /usr/bin/bashwrapper + +# Keyman for Android +RUN apt-get -q -y install gradle maven pandoc sdkmanager jq && \ + sdkmanager platform-tools && \ + yes | sdkmanager --licenses && \ + chown -R build:build /opt/android-sdk/ && \ + echo "export ANDROID_HOME=/opt/android-sdk" >> /usr/bin/bashwrapper && \ + echo "export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64" >> /usr/bin/bashwrapper + +# Finish bashwrapper script and adjust permissions +RUN echo "\${@:-bash}" >> /usr/bin/bashwrapper && \ + chmod +x /usr/bin/bashwrapper && \ + chown -R build:build $HOME + # now, switch to build user USER build + +# Pre-install gradle. This will put files in ~/.gradle which will speed up builds. +RUN mkdir -p $HOME/tmp/gradle/wrapper && \ + # KMEA uses gradle-7.5.1-bin + curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.jar https://raw.githubusercontent.com/keymanapp/keyman/master/android/KMEA/gradle/wrapper/gradle-wrapper.jar && \ + curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.properties https://raw.githubusercontent.com/keymanapp/keyman/master/android/KMEA/gradle/wrapper/gradle-wrapper.properties && \ + curl --location --output $HOME/tmp/gradlew https://raw.githubusercontent.com/keymanapp/keyman/master/android/KMEA/gradlew && \ + chmod +x $HOME/tmp/gradlew && \ + $HOME/tmp/gradlew --quiet && \ + # Some projects use gradle-7.5.1-all, so we pre-install that as well + curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.jar https://raw.githubusercontent.com/keymanapp/keyman/master/android/Samples/KMSample1/gradle/wrapper/gradle-wrapper.jar && \ + curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.properties https://raw.githubusercontent.com/keymanapp/keyman/master/android/Samples/KMSample1/gradle/wrapper/gradle-wrapper.properties && \ + curl --location --output $HOME/tmp/gradlew https://raw.githubusercontent.com/keymanapp/keyman/master/android/Samples/KMSample1/gradlew && \ + chmod +x $HOME/tmp/gradlew && \ + $HOME/tmp/gradlew --quiet && \ + rm -rf $HOME/tmp + +ENTRYPOINT [ "/usr/bin/bashwrapper" ] diff --git a/linux/Jenkinsfile b/linux/Jenkinsfile deleted file mode 100644 index b70cb95d70..0000000000 --- a/linux/Jenkinsfile +++ /dev/null @@ -1,11 +0,0 @@ -#!groovy -// Copyright (c) 2019-2022 SIL International -// This software is licensed under the MIT license (http://opensource.org/licenses/MIT) - -@Library('lsdev-pipeline-library') _ - -keymanPackaging { - distributionsToPackage = 'focal jammy kinetic lunar' - arches = 'amd64 i386' - packagesToBuild = ['keyman'] -} diff --git a/linux/debian/changelog b/linux/debian/changelog index 9f69cc9a12..1ad0f861fb 100644 --- a/linux/debian/changelog +++ b/linux/debian/changelog @@ -1,3 +1,18 @@ +keyman (16.0.141-1) unstable; urgency=medium + + * Work around mips64el build failure (#1041499) + * New upstream release. + * Re-release to Debian + + -- Eberhard Beilharz Thu, 27 Jul 2023 16:30:04 +0200 + +keyman (16.0.140-1) unstable; urgency=medium + + * New upstream release (closes: #1037707). + * Re-release to Debian + + -- Eberhard Beilharz Mon, 24 Jul 2023 11:41:07 +0200 + keyman (16.0.139-4) unstable; urgency=medium * debian/tests: Revert previous change and ignore s390x from autopkgtests diff --git a/linux/ibus-keyman/build.sh b/linux/ibus-keyman/build.sh index 1b8ef97823..58bc057691 100755 --- a/linux/ibus-keyman/build.sh +++ b/linux/ibus-keyman/build.sh @@ -61,7 +61,7 @@ fi if builder_start_action test; then cd "$THIS_SCRIPT_PATH/$MESON_PATH" if builder_has_option --no-integration; then - meson test --print-errorlogs $builder_verbose keymanutil-tests print-kmpdetails-test print-kmp-test + meson test --print-errorlogs $builder_verbose setup-src-test keymanutil-tests print-kmpdetails-test print-kmp-test bcp47-util-tests teardown-src-test else meson test --print-errorlogs $builder_verbose fi diff --git a/linux/ibus-keyman/meson.build b/linux/ibus-keyman/meson.build index e4b31df7db..2e402b868e 100644 --- a/linux/ibus-keyman/meson.build +++ b/linux/ibus-keyman/meson.build @@ -13,6 +13,7 @@ ibus = dependency('ibus-1.0', version: '>= 1.2.0') gtk = dependency('gtk+-3.0', version: '>= 2.4') json_glib = dependency('json-glib-1.0', version: '>= 1.0') systemd = dependency('libsystemd') +icu = dependency('icu-i18n') core_dir = meson.current_source_dir() / '../../core' common_dir = meson.current_source_dir() / '../../common' diff --git a/linux/ibus-keyman/src/bcp47util.c b/linux/ibus-keyman/src/bcp47util.c new file mode 100644 index 0000000000..0b2013038a --- /dev/null +++ b/linux/ibus-keyman/src/bcp47util.c @@ -0,0 +1,99 @@ +#include +#include +#include + +/// Minimize the BCP-47 `tag` so that unnecessary parts get ommitted. +/// The result gets stored in `minimzedTag`. +/// +/// @param tag The tag to process +/// @param minimizedTag Caller-provided character array for the +/// resulting minimized tag. +/// @param tagCapacity Array size of `minimizedTag` +/// @return The length of the minimized tag, or -1 in error case +int bcp47_minimize(const char* tag, char* minimizedTag, int tagCapacity) { + UErrorCode status = U_ZERO_ERROR; + if (!tag || strlen(tag) == 0) { + strncpy(minimizedTag, "", tagCapacity); + return -1; + } + + // special treatment for `und-Latn` which is used by sil_ipa keyboard + if (strcmp(tag, "und-Latn") == 0) { + strncpy(minimizedTag, "und-Latn", tagCapacity); + return strlen(minimizedTag); + } + + int capacity = 255; + char workingTag[capacity]; + + // special treatment for tags that start with `und`: replace `und` with `en`. + // ICU 70 doesn't properly treat `und`. + int isUnd = strncmp(tag, "und", 3) == 0; + if (isUnd) { + strcpy(workingTag, "en"); + strncat(&workingTag[2], &tag[3], capacity - 3); + } else { + strncpy(workingTag, tag, capacity - 1); + } + workingTag[capacity - 1] = 0; + + char localeId[capacity]; + uloc_forLanguageTag(workingTag, localeId, capacity, NULL, &status); + if (U_FAILURE(status)) { + g_error("%s: uloc_forLanguageTag returned %0x", __FUNCTION__, status); + return -1; + } + + char minimizedLocaleId[capacity]; + uloc_minimizeSubtags(localeId, minimizedLocaleId, capacity, &status); + if (U_FAILURE(status)) { + g_error("%s: uloc_minimizeSubtags returned %0x", __FUNCTION__, status); + return -1; + } + + int taglen = uloc_toLanguageTag(minimizedLocaleId, minimizedTag, tagCapacity, FALSE, &status); + if (U_FAILURE(status)) { + g_error("%s: uloc_toLanguageTag returned %0x", __FUNCTION__, status); + return -1; + } + + if (isUnd) { + // Replace 'en' with 'und' again + strncpy(workingTag, &minimizedTag[2], capacity - 1); + workingTag[capacity - 1] = 0; + strcpy(minimizedTag, "und"); + strncat(minimizedTag, workingTag, tagCapacity - 4); + minimizedTag[tagCapacity - 1] = 0; + taglen = strlen(minimizedTag); + } + return taglen; +} + +/// Extract the language code from the BCP-47 `tag` +/// +/// @param tag The BCP-47 tag +/// @param lang_code Caller-provided character array that will receive +/// the language code extracted from `tag` +/// @param capacity Array size of `lang_code` +/// @return TRUE if successful, otherwise FALSE +int bcp47_get_language_code(const char* tag, char* lang_code, int capacity) { + UErrorCode status = U_ZERO_ERROR; + if (!tag || strlen(tag) == 0) { + strncpy(lang_code, "", capacity); + return FALSE; + } + + // ICU 70 doesn't properly treat `und` + if (strncmp(tag, "und", 3) == 0) { + strncpy(lang_code, "und", capacity); + return TRUE; + } + + uloc_getLanguage(tag, lang_code, capacity, &status); + if (U_FAILURE(status)) { + g_error("%s: uloc_getLanguage returned %0x", __FUNCTION__, status); + return FALSE; + } + + return TRUE; +} diff --git a/linux/ibus-keyman/src/bcp47util.h b/linux/ibus-keyman/src/bcp47util.h new file mode 100644 index 0000000000..8a3a057f85 --- /dev/null +++ b/linux/ibus-keyman/src/bcp47util.h @@ -0,0 +1,7 @@ +#ifndef __BCP47UTIL_H__ +#define __BCP47UTIL_H__ + +int bcp47_minimize(const char* tag, char* minimizedTag, int capacity); +int bcp47_get_language_code(const char* tag, char* lang_code, int capacity); + +#endif // __BCP47UTIL_H__ diff --git a/linux/ibus-keyman/src/keymanutil.c b/linux/ibus-keyman/src/keymanutil.c index 4efd941bec..6726aee4e3 100644 --- a/linux/ibus-keyman/src/keymanutil.c +++ b/linux/ibus-keyman/src/keymanutil.c @@ -57,6 +57,7 @@ #include #include +#include "bcp47util.h" #include "keymanutil.h" #include "kmpdetails.h" #include "keyman-version.h" @@ -163,8 +164,6 @@ ibus_keyman_add_engines(GList * engines, GList * kmpdir_list) get_kmp_details(kmp_dir, details); for (k=details->keyboards; k != NULL; k = k->next) { - gchar *lang=NULL; - gchar *name_with_lang = NULL; kmp_keyboard *keyboard = (kmp_keyboard *) k->data; gboolean alreadyexists = FALSE; @@ -191,35 +190,50 @@ ibus_keyman_add_engines(GList * engines, GList * kmpdir_list) for (l=keyboard->languages; l != NULL; l = l->next) { kmp_language *language = (kmp_language *) l->data; if (language->id != NULL) { - gchar **tagparts = g_strsplit(language->id, "-", 2); - lang = g_strdup(tagparts[0]); - g_strfreev(tagparts); - // If ibus doesn't know about the language then append the - // language name to the keyboard name - if (language->name != NULL) { - if (g_strcmp0(ibus_get_untranslated_language_name (lang), "Other") == 0) { - name_with_lang = g_strjoin(" - ", keyboard->name, language->name, NULL); - } + int capacity = 255; + gchar *name_with_lang = NULL; + gchar *minimized_tag = g_new0(gchar, capacity); + int result = bcp47_minimize(language->id, minimized_tag, capacity); + if (result < 0) { + g_strlcpy(minimized_tag, language->id, capacity); + } + + gchar *lang_code = g_new0(gchar, capacity); + if (!bcp47_get_language_code(minimized_tag, lang_code, capacity)) { + g_strlcpy(lang_code, minimized_tag, capacity); + } + + // If ibus doesn't know about the language then append the + // language name to the keyboard name + if (language->name != NULL) { + gchar *ibus_lang = ibus_get_untranslated_language_name(lang_code); + g_debug("%s: untranslated ibus language for %s: %s", __FUNCTION__, minimized_tag, ibus_lang); + if (g_strcmp0(ibus_lang, "Other") == 0) { + name_with_lang = g_strjoin(" - ", keyboard->name, language->name, NULL); } + g_free(ibus_lang); + } - gchar *id_with_lang = g_strjoin(":", language->id, abs_kmx, NULL); + gchar *id_with_lang = g_strjoin(":", minimized_tag, abs_kmx, NULL); - g_message("adding engine %s", id_with_lang); - engines = g_list_append (engines, - ibus_keyman_engine_desc_new (id_with_lang, // lang:kmx full path - name_with_lang ? name_with_lang : keyboard->name, // longname - kbd_details->description, // description - details->info.copyright, // copyright if available - lang, // language, most are ignored by ibus except major languages - kbd_details->license, // license - details->info.author_desc, // author name only, not email - keyman_get_icon_file(abs_kmx), // icon full path - "us", // layout defaulting to us (en-US) - keyboard->version)); - g_free(lang); - g_free(id_with_lang); - g_free(name_with_lang); - name_with_lang = NULL; + g_message("adding engine %s", id_with_lang); + engines = g_list_append( + engines, + ibus_keyman_engine_desc_new( + id_with_lang, // lang:kmx full path + name_with_lang ? name_with_lang : keyboard->name, // longname + kbd_details->description, // description + details->info.copyright, // copyright if available + lang_code, // language, most are ignored by ibus except major languages + kbd_details->license, // license + details->info.author_desc, // author name only, not email + keyman_get_icon_file(abs_kmx), // icon full path + "us", // layout defaulting to us (en-US) + keyboard->version)); + g_free(lang_code); + g_free(minimized_tag); + g_free(id_with_lang); + g_free(name_with_lang); } } } @@ -230,7 +244,7 @@ ibus_keyman_add_engines(GList * engines, GList * kmpdir_list) keyboard->name, // longname kbd_details->description, // description details->info.copyright, // copyright if available - lang, // language, most are ignored by ibus except major languages + NULL, // language, most are ignored by ibus except major languages kbd_details->license, // license details->info.author_desc, // author name only, not email keyman_get_icon_file(abs_kmx), // icon full path diff --git a/linux/ibus-keyman/src/meson.build b/linux/ibus-keyman/src/meson.build index 2e83a65106..f7fe8ada1d 100644 --- a/linux/ibus-keyman/src/meson.build +++ b/linux/ibus-keyman/src/meson.build @@ -1,6 +1,7 @@ util_files = files( 'keymanutil.c', 'kmpdetails.c', + 'bcp47util.c', ) engine_files = files( @@ -16,7 +17,7 @@ include_dirs = [ include_directories(meson.current_build_dir() / '..'), ] -deps = [ibus, gtk, json_glib, kmnkbp_lib, systemd] +deps = [gtk, ibus, icu, json_glib, kmnkbp_lib, systemd] prefix = get_option('prefix') cfg = configuration_data() diff --git a/linux/ibus-keyman/src/test/bcp47util_tests.c b/linux/ibus-keyman/src/test/bcp47util_tests.c new file mode 100644 index 0000000000..a82d30476f --- /dev/null +++ b/linux/ibus-keyman/src/test/bcp47util_tests.c @@ -0,0 +1,100 @@ +#include +#include +#include +#include "bcp47util.h" + +typedef struct { +} Bcp47UtilFixture; + +typedef struct { + const char *tag; + const char *expected; + int expectedResult; +} TestData; + +static void +test_bcp47_minimize(Bcp47UtilFixture *fixture, gconstpointer user_data) { + TestData *testData = (TestData *)user_data; + int capacity = 255; + char minimizedTag[capacity]; + int result = bcp47_minimize(testData->tag, minimizedTag, capacity); + g_assert_cmpint(result, ==, testData->expectedResult); + g_assert_cmpstr(minimizedTag, ==, testData->expected); +} + +static void +test_bcp47_get_language_code(Bcp47UtilFixture *fixture, gconstpointer user_data) { + TestData *testData = (TestData *)user_data; + int capacity = 255; + char lang_code[capacity]; + int result = bcp47_get_language_code(testData->tag, lang_code, capacity); + g_assert_cmpint(result, ==, testData->expectedResult); + g_assert_cmpstr(lang_code, ==, testData->expected); +} + +int +main(int argc, char *argv[]) { + gtk_init(&argc, &argv); + g_test_init(&argc, &argv, NULL); + g_test_set_nonfatal_assertions(); + + TestData testData1 = {NULL, "", -1}; + g_test_add("/bcp47util/minimize/NULL", Bcp47UtilFixture, &testData1, NULL, test_bcp47_minimize, NULL); + + TestData testData2 = {"", "", -1}; + g_test_add("/bcp47util/minimize/EmptyString", Bcp47UtilFixture, &testData2, NULL, test_bcp47_minimize, NULL); + + TestData testData3 = { "fuf", "fuf", 3 }; + g_test_add("/bcp47util/minimize/fuf", Bcp47UtilFixture, &testData3, NULL, test_bcp47_minimize, NULL); + + TestData testData4 = {"fuf-Latn", "fuf", 3}; + g_test_add("/bcp47util/minimize/fuf-Latn", Bcp47UtilFixture, &testData4, NULL, test_bcp47_minimize, NULL); + + TestData testData5 = {"fuf-Arab", "fuf-Arab", 8}; + g_test_add("/bcp47util/minimize/fuf-Arab", Bcp47UtilFixture, &testData5, NULL, test_bcp47_minimize, NULL); + + TestData testData6 = {"fuf-Adlm-ML", "fuf-Adlm-ML", 11}; + g_test_add("/bcp47util/minimize/fuf-Adlm-ML", Bcp47UtilFixture, &testData6, NULL, test_bcp47_minimize, NULL); + + TestData testData7 = {"und", "und", 3}; + g_test_add("/bcp47util/minimize/und", Bcp47UtilFixture, &testData7, NULL, test_bcp47_minimize, NULL); + + TestData testData8 = {"und-Latn", "und-Latn", 8}; + g_test_add("/bcp47util/minimize/und-Latn", Bcp47UtilFixture, &testData8, NULL, test_bcp47_minimize, NULL); + + TestData testData9 = {"und-fonipa", "und-fonipa", 10}; + g_test_add("/bcp47util/minimize/und-fonipa", Bcp47UtilFixture, &testData9, NULL, test_bcp47_minimize, NULL); + + TestData testData10 = {"und-Latn-fonipa", "und-fonipa", 10}; + g_test_add("/bcp47util/minimize/und-Latn-fonipa", Bcp47UtilFixture, &testData10, NULL, test_bcp47_minimize, NULL); + + TestData testData11 = {"mul", "mul", 3}; + g_test_add("/bcp47util/minimize/mul", Bcp47UtilFixture, &testData11, NULL, test_bcp47_minimize, NULL); + + // bcp47_get_language_code tests + TestData testData21 = {NULL, "", FALSE}; + g_test_add("/bcp47util/langcode/NULL", Bcp47UtilFixture, &testData21, NULL, test_bcp47_get_language_code, NULL); + + TestData testData22 = {"", "", FALSE}; + g_test_add("/bcp47util/langcode/EmptyString", Bcp47UtilFixture, &testData22, NULL, test_bcp47_get_language_code, NULL); + + TestData testData23 = {"fuf", "fuf", TRUE}; + g_test_add("/bcp47util/langcode/fuf", Bcp47UtilFixture, &testData23, NULL, test_bcp47_get_language_code, NULL); + + TestData testData24 = {"fuf-Latn", "fuf", TRUE}; + g_test_add("/bcp47util/langcode/fuf-Latn", Bcp47UtilFixture, &testData24, NULL, test_bcp47_get_language_code, NULL); + + TestData testData25 = {"fuf-Arab", "fuf", TRUE}; + g_test_add("/bcp47util/langcode/fuf-Arab", Bcp47UtilFixture, &testData25, NULL, test_bcp47_get_language_code, NULL); + + TestData testData26 = {"fuf-Adlm-ML", "fuf", TRUE}; + g_test_add("/bcp47util/langcode/fuf-Adlm-ML", Bcp47UtilFixture, &testData26, NULL, test_bcp47_get_language_code, NULL); + + TestData testData27 = {"und", "und", TRUE}; + g_test_add("/bcp47util/langcode/und", Bcp47UtilFixture, &testData27, NULL, test_bcp47_get_language_code, NULL); + + TestData testData28 = {"und-Latn", "und", TRUE}; + g_test_add("/bcp47util/langcode/und-Latn", Bcp47UtilFixture, &testData28, NULL, test_bcp47_get_language_code, NULL); + + return g_test_run(); +} diff --git a/linux/ibus-keyman/src/test/meson.build b/linux/ibus-keyman/src/test/meson.build index 7e5cd21dbe..1ad841c6d6 100644 --- a/linux/ibus-keyman/src/test/meson.build +++ b/linux/ibus-keyman/src/test/meson.build @@ -3,11 +3,13 @@ keymanutil_sources = [ util_files, ] -keymanutil_deps = [ibus, gtk, json_glib, kmnkbp_lib] +keymanutil_deps = [gtk, ibus, icu, json_glib, kmnkbp_lib] test_env = [ 'G_TEST_SRCDIR=' + meson.current_source_dir(), 'G_TEST_BUILDDIR=' + meson.current_build_dir(), + 'TOP_SRCDIR=' + meson.global_source_root(), + 'TOP_BINDIR=' + meson.build_root(), ] test_include_dirs = [ @@ -16,47 +18,102 @@ test_include_dirs = [ include_directories(meson.current_build_dir() / '..'), ] -executable( +env_file = '/tmp/env-src-test.txt' +pid_file = '/tmp/ibus-keyman-src-test-pids' + +setup_src_test_tests = find_program('setup-tests.sh', dirs: [meson.current_source_dir()]) +teardown_tests = find_program('teardown-tests.sh', dirs: [meson.current_source_dir() / '../../tests/scripts']) +run_src_test = find_program('run-single-test.sh', dirs: [meson.current_source_dir()]) + +keymanutil_tests = executable( 'keymanutil-tests', sources: keymanutil_sources, dependencies: keymanutil_deps, include_directories : test_include_dirs ) +print_kmpdetails_test = executable( + 'print_kmpdetails', + sources: [ + 'print_kmpdetails.c', + '../kmpdetails.c' + ], + dependencies: [ json_glib ], + include_directories: test_include_dirs +) + +print_kmp_test = executable( + 'print_kmp', + sources: [ + 'print_kmp.c', + ], + dependencies: [ json_glib ], + include_directories: test_include_dirs +) + +bcp47_util_tests = executable( + 'bcp47-util-tests', + sources: [ + 'bcp47util_tests.c', + '../bcp47util.c' + ], + dependencies: [ gtk, icu ], + include_directories: test_include_dirs +) + +test( + 'setup-src-test', + setup_src_test_tests, + args: ['--x11', env_file, pid_file], + env: test_env, + priority: -1, + is_parallel: false, + protocol: 'exitcode' +) + +test( + 'teardown-src-test', + teardown_tests, + args: [pid_file], + priority: -9, + is_parallel: false, + protocol: 'exitcode' +) + test( 'keymanutil-tests', - find_program('run-tests.sh'), + run_src_test, + args: [ '--tap', '-k', '--env', env_file, '--', keymanutil_tests], env: test_env, + priority: -2, + is_parallel: false, protocol: 'tap', ) test( 'print-kmpdetails-test', - executable( - 'print_kmpdetails', - sources: [ - 'print_kmpdetails.c', - '../kmpdetails.c' - ], - dependencies: [ json_glib ], - include_directories: test_include_dirs - ), - args: [ meson.current_source_dir() ], + run_src_test, + args: [ '--', print_kmpdetails_test, meson.current_source_dir() ], env: test_env, + priority: -2, protocol: 'exitcode', ) test( 'print-kmp-test', - executable( - 'print_kmp', - sources: [ - 'print_kmp.c', - ], - dependencies: [ json_glib ], - include_directories: test_include_dirs - ), - args: [ meson.current_source_dir() / 'kmp.json' ], + run_src_test, + args: [ '--', print_kmp_test, meson.current_source_dir() / 'kmp.json' ], env: test_env, + priority: -2, protocol: 'exitcode', ) + +test( + 'bcp47-util-tests', + run_src_test, + args: [ '--tap', '-k', '--env', env_file, '--', bcp47_util_tests ], + env: test_env, + priority: -2, + is_parallel: false, + protocol: 'tap', +) diff --git a/linux/ibus-keyman/src/test/run-single-test.sh b/linux/ibus-keyman/src/test/run-single-test.sh new file mode 100755 index 0000000000..1ad8714449 --- /dev/null +++ b/linux/ibus-keyman/src/test/run-single-test.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +function help() { + echo "Usage:" + echo " $0 [--env ] [-k] [--tap] [--] TESTFILE [TESTARGS]" + echo + echo "Arguments:" + echo " --help, -h, -? Display this help" + echo " --verbose, -v Run tests verbosely" + echo " --debug debug test logging output" + echo " -k passed to GLib testing framework" + echo " --tap output in TAP format. Passed to GLib testing framework" + echo " --env Name of the file containing environment variables to use" + exit 0 +} + +function run_tests() { + # Output these lines to stderr - the first line on stdout has to be the TAP version number + # which running ${TESTFILE} outputs + echo "# NOTE: When the tests fail check /tmp/ibus-engine-keyman.log and /tmp/ibus-daemon.log!" >&2 + echo "" >&2 + + echo "# Starting tests..." >&2 + + # Note: -k and --tap are consumed by the GLib testing framework + # shellcheck disable=SC2086 + "${TESTFILE}" ${ARG_K-} ${ARG_TAP-} ${ARG_VERBOSE-} ${ARG_DEBUG-} "$@" + echo "# Finished tests." +} + +while (( $# )); do + case $1 in + --help|-h|-\?) help ;; + -k) ARG_K=$1 ;; + --tap) ARG_TAP=$1 ;; + --verbose|-v) ARG_VERBOSE=--verbose;; + --debug) ARG_DEBUG=--debug-log;; + --env) shift ; ARG_ENV=$1 ;; + --) shift ; TESTFILE=$1; shift ; break ;; + *) echo "Error: Unexpected argument \"$1\". Exiting." ; exit 4 ;; + esac + shift || (echo "Error: The last argument is missing a value. Exiting."; false) || exit 5 +done + +# shellcheck source=/dev/null +. "$ARG_ENV" + +run_tests "$@" diff --git a/linux/ibus-keyman/src/test/run-tests.sh b/linux/ibus-keyman/src/test/run-tests.sh index 44a397582b..efeb747ddc 100755 --- a/linux/ibus-keyman/src/test/run-tests.sh +++ b/linux/ibus-keyman/src/test/run-tests.sh @@ -7,8 +7,8 @@ if [ -v KEYMAN_PKG_BUILD ]; then # During package builds we skip these tests - they often fail, e.g. # during Debian reproducibility testing with an error like # "cannot open display: :32" - echo "1..1" - echo "ok 1 # SKIP on package build" + echo "TAP version 14" + echo "1..0 # SKIP on package build" exit 0 fi @@ -58,4 +58,4 @@ glib-compile-schemas "$SCHEMA_DIR" export GSETTINGS_BACKEND=memory -${G_TEST_BUILDDIR:-.}/keymanutil-tests "$@" +"${G_TEST_BUILDDIR:-.}/keymanutil-tests" "$@" diff --git a/linux/ibus-keyman/src/test/setup-tests.sh b/linux/ibus-keyman/src/test/setup-tests.sh new file mode 100755 index 0000000000..736650270d --- /dev/null +++ b/linux/ibus-keyman/src/test/setup-tests.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -eu + +. "$(dirname "$0")/../../tests/scripts/test-helper.inc.sh" + +setup_display_server_only "$1" "$2" "$3" diff --git a/linux/ibus-keyman/tests/meson.build b/linux/ibus-keyman/tests/meson.build index 130e3b8bce..20d82e7e2c 100644 --- a/linux/ibus-keyman/tests/meson.build +++ b/linux/ibus-keyman/tests/meson.build @@ -10,7 +10,7 @@ kmnkbp_tests_lib = cc.find_library( dirs: [ core_dir / 'build/arch' / get_option('buildtype') / 'tests/kmx_test_source' ] ) -test_deps = [ibus, gtk, json_glib, kmnkbp_lib, kmnkbp_tests_lib, systemd] +test_deps = [gtk, ibus, icu, json_glib, kmnkbp_lib, kmnkbp_tests_lib, systemd] dbus_deps = [gtk, systemd] test_env = [ @@ -57,6 +57,10 @@ teardown_tests = find_program('teardown-tests.sh', dirs: [meson.current_source_d run_test = find_program('run-single-test.sh', dirs: [meson.current_source_dir() / 'scripts']) find_tests = find_program('find-tests.sh', dirs: [meson.current_source_dir() / 'scripts']) +# Mutter 40.x added the --headless option wich we need in order to be able to run the Wayland tests +mutter = find_program('mutter', required: false, version: '>=40') +can_build_wayland = mutter.found() + test( 'setup-x11', setup_tests, @@ -67,16 +71,6 @@ test( protocol: 'exitcode' ) -test( - 'setup-wayland', - setup_tests, - args: ['--wayland', env_file, pid_file], - env: test_env, - priority: -20, - is_parallel: false, - protocol: 'exitcode' -) - test( 'teardown-x11', teardown_tests, @@ -86,14 +80,26 @@ test( protocol: 'exitcode' ) -test( - 'teardown-wayland', - teardown_tests, - args: [pid_file], - priority: -29, - is_parallel: false, - protocol: 'exitcode' -) +if can_build_wayland + test( + 'setup-wayland', + setup_tests, + args: ['--wayland', env_file, pid_file], + env: test_env, + priority: -20, + is_parallel: false, + protocol: 'exitcode' + ) + + test( + 'teardown-wayland', + teardown_tests, + args: [pid_file], + priority: -29, + is_parallel: false, + protocol: 'exitcode' + ) +endif kmxtest_files = run_command( find_tests, @@ -130,26 +136,28 @@ foreach kmx: kmxtest_files timeout: 120, protocol: 'tap', ) - test( - 'Wayland-' + testname + '__surrounding-text', - run_test, - args: [ '--wayland', '--surrounding-text', test_args], - env: test_env, - depends: [test_exe], - priority: -21, - is_parallel: false, - timeout: 120, - protocol: 'tap', - ) - test( - 'Wayland-' + testname + '__no-surrounding-text', - run_test, - args: [ '--wayland', '--no-surrounding-text', test_args], - env: test_env, - depends: [test_exe], - priority: -22, - is_parallel: false, - timeout: 120, - protocol: 'tap', - ) + if can_build_wayland + test( + 'Wayland-' + testname + '__surrounding-text', + run_test, + args: [ '--wayland', '--surrounding-text', test_args], + env: test_env, + depends: [test_exe], + priority: -21, + is_parallel: false, + timeout: 120, + protocol: 'tap', + ) + test( + 'Wayland-' + testname + '__no-surrounding-text', + run_test, + args: [ '--wayland', '--no-surrounding-text', test_args], + env: test_env, + depends: [test_exe], + priority: -22, + is_parallel: false, + timeout: 120, + protocol: 'tap', + ) + endif endforeach diff --git a/linux/ibus-keyman/tests/scripts/run-single-test.sh b/linux/ibus-keyman/tests/scripts/run-single-test.sh index 19baca7dd6..b4660f8368 100755 --- a/linux/ibus-keyman/tests/scripts/run-single-test.sh +++ b/linux/ibus-keyman/tests/scripts/run-single-test.sh @@ -9,8 +9,8 @@ if [ -v KEYMAN_PKG_BUILD ]; then # ibus requires to find /var/lib/dbus/machine-id or /etc/machine-id, otherwise it fails with: # "Bail out! IBUS-FATAL-WARNING: Unable to load /var/lib/dbus/machine-id: Failed to open file # “/var/lib/dbus/machine-id”: No such file or directory" - echo "1..1" - echo "ok 1 - Integration tests # SKIP on package build" + echo "TAP version 14" + echo "1..0 # SKIP on package build" exit 0 fi @@ -38,10 +38,13 @@ function help() { } function run_tests() { - echo "# NOTE: When the tests fail check /tmp/ibus-engine-keyman.log and /tmp/ibus-daemon.log!" - echo "" + # Output these lines to stderr - the first line on stdout has to be the TAP version number + # which running ${TESTFILE} outputs + echo "# NOTE: When the tests fail check /tmp/ibus-engine-keyman.log and /tmp/ibus-daemon.log!" >&2 + echo "" >&2 + + echo "# Starting tests..." >&2 - echo "# Starting tests..." # Note: -k and --tap are consumed by the GLib testing framework # shellcheck disable=SC2086 "${G_TEST_BUILDDIR:-.}"/ibus-keyman-tests ${ARG_K-} ${ARG_TAP-} \ diff --git a/linux/ibus-keyman/tests/scripts/run-tests.sh b/linux/ibus-keyman/tests/scripts/run-tests.sh index 92cae307ed..ae58705f11 100755 --- a/linux/ibus-keyman/tests/scripts/run-tests.sh +++ b/linux/ibus-keyman/tests/scripts/run-tests.sh @@ -18,8 +18,7 @@ if [ -v KEYMAN_PKG_BUILD ]; then # ibus requires to find /var/lib/dbus/machine-id or /etc/machine-id, otherwise it fails with: # "Bail out! IBUS-FATAL-WARNING: Unable to load /var/lib/dbus/machine-id: Failed to open file # “/var/lib/dbus/machine-id”: No such file or directory" - echo "1..1" - echo "ok 1 - Integration tests # SKIP on package build" + echo "1..0 # SKIP on package build" exit 0 fi diff --git a/linux/ibus-keyman/tests/scripts/setup-tests.sh b/linux/ibus-keyman/tests/scripts/setup-tests.sh index 9d591869fb..67bbcbb1bf 100755 --- a/linux/ibus-keyman/tests/scripts/setup-tests.sh +++ b/linux/ibus-keyman/tests/scripts/setup-tests.sh @@ -3,4 +3,6 @@ set -eu . "$(dirname "$0")/test-helper.inc.sh" +exit_on_package_build + setup "$1" "$2" "$3" diff --git a/linux/ibus-keyman/tests/scripts/teardown-tests.sh b/linux/ibus-keyman/tests/scripts/teardown-tests.sh index c3b955a13a..c74513f281 100755 --- a/linux/ibus-keyman/tests/scripts/teardown-tests.sh +++ b/linux/ibus-keyman/tests/scripts/teardown-tests.sh @@ -3,4 +3,6 @@ set -eu . "$(dirname "$0")/test-helper.inc.sh" +exit_on_package_build + cleanup "$1" diff --git a/linux/ibus-keyman/tests/scripts/test-helper.inc.sh b/linux/ibus-keyman/tests/scripts/test-helper.inc.sh index 2a3735d708..9ea5ffe3fc 100755 --- a/linux/ibus-keyman/tests/scripts/test-helper.inc.sh +++ b/linux/ibus-keyman/tests/scripts/test-helper.inc.sh @@ -10,7 +10,7 @@ function can_run_wayland() { fi } -function generate_kmpjson() { +function _generate_kmpjson() { local TESTDIR TESTDIR="$1" pushd "$TESTDIR" > /dev/null || exit @@ -78,7 +78,7 @@ EOF popd > /dev/null || exit } -function link_test_keyboards() { +function _link_test_keyboards() { KMX_TEST_DIR=$1 TESTDIR=$2 TESTBASEDIR=$3 @@ -93,22 +93,23 @@ function link_test_keyboards() { fi } -function setup() { - local DISPLAY_SERVER ENV_FILE PID_FILE TOP_SRCDIR TOP_BINDIR TESTBASEDIR TESTDIR - DISPLAY_SERVER=$1 - ENV_FILE=$2 - PID_FILE=$3 +function _setup_init() { + local ENV_FILE PID_FILE + ENV_FILE=$1 + PID_FILE=$2 - TOP_SRCDIR=${G_TEST_SRCDIR:-$(realpath "$(dirname "$0")/..")}/.. - TOP_BINDIR=${G_TEST_BUILDDIR:-$(realpath "$(dirname "$0/..")")}/.. - TESTBASEDIR=${XDG_DATA_HOME:-$HOME/.local/share}/keyman - TESTDIR=${TESTBASEDIR}/test_kmx + if [ -z "${TOP_SRCDIR:-}" ]; then + TOP_SRCDIR=${G_TEST_SRCDIR:-$(realpath "$(dirname "$0")/..")}/.. + fi + if [ -z "${TOP_BINDIR:-}" ]; then + TOP_BINDIR=${G_TEST_BUILDDIR:-$(realpath "$(dirname "$0/..")")}/.. + fi echo > "$ENV_FILE" if [ -f "$PID_FILE" ]; then # kill previous instances - "$(dirname "$0")"/teardown-tests.sh "$PID_FILE" + "$(dirname "$0")"/teardown-tests.sh "$PID_FILE" || true fi echo > "$PID_FILE" @@ -127,19 +128,34 @@ function setup() { exit 2 fi - link_test_keyboards "${TOP_SRCDIR}/../../common/test/keyboards/baseline" "$TESTDIR" "$TESTBASEDIR" + export LD_LIBRARY_PATH=${COMMON_ARCH_DIR}/src:${LD_LIBRARY_PATH-} + echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> "$ENV_FILE" +} - generate_kmpjson "$TESTDIR" +function _setup_test_dbus_server() { + local ENV_FILE PID_FILE + ENV_FILE=$1 + PID_FILE=$2 - # Start test dbus server + # Start test dbus server. This will create `/tmp/km-test-server.env`. "${TOP_BINDIR}/tests/km-dbus-test-server" &> /dev/null & sleep 1 - source /tmp/km-test-server.env + cat /tmp/km-test-server.env >> "$ENV_FILE" cat /tmp/km-test-server.env >> "$PID_FILE" echo "${TOP_BINDIR}/tests/stop-test-server" >> "$PID_FILE" - if [ "$DISPLAY_SERVER" == "wayland" ]; then + source /tmp/km-test-server.env + echo "# DBUS_SESSION_BUS_ADDRESS=$DBUS_SESSION_BUS_ADDRESS" +} + +function _setup_display_server() { + local DISPLAY_SERVER ENV_FILE PID_FILE + ENV_FILE=$1 + PID_FILE=$2 + DISPLAY_SERVER=$3 + + if [ "$DISPLAY_SERVER" == "--wayland" ]; then if ! can_run_wayland; then # support for --headless got added in mutter 40.x echo "ERROR: mutter doesn't support running headless. Can't run Wayland tests." @@ -173,6 +189,12 @@ function setup() { export DISPLAY=:32 echo "export DISPLAY=\"$DISPLAY\"" >> "$ENV_FILE" fi +} + +function _setup_schema_and_gsettings() { + local ENV_FILE PID_FILE + ENV_FILE=$1 + PID_FILE=$2 # Install schema to temporary directory. This removes the build dependency on the keyman package. SCHEMA_DIR=$TEMP_DATA_DIR/glib-2.0/schemas @@ -186,9 +208,6 @@ function setup() { cp "${TOP_SRCDIR}"/../keyman-config/resources/com.keyman.gschema.xml "$SCHEMA_DIR"/ glib-compile-schemas "$SCHEMA_DIR" - export LD_LIBRARY_PATH=${COMMON_ARCH_DIR}/src:${LD_LIBRARY_PATH-} - echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> "$ENV_FILE" - # Ubuntu 18.04 Bionic doesn't have ibus-memconf, and glib is not compiled with the keyfile # backend enabled, so we just use the default backend. Otherwise we use the keyfile # store which interferes less when running on a dev machine. @@ -197,6 +216,12 @@ function setup() { echo "export GSETTINGS_BACKEND=\"$GSETTINGS_BACKEND\"" >> "$ENV_FILE" IBUS_CONFIG=--config=/usr/libexec/ibus-memconf fi +} + +function _setup_ibus() { + local ENV_FILE PID_FILE + ENV_FILE=$1 + PID_FILE=$2 #shellcheck disable=SC2086 ibus-daemon ${ARG_VERBOSE-} --daemonize --panel=disable --address=unix:abstract="${TEMP_DATA_DIR}/test-ibus" ${IBUS_CONFIG-} &> /tmp/ibus-daemon.log @@ -208,11 +233,42 @@ function setup() { echo "export IBUS_ADDRESS=\"$IBUS_ADDRESS\"" >> "$ENV_FILE" - echo "# DBUS_SESSION_BUS_ADDRESS=$DBUS_SESSION_BUS_ADDRESS" #shellcheck disable=SC2086 "${TOP_BINDIR}/src/ibus-engine-keyman" --testing ${ARG_VERBOSE-} &> /tmp/ibus-engine-keyman.log & echo "kill -9 $! || true" >> "$PID_FILE" sleep 1s + +} +function setup() { + local DISPLAY_SERVER ENV_FILE PID_FILE TESTBASEDIR TESTDIR + DISPLAY_SERVER=$1 + ENV_FILE=$2 + PID_FILE=$3 + + _setup_init "${ENV_FILE}" "${PID_FILE}" + + TESTBASEDIR=${XDG_DATA_HOME:-$HOME/.local/share}/keyman + TESTDIR=${TESTBASEDIR}/test_kmx + + _link_test_keyboards "${TOP_SRCDIR}/../../common/test/keyboards/baseline" "$TESTDIR" "$TESTBASEDIR" + + _generate_kmpjson "$TESTDIR" + + _setup_test_dbus_server "${ENV_FILE}" "${PID_FILE}" + _setup_display_server "${ENV_FILE}" "${PID_FILE}" "${DISPLAY_SERVER}" + _setup_schema_and_gsettings "${ENV_FILE}" "${PID_FILE}" + _setup_ibus "${ENV_FILE}" "${PID_FILE}" +} + +function setup_display_server_only() { + local DISPLAY_SERVER ENV_FILE PID_FILE TESTBASEDIR TESTDIR + DISPLAY_SERVER=$1 + ENV_FILE=$2 + PID_FILE=$3 + + _setup_init "${ENV_FILE}" "${PID_FILE}" + _setup_display_server "${ENV_FILE}" "${PID_FILE}" "${DISPLAY_SERVER}" + _setup_schema_and_gsettings "${ENV_FILE}" "${PID_FILE}" } function cleanup() { @@ -227,3 +283,11 @@ function cleanup() { echo "# Finished shutdown of processes." fi } + +function exit_on_package_build() { + if [ -v KEYMAN_PKG_BUILD ]; then + # Skip setup during package builds - can't run headless and we won't + # run the other tests anyway + exit 0 + fi +} diff --git a/linux/keyman-config/keyman_config/install_kmp.py b/linux/keyman-config/keyman_config/install_kmp.py index 06c41a0cac..95d1e294e0 100755 --- a/linux/keyman-config/keyman_config/install_kmp.py +++ b/linux/keyman-config/keyman_config/install_kmp.py @@ -238,9 +238,9 @@ class InstallKmp(): if not language: return language - language = CanonicalLanguageCodeUtils.findBestTag(language, False, True) + language = CanonicalLanguageCodeUtils.findBestTag(language, False, False) for supportedLanguage in supportedLanguages: - tag = CanonicalLanguageCodeUtils.findBestTag(supportedLanguage['id'], False, True) + tag = CanonicalLanguageCodeUtils.findBestTag(supportedLanguage['id'], False, False) if tag == language: return tag return None diff --git a/linux/keyman-config/tests/test_install_kmp.py b/linux/keyman-config/tests/test_install_kmp.py index d1e10bd645..3320c532b5 100644 --- a/linux/keyman-config/tests/test_install_kmp.py +++ b/linux/keyman-config/tests/test_install_kmp.py @@ -199,19 +199,22 @@ class InstallKmpTests(unittest.TestCase): languages = [ {'id': 'de'}, {'id': 'esi-Latn'}, - {'id': 'dyo'} + {'id': 'dyo'}, + {'id': 'fuh-Arab'} ] for testcase in [ {'given': 'de', 'expected': 'de'}, - {'given': 'esi', 'expected': 'esi-Latn'}, - {'given': 'esi-Latn', 'expected': 'esi-Latn'}, + {'given': 'esi', 'expected': 'esi'}, + {'given': 'esi-Latn', 'expected': 'esi'}, {'given': 'es', 'expected': None}, {'given': 'en', 'expected': None}, {'given': None, 'expected': None}, # #3399 - {'given': 'dyo-latn', 'expected': 'dyo-Latn'}, - {'given': 'dyo', 'expected': 'dyo-Latn'}, + {'given': 'dyo-latn', 'expected': 'dyo'}, + {'given': 'dyo', 'expected': 'dyo'}, + {'given': 'fuh-Arab', 'expected': 'fuh-Arab'}, + {'given': 'fuh', 'expected': None}, ]: with self.subTest(data=testcase): # Execute diff --git a/linux/keyman-system-service/README.md b/linux/keyman-system-service/README.md index 358004af8f..8b6ca2eba6 100644 --- a/linux/keyman-system-service/README.md +++ b/linux/keyman-system-service/README.md @@ -1,8 +1,8 @@ # keyman-system-service -A DBus system service that allows to access /dev/input/* devices -to toggle capslock and perform other keyboard related actions when -running under Wayland. +A DBus system service that allows to access `/dev/input/*` devices +to toggle capslock and perform other keyboard related actions. This is +required when running under Wayland, but also used with X11. See , , diff --git a/linux/scripts/cow.sh b/linux/scripts/cow.sh index e175a666ef..201896a7e7 100755 --- a/linux/scripts/cow.sh +++ b/linux/scripts/cow.sh @@ -3,7 +3,7 @@ # If needed set cowbuilder up for building Keyman Debian packages # Then cowbuilder update -distributions='focal jammy kinetic lunar' +distributions='focal jammy lunar mantic' if ! dpkg-query -l cowbuilder; then echo "installing pbuilder and cowbuilder" diff --git a/linux/scripts/deb.sh b/linux/scripts/deb.sh index 3c87af0de8..35a476a55c 100755 --- a/linux/scripts/deb.sh +++ b/linux/scripts/deb.sh @@ -11,7 +11,7 @@ set -e -all_distributions="focal jammy" +all_distributions="focal jammy lunar mantic" distributions="" echo "all_distributions: ${all_distributions}" diff --git a/linux/scripts/dist.sh b/linux/scripts/dist.sh index f8d63b63e3..d19abf7f78 100755 --- a/linux/scripts/dist.sh +++ b/linux/scripts/dist.sh @@ -49,7 +49,7 @@ dpkg-source --tar-ignore=*~ --tar-ignore=.git --tar-ignore=.gitattributes \ --tar-ignore=core/build \ --tar-ignore=developer --tar-ignore=docs --tar-ignore=ios \ --tar-ignore=linux/keyman-config/buildtools/build-langtags.py --tar-ignore=__pycache__ \ - --tar-ignore=linux/help --tar-ignore=linux/Jenkinsfile \ + --tar-ignore=linux/help \ --tar-ignore=mac --tar-ignore=node_modules --tar-ignore=oem \ --tar-ignore=linux/build \ --tar-ignore=linux/builddebs \ diff --git a/linux/scripts/jenkins.sh b/linux/scripts/jenkins.sh deleted file mode 100755 index 5c79625955..0000000000 --- a/linux/scripts/jenkins.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash -# $1 - project name with appended tier, e.g. keyman-alpha -# $2 - GPG key used for signing the source package - -set -e -set -u - -## START STANDARD BUILD SCRIPT INCLUDE -# adjust relative paths as necessary -THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" -. "${THIS_SCRIPT%/*}/../../resources/build/build-utils.sh" -## END STANDARD BUILD SCRIPT INCLUDE - -. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" - -. "$THIS_SCRIPT_PATH/package-build.inc.sh" - -keyman_projects="keyman" - -tier="stable" - -if [[ "$1" =~ "-alpha" ]]; then - tier="alpha" -elif [[ "$1" =~ "-beta" ]]; then - tier="beta" -fi - -proj="$1" -proj=${proj%"-alpha"} -proj=${proj%"-beta"} - -fullsourcename="keyman" -sourcedir="$KEYMAN_ROOT" -sourcename=${fullsourcename%"-alpha"} -sourcename=${sourcename%"-beta"} - -# set Debian/changelog environment -export DEBFULLNAME="${fullsourcename} Package Signing Key" -export DEBEMAIL='jenkins@sil.org' - -checkAndInstallRequirements - -# clean up prev deb builds -builder_heading "cleaning previous builds of $1" - -rm -rf builddebs -rm -rf "$sourcedir/${1}"_*.{dsc,build,buildinfo,changes,tar.?z,log} -rm -rf "$sourcedir/../${1}"_*.{dsc,build,buildinfo,changes,tar.?z,log} - -builder_heading "Make source package for $fullsourcename" -builder_heading "reconfigure" -TIER="$tier" ./scripts/reconf.sh - -builder_heading "Make origdist" -./scripts/dist.sh origdist -builder_heading "Make deb source" -./scripts/deb.sh sourcepackage - -#sign source package -for file in builddebs/*.dsc; do - builder_heading "Signing source package $file" - debsign -k"$2" "$file" -done - -mv builddebs/* .. diff --git a/linux/scripts/launchpad.sh b/linux/scripts/launchpad.sh index 30495d2b0e..7437272c03 100755 --- a/linux/scripts/launchpad.sh +++ b/linux/scripts/launchpad.sh @@ -33,7 +33,7 @@ else fi echo "ppa: ${ppa}" -distributions="${DIST:-focal jammy kinetic lunar}" +distributions="${DIST:-focal jammy lunar mantic}" packageversion="${PACKAGEVERSION:-1~sil1}" BASEDIR=$(pwd) diff --git a/linux/scripts/package-build.inc.sh b/linux/scripts/package-build.inc.sh index 6c249891c1..fcfe492b43 100644 --- a/linux/scripts/package-build.inc.sh +++ b/linux/scripts/package-build.inc.sh @@ -49,6 +49,14 @@ function downloadSource() { sha256sum -c --ignore-missing SHA256SUMS |grep "${proj}" } +function wait_for_apt_deb { + # from https://gist.github.com/hrpatel/117419dcc3a75e46f79a9f1dce99ef52 + while sudo fuser /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock &>/dev/null 2>&1; do + echo "Waiting for apt/dpkg lock to release, sleeping 10s" + sleep 10 + done +} + function checkAndInstallRequirements() { local TOINSTALL="" @@ -63,12 +71,12 @@ function checkAndInstallRequirements() export DEBIAN_FRONTEND=noninteractive if [ -n "$TOINSTALL" ]; then - sudo apt-get update + wait_for_apt_deb && sudo apt-get update # shellcheck disable=SC2086 - sudo apt-get -qy install $TOINSTALL + wait_for_apt_deb && sudo apt-get -qy install $TOINSTALL fi sudo mk-build-deps debian/control - sudo apt-get -qy --allow-downgrades install ./keyman-build-deps_*.deb + wait_for_apt_deb && sudo apt-get -qy --allow-downgrades install ./keyman-build-deps_*.deb sudo rm -f keyman-buid-deps_* } diff --git a/linux/scripts/upload-to-debian.sh b/linux/scripts/upload-to-debian.sh index af443cac08..d7665f29f8 100755 --- a/linux/scripts/upload-to-debian.sh +++ b/linux/scripts/upload-to-debian.sh @@ -105,7 +105,7 @@ git add debian/changelog git commit -m "chore(linux): Update debian changelog" if [ -n "$PUSH" ]; then $NOOP git push --force-with-lease origin chore/linux/changelog - $NOOP gh pr create --draft --base "$stable_branch" --title "chore(linux): Update debian changelog" --body "@keymanapp-test-bot skip" + $NOOP gh pr create --draft --base "${stable_branch#origin/}" --title "chore(linux): Update debian changelog" --body "@keymanapp-test-bot skip" fi if $ISBETA; then @@ -118,7 +118,7 @@ git checkout -B chore/linux/cherry-pick/changelog ${CLBRANCH} git cherry-pick -x chore/linux/changelog if [ -n "$PUSH" ]; then $NOOP git push --force-with-lease origin chore/linux/cherry-pick/changelog - $NOOP gh pr create --draft --base ${CLBRANCH} --title "chore(linux): Update debian changelog 🍒" --body "@keymanapp-test-bot skip" + $NOOP gh pr create --draft --base ${CLBRANCH#origin/} --title "chore(linux): Update debian changelog 🍒" --body "@keymanapp-test-bot skip" fi builder_heading "Finishing" diff --git a/package-lock.json b/package-lock.json index f1b11b6741..49ef815928 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11214,9 +11214,9 @@ } }, "node_modules/word-wrap": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", - "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.4.tgz", + "integrity": "sha512-2V81OA4ugVo5pRo46hAoD2ivUJx8jXmWXfUkY4KFNw0hEptvN0QfH3K4nHiwzGeKl5rFKedV48QVoqYavy4YpA==", "dev": true, "engines": { "node": ">=0.10.0" diff --git a/resources/build/build-utils.sh b/resources/build/build-utils.sh index 7ae8db2ae1..cdb1944246 100755 --- a/resources/build/build-utils.sh +++ b/resources/build/build-utils.sh @@ -74,24 +74,31 @@ function findVersion() { VERSION_TAG= fi - if [ -z "${TEAMCITY_VERSION-}" -a -z "${JENKINS_HOME-}" ]; then - # Local dev machine, not TeamCity + if [ -z "${TEAMCITY_VERSION-}" ] && [ -z "${GITHUB_ACTIONS-}" ]; then + # Local dev machine, not TeamCity or GitHub Action VERSION_TAG="$VERSION_TAG-local" VERSION_ENVIRONMENT=local - else + elif [ -n "${TEAMCITY_PR_NUMBER-}" ]; then # On TeamCity: are we running a pull request build or a master/beta/stable build? - if [ ! -z "${TEAMCITY_PR_NUMBER-}" ]; then - VERSION_ENVIRONMENT=test - # Note TEAMCITY_PR_NUMBER can also be 'master', 'beta', or 'stable-x.y' - # This indicates we are running a Test build. - if [[ $TEAMCITY_PR_NUMBER =~ ^(master|beta|stable(-[0-9]+\.[0-9]+)?)$ ]]; then - VERSION_TAG="$VERSION_TAG-test" - else - VERSION_TAG="$VERSION_TAG-test-$TEAMCITY_PR_NUMBER" - fi + VERSION_ENVIRONMENT="test" + # Note TEAMCITY_PR_NUMBER can also be 'master', 'beta', or 'stable-x.y' + # This indicates we are running a Test build. + if [[ $TEAMCITY_PR_NUMBER =~ ^(master|beta|stable(-[0-9]+\.[0-9]+)?)$ ]]; then + VERSION_TAG="$VERSION_TAG-test" else - VERSION_ENVIRONMENT="$TIER" + VERSION_TAG="$VERSION_TAG-test-$TEAMCITY_PR_NUMBER" fi + elif [ -n "${GITHUB_ACTIONS-}" ] && ${GHA_TEST_BUILD-}; then + VERSION_ENVIRONMENT="test" + # Note GHA_BRANCH can be 'master', 'beta', or 'stable-x.y' + # This indicates we are running a Test build. + if [[ ${GHA_BRANCH-} =~ ^(master|beta|stable(-[0-9]+\.[0-9]+)?)$ ]]; then + VERSION_TAG="${VERSION_TAG}-test" + else + VERSION_TAG="${VERSION_TAG}-test-${GHA_BRANCH-unset}" + fi + else + VERSION_ENVIRONMENT="$TIER" fi VERSION_WITH_TAG="$VERSION$VERSION_TAG" diff --git a/resources/build/increment-version.sh b/resources/build/increment-version.sh index d54990194c..445459da16 100755 --- a/resources/build/increment-version.sh +++ b/resources/build/increment-version.sh @@ -143,7 +143,7 @@ if [ "$action" == "commit" ]; then popd > /dev/null # - # Trigger builds for the previous version on TeamCity, Jenkins and GitHub + # Trigger builds for the previous version on TeamCity and GitHub # triggerBuilds diff --git a/resources/build/run-required-test-builds.sh b/resources/build/run-required-test-builds.sh index 218383d159..300bd83156 100755 --- a/resources/build/run-required-test-builds.sh +++ b/resources/build/run-required-test-builds.sh @@ -54,11 +54,7 @@ function triggerTestBuilds() { eval test_builds='(${'bc_test_$platform'[@]})' for test_build in "${test_builds[@]}"; do if [[ $test_build == "" ]]; then continue; fi - if [ "${test_build:(-8)}" == "_Jenkins" ]; then - local job=${test_build%_Jenkins} - echo " -- Triggering build configuration $job/$branch on Jenkins" - triggerJenkinsBuild "$job" "$branch" "$force" - elif [ "${test_build:(-7)}" == "_GitHub" ]; then + if [ "${test_build:(-7)}" == "_GitHub" ]; then local job=${test_build%_GitHub} echo " -- Triggering GitHub action build $job/$branch" triggerGitHubActionsBuild true "$job" "$branch" diff --git a/resources/build/trigger-builds.inc.sh b/resources/build/trigger-builds.inc.sh index ac8d519d4a..8d6297b9e1 100644 --- a/resources/build/trigger-builds.inc.sh +++ b/resources/build/trigger-builds.inc.sh @@ -17,11 +17,7 @@ function triggerBuilds() { eval builds='(${'bc_${bcbase}_${platform}'[@]})' for build in "${builds[@]}"; do if [[ $build == "" ]]; then continue; fi - if [ "${build:(-8)}" == "_Jenkins" ]; then - local job=${build%_Jenkins} - echo Triggering Jenkins build "$job" "$base" "true" - triggerJenkinsBuild "$job" "$base" "true" - elif [ "${build:(-7)}" == "_GitHub" ]; then + if [ "${build:(-7)}" == "_GitHub" ]; then local job=${build%_GitHub} echo Triggering GitHub action build "$job" "$base" triggerGitHubActionsBuild false "$job" "$base" @@ -69,62 +65,6 @@ function triggerTeamCityBuild() { -d "$command" } -function triggerJenkinsBuild() { - local JENKINS_JOB="$1" - local JENKINS_BRANCH="${2:-master}" - - local JENKINS_SERVER=https://jenkins.lsdev.sil.org - - local FORCE="" - if [ "${3:-false}" == "true" ]; then - FORCE=", \"force\": true" - fi - - local TAG="" - # This will only be true if we created and pushed a tag - if [ "${action:-""}" == "commit" ]; then - TAG=", \"tag\": \"$VERSION_GIT_TAG\", \"tag2\": \"$VERSION_GIT_TAG\"" - fi - - if [[ $JENKINS_BRANCH != stable-* ]] && [[ $JENKINS_BRANCH =~ [0-9]+ ]]; then - JENKINS_BRANCH="PR-${JENKINS_BRANCH}" - fi - - local OUTPUT=$(curl --silent --write-out '\n' \ - -X POST \ - --header "token: $JENKINS_TOKEN" \ - --header "Content-Type: application/json" \ - $JENKINS_SERVER/generic-webhook-trigger/invoke \ - --data "{ \"project\": \"$JENKINS_JOB/$JENKINS_BRANCH\", \"branch\": \"$JENKINS_BRANCH\" $TAG $FORCE }") - - if echo "$OUTPUT" | grep -q "\"triggered\":true"; then - echo -n " job triggered: " - else - echo "##teamcity[buildProblem description='Triggering Jenkins build failed']" - echo -n " triggering failed: " - fi - - # Strip {"jobs":{ from the beginning of OUTPUT - OUTPUT=${OUTPUT#\{\"jobs\":\{} - # Split json string to lines with one job each - local jobs count - count=0 - IFS='|' jobs=(${OUTPUT//\},\"pipeline/\},|\"pipeline}) - # Find job that actually got triggered (or that we should have triggered) - for line in "${jobs[@]}"; do - if [[ $line == \"$JENKINS_JOB/$JENKINS_BRANCH* ]]; then - echo "$line" - count=$((++count)) - fi - done - if [[ $count < 1 ]]; then - # DEBUG - echo -n $OUTPUT - - echo - fi -} - function triggerGitHubActionsBuild() { local IS_TEST_BUILD="$1" local GITHUB_ACTION="$2" diff --git a/resources/build/trigger-definitions.inc.sh b/resources/build/trigger-definitions.inc.sh index 9370dadd82..4698719d4d 100644 --- a/resources/build/trigger-definitions.inc.sh +++ b/resources/build/trigger-definitions.inc.sh @@ -34,9 +34,6 @@ watch_common_linux='common/linux|common/web' # These bc_x_y variables ARE used in trigger-builds.inc.sh by pattern so the names are important, # and you won't find them directly in a grep search. # -# _Jenkins should be appended to any build configuration (pipeline) name that is from Jenkins, -# not TeamCity. -# # _GitHub should be appended to any build configuration name that is from GitHub, not TeamCity. # Test Build Configurations @@ -46,7 +43,7 @@ bc_test_all=() bc_test_android=(KeymanAndroid_TestPullRequests KeymanAndroid_TestSamplesAndTestProjects) bc_test_ios=(Keyman_iOS_TestPullRequests Keyman_iOS_TestSamplesAndTestProjects) -bc_test_linux=(KeymanLinux_TestPullRequests Keyman_Linux_Test_Integration Keyman_Common_KPAPI_TestPullRequests_Linux pipeline-keyman-packaging_Jenkins deb-pr-packaging_GitHub) +bc_test_linux=(KeymanLinux_TestPullRequests Keyman_Linux_Test_Integration Keyman_Common_KPAPI_TestPullRequests_Linux deb-pr-packaging_GitHub) bc_test_mac=(Keyman_KeymanMac_PullRequests Keyman_Common_KPAPI_TestPullRequests_macOS) bc_test_windows=(KeymanDesktop_TestPullRequests KeymanDesktop_TestPrRenderOnScreenKeyboards Keyman_Common_KPAPI_TestPullRequests_Windows) bc_test_web=(Keymanweb_TestPullRequests Keyman_Common_LMLayer_TestPullRequests Keyman_Common_KPAPI_TestPullRequests_WASM) @@ -66,7 +63,7 @@ vcs_test=HttpsGithubComKeymanappKeymanPRs bc_master_android=(KeymanAndroid_Build) bc_master_ios=(Keyman_iOS_Master) -bc_master_linux=(KeymanLinux_Master pipeline-keyman-packaging_Jenkins deb-release-packaging_GitHub) +bc_master_linux=(KeymanLinux_Master deb-release-packaging_GitHub) bc_master_mac=(KeymanMac_Master) bc_master_windows=(Keyman_Build) bc_master_web=(Keymanweb_Build) @@ -78,7 +75,7 @@ vcs_master=HttpsGithubComKeymanappKeyman bc_beta_android=(KeymanAndroid_Build) bc_beta_ios=(Keyman_iOS_Master) -bc_beta_linux=(KeymanLinux_Master pipeline-keyman-packaging_Jenkins deb-release-packaging_GitHub) +bc_beta_linux=(KeymanLinux_Master deb-release-packaging_GitHub) bc_beta_mac=(KeymanMac_Master) bc_beta_windows=(Keyman_Build) bc_beta_web=(Keymanweb_Build) @@ -90,7 +87,7 @@ vcs_beta=HttpsGithubComKeymanappKeyman bc_stable_14_0_android=(KeymanAndroid_Build) bc_stable_14_0_ios=(Keyman_iOS_Master) -bc_stable_14_0_linux=(KeymanLinux_Master pipeline-keyman-packaging_Jenkins deb-release-packaging_GitHub) +bc_stable_14_0_linux=(KeymanLinux_Master deb-release-packaging_GitHub) bc_stable_14_0_mac=(KeymanMac_Master) bc_stable_14_0_windows=(Keyman_Build) bc_stable_14_0_web=(Keymanweb_Build) @@ -105,7 +102,7 @@ vcs_stable_14_0=HttpsGithubComKeymanappKeyman bc_stable_15_0_android=(KeymanAndroid_Build) bc_stable_15_0_ios=(Keyman_iOS_Master) -bc_stable_15_0_linux=(KeymanLinux_Master pipeline-keyman-packaging_Jenkins deb-release-packaging_GitHub) +bc_stable_15_0_linux=(KeymanLinux_Master deb-release-packaging_GitHub) bc_stable_15_0_mac=(KeymanMac_Master) bc_stable_15_0_windows=(Keyman_Build) bc_stable_15_0_web=(Keymanweb_Build) @@ -116,7 +113,7 @@ vcs_stable_15_0=HttpsGithubComKeymanappKeyman bc_stable_16_0_android=(KeymanAndroid_Build) bc_stable_16_0_ios=(Keyman_iOS_Master) -bc_stable_16_0_linux=(KeymanLinux_Master pipeline-keyman-packaging_Jenkins) +bc_stable_16_0_linux=(KeymanLinux_Master) bc_stable_16_0_mac=(KeymanMac_Master) bc_stable_16_0_windows=(Keyman_Build) bc_stable_16_0_web=(Keymanweb_Build) diff --git a/resources/standards-data/ldml-keyboards/techpreview/3.0/fr-t-k0-azerty.xml b/resources/standards-data/ldml-keyboards/techpreview/3.0/fr-t-k0-azerty.xml index 7a1916d1df..156a195c05 100644 --- a/resources/standards-data/ldml-keyboards/techpreview/3.0/fr-t-k0-azerty.xml +++ b/resources/standards-data/ldml-keyboards/techpreview/3.0/fr-t-k0-azerty.xml @@ -63,12 +63,9 @@ - diff --git a/web/src/app/browser/src/contextManager.ts b/web/src/app/browser/src/contextManager.ts index 42e7edd729..492c627146 100644 --- a/web/src/app/browser/src/contextManager.ts +++ b/web/src/app/browser/src/contextManager.ts @@ -211,6 +211,12 @@ export default class ContextManager extends ContextManagerBase) => void +} + +export default class CloudQueryEngine extends EventEmitter { private cloudResolutionPromises: Record>> = {}; private _languageListPromise: ManagedPromise; @@ -65,6 +71,8 @@ export default class CloudQueryEngine { private pathConfig: PathConfiguration; constructor(requestEngine: CloudRequesterInterface, pathConfig: PathConfiguration) { + super(); + this.requestEngine = requestEngine; this.pathConfig = pathConfig; @@ -138,10 +146,16 @@ export default class CloudQueryEngine { result = new Error(CLOUD_REGISTRATION_ERR + err); } - if(promiseid) { + if(!promiseid) { + this.emit('unboundregister', result); + return; + } else { const promise: ManagedPromise | ManagedPromise = this.cloudResolutionPromises[promiseid]; - if(promise) { + if(!promise) { + this.emit('unboundregister', result); + return; + } else { try { if(result instanceof Error) { promise.reject(result as Error); diff --git a/web/src/engine/package-cache/src/keyboardRequisitioner.ts b/web/src/engine/package-cache/src/keyboardRequisitioner.ts index 6a3f060510..7bed7b12ac 100644 --- a/web/src/engine/package-cache/src/keyboardRequisitioner.ts +++ b/web/src/engine/package-cache/src/keyboardRequisitioner.ts @@ -97,6 +97,19 @@ export default class KeyboardRequisitioner { this.pathConfig = pathConfig; this.cache = new StubAndKeyboardCache(keyboardLoader); this.cloudQueryEngine = new CloudQueryEngine(keyboardRequester, this.pathConfig); + + // Handles keymanweb.com's precached keyboard array. There is no associated promise, + // so there's nothing handling the `register` call's results otherwise. + this.cloudQueryEngine.on('unboundregister', (registration) => { + // Internal, undocumented use-case of `keyman.register`: precached keyboard loading + // Other uses may trigger errors, especially if there's a type-structure mismatch. + // Those errors should not be handled here; let them surface. + if(Array.isArray(registration)) { + registration.forEach((entry) => { + this.cache.addStub(entry); + }); + } + }); } addKeyboardArray(x: (string|RawKeyboardMetadata)[]): Promise<(KeyboardStub | ErrorStub)[]> { diff --git a/windows/src/README.md b/windows/src/README.md index 9277b9c603..d8ca3b3901 100644 --- a/windows/src/README.md +++ b/windows/src/README.md @@ -59,13 +59,13 @@ will need to uninstall and reinstall. ## Certificates -In order to create a release build, you will need a code signing certiicate. +In order to create a release build, you will need a code signing certificate. You can use your own certificate, or you can use test certificates which are not globally trusted. The environment variables `SC_PFX_SHA1` and `SC_PFX_SHA256` can be set to custom certificate paths. The Keyman repo no longer includes test certificates. To build your own, run -`nmake test-certificates` from **common/windows/delphi/tools/certificates** to +`nmake test-certificate` from **common/windows/delphi/tools/certificates** to build and install your own local root CA "**KeymanTestCA**" certificates. If you specify a password for the certificate, you'll need to set that in the environment variable `SC_PWD`. diff --git a/windows/src/desktop/help/about/requirements.md b/windows/src/desktop/help/about/requirements.md index b9dd275555..3c241bc858 100644 --- a/windows/src/desktop/help/about/requirements.md +++ b/windows/src/desktop/help/about/requirements.md @@ -4,20 +4,9 @@ title: System Requirements ## Supported Windows Operating Systems -Keyman fully supports 32-bit *and* 64-bit versions of the following -Windows operating systems: - -- Windows 7 -- Windows 8 -- Windows 8.1 -- Windows 10 -- Windows 11 -- Windows Server 2008 and 2008 R2 -- Windows Server 2012 and 2012 R2 - -**Note:** Keyman works slightly differently in different versions of Windows. Older versions of Windows have more language limitations and need extra configuration. +* See [What operating systems does Keyman support?](../common/os) topic. ## Resource Requirements Keyman has minimal resource requirements. Any computer that can run -Windows 7 should be able to run Keyman without trouble. +Windows 10 should be able to run Keyman without trouble. diff --git a/windows/src/desktop/help/advanced/text_services_framework.md b/windows/src/desktop/help/advanced/text_services_framework.md index 7b3ed4f994..e7a977a062 100644 --- a/windows/src/desktop/help/advanced/text_services_framework.md +++ b/windows/src/desktop/help/advanced/text_services_framework.md @@ -10,14 +10,6 @@ supports features such as keyboard drivers, handwriting recognition, speech recognition, as well as spell checking and other text processing functions. -In Windows 7, the Language Bar is the core user interface for TSF. From -the Language Bar, you can select the input language, and control -keyboard input, handwriting recognition and speech recognition. In -Windows 8, the Language Bar is no longer used, and the interface is -tightly integrated into the taskbar. - -![](../desktop_images/language-bar.png) - With Keyman Desktop 9 and later versions, all keyboards are registered through the Windows interfaces, and the key advantage is that Keyman now automatically detects applications that have support for TSF and diff --git a/windows/src/desktop/help/basic/config/options.md b/windows/src/desktop/help/basic/config/options.md index e19d51c24d..3b4bbd86b0 100644 --- a/windows/src/desktop/help/basic/config/options.md +++ b/windows/src/desktop/help/basic/config/options.md @@ -60,23 +60,6 @@ To open the Options tab of Keyman Configuration: Click the Reset Hints button to switch all hint messages on again, even those that you have switched off on a case-by-case basis. -- Select keyboard layout for all applications (Windows 7) - - Unlike Windows 7 default behaviour, Keyman allows you to select one - Windows language and Keyman keyboard for all open applications and - text fields across your entire system. Tick this option to select - one Windows language and Keyman keyboard across your entire system. - Untick this option to select Windows language and Keyman keyboards - independently for different programs. - - On Windows 8, 8.1, 10 and later versions, this checkbox is disabled - because Windows has this functionality built in. The setting can be - changed in Windows, with the following steps: - - - Search for "Advanced Keyboard Settings" - - Check the \"Let me set a different input method for each app - window\" checkbox. - - Automatically report errors to keyman.com If Keyman crashes, then it can automatically send a report to the diff --git a/windows/src/desktop/help/basic/make-taskbar-icon-visible.md b/windows/src/desktop/help/basic/make-taskbar-icon-visible.md index 9c5bbd66a4..62b90be5d4 100644 --- a/windows/src/desktop/help/basic/make-taskbar-icon-visible.md +++ b/windows/src/desktop/help/basic/make-taskbar-icon-visible.md @@ -37,30 +37,3 @@ Here's how to make the change with Windows Settings: 5. The Keyman icon will now always appear in the Windows Taskbar near the clock, if Keyman is on. 6. Continue on to [Step 5](../start/tutorial#step-5-) of this guide. - -- On Windows 8: - - 1. Right-click on the Windows Taskbar. - ![](../desktop_images/win8-taskbar1.png) - 2. Select 'Properties'. - 3. Next to 'Notification area', click Customize…. - ![](../desktop_images/win8-taskbar2.png) - 4. From the dropdown menu beside 'Keyman - Engine x86', select 'Show icon and notifications'. - ![](../desktop_images/win8-taskbar3.png) - 5. Click OK to apply changes. The Keyman icon will now always - appear in the Windows Taskbar near the clock, if Keyman is - on. - 6. Continue on to [Step 5](../start/tutorial#step-5-) of this guide. - -- On Windows 7: - - 1. Open the Windows Start menu. - 2. In the search field, type and enter: Notification Area Icons - 3. From the dropdown menu beside Keyman - Engine x86, select 'Show icon and notifications'. - ![](../desktop_images/7-taskbar.png) - 4. Click OK to apply changes. The Keyman icon will now always - appear in the Windows Taskbar near the clock, if Keyman is - on. - 5. Continue on to [Step 5](../start/tutorial#step-5-) of this guide. diff --git a/windows/src/desktop/help/basic/select-keyboard.md b/windows/src/desktop/help/basic/select-keyboard.md index 203866d545..13e6bbc016 100644 --- a/windows/src/desktop/help/basic/select-keyboard.md +++ b/windows/src/desktop/help/basic/select-keyboard.md @@ -120,4 +120,3 @@ reach the desired keyboard, then release spacebar. - [Keyman Configuration - Keyboard Layouts Tab](config/keyboards) - [How To - Download and Install a Keyman Keyboard](../start/download-and-install-keyboard) - [Keyboard Task - Enable or Disable a Keyboard](enable-or-disable-keyboard) -- [How To - Fix A Problem with an Active Keyman Keyboard Not Typing](../troubleshooting/hidden) diff --git a/windows/src/desktop/help/basic/uninstall.md b/windows/src/desktop/help/basic/uninstall.md index d9952a4279..7e7cb2274c 100644 --- a/windows/src/desktop/help/basic/uninstall.md +++ b/windows/src/desktop/help/basic/uninstall.md @@ -15,18 +15,3 @@ title: Software Task - Uninstall Keyman 5. Click Uninstall. 6. Follow the prompts to complete the uninstall. - -## Uninstall Keyman from Windows 7 or 8 - -1. Exit Keyman. - -2. Open Windows Control Panel. - -3. Select \'Add or Remove Programs\' or \'Programs and Features\' or - \'Uninstall a program\'. - -4. Click Keyman in the list. - -5. Click Remove or Uninstall or right-click \'Uninstall\'. - -6. Follow the prompts to complete the uninstall. diff --git a/windows/src/desktop/help/common/os.md b/windows/src/desktop/help/common/os.md index ca1ead7b27..32bf778813 100644 --- a/windows/src/desktop/help/common/os.md +++ b/windows/src/desktop/help/common/os.md @@ -2,26 +2,16 @@ title: What operating systems does Keyman support? --- -Keyman fully supports 32-bit *and* 64-bit versions of the following +Keyman for Windows fully supports the following versions of the Windows operating systems: -- Windows Server 2008 - -- Windows 7 - -- Windows 8 - -- Windows 8.1 - -- Windows 10 +- Windows 10 (32-bit and 64-bit editions) - Windows 11 -- Windows Server 2012 and 2012 R2 +- Windows Server 2019 and 2022 -**Note:** -Keyman works slightly differently in different versions of Windows. -Older versions of Windows have more language limitations and need extra -configuration. Windows Vista and later versions of Windows include added -security measures and some configuration actions will require you to -confirm security prompts. +Keyman also runs on Android, iOS, Linux, macOS, and in websites. Visit https://keyman.com for more downloads. + +* **Note:** Keyman does not yet support Windows Insider Preview ARM64. +* Keyman works slightly differently in different versions of Windows. diff --git a/windows/src/desktop/help/common/requirements.md b/windows/src/desktop/help/common/requirements.md index cdd3d81c23..11fefc6dfd 100644 --- a/windows/src/desktop/help/common/requirements.md +++ b/windows/src/desktop/help/common/requirements.md @@ -3,4 +3,4 @@ title: What are Keyman's hardware requirements? --- Keyman has minimal resource requirements. Any computer that can run -Windows 7 or later should be able to run Keyman without trouble. +Windows 10 or later should be able to run Keyman without trouble. diff --git a/windows/src/desktop/help/troubleshooting/hidden.md b/windows/src/desktop/help/troubleshooting/hidden.md deleted file mode 100644 index 11d35a151f..0000000000 --- a/windows/src/desktop/help/troubleshooting/hidden.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -title: How To - Fix A Problem with an Active Keyman Keyboard Not Typing ---- - -## Symptoms -You may experience an issue in Windows 7 where: -* Keyman is on; and -* your Keyman keyboard is active, your Keyman keyboard is active, with its icon showing in the Keyman menu; but -* your Keyman keyboard does not work. - -## Resolution -Unhide the Keyman menu as described in [Step 2](../start/tutorial#step-2-) of the Getting Started tutorial. - -This can also be resolved by switching language using the Windows Language Bar or the Language Icon - -## Background -When the Keyman icon is in the hidden notification area of the Windows taskbar, opening the notification area and selecting the icon causes it to change the keyboard for the taskbar, not for your active application. Clicking back to the active application results in the keyboard turning off again. - -## Related Topics -- [Keyboard Task - Turn on a Keyboard](../basic/select-keyboard) diff --git a/windows/src/desktop/help/troubleshooting/index.md b/windows/src/desktop/help/troubleshooting/index.md index 03e9745fec..54416ad9ba 100644 --- a/windows/src/desktop/help/troubleshooting/index.md +++ b/windows/src/desktop/help/troubleshooting/index.md @@ -10,6 +10,4 @@ title: Troubleshooting * [How To - Resolve Security Software Conflicts with keyman32.dll](securitysoftware) * [How To - Use the Keyman Setup Bootstrapper](bootstrapper) -* [How To - Fix A Problem with an Active Keyman Keyboard Not Typing](hidden) - * [How To - Allow Windows 10 to Install Apps From Anywhere](install-app-from-anywhere)