diff --git a/.github/workflows/api-verification.yml b/.github/workflows/api-verification.yml index 443adb471f..8f37141c08 100644 --- a/.github/workflows/api-verification.yml +++ b/.github/workflows/api-verification.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Restore artifacts - uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 with: path: | artifacts diff --git a/.github/workflows/close-linked-issues-for-merged-prs.yml b/.github/workflows/close-linked-issues-for-merged-prs.yml new file mode 100644 index 0000000000..2a0ce0a793 --- /dev/null +++ b/.github/workflows/close-linked-issues-for-merged-prs.yml @@ -0,0 +1,20 @@ +name: Close linked issues for merged pull requests + +on: + pull_request: + types: [closed] + branches: + - beta + - 'stable-*' + - 'epic/*' + +jobs: + closeIssueOnPrMergeTrigger: + + runs-on: ubuntu-latest + + steps: + - name: Closes issues related to a merged pull request. + uses: ldez/gha-mjolnir@5574ed1f1151e4d2f11e3513cd85920a3a46bb7b # v1.5.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/deb-packaging.yml b/.github/workflows/deb-packaging.yml index bf1bfca2fa..00468125c7 100644 --- a/.github/workflows/deb-packaging.yml +++ b/.github/workflows/deb-packaging.yml @@ -221,7 +221,7 @@ jobs: export DEBIAN_PRIORITY=critical export DEBCONF_NOWARNINGS=yes sudo apt-get update - sudo apt-get install -q -y dput + sudo apt-get install -q -y dput rsync - name: Setup .dput.cf run: | @@ -315,7 +315,7 @@ jobs: echo "GIT_USER=${{ github.event.client_payload.user }}" >> artifacts/env - name: Cache artifacts - uses: actions/cache/save@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 with: path: | artifacts diff --git a/HISTORY.md b/HISTORY.md index 544a275dea..9de791f041 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,38 @@ # Keyman Version History +## 19.0.9 alpha 2025-02-27 + +* docs: move node.js before Emscripten (#13366) +* chore: add a workflow to automatically close linked issues (#13368) + +## 19.0.8 alpha 2025-02-21 + +* feat(windows): hack in some fun features for kmdevlink (#13237) +* feat(developer): use ABNF to validate LDML transform (#13236) +* fix(linux): remove `--platform amd64` from docker build script (#13318) + +## 19.0.7 alpha 2025-02-20 + +* chore(linux): allow to build and use Debian docker image (#13284) + +## 19.0.6 alpha 2025-02-19 + +* chore: merge beta to master B18S1 (#13239) +* fix(linux): add missing dependency for uploading to llso (#13280) + +## 19.0.5 alpha 2025-02-15 + +* fix(linux): update location of lcov.deb for Jammy (#13253) + +## 19.0.4 alpha 2025-02-14 + +* feat(developer): serialize KMXPlus into XML (#13174) + +## 19.0.3 alpha 2025-02-13 + +* docs: update keyboard processor build source (#13221) +* chore(linux): additional code cleanup after Focal removal (#13206) + ## 19.0.2 alpha 2025-02-12 * chore(linux): remove support of Ubuntu 20.04 Focal (#13203) @@ -9,6 +42,30 @@ * refactor(windows): rename `TKeymanMutex.MutexOwned` to `TakeOwnership` and add `ReleaseOwnership` (#13168) * chore: increment to alpha 19.0 (#13187) +## 18.0.191 beta 2025-02-12 + +* feat(windows): handle a hard windows reset occurring while downloading updated keyman files (#13128) +* docs(developer): update what's new for 18.0 (#13198) +* chore(mac): update whats new for Keyman 18 (#13197) +* chore(ios): update whats new for Keyman 18 (#13199) +* fix: use tier and version from branch when merging history from another branch (#13170) +* fix(linux): start system service when switching keyboards (#13172) +* chore(linux): update actions/cache to non-deprecated version (#13193) +* chore(linux): update branch that's used for Debian packaging (#13192) +* chore(linux): remove support of Ubuntu 20.04 Focal (#13202) + +## 18.0.190 beta 2025-02-11 + +* fix(developer): bundle ttfmeta library internally with kmc-keyboard-info (#11631) +* fix(developer): support Windows and Unicode names in .ttf (#11633) +* chore(developer): increase timeout for kmc-ldml compiler test (#11635) +* chore(common): deps: update eslint typescript plugins (#11842) +* refactor(windows): rename `TKeymanMutex.MutexOwned` to `TakeOwnership` and add `ReleaseOwnership` (#13168) + +## 18.0.189 beta 2025-02-11 + +* chore: move to beta + ## 18.0.188 alpha 2025-02-10 * fix(windows): check the params status flag equals ucrsUpdateReady before attempting to download the keyman setup file (#13154) @@ -1321,7 +1378,7 @@ * chore(common): move to 18.0 alpha (#10713) * chore: move to 18.0 alpha -## 17.0.335 alpha 2025-02-06 +## 17.0.335 stable 2025-02-06 * fix(android): improve resource-update tool handling of host Activity's closure (#13057) * fix(ios): prevent message-handler collision (#13058) diff --git a/VERSION.md b/VERSION.md index dd7d796646..0394531a29 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -19.0.3 \ No newline at end of file +19.0.10 \ No newline at end of file diff --git a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts index d0de2989d0..24538c0108 100644 --- a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts +++ b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts @@ -385,15 +385,11 @@ export class UnicodeSetItem extends VarsItem { }; export class SetVarItem extends VarsItem { - constructor(id: string, value: string[], sections: DependencySections, rawItems: string[]) { + constructor(id: string, value: string[], sections: DependencySections) { super(id, value.join(' '), sections); this.items = sections.elem.allocElementString(sections, value); - this.rawItems = rawItems; } - // element string array - items: ElementString; - // like items, but with unprocessed marker strings - rawItems: string[]; + items: ElementString; // element string array valid() : boolean { return !!this.items; } @@ -409,10 +405,12 @@ export class StringVarItem extends VarsItem { // 'tran' export class TranTransform { - from: StrsItem; - to: StrsItem; - mapFrom: StrsItem; // var name - mapTo: StrsItem; // var name + from: StrsItem; // "from" computed regex string + to: StrsItem; // "to" (replacement) computed regex string + mapFrom: StrsItem; // var name for map + mapTo: StrsItem; // var name for map + _from?: string; // Not part of binary file: for use in the XML serializer. If present, sets the from= attribute for XML. + _to?: string; // Not part of binary file: for use in the XML serializer. If present, sets the to= attribute for XML. } export class TranGroup { @@ -424,6 +422,9 @@ export class TranGroup { export class TranReorder { elements: ElementString; before: ElementString; + _before?: string; // Not part of binary file: for use in the XML serializer. If present, sets the before= attribute for XML. + _from?: string; // Not part of binary file: for use in the XML serializer. If present, sets the from= attribute for XML. + _order?: string; // Not part of binary file: for use in the XML serializer. If present, sets the order= attribute for XML. }; export class Tran extends Section { diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 022328ff94..69a039aa00 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -114,6 +114,13 @@ const GENERATOR_OPTIONS: KeymanXMLOptionsBag = { textNodeName: '_', suppressEmptyNode: true, }, + keyboard3: { + attributeNamePrefix: '$', + ignoreAttributes: false, + format: true, + textNodeName: '_', + suppressEmptyNode: true, + }, }; /** wrapper for XML parsing support */ diff --git a/developer/src/kmc-ldml/.gitignore b/developer/src/kmc-ldml/.gitignore new file mode 100644 index 0000000000..3745d0bfbc --- /dev/null +++ b/developer/src/kmc-ldml/.gitignore @@ -0,0 +1,3 @@ +src/util/abnf/**/*.pegjs +src/util/abnf/**/*.d.ts +src/util/abnf/**/*.js diff --git a/developer/src/kmc-ldml/build.sh b/developer/src/kmc-ldml/build.sh index a961fea6fd..b48e851c71 100755 --- a/developer/src/kmc-ldml/build.sh +++ b/developer/src/kmc-ldml/build.sh @@ -27,24 +27,45 @@ builder_describe "Keyman kmc Keyboard Compiler module" \ "--dry-run,-n don't actually publish, just dry run" builder_describe_outputs \ - configure /node_modules \ + configure /developer/src/kmc-ldml/src/util/abnf/46/transform-from-required.js \ build /developer/src/kmc-ldml/build/src/main.js \ api /developer/build/api/kmc-ldml.api.json builder_parse "$@" function do_clean() { - rm -rf ./build/ ./tsconfig.tsbuildinfo + rm -rf ./build/ ./tsconfig.tsbuildinfo ./src/util/abnf/*/*.pegjs ./src/util/abnf/*/*.ts ./src/util/abnf/*/*.js } function do_configure() { verify_npm_setup + do_build_abnf } function do_build() { npm run build } +function do_build_abnf() { + # we convert over all abnf files found. + local file + for file in "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards"/*/abnf/*.abnf; do + local cldrver="$(basename $(dirname $(dirname "$file")))" + local base="$(basename "$file" .abnf)" + local peg="$base.pegjs" + local outdir="./src/util/abnf/$cldrver" + local outfile="$outdir/$peg" + local outjs="$outdir/$base.js" + if [ ! -f "$outjs" ]; then + mkdir -p "$outdir" + printf "${COLOR_GREY}abnf_gen ${COLOR_PURPLE}${cldrver}/${base}.abnf -> ${peg}${COLOR_RESET}\n" + "$KEYMAN_ROOT/node_modules/.bin/abnf_gen" "$file" -o "$outfile" + printf "${COLOR_GREY}peggy ${COLOR_PURPLE}${cldrver}/${peg} -> ${base}.js${COLOR_RESET}\n" + "$KEYMAN_ROOT/node_modules/.bin/peggy" "$outfile" -o "$outjs" --format es --dts + fi + done +} + function do_build_fixtures() { # Build basic.kmx and emit its checksum mkdir -p ./build/test/fixtures diff --git a/developer/src/kmc-ldml/package.json b/developer/src/kmc-ldml/package.json index cecbc8e15b..5375e2673a 100644 --- a/developer/src/kmc-ldml/package.json +++ b/developer/src/kmc-ldml/package.json @@ -38,10 +38,12 @@ "@types/mocha": "^5.2.7", "@types/node": "^20.4.1", "@types/semver": "^7.3.12", + "abnf": "^4.3.1", "c8": "^7.12.0", "chalk": "^2.4.2", "common-tags": "^1.8.2", "mocha": "^8.4.0", + "peggy": "^4.2.0", "typescript": "^5.4.5" }, "mocha": { diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index e341f3ac87..d86d7e2bf5 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -15,8 +15,9 @@ import { SubstitutionUse, Substitutions } from './substitution-tracker.js'; /** reserved name for the special gap key. space is not allowed in key ids. */ const reserved_gap = "gap (reserved)"; - export class KeysCompiler extends SectionCompiler { + /** keys that are of a reserved type */ + public static RESERVED_KEY = Symbol('Reserved Key'); static validateSubstitutions( keyboard: LDMLKeyboard.LKKeyboard, st: Substitutions @@ -222,6 +223,19 @@ export class KeysCompiler extends SectionCompiler { /** count of reserved keys, for tests */ public static readonly reserved_count = KeysCompiler.reserved_keys.length; + /** mark as reserved */ + private static asReserved(k : KeysKeys) : KeysKeys { + const o = k as any; + o[KeysCompiler.RESERVED_KEY] = true; + return k; + } + + /** true if a reserved key */ + public static isReserved(k : KeysKeys) : boolean { + const o = k as any; + return !!o[KeysCompiler.RESERVED_KEY]; + } + /** load up all reserved keys */ getReservedKeys(sections: KMXPlus.DependencySections) : Map { const r = new Map(); @@ -231,7 +245,7 @@ export class KeysCompiler extends SectionCompiler { const no_list = sections.list.allocList([], {}, sections); // now add the reserved key(s). - r.set(reserved_gap, { + r.set(reserved_gap, KeysCompiler.asReserved({ flags: constants.keys_key_flags_gap | constants.keys_key_flags_extend, id: sections.strs.allocString(reserved_gap), flicks: '', @@ -241,7 +255,7 @@ export class KeysCompiler extends SectionCompiler { switch: no_string, to: no_string, width: 10.0, // 10 * .1 - }); + })); if (r.size !== KeysCompiler.reserved_count) { throw Error(`Internal Error: KeysCompiler.reserved_count=${KeysCompiler.reserved_count} != ${r.size} actual reserved keys.`); @@ -357,7 +371,7 @@ export class KeysCompiler extends SectionCompiler { flags |= constants.keys_key_flags_extend; } const width = Math.ceil((key.width || 1) * 10.0); // default, width=1 - sect.keys.push({ + sect.keys.push(SectionCompiler.copySymbols({ flags, flicks: flickId, id, @@ -367,7 +381,7 @@ export class KeysCompiler extends SectionCompiler { switch: keySwitch, // 'switch' is a reserved word to, width, - }); + }, key)); } } diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 3577a2e110..1a84b10937 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -258,4 +258,11 @@ export class LdmlCompilerMessages { \`\`. `); + static ERROR_UnparseableTransformTo = SevErrorTransform | 0x06; + static Error_UnparseableTransformTo = (o: {to: string, message: string}) => m( + this.ERROR_UnparseableTransformTo, + `Invalid transform to="${def(o.to)}": "${def(o.message)}"`, + ); + + } diff --git a/developer/src/kmc-ldml/src/compiler/section-compiler.ts b/developer/src/kmc-ldml/src/compiler/section-compiler.ts index e33eda1086..dd83a9deda 100644 --- a/developer/src/kmc-ldml/src/compiler/section-compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/section-compiler.ts @@ -55,4 +55,20 @@ export abstract class SectionCompiler { ]); return defaults; } + + /** + * Copy symbols from 'from' onto 'onto' + * This is used to propagate special symbols such as ImportStatus + * and XML + * @param onto object to copy onto + * @param from source for symbols + * @returns the onto object + */ + public static copySymbols(onto: T, from: any) : T { + const o = onto as any; + for (const sym of Object.getOwnPropertySymbols(from)) { + o[sym] = from[sym]; + } + return onto; + } } diff --git a/developer/src/kmc-ldml/src/compiler/tran.ts b/developer/src/kmc-ldml/src/compiler/tran.ts index e8d7b9eb37..95d79dd468 100644 --- a/developer/src/kmc-ldml/src/compiler/tran.ts +++ b/developer/src/kmc-ldml/src/compiler/tran.ts @@ -17,6 +17,7 @@ import LKTransforms = LDMLKeyboard.LKTransforms; import { verifyValidAndUnique } from "../util/util.js"; import { LdmlCompilerMessages } from "./ldml-compiler-messages.js"; import { Substitutions, SubstitutionUse } from "./substitution-tracker.js"; +import { transform_from_parse, transform_to_parse } from "../util/abnf/abnf.js"; type TransformCompilerType = 'simple' | 'backspace'; @@ -138,11 +139,20 @@ export abstract class TransformCompiler result.substituteMarkerString(v, false)); - result.sets.push(new SetVarItem(id, cookedItems, sections, rawItems)); + result.sets.push(new SetVarItem(id, cookedItems, sections)); } addUnicodeSet(result: Vars, e: LDMLKeyboard.LKUSet, sections: DependencySections): void { const { id } = e; diff --git a/developer/src/kmc-ldml/src/util/abnf/abnf.ts b/developer/src/kmc-ldml/src/util/abnf/abnf.ts new file mode 100644 index 0000000000..17506314f7 --- /dev/null +++ b/developer/src/kmc-ldml/src/util/abnf/abnf.ts @@ -0,0 +1,6 @@ +/** + * Re-export the generated parsers. At present, we hard code CLDR version. + */ + +export { parse as transform_to_parse } from './46/transform-to-required.js'; +export { parse as transform_from_parse } from './46/transform-from-required.js'; diff --git a/developer/src/kmc-ldml/src/util/serialize.ts b/developer/src/kmc-ldml/src/util/serialize.ts new file mode 100644 index 0000000000..2a7391d0c6 --- /dev/null +++ b/developer/src/kmc-ldml/src/util/serialize.ts @@ -0,0 +1,274 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * This module contains routines for serializing from a KMXPlus file into XML. + */ + +import { KMXPlus } from "@keymanapp/common-types"; +import { KeymanXMLWriter, LDMLKeyboard } from "@keymanapp/developer-utils"; +import { constants } from "@keymanapp/ldml-keyboard-constants"; +import { KeysCompiler } from "../compiler/keys.js"; + +/** + * Serialize a KMXPlusFile back to XML. + * This is implemented for the LDML editor to be able to mutate LDML (XML) content by: + * 1. reading the original XML + * 2. compiling to KMXPlusFile + * 3. modifying the KMXPlusFile + * 4. serializing back to XML + * + * There are limitations: + * - TODO-LDML-EDITOR: Transforms would not be serialized properly, due to + * regex munging around markers and such. + * + * To work around this, fields with underscores such as _from and _to are added to + * the KMXPlusFile classes. These provide hints as to what the output XML should be, + * and are populated by the tran compiler. + * + * - TODO-LDML-EDITOR: Comments, whitespace, etc. are not preserved by this + * approach. Updates to the XML parsing will support this, see #10622. + * + * @param kmx input KMXPlusFile + * @returns XML String + */ +export function kmxToXml(kmx: KMXPlus.KMXPlusFile): string { + const writer = new KeymanXMLWriter("keyboard3"); + const { kmxplus } = kmx; + const { + // sect, + bksp, + disp, + // elem, + keys, + layr, + // list, + loca, + meta, + // strs, + tran, + // uset, + vars, + } = kmxplus; + const data = { + keyboard3: { + ...getRootAttributes(), + ...getLocales(), + version: getVersion(), + info: getInfo(), + ...getDisplays(), + ...getKeys(), + ...getFlicks(), + ...getLayers(), + ...getVariables(), + ...getTransforms(), + } + }; + + return writer.write(data); + + function getRootAttributes() { + const conform = meta.conform.value; + return { + '$xmlns': `https://schemas.unicode.org/cldr/${conform}/keyboard3`, + '$locale': kmx.kmxplus.loca.locales[0].value, + '$conformsTo': conform, + }; + } + + function getLocales() { + if (loca?.locales?.length < 2) { + return {}; // no additional locales + } else { + return { + locales: + loca.locales.map(({ value }) => ({ '$id': value })), + } + } + } + + function getInfo() { + return { + '$author': meta.author.value, + '$name': meta.name.value, + '$layout': meta.layout.value, + '$indicator': meta.indicator.value, + }; + } + + function getVersion() { + return { '$number': kmx.kmxplus.meta.version.value }; + } + + function getDisplays() { + const displays = { + display: disp?.disps.map(disp => getDisplay(disp)) || [], + ...getDisplaySettings(), + }; + if (displays?.display?.length || displays?.displayOptions) { + return { displays } + } else { + return {}; + } + } + + function stringToAttr(attr: string, s?: KMXPlus.StrsItem, override?: string) { + if (override) return asAttr(attr, override); + if (!s || !s?.value?.length) return {}; + return Object.fromEntries([[`\$${attr}`, s.value]]); + } + + function asAttr(attr: string, s?: any) { + if (s === undefined) return {}; + return Object.fromEntries([[`\$${attr}`, s]]); + } + + function numberToAttr(attr: string, s?: number) { + if (s === undefined) return {}; + return Object.fromEntries([[`\$${attr}`, s.toString()]]); + } + + function getDisplay(disp: KMXPlus.DispItem) { + return { + ...stringToAttr('output', disp?.to), + ...stringToAttr('keyId', disp?.id), + ...stringToAttr('display', disp?.display), + }; + } + + function getDisplaySettings() { + if (!disp?.baseCharacter?.value) return {}; + return { + displayOptions: { + '$baseCharacter': disp?.baseCharacter?.value, + } + }; + } + + function getKeys() { + if (!keys?.keys?.length) { + return {}; + } + return { + keys: { + key: keys.keys + // skip reserved keys (gap) + .filter((key: KMXPlus.KeysKeys) => + !KeysCompiler.isReserved(key) && + !LDMLKeyboard.ImportStatus.isImpliedImport(key)) + .map((key: KMXPlus.KeysKeys) => ({ + ...stringToAttr('id', key.id), + ...stringToAttr('output', key.to), + ...asAttr('longPressKeyIds', key?.longPress?.join(' ') || undefined), + })), + }, + }; + } + + function getFlicks() { + // skip the null flicks + if (keys?.flicks?.length < 2) { + return {}; + } + return { + flicks: { + // keys.key.. + } + }; + } + + function getLayers() { + if (!layr?.lists?.length) { + return {}; + } + return { + layers: layr.lists.map(({ hardware, minDeviceWidth, layers }) => ({ + ...stringToAttr('formId', hardware), + ...numberToAttr('minDeviceWidth', minDeviceWidth), + layer: layers.map(({ id, mod, rows }) => ({ + ...stringToAttr('id', id), + ...asAttr('modifiers', modToString(mod)), + row: rows.map(({ keys }) => ({ + ...asAttr('keys', keys.map(({ value }) => value).join(' ')), + })), + })), + })), + }; + } + + function getVariables() { + if (!vars?.strings.length && !vars?.sets.length && !vars?.usets.length) { + return {}; + } + function varToObj(v: KMXPlus.VarsItem): any { + const { id, value } = v; + return { + ...stringToAttr('id', id), + ...stringToAttr('value', value), + }; + } + function varsToArray(vars: KMXPlus.VarsItem[]): any[] { + return vars.map(varToObj); + } + const { strings, sets, usets } = vars; + + return { + variables: { + string: varsToArray(strings), + set: varsToArray(sets), + uset: varsToArray(usets), + }, + }; + } + + function getTransforms() { + return { + transforms: [ + ...getTransformType("simple", tran), + ...getTransformType("backspace", bksp), + ], + }; + } + + /** NB: Bksp is a child class of Tran */ + function getTransformType(type: string, t: KMXPlus.Tran) { + if (!t?.groups?.length) { + return []; + } + const { groups } = t; + return [{ + ...asAttr('type', type), + transformGroup: groups.map((group) => { + if (group.type === constants.tran_group_type_transform) { + return { + transform: group.transforms.map(({from, to, _from, _to}) => ({ + ...stringToAttr('from', from, _from), + ...stringToAttr('to', to, _to), + })), + }; + } else if(group.type === constants.tran_group_type_reorder) { + return { + reorder: group.reorders.map(({before, elements, _before, _from, _order}) => ({ + ...asAttr('before', _before || before.toString()), + ...asAttr('from', _from || elements.toString()), + ...asAttr('order', _order), + })), + }; + } else { + throw Error(`Invalid tran.group.type ${group.type}`); + } + }), + }]; + } +} + + +/** convert a keys_mod value to a space-separated string list */ +function modToString(mod: number) { + // first try exact match + const matches: string[] = []; + for (const [name, value] of constants.keys_mod_map.entries()) { + if (mod === value) return name; // exact match + if (mod & value) matches.push(name); + } + return matches.sort().join(' '); +} diff --git a/developer/src/kmc-ldml/test/compiler-e2e.tests.ts b/developer/src/kmc-ldml/test/compiler-e2e.tests.ts index 1f87e55a22..c1bafeedb2 100644 --- a/developer/src/kmc-ldml/test/compiler-e2e.tests.ts +++ b/developer/src/kmc-ldml/test/compiler-e2e.tests.ts @@ -3,7 +3,10 @@ import {assert} from 'chai'; import hextobin from '@keymanapp/hextobin'; import { KMXBuilder } from '@keymanapp/developer-utils'; import {checkMessages, compileKeyboard, compilerTestCallbacks, compilerTestOptions, makePathToFixture} from './helpers/index.js'; +import { compareXml } from './helpers/compareXml.js'; import { LdmlKeyboardCompiler } from '../src/compiler/compiler.js'; +import { kmxToXml } from '../src/util/serialize.js'; +import { writeFileSync } from 'node:fs'; /** Overall compiler tests */ describe('compiler-tests', function() { @@ -35,8 +38,34 @@ describe('compiler-tests', function() { let expected = await hextobin(binaryFilename, undefined, {silent:true}); assert.deepEqual(code, expected); + + // now output it again as XML + const outputFilename = makePathToFixture('basic-serialized.xml'); + const asXml = kmxToXml(kmx); + writeFileSync(outputFilename, asXml, 'utf-8'); + }); + it('should-serialize-kmx', async function() { + this.timeout(4000); + // Let's build basic.xml + // It should match basic.kmx (built from basic.txt) + + const inputFilename = makePathToFixture('basic.xml'); + + // Compile the keyboard + const kmx = await compileKeyboard(inputFilename, {...compilerTestOptions, saveDebug: true, shouldAddCompilerVersion: false}); + assert.isNotNull(kmx); + + // now output it as XML + const outputFilename = makePathToFixture('basic-serialized.xml'); + const asXml = kmxToXml(kmx); + writeFileSync(outputFilename, asXml, 'utf-8'); + + compareXml(outputFilename, inputFilename); + }); + + it('should handle non existent files', async () => { const filename = 'DOES_NOT_EXIST.xml'; const k = new LdmlKeyboardCompiler(); diff --git a/developer/src/kmc-ldml/test/fixtures/basic-serialized.xml b/developer/src/kmc-ldml/test/fixtures/basic-serialized.xml new file mode 100644 index 0000000000..aa570fd6f0 --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/basic-serialized.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/fixtures/sections/tran/fail-bad-tran-1.xml b/developer/src/kmc-ldml/test/fixtures/sections/tran/fail-bad-tran-1.xml index a644d5875f..ffa6e981d0 100644 --- a/developer/src/kmc-ldml/test/fixtures/sections/tran/fail-bad-tran-1.xml +++ b/developer/src/kmc-ldml/test/fixtures/sections/tran/fail-bad-tran-1.xml @@ -7,6 +7,7 @@ + diff --git a/developer/src/kmc-ldml/test/fixtures/sections/tran/fail-bad-tran-3.xml b/developer/src/kmc-ldml/test/fixtures/sections/tran/fail-bad-tran-3.xml new file mode 100644 index 0000000000..4dcb78c98b --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/sections/tran/fail-bad-tran-3.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/fixtures/sections/tran/fail-bad-tran-4.xml b/developer/src/kmc-ldml/test/fixtures/sections/tran/fail-bad-tran-4.xml new file mode 100644 index 0000000000..65d3af4c67 --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/sections/tran/fail-bad-tran-4.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/fixtures/sections/tran/fail-bad-tran-5.xml b/developer/src/kmc-ldml/test/fixtures/sections/tran/fail-bad-tran-5.xml new file mode 100644 index 0000000000..d3886523b5 --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/sections/tran/fail-bad-tran-5.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/fixtures/sections/tran/fail-bad-tran-6.xml b/developer/src/kmc-ldml/test/fixtures/sections/tran/fail-bad-tran-6.xml new file mode 100644 index 0000000000..09dc88a6c8 --- /dev/null +++ b/developer/src/kmc-ldml/test/fixtures/sections/tran/fail-bad-tran-6.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/developer/src/kmc-ldml/test/helpers/compareXml.ts b/developer/src/kmc-ldml/test/helpers/compareXml.ts new file mode 100644 index 0000000000..f245da51a9 --- /dev/null +++ b/developer/src/kmc-ldml/test/helpers/compareXml.ts @@ -0,0 +1,25 @@ +import {assert} from 'chai'; +import {readFileSync} from 'node:fs'; +import { KeymanXMLReader } from "@keymanapp/developer-utils"; + +/** + * + * @param actual path to actual XML + * @param expect path to expected XML + * @param mutator optional function that will be applied to the parsed object + */ +export function compareXml(actual : string, expect: string, mutator?: (input: any) => any) { + if (!mutator) { + // no-op + mutator = (x: any) => x; + } + const reader = new KeymanXMLReader('keyboard3'); + + const actualStr = readFileSync(actual, 'utf-8'); + const expectStr = readFileSync(expect, 'utf-8'); + + const actualParsed = mutator(reader.parse(actualStr)); + const expectParsed = mutator(reader.parse(expectStr)); + + assert.deepEqual(actualParsed, expectParsed); +} diff --git a/developer/src/kmc-ldml/test/helpers/index.ts b/developer/src/kmc-ldml/test/helpers/index.ts index bd438c4ca1..7b67ee4e4c 100644 --- a/developer/src/kmc-ldml/test/helpers/index.ts +++ b/developer/src/kmc-ldml/test/helpers/index.ts @@ -275,12 +275,6 @@ export function testCompilationCases(compiler: SectionCompilerNew, cases : Compi return; } let section = await loadSectionFixture(compiler, testcase.subpath, callbacks, testcase.dependencies || dependencies); - if (expectFailure) { - assert.isNull(section, 'expected compilation result failure (null)'); - } else { - assert.isNotNull(section, `failed with ${compilerEventFormat(callbacks.messages)}`); - } - const testcaseErrors = matchCompilerEventsOrBoolean(callbacks.messages, testcase.errors); const testcaseWarnings = matchCompilerEvents(callbacks.messages, testcase.warnings); // if we expected errors or warnings, show them @@ -296,6 +290,12 @@ export function testCompilationCases(compiler: SectionCompilerNew, cases : Compi // no warnings, so expect zero messages assert.sameDeepMembers(callbacks.messages, [], 'expected zero messages but got ' + callbacks.messages); } + + if (expectFailure) { + assert.isNull(section, 'expected compilation result failure (null)'); + } else { + assert.isNotNull(section, `failed with ${compilerEventFormat(callbacks.messages)}`); + } // run the user-supplied callback if any if (testcase.callback) { diff --git a/developer/src/kmc-ldml/test/tran.tests.ts b/developer/src/kmc-ldml/test/tran.tests.ts index 2fc401e66b..d64e26525d 100644 --- a/developer/src/kmc-ldml/test/tran.tests.ts +++ b/developer/src/kmc-ldml/test/tran.tests.ts @@ -287,8 +287,8 @@ describe('tran', function () { subpath: `sections/tran/fail-bad-tran-1.xml`, errors: [ { code: LdmlCompilerMessages.ERROR_UnparseableTransformFrom, - matchMessage: /Invalid regular expression.*Unterminated group/, - } + matchMessage: /.*Unterminated group.*/, + }, ], }, { @@ -298,6 +298,15 @@ describe('tran', function () { LdmlCompilerMessages.Error_InvalidQuadEscape({ cp: 295 }), ], }, + { + subpath: `sections/tran/fail-bad-tran-3.xml`, + errors: [ + { + code: LdmlCompilerMessages.ERROR_UnparseableTransformFrom, + matchMessage: /.*Syntax.*0-9.*/, + } + ], + }, { subpath: `sections/tran/fail-missing-var-1.xml`, errors: [ @@ -334,9 +343,35 @@ describe('tran', function () { LdmlCompilerMessages.Error_MissingStringVariable({ id: "missingstr" }), ], }, - // cases that share the same error code - ...[1, 2, 3].map(n => ({ - subpath: `sections/tran/fail-IllegalTransformDollarsign-${n}.xml`, + // cases that are now caught by the abnf + ...[ + 'fail-bad-tran-5', + ].map(s => ({ + subpath: `sections/tran/${s}.xml`, + errors: [ + { + code: LdmlCompilerMessages.ERROR_UnparseableTransformFrom, + matchMessage: /.*/, + } + ], + })), + ...[ + 'fail-bad-tran-4', + ].map(s => ({ + subpath: `sections/tran/${s}.xml`, + errors: [ + { + code: LdmlCompilerMessages.ERROR_UnparseableTransformTo, + matchMessage: /.*/, + } + ], + })), + ...[ + 'fail-IllegalTransformDollarsign-1', + 'fail-IllegalTransformDollarsign-2', + 'fail-IllegalTransformDollarsign-3', + ].map(s => ({ + subpath: `sections/tran/${s}.xml`, errors: [ { code: LdmlCompilerMessages.ERROR_IllegalTransformDollarsign, @@ -344,8 +379,11 @@ describe('tran', function () { } ], })), - ...[1, 2].map(n => ({ - subpath: `sections/tran/fail-IllegalTransformAsterisk-${n}.xml`, + ...[ + 'fail-IllegalTransformAsterisk-1', + 'fail-IllegalTransformAsterisk-2', + ].map(s => ({ + subpath: `sections/tran/${s}.xml`, errors: [ { code: LdmlCompilerMessages.ERROR_IllegalTransformAsterisk, @@ -353,8 +391,11 @@ describe('tran', function () { } ], })), - ...[1, 2].map(n => ({ - subpath: `sections/tran/fail-IllegalTransformPlus-${n}.xml`, + ...[ + 'fail-IllegalTransformPlus-1', + 'fail-IllegalTransformPlus-2', + ].map(n => ({ + subpath: `sections/tran/${n}.xml`, errors: [ { code: LdmlCompilerMessages.ERROR_IllegalTransformPlus, diff --git a/developer/src/kmc-ldml/test/tsconfig.json b/developer/src/kmc-ldml/test/tsconfig.json index 354f5235f1..5f6d6e93b6 100644 --- a/developer/src/kmc-ldml/test/tsconfig.json +++ b/developer/src/kmc-ldml/test/tsconfig.json @@ -10,7 +10,7 @@ }, "include": [ "**/*.tests.ts", - "./helpers/index.ts" + "./helpers/*.ts", ], "references": [ { "path": "../../../../common/web/keyman-version" }, diff --git a/developer/src/kmc-ldml/tsconfig.json b/developer/src/kmc-ldml/tsconfig.json index 36d915f074..785b30125a 100644 --- a/developer/src/kmc-ldml/tsconfig.json +++ b/developer/src/kmc-ldml/tsconfig.json @@ -7,7 +7,8 @@ "baseUrl": ".", }, "include": [ - "src/**/*.ts" + "src/**/*.ts", + "src/util/abnf/**/*.js", ], "references": [ { "path": "../../../common/web/keyman-version" }, diff --git a/docs/build/windows.md b/docs/build/windows.md index 63a77d1c41..ccf74be55d 100644 --- a/docs/build/windows.md +++ b/docs/build/windows.md @@ -162,8 +162,19 @@ SETX KEYMAN_ROOT "c:\Projects\keyman\keyman" * KeymanWeb **Requirements**: -* Emscripten * node.js +* Emscripten + +#### node.js + +Our recommended way to install node.js is to use +[nvm-windows](https://github.com/coreybutler/nvm-windows). This makes it +easy to switch between versions of node.js. + +```bat +nvm install 20.16.0 +nvm use 20.16.0 +``` #### Emscripten @@ -216,17 +227,6 @@ installed on your computer: SETX KEYMAN_USE_EMSDK 1 ``` -#### node.js - -Our recommended way to install node.js is to use -[nvm-windows](https://github.com/coreybutler/nvm-windows). This makes it -easy to switch between versions of node.js. - -```bat -nvm install 20.16.0 -nvm use 20.16.0 -``` - **Optional environment variables**: To let the Keyman build scripts control the version of node.js installed diff --git a/linux/debian/control b/linux/debian/control index e12da18aa3..7e71df1b7b 100644 --- a/linux/debian/control +++ b/linux/debian/control @@ -43,8 +43,8 @@ Build-Depends: xserver-xephyr, xvfb, Standards-Version: 4.7.0 -Vcs-Git: https://github.com/keymanapp/keyman.git -b stable-17.0 [linux/debian] -Vcs-Browser: https://github.com/keymanapp/keyman/tree/stable-17.0/linux/debian +Vcs-Git: https://github.com/keymanapp/keyman.git -b beta [linux/debian] +Vcs-Browser: https://github.com/keymanapp/keyman/tree/beta/linux/debian Homepage: https://www.keyman.com Rules-Requires-Root: binary-targets diff --git a/linux/ibus-keyman/src/KeymanSystemServiceClient.cpp b/linux/ibus-keyman/src/KeymanSystemServiceClient.cpp index b411e808f7..928d929dd0 100644 --- a/linux/ibus-keyman/src/KeymanSystemServiceClient.cpp +++ b/linux/ibus-keyman/src/KeymanSystemServiceClient.cpp @@ -25,6 +25,7 @@ public: void SetCapsLockIndicator(guint32 capsLock); gint32 GetCapsLockIndicator(); void CallOrderedOutputSentinel(); + void Ping(); }; KeymanSystemServiceClient::KeymanSystemServiceClient() { @@ -111,6 +112,23 @@ KeymanSystemServiceClient::CallOrderedOutputSentinel() { } } +void KeymanSystemServiceClient::Ping() { + if (!bus) { + // we already reported the error in the c'tor, so just return + return; + } + + sd_bus_error *error = NULL; + int result = sd_bus_call_method(bus, KEYMAN_BUS_NAME, KEYMAN_OBJECT_PATH, + KEYMAN_INTERFACE_NAME, "Ping", error, &msg, ""); + if (result < 0) { + g_error("%s: Failed to call method Ping: %s. %s. %s.", + __FUNCTION__, strerror(-result), error ? error->name : "-", error ? error->message : "-"); + sd_bus_error_free(error); + return; + } +} + void set_capslock_indicator(guint32 capsLock) { KeymanSystemServiceClient client; @@ -128,3 +146,10 @@ call_ordered_output_sentinel() { KeymanSystemServiceClient client; client.CallOrderedOutputSentinel(); } + +void +ping_keyman_system_service() { + g_message("%s: Pinging keyman-system-service", __FUNCTION__); + KeymanSystemServiceClient client; + client.Ping(); +} diff --git a/linux/ibus-keyman/src/KeymanSystemServiceClient.h b/linux/ibus-keyman/src/KeymanSystemServiceClient.h index bf5766434f..a10cab4717 100644 --- a/linux/ibus-keyman/src/KeymanSystemServiceClient.h +++ b/linux/ibus-keyman/src/KeymanSystemServiceClient.h @@ -10,6 +10,7 @@ extern "C" { void set_capslock_indicator(guint32 capsLockState); gint32 get_capslock_indicator(); void call_ordered_output_sentinel(); +void ping_keyman_system_service(); #ifdef __cplusplus } diff --git a/linux/ibus-keyman/src/engine.c b/linux/ibus-keyman/src/engine.c index 29d1be00d2..c4511ee3b2 100644 --- a/linux/ibus-keyman/src/engine.c +++ b/linux/ibus-keyman/src/engine.c @@ -553,6 +553,8 @@ ibus_keyman_engine_constructor( return NULL; } + ping_keyman_system_service(); + set_context_if_needed(engine); return (GObject *) keyman; diff --git a/linux/keyman-system-service/src/KeymanSystemService.cpp b/linux/keyman-system-service/src/KeymanSystemService.cpp index 5e5b4f6e3e..56d1e558b3 100644 --- a/linux/keyman-system-service/src/KeymanSystemService.cpp +++ b/linux/keyman-system-service/src/KeymanSystemService.cpp @@ -76,11 +76,22 @@ on_call_ordered_output_sentinel( return sd_bus_reply_method_return(msg, ""); } +static int32_t +on_ping( + sd_bus_message *msg, + void *user_data, + sd_bus_error *ret_error +) { + *ret_error = SD_BUS_ERROR_NULL; + return sd_bus_reply_method_return(msg, "s", "pong"); +} + static const sd_bus_vtable system_service_vtable[] = { SD_BUS_VTABLE_START(0), SD_BUS_METHOD("SetCapsLockIndicator", "b", "", on_set_caps_lock_indicator, SD_BUS_VTABLE_UNPRIVILEGED), SD_BUS_METHOD("GetCapsLockIndicator", "", "b", on_get_caps_lock_indicator, SD_BUS_VTABLE_UNPRIVILEGED), SD_BUS_METHOD("CallOrderedOutputSentinel", "", "", on_call_ordered_output_sentinel, SD_BUS_VTABLE_UNPRIVILEGED), + SD_BUS_METHOD("Ping", "", "s", on_ping, SD_BUS_VTABLE_UNPRIVILEGED), SD_BUS_VTABLE_END }; diff --git a/linux/keyman-system-service/src/com.keyman.SystemService1.System.xml b/linux/keyman-system-service/src/com.keyman.SystemService1.System.xml index b93bcffa3a..70ca44cba2 100644 --- a/linux/keyman-system-service/src/com.keyman.SystemService1.System.xml +++ b/linux/keyman-system-service/src/com.keyman.SystemService1.System.xml @@ -34,5 +34,13 @@ + + + + diff --git a/package-lock.json b/package-lock.json index 0528b2c814..4e340f8af4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1290,10 +1290,12 @@ "@types/mocha": "^5.2.7", "@types/node": "^20.4.1", "@types/semver": "^7.3.12", + "abnf": "^4.3.1", "c8": "^7.12.0", "chalk": "^2.4.2", "common-tags": "^1.8.2", "mocha": "^8.4.0", + "peggy": "^4.2.0", "typescript": "^5.4.5" } }, @@ -3658,6 +3660,30 @@ "@octokit/openapi-types": "^11.2.0" } }, + "node_modules/@peggyjs/from-mem": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@peggyjs/from-mem/-/from-mem-1.3.5.tgz", + "integrity": "sha512-oRyzXE7nirAn+5yYjCdWQHg3EG2XXcYRoYNOK8Quqnmm+9FyK/2YWVunwudlYl++M3xY+gIAdf0vAYS+p0nKfQ==", + "dev": true, + "dependencies": { + "semver": "7.6.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@peggyjs/from-mem/node_modules/semver": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", @@ -5765,6 +5791,35 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, + "node_modules/abnf": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/abnf/-/abnf-4.3.1.tgz", + "integrity": "sha512-j4A8wWqKqkcSjx5xFESo9GtW2EUvlUZutcWB1knhxSP9kaXJ/YwL0g6dvMhHRjCPCNsIWwNGoKMHzPwemSpCvw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "commander": "^13.0.0", + "peggy": "^4.2.0" + }, + "bin": { + "abnf_ast": "bin/abnf_ast.js", + "abnf_check": "bin/abnf_check.js", + "abnf_gen": "bin/abnf_gen.js", + "abnf_test": "bin/abnf_test.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/abnf/node_modules/commander": { + "version": "13.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-13.1.0.tgz", + "integrity": "sha512-/rFeCpNJQbhSZjGVwO9RFV3xPqbnERS8MmIQzCtD/zl6gpJuV/bMLuN92oG3F7d8oDEHHRrujSXNUr8fpjntKw==", + "dev": true, + "engines": { + "node": ">=18" + } + }, "node_modules/accepts": { "version": "1.3.8", "license": "MIT", @@ -12376,6 +12431,33 @@ "through": "~2.3" } }, + "node_modules/peggy": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/peggy/-/peggy-4.2.0.tgz", + "integrity": "sha512-ZjzyJYY8NqW8JOZr2PbS/J0UH/hnfGALxSDsBUVQg5Y/I+ZaPuGeBJ7EclUX2RvWjhlsi4pnuL1C/K/3u+cDeg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peggyjs/from-mem": "1.3.5", + "commander": "^12.1.0", + "source-map-generator": "0.8.0" + }, + "bin": { + "peggy": "bin/peggy.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/peggy/node_modules/commander": { + "version": "12.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", + "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", + "dev": true, + "engines": { + "node": ">=18" + } + }, "node_modules/pend": { "version": "1.2.0", "license": "MIT" @@ -13533,6 +13615,15 @@ "node": ">=0.10.0" } }, + "node_modules/source-map-generator": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/source-map-generator/-/source-map-generator-0.8.0.tgz", + "integrity": "sha512-psgxdGMwl5MZM9S3FWee4EgsEaIjahYV5AzGnwUvPhWeITz/j6rKpysQHlQ4USdxvINlb8lKfWGIXwfkrgtqkA==", + "dev": true, + "engines": { + "node": ">= 10" + } + }, "node_modules/source-map-support": { "version": "0.5.21", "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", diff --git a/resources/build/version/src/fixupHistory.ts b/resources/build/version/src/fixupHistory.ts index a8ceda3c50..1d6a290957 100644 --- a/resources/build/version/src/fixupHistory.ts +++ b/resources/build/version/src/fixupHistory.ts @@ -6,6 +6,7 @@ import { GitHub } from '@actions/github'; import { readFileSync, writeFileSync } from 'fs'; import { gt } from 'semver'; import { reportHistory } from './reportHistory.js'; +import { spawnChild } from './util/spawnAwait.js'; interface PRInformation { title: string; @@ -16,16 +17,28 @@ interface PRInformation { // splitPullsIntoHistory // ------------------------------------------------------------------------------------ -const splicePullsIntoHistory = async (pulls: PRInformation[]): Promise<{count: number, pulls: number[]}> => { +const splicePullsIntoHistory = async (pulls: PRInformation[], base?: string): Promise<{count: number, pulls: number[]}> => { let currentPulls: number[] = []; // - // Get current version and history from VERSION.md and TIER.md + // Get current version and history from VERSION.md and TIER.md. This may not + // yet be committed, so read the details from the worktree. // - const version = readFileSync('./VERSION.md', 'utf8').trim(); - const tier = readFileSync('./TIER.md', 'utf8').trim(); + let version = readFileSync('./VERSION.md', 'utf8').trim(); + let tier = readFileSync('./TIER.md', 'utf8').trim(); + + if(base) { + // If we are merging history from another branch, we need to use the data + // from that branch. In this case, we must assume that the version and tier + // data are definitely already committed. + const currentBase = (await spawnChild('git', ['branch', '--show-current'])).trim(); + if(currentBase != base) { + version = (await spawnChild('git', ['show', base+':VERSION.md'])).trim(); + tier = (await spawnChild('git', ['show', base+':TIER.md'])).trim(); + } + } //logInfo(`VERSION="${version}"`); // @@ -208,7 +221,7 @@ export const fixupHistory = async ( // Splice these into HISTORY.md // - const historyResult = await splicePullsIntoHistory(pulls); + const historyResult = await splicePullsIntoHistory(pulls, base); // // Write a comment to GitHub for each of the pulls diff --git a/resources/docker-images/base/Dockerfile b/resources/docker-images/base/Dockerfile index 87516b3f96..ae0d098acf 100644 --- a/resources/docker-images/base/Dockerfile +++ b/resources/docker-images/base/Dockerfile @@ -1,7 +1,8 @@ # Keyman is copyright (C) SIL Global. MIT License. -ARG UBUNTU_VERSION=latest -FROM ubuntu:${UBUNTU_VERSION} +ARG DISTRO=ubuntu +ARG DISTRO_VERSION=latest +FROM ${DISTRO}:${DISTRO_VERSION} LABEL org.opencontainers.image.authors="SIL Global." LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git" @@ -21,8 +22,10 @@ ENV DEBCONF_NOWARNINGS=yes # Update to the latest RUN apt-get -q -y update && \ apt-get -q -y install ca-certificates curl gnupg meson software-properties-common sudo && \ - add-apt-repository ppa:keymanapp/keyman && \ - add-apt-repository ppa:keymanapp/keyman-alpha + if [[ "$(lsb_release -is)" == "Ubuntu" ]]; then \ + add-apt-repository ppa:keymanapp/keyman && \ + add-apt-repository ppa:keymanapp/keyman-alpha ; \ + fi RUN apt-get -q -y update && \ apt-get -q -y upgrade diff --git a/resources/docker-images/build.sh b/resources/docker-images/build.sh index 8100459d47..d47af7454c 100755 --- a/resources/docker-images/build.sh +++ b/resources/docker-images/build.sh @@ -18,7 +18,8 @@ builder_describe \ ":core" \ ":linux" \ ":web" \ - "--ubuntu-version=UBUNTU_VERSION The Ubuntu version (default: ${KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER})" \ + "--distro=DISTRO The distribution to use for the base image (debian or ubuntu, default: ubuntu)" \ + "--distro-version=DISTRO_VERSION The Ubuntu/Debian version (default: ${KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER})" \ "--no-cache Force rebuild of docker images" \ "build Build docker images" \ "test Test the docker images by running configure,build,test for all or the specified platforms" @@ -31,11 +32,7 @@ _add_build_args() { local name=$3 local value - if [[ -n "${!var:-}" ]]; then - value="${!var}" - else - value="${!default_var:-}" - fi + value="${!var:=${!default_var:-}}" build_args+=(--build-arg="${var}=${value}") @@ -49,11 +46,14 @@ _add_build_args() { _convert_parameters_to_build_args() { build_args=() build_version= - local required_node_version + local required_node_version keyman_default_distro # shellcheck disable=SC2034 required_node_version="$(_print_expected_node_version)" + # shellcheck disable=SC2034 + keyman_default_distro="ubuntu" - _add_build_args UBUNTU_VERSION KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER "" + _add_build_args DISTRO keyman_default_distro "" + _add_build_args DISTRO_VERSION KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER "" _add_build_args JAVA_VERSION KEYMAN_VERSION_JAVA java _add_build_args REQUIRED_NODE_VERSION required_node_version node _add_build_args REQUIRED_EMSCRIPTEN_VERSION KEYMAN_MIN_VERSION_EMSCRIPTEN emsdk @@ -63,8 +63,16 @@ _convert_parameters_to_build_args() { fi } +_check_for_default_values() { + if [[ -z "${DISTRO_VERSION:-}" ]] && [[ -z "${JAVA_VERSION:-}" ]]; then + is_default_values=true + else + is_default_values=false + fi +} + _is_default_values() { - [[ -z "${UBUNTU_VERSION:-}" ]] && [[ -z "${JAVA_VERSION:-}" ]] + ${is_default_values} } build_action() { @@ -72,10 +80,9 @@ build_action() { builder_echo debug "Building image for ${platform}" - _convert_parameters_to_build_args - if [[ "${platform}" == "base" ]]; then - docker pull --platform "amd64" "ubuntu:${UBUNTU_VERSION:-${KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER}}" + # shellcheck disable=SC2154 # set by _convert_parameters_to_build_args + docker pull "${DISTRO}:${DISTRO_VERSION}" elif [[ "${platform}" == "linux" ]]; then cp "${KEYMAN_ROOT}/linux/debian/control" "${platform}" fi @@ -87,12 +94,12 @@ build_action() { # shellcheck disable=SC2164 cd "${platform}" # shellcheck disable=SC2248,SC2086 - docker build ${OPTION_NO_CACHE:-} --platform amd64 -t "keymanapp/keyman-${platform}-ci:${build_version}" "${build_args[@]}" . + docker build ${OPTION_NO_CACHE:-} -t "keymanapp/keyman-${platform}-ci:${build_version}" "${build_args[@]}" . # If the user didn't specify particular versions we will additionaly create an image # with the tag 'default'. if _is_default_values; then builder_echo debug "Setting default tag for ${platform}" - docker build --platform amd64 -t "keymanapp/keyman-${platform}-ci:default" "${build_args[@]}" . + docker build -t "keymanapp/keyman-${platform}-ci:default" "${build_args[@]}" . fi # shellcheck disable=SC2164,SC2103 cd - @@ -103,9 +110,13 @@ test_action() { local platform=$1 builder_echo debug "Testing image for ${platform}" - ./run.sh "${platform}" -- ./build.sh configure,build,test:"${platform}" + ./run.sh --distro "${DISTRO}" --distro-version "${DISTRO_VERSION}" \ + "${platform}" -- ./build.sh configure,build,test:"${platform}" } +_check_for_default_values +_convert_parameters_to_build_args + if builder_has_action build; then build_action base BASE_VERSION="${build_version}" diff --git a/resources/docker-images/linux/Dockerfile b/resources/docker-images/linux/Dockerfile index 2083ce8c55..061e9f619b 100644 --- a/resources/docker-images/linux/Dockerfile +++ b/resources/docker-images/linux/Dockerfile @@ -23,7 +23,7 @@ RUN apt-get update && \ # version 2.0-1 from Mantic. RUN LCOV_VERSION=$(dpkg -s lcov | grep Version | cut -d' ' -f2) && \ if dpkg --compare-versions "${LCOV_VERSION}" lt 2.0; then \ - curl -sS -o /tmp/lcov.deb --location http://mirrors.kernel.org/ubuntu/pool/universe/l/lcov/lcov_2.0-1_all.deb && \ + curl -sS -o /tmp/lcov.deb --location https://old-releases.ubuntu.com/ubuntu/pool/universe/l/lcov/lcov_2.0-1ubuntu0.2_all.deb && \ apt-get -qy install /tmp/lcov.deb && \ rm /tmp/lcov.deb ; \ fi diff --git a/resources/docker-images/run.sh b/resources/docker-images/run.sh index 63d299f7e3..f85c41b238 100755 --- a/resources/docker-images/run.sh +++ b/resources/docker-images/run.sh @@ -7,6 +7,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" ## END STANDARD BUILD SCRIPT INCLUDE . "${KEYMAN_ROOT}/resources/build/minimum-versions.inc.sh" +. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" ################################ Main script ################################ @@ -16,44 +17,53 @@ builder_describe \ "core" \ "linux" \ "web" \ - "--ubuntu-version=UBUNTU_VERSION The Ubuntu version (default: ${KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER})" + "--distro=DISTRO The distribution (debian or ubuntu, default: ubuntu)" \ + "--distro-version=DISTRO_VERSION The Ubuntu/Debian version (default: ${KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER})" builder_parse "$@" run_android() { docker run -it --rm -v "${KEYMAN_ROOT}":/home/build/build \ - -v "${KEYMAN_ROOT}/core/build/docker-core":/home/build/build/core/build \ - keymanapp/keyman-android-ci:default \ + -v "${KEYMAN_ROOT}/core/build/docker-core/${build_dir}":/home/build/build/core/build \ + "keymanapp/keyman-android-ci:${image_version}" \ "${builder_extra_params[@]}" } run_core() { docker run -it --rm -v "${KEYMAN_ROOT}":/home/build/build \ - -v "${KEYMAN_ROOT}/core/build/docker-core":/home/build/build/core/build \ - keymanapp/keyman-core-ci:default \ + -v "${KEYMAN_ROOT}/core/build/docker-core/${build_dir}":/home/build/build/core/build \ + "keymanapp/keyman-core-ci:${image_version}" \ "${builder_extra_params[@]}" } run_linux() { - mkdir -p "${KEYMAN_ROOT}/linux/build/docker-linux" - mkdir -p "${KEYMAN_ROOT}/linux/keyman-system-service/build/docker-linux" + mkdir -p "${KEYMAN_ROOT}/linux/build/docker-linux/${build_dir}" + mkdir -p "${KEYMAN_ROOT}/linux/keyman-system-service/build/docker-linux/${build_dir}" docker run -it --privileged --rm -v "${KEYMAN_ROOT}":/home/build/build \ - -v "${KEYMAN_ROOT}/core/build/docker-core":/home/build/build/core/build \ - -v "${KEYMAN_ROOT}/linux/build/docker-linux":/home/build/build/linux/build \ - -v "${KEYMAN_ROOT}/linux/keyman-system-service/build/docker-linux":/home/build/build/linux/keyman-system-service/build \ + -v "${KEYMAN_ROOT}/core/build/docker-core/${build_dir}":/home/build/build/core/build \ + -v "${KEYMAN_ROOT}/linux/build/docker-linux/${build_dir}":/home/build/build/linux/build \ + -v "${KEYMAN_ROOT}/linux/keyman-system-service/build/docker-linux/${build_dir}":/home/build/build/linux/keyman-system-service/build \ -e DESTDIR=/tmp \ - keymanapp/keyman-linux-ci:default \ + "keymanapp/keyman-linux-ci:${image_version}" \ "${builder_extra_params[@]}" } run_web() { docker run -it --privileged --rm -v "${KEYMAN_ROOT}":/home/build/build \ - -v "${KEYMAN_ROOT}/core/build/docker-core":/home/build/build/core/build \ - keymanapp/keyman-web-ci:default \ + -v "${KEYMAN_ROOT}/core/build/docker-core/${build_dir}":/home/build/build/core/build \ + "keymanapp/keyman-web-ci:${image_version}" \ "${builder_extra_params[@]}" } -mkdir -p "${KEYMAN_ROOT}/core/build/docker-core" +if [[ -z "${DISTRO_VERSION:-}" ]]; then + image_version=default + build_dir=default +else + image_version="${DISTRO:-}-${DISTRO_VERSION}-java${KEYMAN_VERSION_JAVA}-node$(_print_expected_node_version)-emsdk${KEYMAN_MIN_VERSION_EMSCRIPTEN}" + build_dir="${DISTRO:-}-${DISTRO_VERSION}" +fi + +mkdir -p "${KEYMAN_ROOT}/core/build/docker-core/${build_dir}" builder_run_action android run_android builder_run_action core run_core diff --git a/resources/standards-data/ldml-keyboards/46/abnf/transform-from-required.abnf b/resources/standards-data/ldml-keyboards/46/abnf/transform-from-required.abnf new file mode 100644 index 0000000000..9e58904e02 --- /dev/null +++ b/resources/standards-data/ldml-keyboards/46/abnf/transform-from-required.abnf @@ -0,0 +1,143 @@ +; Copyright (c) 2025 Unicode, Inc. +; For terms of use, see http://www.unicode.org/copyright.html +; SPDX-License-Identifier: Unicode-3.0 +; CLDR data files are interpreted according to the LDML specification (http://unicode.org/reports/tr35/) + +; This is an ABNF grammar for the CLDR Keyboard spec transform match syntax. +; Note that there are sample matching/failing data files in tools/scripts/keyboard-abnf-tests/ + +; An entire string. +; Note that the empty string is not a match. +; Also note that a string may match this ABNF but be invalid according to the spec - which see. + +from-match = start-context atoms / atoms + +; special marker anchoring to the start of context +start-context = "^" + +; sequence of items for input match. note that empty is not allowed, must be at least one atom. +atoms = atom *(disjunction atom / atom) + +; for use with or +disjunction = "|" + +; a 'quark' is the matching part of an atom, and then a quantifier +atom = quark quantifier / quark + +; quark can be a grouping or non grouping +quark = non-group / group + +non-group = simple-matcher / escaped-codepoints / variable + +variable = string-variable / set-variable + +string-variable = "${" var-id "}" + +set-variable = "$[" var-id "]" + +; variable ID +var-id = 1*32IDCHAR + +group = capturing-group / non-capturing-group + +quantifier = bounded-quantifier / optional-quantifier + +escaped-codepoints = backslash "u" "{" codepoints-hex "}" +escaped-codepoint = backslash "u" "{" codepoint-hex "}" + +bounded-quantifier = "{" DIGIT "," DIGIT "}" +optional-quantifier = "?" + +non-capturing-group = "(" "?" ":" atoms ")" + +; a capturing group may not contain other capturing groups. +capturing-group = "(" catoms ")" + +; capturing atoms can't include any groups +catoms = catom *(catom) +; capturing atoms can't include any groups +catom = cquark / cquark quantifier + +; capturing atoms can't include groups +cquark = non-group + +; multiple hex codepoints +codepoints-hex = codepoint-hex *(SP codepoint-hex) + +; one hex codepoint (1-6 digits) +codepoint-hex = 1*6LHEXDIG + +simple-matcher = text-char / class / match-any-codepoint / match-marker + +match-any-codepoint = "." + +match-marker = match-any-marker / match-named-marker +match-any-marker = "\m{.}" +match-named-marker = "\m{" marker-id "}" +; marker id is nmtoken, but may be UAX31 in the future. +marker-id = NMTOKEN + +class = fixed-class / set-class + +fixed-class = backslash fixed-class-char + +fixed-class-char = "s" / "S" / "t" / "r" / "n" / "f" / "v" / "d" / "w" / "D" / "W" + +set-class = "[" set-negator set-members "]" +set-members = set-member *(set-member) +set-member = char-range / range-char / match-marker / escaped-codepoint +char-range = range-edge "-" range-edge +range-edge = escaped-codepoint / range-char +set-negator = "^" / "" + +; Restrictions on characters in various contexts + +; normal text +text-char = content-char / ws / escaped-char / "-" / ":" +; text in a range sequence +range-char = content-char / ws / escaped-range-char / "." / "|" / "{" / "}" +; group for everything BUT syntax chars. +content-char = ASCII-PUNCT / ALPHA / DIGIT / NON-ASCII + +; Character escapes +escaped-char = backslash ( escapable-char ) +escapable-char = backslash / "$" / "{" / "|" / "}" / "(" / ")" / "*" / "+" / "." / "/" / "?" / "[" / "]" / "^" + +escaped-range-char = backslash escapable-range-char +escapable-range-char = escapable-char / "-" + +backslash = %x5C ; U+005C REVERSE SOLIDUS "\" +ws = SP / HTAB / CR / LF / %x3000 + +IDCHAR = ALPHA / DIGIT / "_" +; ASCII-CTRLS = %x01-08 ; omit NULL (%x00), HTAB (%x09) and LF (%x0A) +; / %x0B-0C ; omit CR (%x0D) +; / %x0E-1F ; omit SP (%x20) +ASCII-PUNCT = %x21-23 ; omit DOLLAR + / %x25-27 ; omit () * + + / %x2C ; omit . (%x2E) and - (%x2D) + / %x2F ; skip over digits and : + / %x3B-3E ; omit ? 3f + / %x5F ; omit upper A-Z and [\]^ + / %x60 ; omit a-z {|} + / %x7E-7F ; just for completeness +NON-ASCII = %x7E-D7FF ; omit surrogates + / %xE000-10FFFF ; that's the rest. (TODO: omit other non-characters) + +; from STD-68 +DIGIT = %x30-39 ; 0-9 +ALPHA = %x41-5A / %x61-7A ; A-Z / a-z +SP = %x20 +HTAB = %xF900 ; horizontal tab +LF = %x0A ; linefeed +CR = %x0D ; carriage return +HEXDIG = DIGIT / "A" / "B" / "C" / "D" / "E" / "F" +; like HEXDIG but lowercase also +LHEXDIG = HEXDIG / "a" / "b" / "c" / "d" / "e" / "f" + +; from XML +NAMESTARTCHAR = ":" / ALPHA / "_" / %xC0-D6 / %xD8-F6 / %xF8-2FF / %x370-37D / %x37F-1FFF / %x200C-200D / %x2070-218F / %x2C00-2FEF / %x3001-D7FF / %xF900-FDCF / %xFDF0-FFFD +NAMESTARTCHAR =/ %x10000-10FFFF ; SKIP-NODE-ABNF: TODO: + +NAMECHAR = NAMESTARTCHAR / "-" / "." / DIGIT / %xB7 / %x0300-036F / %x203F-2040 +NMTOKEN = 1*NAMECHAR diff --git a/resources/standards-data/ldml-keyboards/46/abnf/transform-to-required.abnf b/resources/standards-data/ldml-keyboards/46/abnf/transform-to-required.abnf new file mode 100644 index 0000000000..25b16d74db --- /dev/null +++ b/resources/standards-data/ldml-keyboards/46/abnf/transform-to-required.abnf @@ -0,0 +1,95 @@ +; Copyright (c) 2025 Unicode, Inc. +; For terms of use, see http://www.unicode.org/copyright.html +; SPDX-License-Identifier: Unicode-3.0 +; CLDR data files are interpreted according to the LDML specification (http://unicode.org/reports/tr35/) + +; This is an ABNF grammar for the CLDR Keyboard spec transform to= (replacement) match syntax. +; Note that there are sample matching/failing data files in tools/scripts/keyboard-abnf-tests/ + +; An entire string. +; An empty string is valid, meaning deletion. +; Also note that a string may match this ABNF but be invalid according to the spec - which see. +to-replacement = atoms + +; a sequence of items for the output production +atoms = *(atom) + +; each atom can be one of several things +atom = replacement-char / escaped-char / group-reference / escaped-codepoints / named-marker / string-variable / mapped-set + +; normal text being output +replacement-char = content-char / ws / "-" / ":" / "(" / ")" / "." / "*" / "+" / "?" / "[" / "]" / "^" / "{" / "}" / "|" + +; Character escapes +escaped-char = backslash ( backslash / "$" ) / "$$" + +; reference to a capture group +group-reference = "$" DIGIT + +; hex codepoint such as \u{01234} +escaped-codepoints = backslash "u" "{" codepoints-hex "}" + +; multiple hex codepoints +codepoints-hex = codepoint-hex *(SP codepoint-hex) + +; one hex codepoint (1-6 digits) +codepoint-hex = 1*6LHEXDIG + +; a specific marker ID. +named-marker = "\m{" marker-id "}" + +; marker id is nmtoken, but may be UAX31 in the future. +marker-id = NMTOKEN + + +; substitution of a string variable +string-variable = "${" var-id "}" + +; variable ID +var-id = 1*32IDCHAR + +; special case for a mapped set variable +mapped-set = "$[1:" var-id "]" + +; group for everything BUT syntax chars. +content-char = ASCII-PUNCT / ALPHA / DIGIT / NON-ASCII + +; \ +backslash = %x5C ; U+005C REVERSE SOLIDUS "\" + +; whitespace +ws = SP / HTAB / CR / LF / %x3000 + +IDCHAR = ALPHA / DIGIT / "_" +; below is same as transform-from for maintenance +; ASCII-CTRLS = %x01-08 ; omit NULL (%x00), HTAB (%x09) and LF (%x0A) +; / %x0B-0C ; omit CR (%x0D) +; / %x0E-1F ; omit SP (%x20) +ASCII-PUNCT = %x21-23 ; omit DOLLAR + / %x25-27 ; omit () * + + / %x2C ; omit . (%x2E) and - (%x2D) + / %x2F ; skip over digits and : + / %x3B-3E ; omit ? 3f + / %x5F ; omit upper A-Z and [\]^ + / %x60 ; omit a-z {|} + / %x7E-7F ; just for completeness +NON-ASCII = %x7E-D7FF ; omit surrogates + / %xE000-10FFFF ; that's the rest. (TODO: omit other non-characters) + +; from STD-68 +DIGIT = %x30-39 ; 0-9 +ALPHA = %x41-5A / %x61-7A ; A-Z / a-z +SP = %x20 +HTAB = %xF900 ; horizontal tab +LF = %x0A ; linefeed +CR = %x0D ; carriage return +HEXDIG = DIGIT / "A" / "B" / "C" / "D" / "E" / "F" +; like HEXDIG but lowercase also +LHEXDIG = HEXDIG / "a" / "b" / "c" / "d" / "e" / "f" + +; from XML +NAMESTARTCHAR = ":" / ALPHA / "_" / %xC0-D6 / %xD8-F6 / %xF8-2FF / %x370-37D / %x37F-1FFF / %x200C-200D / %x2070-218F / %x2C00-2FEF / %x3001-D7FF / %xF900-FDCF / %xFDF0-FFFD +NAMESTARTCHAR =/ %x10000-10FFFF ; SKIP-NODE-ABNF: TODO: + +NAMECHAR = NAMESTARTCHAR / "-" / "." / DIGIT / %xB7 / %x0300-036F / %x203F-2040 +NMTOKEN = 1*NAMECHAR diff --git a/resources/standards-data/ldml-keyboards/fetch-latest-cldr.sh b/resources/standards-data/ldml-keyboards/fetch-latest-cldr.sh index 7234c69fb9..13037eed84 100755 --- a/resources/standards-data/ldml-keyboards/fetch-latest-cldr.sh +++ b/resources/standards-data/ldml-keyboards/fetch-latest-cldr.sh @@ -49,10 +49,12 @@ DTD_DIR="${KEYBOARDS_DIR}/dtd" IMPORT_DIR="${KEYBOARDS_DIR}/import" DATA_DIR="${KEYBOARDS_DIR}/3.0" TEST_DIR="${KEYBOARDS_DIR}/test" +ABNF_DIR="${KEYBOARDS_DIR}/abnf" # a file to check -CHECK_1="${DTD_DIR}/ldmlKeyboard3.dtd" # Critical, present in prior CLDR -CHECK_2="${DTD_DIR}/ldmlKeyboardTest3.dtd" # Only in Keyboard 3.0+ +CHECK_1="${DTD_DIR}/ldmlKeyboard3.dtd" # Critical, present in prior CLDR +CHECK_2="${DTD_DIR}/ldmlKeyboardTest3.dtd" # Only in Keyboard 3.0+ +CHECK_3="${ABNF_DIR}/transform-from-required.abnf" # Present in v47+ if [[ ! -f "${CHECK_1}" ]]; then @@ -64,6 +66,12 @@ then builder_die "${CHECK_2} did not exist: is ${CLDR_DIR} a valid CLDR keyboard directory?" fi +if [[ ! -f "${CHECK_3}" ]]; +then + builder_die "${CHECK_3} did not exist: does ${CLDR_DIR} contain CLDR 47+? Or did ABNF change?" +fi + + # collect git info GIT_DESCRIBE=$(cd "${CLDR_DIR}" && git describe HEAD || echo unknown) GIT_SHA=$(cd "${CLDR_DIR}" && git rev-parse HEAD || echo unknown) @@ -82,7 +90,7 @@ pwd # delete the old files in case some were removed from CLDR rm -rf ./import ./3.0 ./dtd ./test # copy over everything -cp -Rv "${IMPORT_DIR}" "${DATA_DIR}" "${DTD_DIR}" "${TEST_DIR}" . +cp -Rv "${IMPORT_DIR}" "${DATA_DIR}" "${DTD_DIR}" "${TEST_DIR}" "${ABNF_DIR}" . # delete old files, no reason to keep them rm -vf dtd/{ldmlKeyboard,ldmlPlatform}.{xsd,dtd} diff --git a/windows/src/desktop/kmshell/main/Keyman.System.UpdateStateMachine.pas b/windows/src/desktop/kmshell/main/Keyman.System.UpdateStateMachine.pas index 750f2a4616..afce261cfa 100644 --- a/windows/src/desktop/kmshell/main/Keyman.System.UpdateStateMachine.pas +++ b/windows/src/desktop/kmshell/main/Keyman.System.UpdateStateMachine.pas @@ -118,6 +118,7 @@ uses GlobalProxySettings, kmint, keymanapi_TLB, + KeymanMutex, Keyman.System.KeymanSentryClient, Keyman.System.DownloadUpdate, Keyman.System.RemoteUpdateCheck, @@ -131,6 +132,7 @@ const SPackageUpgradeFilename = 'upgrade_packages.inf'; kmShellContinue = 0; kmShellExit = 1; + KeymanDownloadMutexName = 'KeymanDownloading'; { State Class Memebers } @@ -753,18 +755,29 @@ procedure DownloadingState.Enter; var DownloadResult: Boolean; RetryCount: Integer; + FMutex: TKeymanMutex; begin // Enter DownloadingState bucStateContext.SetRegistryState(usDownloading); - RetryCount := 0; - DownloadResult := False; + FMutex := TKeymanMutex.Create(KeymanDownloadMutexName); - while (not DownloadResult) and (RetryCount < 3) do - begin - DownloadResult := DownloadUpdatesBackground; - if not DownloadResult then + try + // Should be impossible but just exit anyway and let the process current + // downloading process finish. + if not FMutex.TakeOwnership then + begin + Exit; + end; + + RetryCount := 0; + repeat + DownloadResult := DownloadUpdatesBackground; Inc(RetryCount); + until DownloadResult or (RetryCount >= 3); + FMutex.ReleaseOwnership; + finally + FreeAndNil(FMutex); end; if (not DownloadResult) then @@ -805,14 +818,45 @@ begin end; function DownloadingState.HandleKmShell; +var + FMutex: TKeymanMutex; begin - // Downloading state, in other process, so continue + // Whether already downloading in another process or download has failed and + // this function has clean up and force a restart, kmshell should continue processing Result := kmShellContinue; + // Check to ensure a download process is running if not + // clean up return to the the idle state and check for updates + FMutex := TKeymanMutex.Create(KeymanDownloadMutexName); + try + if FMutex.TakeOwnership then + begin + bucStateContext.RemoveCachedFiles; + FMutex.ReleaseOwnership; // Mutex must be freed before changing state + ChangeState(IdleState); + bucStateContext.CurrentState.HandleCheck; + end; + finally + FreeAndNil(FMutex); + end; end; procedure DownloadingState.HandleDownload; +var + FMutex: TKeymanMutex; begin - // Enter Already Downloading + // If downloading process is not running clean files and return to idle + FMutex := TKeymanMutex.Create(KeymanDownloadMutexName); + try + if FMutex.TakeOwnership then + begin + bucStateContext.RemoveCachedFiles; + FMutex.ReleaseOwnership; // Mutex must be freed before changing state + ChangeState(IdleState); + bucStateContext.CurrentState.HandleCheck; + end; + finally + FreeAndNil(FMutex); + end; end; procedure DownloadingState.HandleAbort; diff --git a/windows/src/support/kmdevlink/UfrmMain.pas b/windows/src/support/kmdevlink/UfrmMain.pas index 141596c4ca..6dcf14f575 100644 --- a/windows/src/support/kmdevlink/UfrmMain.pas +++ b/windows/src/support/kmdevlink/UfrmMain.pas @@ -12,7 +12,9 @@ uses const SStatusSiteURL = 'https://status.keyman.com'; - SSearchURL = 'https://github.com/keymanapp/%s/issues/%s'; + SOpenIssueURL = 'https://github.com/keymanapp/%s/issues/%d'; + SOpenAllIssuesURL = 'https://github.com/keymanapp/%s/issues'; + SIssueSearchURL = 'https://github.com/keymanapp/%s/issues?q=state%%3Aopen%%20%s'; SAddIssueURL = 'https://github.com/keymanapp/keyman/issues/new'; type @@ -81,6 +83,7 @@ var implementation uses + System.NetEncoding, ErrorControlledRegistry, ShellApi, UfrmCharacterIdentifier, @@ -228,8 +231,8 @@ end; procedure TfrmMain.cmdOpenIssueOrPRClick(Sender: TObject); var - parts: TArray; - repo, number: string; + URL: string; + iq: TIssueQuery; begin with TfrmOpenCRMRecord.Create(Self) do try @@ -240,20 +243,21 @@ begin Free; end; - parts := CustomerText.Split(['#']); - if Length(parts) = 1 then + iq := SearchTextToQuery(CustomerText); + if iq.searchString <> '' then begin - repo := 'keyman'; - number := parts[0]; + URL := Format(SIssueSearchURL, [iq.repo, TNetEncoding.URL.Encode(iq.searchString)]); + end + else if iq.issueNumber > 0 then + begin + URL := Format(SOpenIssueURL, [iq.repo, iq.issueNumber]); end else begin - repo := parts[0]; - if repo = '' then repo := 'keyman' else repo := RepoShortNameToFullName(repo); - - number := parts[1]; + URL := Format(SOpenAllIssuesURL, [iq.repo]); end; - if not TUtilExecute.URL(Format(SSearchURL, [repo, number])) then // I3349 + + if not TUtilExecute.URL(URL) then // I3349 ShowMessage(SysErrorMessage(GetLastError)); end; diff --git a/windows/src/support/kmdevlink/UfrmOpenCRMRecord.dfm b/windows/src/support/kmdevlink/UfrmOpenCRMRecord.dfm index 1914b23468..69f9f1dd3f 100644 --- a/windows/src/support/kmdevlink/UfrmOpenCRMRecord.dfm +++ b/windows/src/support/kmdevlink/UfrmOpenCRMRecord.dfm @@ -20,9 +20,9 @@ object frmOpenCRMRecord: TfrmOpenCRMRecord object TntLabel1: TLabel Left = 20 Top = 20 - Width = 62 + Width = 115 Height = 19 - Caption = '&Issue/PR' + Caption = '&Issue/PR/Search' Font.Charset = DEFAULT_CHARSET Font.Color = clWindowText Font.Height = -16 @@ -45,9 +45,9 @@ object frmOpenCRMRecord: TfrmOpenCRMRecord ParentFont = False end object editSearchFor: TEdit - Left = 108 + Left = 156 Top = 17 - Width = 341 + Width = 293 Height = 27 Font.Charset = DEFAULT_CHARSET Font.Color = clWindowText @@ -66,7 +66,7 @@ object frmOpenCRMRecord: TfrmOpenCRMRecord Caption = 'OK' Default = True ModalResult = 1 - TabOrder = 1 + TabOrder = 3 end object cmdCancel: TButton Left = 240 @@ -76,12 +76,12 @@ object frmOpenCRMRecord: TfrmOpenCRMRecord Cancel = True Caption = 'Cancel' ModalResult = 2 - TabOrder = 2 + TabOrder = 4 end object cbRepository: TComboBox - Left = 108 + Left = 156 Top = 69 - Width = 341 + Width = 293 Height = 27 Style = csDropDownList Font.Charset = DEFAULT_CHARSET @@ -90,7 +90,16 @@ object frmOpenCRMRecord: TfrmOpenCRMRecord Font.Name = 'Tahoma' Font.Style = [] ParentFont = False - TabOrder = 3 + TabOrder = 1 OnClick = cbRepositoryClick end + object cmdCopyHTML: TButton + Left = 156 + Top = 110 + Width = 159 + Height = 25 + Caption = 'Copy short form as &HTML link' + TabOrder = 2 + OnClick = cmdCopyHTMLClick + end end diff --git a/windows/src/support/kmdevlink/UfrmOpenCRMRecord.pas b/windows/src/support/kmdevlink/UfrmOpenCRMRecord.pas index e0fcf2fb74..3ca8cfedbd 100644 --- a/windows/src/support/kmdevlink/UfrmOpenCRMRecord.pas +++ b/windows/src/support/kmdevlink/UfrmOpenCRMRecord.pas @@ -17,9 +17,11 @@ type cmdCancel: TButton; lblRepo: TLabel; cbRepository: TComboBox; + cmdCopyHTML: TButton; procedure cbRepositoryClick(Sender: TObject); procedure editSearchForChange(Sender: TObject); procedure FormCreate(Sender: TObject); + procedure cmdCopyHTMLClick(Sender: TObject); private Changing: Boolean; function GetSearchText: WideString; @@ -33,8 +35,21 @@ type function RepoFullNameToShortName(name: string): string; function RepoShortNameToFullName(name: string): string; + +type + TIssueQuery = record + searchString: string; + repo: string; + issueNumber: Integer; + end; + +function SearchTextToQuery(s: string): TIssueQuery; + implementation +uses + Vcl.Clipbrd; + {$R *.dfm} const repos: TArray> = [ @@ -74,28 +89,131 @@ begin Result := name; end; +function SearchTextToQuery(s: string): TIssueQuery; +var + parts: TArray; +begin + Result.repo := 'keyman'; + Result.issueNumber := 0; + Result.searchString := ''; + + s := s.Trim; + parts := s.Split(['#']); + if Length(parts) = 0 then + begin + Exit; + end + else if Length(parts) = 1 then + begin + Result.issueNumber := StrToIntDef(s, 0); + if IntToStr(Result.issueNumber) <> s then + begin + Result.issueNumber := 0; + Result.searchString := s; + end; + end + else + begin + Result.repo := RepoShortNameToFullName(parts[0]); + Result.issueNumber := StrToIntDef(parts[1], 0); + if IntToStr(Result.issueNumber) <> parts[1] then + begin + Result.issueNumber := 0; + Result.searchString := parts[1]; + end; + end; +end; + +type + TMyClipboard = class(TClipboard); + +procedure TfrmOpenCRMRecord.cmdCopyHTMLClick(Sender: TObject); +var + c: TMyClipboard; + m: TMemoryStream; + ss: TStream; + displayRepo, html, s: string; + iq: TIssueQuery; + CF_HTML: UINT; + header: string; +const + // https://learn.microsoft.com/en-us/windows/win32/dataxchg/html-clipboard-format + // yeesh what a format + header_template = + 'Version:0.9'#$D#$A+ + 'StartHTML:%0.09d'#$D#$A+ + 'EndHTML:%0.09d'#$D#$A+ + 'StartFragment:%0.09d'#$D#$A+ + 'EndFragment:%0.09d'#$D#$A; + start_fragment = ''; + end_fragment = ''; + context_start = ''#$D#$A''#$D#$A; + context_end = #$D#$A''#$D#$A''; +begin + iq := SearchTextToQuery(SearchText); + if iq.repo = 'keyman' then + displayRepo := '' + else + displayRepo := iq.repo; + html := Format('%2:s#%1:d', [ + iq.repo, iq.issueNumber, displayRepo + ]); + + // Warning, this will go sadly badly with non-ascii letters + // because I am lazily not using UTF8Strings at this point + + header := Format(header_template, [0,0,0,0]); + s := Format(header_template, [ + header.Length, + header.Length + context_start.Length + start_fragment.Length + html.Length + end_fragment.Length + context_end.Length, + header.Length + context_start.Length + start_fragment.Length, + header.Length + context_start.Length + start_fragment.Length + html.Length + ]) + context_start + start_fragment + html + end_fragment + context_end + #0; + + ss := TStringStream.Create(s, TEncoding.UTF8); + try + CF_HTML := RegisterClipboardFormat('HTML Format'); + m := TMemoryStream.Create; + try + m.CopyFrom(ss, 0); + c := TMyClipboard(Clipboard); // access protected members yay delphi + c.SetBuffer(CF_HTML, m.Memory^, m.Size); + finally + m.Free; + end; + finally + ss.Free; + end; +end; + procedure TfrmOpenCRMRecord.editSearchForChange(Sender: TObject); var - s: string; - parts: TArray; - repo: string; + iq: TIssueQuery; begin if Changing then Exit; Changing := True; - s := editSearchFor.Text; - parts := s.Split(['#']); - if Length(parts) = 1 then + iq := SearchTextToQuery(SearchText); + + if iq.searchString <> '' then begin - cbRepository.ItemIndex := cbRepository.Items.IndexOf('keyman'); + cmdCopyHTML.Enabled := False; + cmdOK.Caption := '&Search'; + end + else if iq.issueNumber > 0 then + begin + cmdCopyHTML.Enabled := True; + cmdOK.Caption := '&Open issue'; end else begin - repo := RepoShortNameToFullName(parts[0]); - cbRepository.ItemIndex := cbRepository.Items.IndexOf(repo); + cmdCopyHTML.Enabled := False; + cmdOK.Caption := '&All issues'; end; + cbRepository.ItemIndex := cbRepository.Items.IndexOf(iq.repo); + Changing := False; end; @@ -106,11 +224,12 @@ begin for i := 0 to High(repos) do cbRepository.Items.Add(repos[i][0]); cbRepository.ItemIndex := cbRepository.Items.IndexOf('keyman'); + editSearchForChange(nil); end; function TfrmOpenCRMRecord.GetSearchText: WideString; begin - Result := editSearchFor.Text; + Result := Trim(editSearchFor.Text); end; procedure TfrmOpenCRMRecord.cbRepositoryClick(Sender: TObject);