diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..cd9921d963 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,9 @@ +# Always use LF for Dockerfile - otherwise HEREDOCs in Dockerfile +# will not work on Windows - we end up with a filename `foo\r` for +# the command +# ```Dockerfile +# RUN < foo +# echo "Hello world" +# EOF +# ``` +Dockerfile text eol=lf diff --git a/.github/actions/build-binary-packages/action.yml b/.github/actions/build-binary-packages/action.yml index 09e23e3f7a..e764a26ac7 100644 --- a/.github/actions/build-binary-packages/action.yml +++ b/.github/actions/build-binary-packages/action.yml @@ -31,7 +31,7 @@ runs: path: artifacts/keyman-srcpkg - name: Build - uses: sillsdev/gha-ubuntu-packaging@1ab4a5967afbadab82a480936b9e53d7190acdf2 # v2.0 + uses: sillsdev/gha-ubuntu-packaging@556b268762be9bea0f39861a7392587211ef6a58 # v2.0.1 with: dist: "${{ inputs.dist }}" platform: "${{ inputs.arch }}" diff --git a/HISTORY.md b/HISTORY.md index 9048b14d87..4a1bc00c6f 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,44 @@ # Keyman Version History +## 19.0.61 alpha 2025-06-10 + +* chore(developer): extend unit test timeout for getLastGitCommitDate (#14147) +* fix(common): Use zip.inc.sh for common test keyboards (#14145) +* maint(developer): add docker image to build developer node modules (#14058) + +## 19.0.60 alpha 2025-06-06 + +* maint(developer): allow to build locally on Linux (#14059) + +## 19.0.59 alpha 2025-06-05 + +* chore(deps): bump multer from 2.0.0 to 2.0.1 (#14127) +* maint(web): fix working directory for zipping artifacts for Web release (#14128) +* chore(linux): Update debian changelog (#14119) +* maint(web): improve TC build script for Keyman Web/Test (#14051) +* maint(developer): move TC configuration for Keyman Developer to scripts (#14052) + +## 19.0.58 alpha 2025-06-04 + +* feat(developer): update strs compiler to collect context (#14064) +* feat(developer): line numbers for UsetCompiler, others (#14090) +* feat(developer): remaining line numbers for tran/bksp compilers (#14097) +* feat(developer): ldml: update errors for unparseable reorder (#14098) +* chore(developer): ldml: messages cleanup (#14100) +* maint(android): Create "archive" builder step to prep release artifacts (#14093) +* maint(web): move TC configuration for Web to script (#14049) + +## 19.0.57 alpha 2025-06-03 + +* fix(windows): change from error to breadcrumb and log (#14069) +* maint(mac): make make-km-dmg.sh less quiet (#14105) + +## 19.0.56 alpha 2025-06-02 + +* maint(ios): Use zip.inc.sh to prep release artifacts (#13919) +* maint(linux): update to new version of gha-ubuntu-packaging (#14094) +* chore(linux): add Python TC test runner (#14062) + ## 19.0.55 alpha 2025-05-31 * maint(linux): `sudo` not required for removing temporary files (#14086) @@ -334,6 +373,13 @@ * refactor(windows): rename `TKeymanMutex.MutexOwned` to `TakeOwnership` and add `ReleaseOwnership` (#13168) * chore: increment to alpha 19.0 (#13187) +## 18.0.238 stable 2025-06-04 + +* fix(developer): ldml: make sure .run() calls compile() for validation (#14071) +* maint(linux): fix typo of temporary dependency package (#14085) +* maint(linux): update to new version of gha-ubuntu-packaging (#14096) +* maint(mac): make make-km-dmg.sh less quiet (#14106) + ## 18.0.237 stable 2025-05-30 * maint(android): Update FirstVoices launcher icon and red theme (#13917) diff --git a/VERSION.md b/VERSION.md index 4073c70600..9f9ccefe21 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -19.0.56 \ No newline at end of file +19.0.62 \ No newline at end of file diff --git a/android/.gitignore b/android/.gitignore index a5720bccc7..e822a25c2b 100644 --- a/android/.gitignore +++ b/android/.gitignore @@ -57,6 +57,9 @@ KMAPro/**/assets/*.kmp # Play Store release notes (copied from whatsnew.md) KMAPro/kMAPro/src/main/play/release-notes/**/*.txt +# Release artifacts +upload/ + # Eclipse files .project .settings/ diff --git a/android/build.sh b/android/build.sh index 27ba447b4c..12017e2c6b 100755 --- a/android/build.sh +++ b/android/build.sh @@ -15,6 +15,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" . "$KEYMAN_ROOT/resources/build/build-utils-ci.inc.sh" +. "$KEYMAN_ROOT/resources/build/zip.inc.sh" ################################ Main script ################################ @@ -37,6 +38,7 @@ builder_describe \ build \ test \ "publish Publishes symbols to Sentry and the APKs to the Play Store." \ + "archive Copy release artifacts to upload/ and rsync to downloads.keyman" \ "--ci+ Deprecated build option. Remove in 20.0" \ --upload-sentry+ \ ":engine=KMEA Keyman Engine for Android" \ @@ -70,4 +72,51 @@ function do_test_help() { builder_run_action test:help do_test_help -builder_run_child_actions publish \ No newline at end of file +builder_run_child_actions publish + +# Copy release artifacts to upload/ and rsync to downloads.keyman.com +if builder_start_action archive; then + UPLOAD_PATH="$KEYMAN_ROOT/android/upload/${KEYMAN_VERSION}" + KEYMAN_ENGINE_ANDROID_ZIP="keyman-engine-android-${KEYMAN_VERSION}.zip" + KEYMAN_APK="keyman-${KEYMAN_VERSION}.apk" + FIRSTVOICES_APK="firstvoices-${KEYMAN_VERSION}.apk" + ZIP_FILE="${UPLOAD_PATH}/${KEYMAN_ENGINE_ANDROID_ZIP}" + ZIP_FLAGS=("-q" "-r") # quiet, recursive + + mkdir -p "${UPLOAD_PATH}" + + # Create Keyman Engine for Android archive + builder_echo "Copying Keyman Engine for Android into ${UPLOAD_PATH}..." + cd "${UPLOAD_PATH}" + cp "${KEYMAN_ROOT}/android/KMAPro/kMAPro/libs/keyman-engine.aar" ./ + add_zip_files "${ZIP_FILE}" "${ZIP_FLAGS[@]}" "keyman-engine.aar" + rm -f "keyman-engine.aar" + + builder_echo "Copying Keyman Engine for Android Sample projects into ${UPLOAD_PATH}..." + cp -rf "${KEYMAN_ROOT}/android/Samples" ./ + add_zip_files "${ZIP_FILE}" "-x@../../zip-excludes" "${ZIP_FLAGS[@]}" "Samples" + rm -rf "Samples" + + # Copy release APK + cp "${KEYMAN_ROOT}/android/KMAPro/kMAPro/build/outputs/apk/release/${KEYMAN_APK}" ./ + + # FirstVoices app + + if [ "${RELEASE_OEM_FIRSTVOICES-false}" = true ]; then + cp "${KEYMAN_ROOT}/oem/firstvoices/android/app/build/outputs/apk/release/${FIRSTVOICES_APK}" ./ + fi + + # + # Write download info files + # + + cd "${UPLOAD_PATH}" + write_download_info "Keyman Engine for Android" "${KEYMAN_ENGINE_ANDROID_ZIP}" "${KEYMAN_VERSION}" "${KEYMAN_TIER}" "android" + write_download_info "Keyman for Android" "${KEYMAN_APK}" "${KEYMAN_VERSION}" "${KEYMAN_TIER}" "android" + + if [ "${RELEASE_OEM_FIRSTVOICES-false}" = true ]; then + write_download_info "FirstVoices Keyboards" "${FIRSTVOICES_APK}" "${KEYMAN_VERSION}" "${KEYMAN_TIER}" "android" + fi + + builder_finish_action success archive +fi diff --git a/android/zip-excludes b/android/zip-excludes new file mode 100644 index 0000000000..1c2b62205d --- /dev/null +++ b/android/zip-excludes @@ -0,0 +1 @@ +Samples/*/build.sh diff --git a/common/test/keyboards/build.sh b/common/test/keyboards/build.sh index 1a1c9c2675..2d6a986b87 100755 --- a/common/test/keyboards/build.sh +++ b/common/test/keyboards/build.sh @@ -5,6 +5,8 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" . "${THIS_SCRIPT%/*}/../../../resources/build/builder.inc.sh" ## END STANDARD BUILD SCRIPT INCLUDE +. "$KEYMAN_ROOT/resources/build/zip.inc.sh" + targets=() # Build list of available targets from subfolders, if none specified for d in */; do @@ -29,7 +31,7 @@ builder_parse "$@" function zipsource() { local target="$1" pushd "$1" > /dev/null - 7z a -r -x!build -x"!$target.kpj.user" "${target}_source.zip" . + add_zip_files "${target}_source.zip" -x@../zip-excludes -q -r . # -q quiet, -r recursive popd > /dev/null } diff --git a/common/test/keyboards/zip-excludes b/common/test/keyboards/zip-excludes new file mode 100644 index 0000000000..1a27385042 --- /dev/null +++ b/common/test/keyboards/zip-excludes @@ -0,0 +1,3 @@ +build/ +*.kpj.user +*_source.zip diff --git a/common/web/types/src/kmx/kmx-plus/element-string.ts b/common/web/types/src/kmx/kmx-plus/element-string.ts index 49af2e1f49..e65d996f34 100644 --- a/common/web/types/src/kmx/kmx-plus/element-string.ts +++ b/common/web/types/src/kmx/kmx-plus/element-string.ts @@ -1,5 +1,5 @@ import { constants } from '@keymanapp/ldml-keyboard-constants'; -import { DependencySections, StrsItem, UsetItem } from './kmx-plus.js'; +import { DependencySections, StrsItem, StrsOptions, UsetItem } from './kmx-plus.js'; import { ElementParser, ElementSegment, ElementType } from '../../ldml-keyboard/pattern-parser.js'; import * as util from '../../util/util.js'; import MATCH_HEX_ESCAPE = util.MATCH_HEX_ESCAPE; @@ -33,7 +33,7 @@ export class ElementString extends Array { * @param source if a string array, does not get reinterpreted as UnicodeSet. This is used with vars, etc. Or pass `["str"]` for an explicit 1-element elem. * If it is a string, will be interpreted per reorder element rules. */ - static fromStrings(sections: DependencySections, source: string | string[], order?: string, tertiary?: string, tertiary_base?: string, prebase?: string) : ElementString { + static fromStrings(sections: DependencySections, options: StrsOptions, source: string | string[], order?: string, tertiary?: string, tertiary_base?: string, prebase?: string) : ElementString { // the returned array const array = new ElementString(); if(!source) { @@ -81,21 +81,21 @@ export class ElementString extends Array { // error. So we can just exit here. return null; // UnicodeSet error } - const uset = sections.usetparser.parseUnicodeSet(item.segment, needRanges); + const uset = sections.usetparser.parseUnicodeSet(item.segment, needRanges, options?.x); if (!uset) { - return null; // UnicodeSet error already thrown + return null; // UnicodeSet error already added to callback } - elem.uset = sections.uset.allocUset(uset, sections); - elem.value = sections.strs.allocString('', {singleOk: true}); // no string + elem.uset = sections.uset.allocUset(uset, sections, options?.x); + elem.value = sections.strs.allocString('', {...options, singleOk: true}); // no string } else if (item.type === ElementType.codepoint || item.type === ElementType.escaped || item.type === ElementType.string) { // some kind of a string let str = item.segment; if (item.type === ElementType.escaped && !MATCH_HEX_ESCAPE.test(str)) { str = unescapeOneQuadString(str); // TODO-LDML: any other escape forms here? - elem.value = sections.strs.allocString(str, { singleOk: true }); + elem.value = sections.strs.allocString(str, { ...options, singleOk: true }); } else { - elem.value = sections.strs.allocString(str, { unescape: true, singleOk: true }); + elem.value = sections.strs.allocString(str, { ...options, unescape: true, singleOk: true }); } // Now did we end up with one char or no? if (elem.value.isOneChar) { diff --git a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts index 8d6cd57003..373cc57b27 100644 --- a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts +++ b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts @@ -41,14 +41,14 @@ export class Elem extends Section { strings: ElementString[] = []; constructor(sections: DependencySections) { super(); - this.strings.push(ElementString.fromStrings(sections, '')); // C7043: null element string + this.strings.push(ElementString.fromStrings(sections, {}, '')); // C7043: null element string } /** * @param source if a string array, does not get reinterpreted as UnicodeSet. This is used with vars, etc. Or pass `["str"]` for an explicit 1-element elem. * If it is a string, will be interpreted per reorder element ruls. */ - allocElementString(sections: DependencySections, source: string | string[], order?: string, tertiary?: string, tertiary_base?: string, prebase?: string): ElementString { - const s = ElementString.fromStrings(sections, source, order, tertiary, tertiary_base, prebase); + allocElementString(sections: DependencySections, options: StrsOptions, source: string | string[], order?: string, tertiary?: string, tertiary_base?: string, prebase?: string): ElementString { + const s = ElementString.fromStrings(sections, options, source, order, tertiary, tertiary_base, prebase); if (!s) return s; let result = this.strings.find(item => item.isEqual(s)); if(result === undefined) { @@ -135,6 +135,18 @@ export class StrsItem { isEqual(a: StrsItem): boolean { return a.value === this.value && a.char === this.char; } + + private _context: any = undefined; + + /** add any context from the options to this strsitem */ + setContext(opts?: StrsOptions) { + // At present, there's only a single piece of context available + this._context = this._context || opts?.x; + } + + get context() : any { + return this._context; + } }; /** @@ -161,6 +173,8 @@ export interface StrsOptions { nfd?: boolean; /** string can be stored as a single CharStrsItem, not in strs table. */ singleOk?: boolean; + /** optional context */ + x?: any; }; export class Strs extends Section { @@ -192,6 +206,8 @@ export class Strs extends Section { result = new StrsItem(s); this.strings.push(result); } + // give an option to set the context + result.setContext(opts); return result; } @@ -200,7 +216,7 @@ export class Strs extends Section { s = s ?? ''; // type check everything else if (typeof s !== 'string') { - throw new Error('alloc_string: s must be a string, undefined, or null.'); + throw new Error(`Internal Error: processString: s must be a string, undefined, or null, not ${typeof s} ${s}`); } // substitute variables if (opts?.stringVariables) { @@ -358,11 +374,13 @@ export class Vars extends Section { export class VarsItem extends Section { id: StrsItem; value: StrsItem; + x?: any; - constructor(id: string, value: string, sections: DependencySections) { + constructor(id: string, value: string, sections: DependencySections, x?: any) { super(); this.id = sections.strs.allocString(id); this.value = sections.strs.allocString(value, {unescape: true}); + this.x = x; } valid() : boolean { @@ -371,8 +389,8 @@ export class VarsItem extends Section { }; export class UnicodeSetItem extends VarsItem { - constructor(id: string, value: string, sections: DependencySections, usetparser: UnicodeSetParser) { - super(id, value, sections); + constructor(id: string, value: string, sections: DependencySections, usetparser: UnicodeSetParser, x?: any) { + super(id, value, sections, x); const needRanges = sections.usetparser.sizeUnicodeSet(value); if (needRanges >= 0) { this.unicodeSet = sections.usetparser.parseUnicodeSet(value, needRanges); @@ -385,9 +403,9 @@ export class UnicodeSetItem extends VarsItem { }; export class SetVarItem extends VarsItem { - constructor(id: string, value: string[], sections: DependencySections) { - super(id, value.join(' '), sections); - this.items = sections.elem.allocElementString(sections, value); + constructor(id: string, value: string[], sections: DependencySections, x?: any) { + super(id, value.join(' '), sections, x); + this.items = sections.elem.allocElementString(sections, {x}, value); } items: ElementString; // element string array valid() : boolean { @@ -396,8 +414,8 @@ export class SetVarItem extends VarsItem { }; export class StringVarItem extends VarsItem { - constructor(id: string, value: string, sections: DependencySections) { - super(id, value, sections); + constructor(id: string, value: string, sections: DependencySections, x?: any) { + super(id, value, sections, x); } // no added fields }; @@ -435,7 +453,7 @@ export class Tran extends Section { }; export class UsetItem { - constructor(public uset: UnicodeSet, public str: StrsItem) { + constructor(public uset: UnicodeSet, public str: StrsItem, public x?: any) { } compareTo(other: UsetItem) : number { return this.str.compareTo(other.str); @@ -444,11 +462,11 @@ export class UsetItem { export class Uset extends Section { usets: UsetItem[] = []; - allocUset(set: UnicodeSet, sections: DependencySections) : UsetItem { + allocUset(set: UnicodeSet, sections: DependencySections, x?: any) : UsetItem { // match the same pattern let result = this.usets.find(s => set.pattern == s.uset.pattern); if (result === undefined) { - result = new UsetItem(set, sections.strs.allocString(set.pattern)); + result = new UsetItem(set, sections.strs.allocString(set.pattern), x); this.usets.push(result); } return result; diff --git a/common/web/types/src/ldml-keyboard/unicodeset-parser-api.ts b/common/web/types/src/ldml-keyboard/unicodeset-parser-api.ts index 0c890aa95a..797a5415b9 100644 --- a/common/web/types/src/ldml-keyboard/unicodeset-parser-api.ts +++ b/common/web/types/src/ldml-keyboard/unicodeset-parser-api.ts @@ -8,13 +8,13 @@ export interface UnicodeSetParser { * @param pattern string to parse such as `[a-z]` * @param rangeCount number of ranges to allow for */ - parseUnicodeSet(pattern: string, rangeCount: number) : UnicodeSet | null; + parseUnicodeSet(pattern: string, rangeCount: number, x?: any) : UnicodeSet | null; /** * Calculate the number of ranges in a UnicodeSet * @param pattern string to parse such as `[a-z]` * @returns number of ranges, or -1 (with callback-reported err) on err */ - sizeUnicodeSet(pattern: string) : number; + sizeUnicodeSet(pattern: string, x?: any) : number; } /** diff --git a/common/web/types/tests/kmx/kmx-plus/element-string.tests.ts b/common/web/types/tests/kmx/kmx-plus/element-string.tests.ts index 49a4c2c954..3818f7022b 100644 --- a/common/web/types/tests/kmx/kmx-plus/element-string.tests.ts +++ b/common/web/types/tests/kmx/kmx-plus/element-string.tests.ts @@ -91,12 +91,12 @@ describe('Test of ElementString file', () => { }); describe('Test of fromStrings()', () => { it('returns an empty ElementString if source is null', () => { - const es = ElementString.fromStrings({}, null); + const es = ElementString.fromStrings({}, {}, null); assert.deepEqual(es, new ElementString()); }); it('can create an ElementString from a string array', () => { sections.strs.allocString = stubStrsAllocString_Char; - const actual = ElementString.fromStrings(sections, ["𐌰", "𐌱", "𐌲"]); + const actual = ElementString.fromStrings(sections, {}, ["𐌰", "𐌱", "𐌲"]); const expected = [ initElemElement(GOTHIC_A), initElemElement(GOTHIC_B), @@ -106,7 +106,7 @@ describe('Test of ElementString file', () => { }); it('can create an ElementString from a string', () => { sections.strs.allocString = stubStrsAllocString_Char; - const actual = ElementString.fromStrings(sections, "𐌰𐌱𐌲"); + const actual = ElementString.fromStrings(sections, {}, "𐌰𐌱𐌲"); const expected = [ initElemElement(GOTHIC_A), initElemElement(GOTHIC_B), @@ -118,6 +118,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", "1 2 3", ); @@ -132,6 +133,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", "1", ); @@ -146,6 +148,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", "1 2", ); @@ -160,6 +163,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", "1 A 3", ); @@ -174,6 +178,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, "1 2 3", @@ -189,6 +194,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, "1", @@ -204,6 +210,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, "1 2", @@ -219,6 +226,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, "1 A 3", @@ -234,6 +242,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, null, @@ -250,6 +259,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, null, @@ -266,6 +276,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, null, @@ -282,6 +293,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, null, @@ -299,6 +311,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, null, @@ -316,6 +329,7 @@ describe('Test of ElementString file', () => { sections.strs.allocString = stubStrsAllocString_Char; const actual = ElementString.fromStrings( sections, + {}, "𐌰𐌱𐌲", null, null, @@ -333,7 +347,7 @@ describe('Test of ElementString file', () => { ElementParser.segment = stubElementParserSegment_Uset; sections.strs.allocString = stubStrsAllocString_Str; sections.uset.allocUset = stubUsetAllocUset; - const actual = ElementString.fromStrings(sections, "[𐌰-𐍊]"); + const actual = ElementString.fromStrings(sections, {}, "[𐌰-𐍊]"); const expected = [ initElemElement( new StrsItem(''), @@ -348,17 +362,17 @@ describe('Test of ElementString file', () => { it('returns null for an invalid unicode set size', () => { ElementParser.segment = stubElementParserSegment_Uset; sections.usetparser.sizeUnicodeSet = (pattern: string) : number => -1; - assert.isNull(ElementString.fromStrings(sections, "[𐌰-𐍊]")); + assert.isNull(ElementString.fromStrings(sections, {}, "[𐌰-𐍊]")); }); it('returns null if it cannot parse the unicode set', () => { ElementParser.segment = stubElementParserSegment_Uset; sections.usetparser.parseUnicodeSet = (pattern: string, rangeCount: number) : UnicodeSet | null => null; - assert.isNull(ElementString.fromStrings(sections, "[𐌰-𐍊]")); + assert.isNull(ElementString.fromStrings(sections, {}, "[𐌰-𐍊]")); }); it('can handle quad strings', () => { sections.strs.allocString = stubStrsAllocString_Char; ElementParser.segment = stubElementParserSegment_Escaped; - const actual = ElementString.fromStrings(sections, "\\ud800\\udf30"); + const actual = ElementString.fromStrings(sections, {}, "\\ud800\\udf30"); const expected = [ initElemElement(HI_GOTHIC_A), initElemElement(LO_GOTHIC_A), @@ -367,7 +381,7 @@ describe('Test of ElementString file', () => { }); it('can handle ElemElement of string type', () => { sections.strs.allocString = stubStrsAllocString_Str; - const actual = ElementString.fromStrings(sections, ["𐌰𐌱𐌲",]); + const actual = ElementString.fromStrings(sections, {}, ["𐌰𐌱𐌲",]); const expected = [ initElemElement( new StrsItem("𐌰𐌱𐌲"), diff --git a/developer/src/common/web/utils/src/compiler-interfaces.ts b/developer/src/common/web/utils/src/compiler-interfaces.ts index aaeffc8d8c..3cce4e75ef 100644 --- a/developer/src/common/web/utils/src/compiler-interfaces.ts +++ b/developer/src/common/web/utils/src/compiler-interfaces.ts @@ -1,4 +1,6 @@ import { CompilerCallbacks } from "./compiler-callbacks.js"; +import { ObjectWithMetadata } from "./symbol-utils.js"; +import { KeymanXMLReader, XML_FILENAME_SYMBOL } from "./xml-utils.js"; /** * Abstract interface for compiler error and warning messages @@ -206,6 +208,27 @@ export class CompilerError { } return null; } + + /** + * Get an offset from o and set event's offset field + * @param event a compiler event, such as from functions in this class + * @param x any object parsed from XML or with the XML_META_DATA_SYMBOL symbol copied over + * @returns modified event object + */ + public static setFromMetadata(event: CompilerEvent, x?: ObjectWithMetadata): CompilerEvent { + if(x) { + const metadata = KeymanXMLReader.getMetaData(x) || {}; + const offset = metadata?.startIndex; + if (offset) { + event.offset = offset; + } + const filename = event.filename || metadata[XML_FILENAME_SYMBOL]; + if (filename) { + event.filename = filename; + } + } + return event; + } }; /** @deprecated use `CompilerError.severity` instead */ @@ -423,6 +446,40 @@ export function dedentCompilerMessageDetail(event: CompilerEvent) { return (event.detail ?? '').replace(/^[ ]+/gm, ''); } +/** + * Convenience function for constructing CompilerEvents with line numbers. + * Use it as below: (abbreviated as mx()) + * + * ```js + * // Note: Indentation makes "InvalidScanCode" line up thrice + * static ERROR_InvalidScanCode = SevError | 0x0009; + * // Note: + * // 1. All parameters are passed in 'o', the context object is only used for context even if + * // it contains redundant info. + * // 2. No code execution within the arrow function other than the 'mx' call, string interpolation, + * // with `${def(o.property)}` as the max complexity of interpolation. + * static Error_InvalidScanCode = (o:{id: string, invalidCodeList: string}, x: ObjectWithMetadata) => mx( + * this.ERROR_InvalidScanCode, x, + * `Form '${def(o.id)}' has invalid/unknown scancodes '${def(o.codes)}'`, + * // Note: If detail is omitted, leave the trailing comma on the prior line to leave room for it + * `…additional markdown detail…` + * ); + * ``` + * + * @param code Unique numeric value of the event + * @param message A short description of the error presented to the user + * @param context Object to be used as a source for line number information + * @param detail Detailed Markdown-formatted description of the error + * including references to documentation, remediation options. + * @see CompilerMessageSpec + * @returns the event + */ +export function CompilerMessageObjectSpec(code: number, context: ObjectWithMetadata, message: string, detail?: string): CompilerEvent { + let evt = CompilerMessageSpec(code, message, detail); // constructs raw message + evt = CompilerError.setFromMetadata(evt, context); // updates with offset from context + return evt; +}; + export const CompilerMessageDef = (param: any) => String(param ?? ``); export const CompilerMessageSpecWithException = (code: number, message: string, exceptionVar: any, detail?: string) : CompilerEvent => ({ diff --git a/developer/src/common/web/utils/src/index.ts b/developer/src/common/web/utils/src/index.ts index ef6b489b67..c39afa8bff 100644 --- a/developer/src/common/web/utils/src/index.ts +++ b/developer/src/common/web/utils/src/index.ts @@ -48,7 +48,7 @@ export { export { defaultCompilerOptions, CompilerBaseOptions, CompilerOptions, CompilerEvent, CompilerErrorNamespace, CompilerErrorSeverity, CompilerCallbackOptions, - CompilerError, CompilerMessageSpec, CompilerMessageSpecWithException, compilerErrorSeverity, CompilerErrorMask, compilerErrorSeverityName, + CompilerError, CompilerMessageSpec, CompilerMessageObjectSpec, CompilerMessageSpecWithException, compilerErrorSeverity, CompilerErrorMask, compilerErrorSeverityName, compilerErrorFormatCode, CompilerMessageDef, compilerLogLevelToSeverity, CompilerLogLevel, compilerEventFormat, ALL_COMPILER_LOG_LEVELS, ALL_COMPILER_LOG_FORMATS, CompilerLogFormat, diff --git a/developer/src/kmc-kmn/src/compiler/compiler.ts b/developer/src/kmc-kmn/src/compiler/compiler.ts index d251bc2c5b..21985ad09a 100644 --- a/developer/src/kmc-kmn/src/compiler/compiler.ts +++ b/developer/src/kmc-kmn/src/compiler/compiler.ts @@ -9,7 +9,8 @@ TODO: implement additional interfaces: import { VisualKeyboard, KvkFileReader, LdmlKeyboardTypes, KeymanFileTypes, KvkFileWriter } from '@keymanapp/common-types'; import { CompilerCallbacks, CompilerEvent, CompilerOptions, KeymanCompiler, KeymanCompilerArtifacts, - KeymanCompilerArtifactOptional, KeymanCompilerResult, KeymanCompilerArtifact, KvksFileReader + KeymanCompilerArtifactOptional, KeymanCompilerResult, KeymanCompilerArtifact, KvksFileReader, + CompilerError } from '@keymanapp/developer-utils'; import * as Osk from './osk.js'; import loadWasmHost from '../import/kmcmplib/wasm-host.js'; @@ -529,7 +530,7 @@ export class KmnCompiler implements KeymanCompiler, LdmlKeyboardTypes.UnicodeSet * @param rangeCount - number of ranges to allocate * @returns UnicodeSet accessor object, or null on failure */ - public parseUnicodeSet(pattern: string, rangeCount: number) : LdmlKeyboardTypes.UnicodeSet | null { + public parseUnicodeSet(pattern: string, rangeCount: number, x?: any) : LdmlKeyboardTypes.UnicodeSet | null { if(!this.verifyInitialized()) { /* c8 ignore next 2 */ // verifyInitialized will set a callback if needed @@ -562,7 +563,7 @@ export class KmnCompiler implements KeymanCompiler, LdmlKeyboardTypes.UnicodeSet // rc is negative: it's an error code. this.wasmExports.free(buf); // translate error code into callback - this.callbacks.reportMessage(getUnicodeSetError(rc)); + this.callbacks.reportMessage(getUnicodeSetError(rc, x)); return null; } } @@ -570,7 +571,7 @@ export class KmnCompiler implements KeymanCompiler, LdmlKeyboardTypes.UnicodeSet /** * @internal */ - public sizeUnicodeSet(pattern: string) : number { + public sizeUnicodeSet(pattern: string, x?: any) : number { if(!this.verifyInitialized()) { /* c8 ignore next 2 */ return null; @@ -582,7 +583,7 @@ export class KmnCompiler implements KeymanCompiler, LdmlKeyboardTypes.UnicodeSet if (rc >= 0) { return rc; } else { - this.callbacks.reportMessage(getUnicodeSetError(rc)); + this.callbacks.reportMessage(getUnicodeSetError(rc, x)); return -1; } } @@ -593,7 +594,7 @@ export class KmnCompiler implements KeymanCompiler, LdmlKeyboardTypes.UnicodeSet * @param rc parseUnicodeSet error code * @returns the compiler event */ -function getUnicodeSetError(rc: number) : CompilerEvent { +function getUnicodeSetError(rc: number, x?: any) : CompilerEvent { // from kmcmplib.h const KMCMP_ERROR_SYNTAX_ERR = -1; const KMCMP_ERROR_HAS_STRINGS = -2; @@ -601,16 +602,16 @@ function getUnicodeSetError(rc: number) : CompilerEvent { const KMCMP_FATAL_OUT_OF_RANGE = -4; switch(rc) { case KMCMP_ERROR_SYNTAX_ERR: - return KmnCompilerMessages.Error_UnicodeSetSyntaxError(); + return KmnCompilerMessages.Error_UnicodeSetSyntaxError(x); case KMCMP_ERROR_HAS_STRINGS: - return KmnCompilerMessages.Error_UnicodeSetHasStrings(); + return KmnCompilerMessages.Error_UnicodeSetHasStrings(x); case KMCMP_ERROR_UNSUPPORTED_PROPERTY: - return KmnCompilerMessages.Error_UnicodeSetHasProperties(); + return KmnCompilerMessages.Error_UnicodeSetHasProperties(x); case KMCMP_FATAL_OUT_OF_RANGE: - return KmnCompilerMessages.Fatal_UnicodeSetOutOfRange(); + return KmnCompilerMessages.Fatal_UnicodeSetOutOfRange(x); default: /* c8 ignore next */ - return KmnCompilerMessages.Fatal_UnexpectedException({e: `Unexpected UnicodeSet error code ${rc}`}); + return CompilerError.setFromMetadata(KmnCompilerMessages.Fatal_UnexpectedException({e: `Unexpected UnicodeSet error code ${rc}`}), x); } } diff --git a/developer/src/kmc-kmn/src/compiler/kmn-compiler-messages.ts b/developer/src/kmc-kmn/src/compiler/kmn-compiler-messages.ts index 591083a85c..3274a62ca1 100644 --- a/developer/src/kmc-kmn/src/compiler/kmn-compiler-messages.ts +++ b/developer/src/kmc-kmn/src/compiler/kmn-compiler-messages.ts @@ -1,6 +1,6 @@ import { KeyAddress } from "../kmw-compiler/validate-layout-file.js"; import { kmnfile } from "../kmw-compiler/compiler-globals.js"; -import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerEvent, CompilerMessageSpec as m, CompilerMessageDef as def, CompilerMessageSpecWithException, KeymanUrls } from "@keymanapp/developer-utils"; +import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerEvent, CompilerMessageSpec as m, CompilerMessageObjectSpec as mx, CompilerMessageDef as def, CompilerMessageSpecWithException, KeymanUrls, ObjectWithMetadata } from "@keymanapp/developer-utils"; const Namespace = CompilerErrorNamespace.KmnCompiler; const SevInfo = CompilerErrorSeverity.Info | Namespace; @@ -12,6 +12,9 @@ const SevFatal = CompilerErrorSeverity.Fatal | Namespace; // For messages from the KeymanWeb compiler, we need to construct our messages // slightly differently. This could be refactored in the future, as it is not // obvious which messages should use which function. +// Specifically, this could perhaps use CompilerFileCallbacks which provides a +// default filename. +// Then, there could be a CompilerMessageSpecWithLine const mw = (code: number, message: string, o?: {filename?: string, line?: number}) : CompilerEvent => ({ ...m(code, message), filename: o?.filename ?? kmnfile, @@ -104,9 +107,8 @@ export class KmnCompilerMessages { ); static FATAL_UnicodeSetOutOfRange = SevFatal | 0x904; - static Fatal_UnicodeSetOutOfRange = () => CompilerMessageSpecWithException( - this.FATAL_UnicodeSetOutOfRange, - null, + static Fatal_UnicodeSetOutOfRange = (x?: ObjectWithMetadata) => mx( + this.FATAL_UnicodeSetOutOfRange, x, `UnicodeSet buffer was too small`, `Raised when caller to UnicodeSet functions provides an invalid buffer. If you experience this error, it should be reported to the Keyman team for @@ -116,8 +118,8 @@ export class KmnCompilerMessages { // TODO: rename the following functions to Error_UsetHasStrings etc static ERROR_UnicodeSetHasStrings = SevError | 0x905; - static Error_UnicodeSetHasStrings = () => m( - this.ERROR_UnicodeSetHasStrings, + static Error_UnicodeSetHasStrings = (x?: ObjectWithMetadata) => mx( + this.ERROR_UnicodeSetHasStrings, x, `uset contains strings, not allowed`, `The provided uset uses multi-character strings, (\`{}\` notation, e.g. \`[ΕΌΔ‘Δ§{ie}{gΔ§}]\`. ). Although full UnicodeSets support strings, LDML @@ -128,8 +130,8 @@ export class KmnCompilerMessages { ); static ERROR_UnicodeSetHasProperties = SevError | 0x906; - static Error_UnicodeSetHasProperties = () => m( - this.ERROR_UnicodeSetHasProperties, + static Error_UnicodeSetHasProperties = (x?: ObjectWithMetadata) => mx( + this.ERROR_UnicodeSetHasProperties, x, `uset contains properties, not allowed`, `The provided uset uses property notation (\`\\p{…}\` or \`[:…:]\`). LDML keyboards do not support Unicode properties in usets, because that would @@ -140,8 +142,8 @@ export class KmnCompilerMessages { ); static ERROR_UnicodeSetSyntaxError = SevError | 0x907; - static Error_UnicodeSetSyntaxError = () => m( - this.ERROR_UnicodeSetSyntaxError, + static Error_UnicodeSetSyntaxError = (x?: ObjectWithMetadata) => mx( + this.ERROR_UnicodeSetSyntaxError, x, `uset had a Syntax Error while parsing`, `The provided uset has a syntax error and could not be parsed. Verify the format of the uset against the specification. diff --git a/developer/src/kmc-ldml/src/compiler/disp.ts b/developer/src/kmc-ldml/src/compiler/disp.ts index 5b63a99a8f..fd15c8ace7 100644 --- a/developer/src/kmc-ldml/src/compiler/disp.ts +++ b/developer/src/kmc-ldml/src/compiler/disp.ts @@ -12,9 +12,10 @@ import { SubstitutionUse, Substitutions } from "./substitution-tracker.js"; export class DispCompiler extends SectionCompiler { static validateSubstitutions(keyboard: LDMLKeyboard.LKKeyboard, st : Substitutions): boolean { - keyboard.displays?.display?.forEach(({ display, output }) => { - st.addStringAndMarkerSubstitution(SubstitutionUse.match, output); - st.addStringSubstitution(SubstitutionUse.emit, display); + keyboard.displays?.display?.forEach((e) => { + const { display, output } = e; + st.addStringAndMarkerSubstitution(SubstitutionUse.match, output, e); + st.addStringSubstitution(SubstitutionUse.emit, display, e); }); // no marker references in 'id' return true; @@ -34,18 +35,18 @@ export class DispCompiler extends SectionCompiler { for (const display of this.keyboard3.displays?.display) { const { output, keyId } = display; if ((output && keyId) || (!output && !keyId)) { - this.callbacks.reportMessage(LdmlCompilerMessages.Error_DisplayNeedsToOrId({ output, keyId }, display)); + this.callbacks.reportMessage(LdmlCompilerMessages.Error_DisplayNeedsToOrId({ display: display.display }, display)); return false; } else if (output) { if (tos.has(output)) { - this.callbacks.reportMessage(LdmlCompilerMessages.Error_DisplayIsRepeated({ output }, display)); + this.callbacks.reportMessage(LdmlCompilerMessages.Error_DisplayIsRepeated({ display: display.display }, display)); return false; } else { tos.add(output); } } else if (keyId) { if (ids.has(keyId)) { - this.callbacks.reportMessage(LdmlCompilerMessages.Error_DisplayIsRepeated({ keyId }, display)); + this.callbacks.reportMessage(LdmlCompilerMessages.Error_DisplayIsRepeated({ display: display.display }, display)); return false; } else { ids.add(keyId); @@ -61,7 +62,7 @@ export class DispCompiler extends SectionCompiler { const result = new Disp(); // displayOptions - result.baseCharacter = sections.strs.allocString(this.keyboard3.displays?.displayOptions?.baseCharacter, {unescape: true}); + result.baseCharacter = sections.strs.allocString(this.keyboard3.displays?.displayOptions?.baseCharacter, { unescape: true, x: this.keyboard3?.displays?.displayOptions }); // displays result.disps = this.keyboard3.displays?.display.map(display => ({ @@ -69,11 +70,13 @@ export class DispCompiler extends SectionCompiler { stringVariables: true, markers: true, unescape: true, + x: display, }, sections), - id: sections.strs.allocString(display.keyId), // not escaped, not substituted + id: sections.strs.allocString(display.keyId, { x: display }), // not escaped, not substituted display: sections.strs.allocString(display.display, { stringVariables: true, unescape: true, + x: display, }, sections), })) || []; // TODO-LDML: need coverage for the [] diff --git a/developer/src/kmc-ldml/src/compiler/empty-compiler.ts b/developer/src/kmc-ldml/src/compiler/empty-compiler.ts index 4b9aec7613..79015dca6b 100644 --- a/developer/src/kmc-ldml/src/compiler/empty-compiler.ts +++ b/developer/src/kmc-ldml/src/compiler/empty-compiler.ts @@ -1,7 +1,7 @@ import { SectionIdent, constants } from '@keymanapp/ldml-keyboard-constants'; import { SectionCompiler } from "./section-compiler.js"; import { util, KMXPlus, LdmlKeyboardTypes } from "@keymanapp/common-types"; -import { CompilerCallbacks, LDMLKeyboard } from "@keymanapp/developer-utils"; +import { CompilerCallbacks, LDMLKeyboard, ObjectWithMetadata } from "@keymanapp/developer-utils"; import { VarsCompiler } from './vars.js'; import { LdmlCompilerMessages } from './ldml-compiler-messages.js'; @@ -33,13 +33,26 @@ export class StrsCompiler extends EmptyCompiler { public postValidate(section?: KMXPlus.Section): boolean { const strs = section; + /** attempt to find a context object for the string */ + function findContextForString(s: string): ObjectWithMetadata { + // try exact match + for(const str of strs.strings) { + if (str.value == s) return str.context; + } + // try substring match + for(const str of strs.strings) { + if (str.value.includes(s)) return str.context; + } + return null; + } + if (strs) { const badStringAnalyzer = new util.BadStringAnalyzer(); const CONTAINS_MARKER_REGEX = new RegExp(LdmlKeyboardTypes.MarkerParser.ANY_MARKER_MATCH); for (let s of strs.allProcessedStrings.values()) { // stop at the first denormalized string if (!util.isNormalized(s)) { - this.callbacks.reportMessage(LdmlCompilerMessages.Warn_StringDenorm({s})); + this.callbacks.reportMessage(LdmlCompilerMessages.Warn_StringDenorm({s}, findContextForString(s))); } // replace all \\uXXXX with the actual code point. // this lets us analyze whether there are PUA, unassigned, etc. @@ -63,16 +76,19 @@ export class StrsCompiler extends EmptyCompiler { const illegals = m.get(util.BadStringType.illegal); if (puas) { const [count, lowestCh] = [puas.size, Array.from(puas.values()).sort((a, b) => a - b)[0]]; - this.callbacks.reportMessage(LdmlCompilerMessages.Hint_PUACharacters({ count, lowestCh })) + this.callbacks.reportMessage(LdmlCompilerMessages.Hint_PUACharacters({ count, lowestCh: util.describeCodepoint(lowestCh) }, + findContextForString(String.fromCodePoint(lowestCh)))); } if (unassigneds) { const [count, lowestCh] = [unassigneds.size, Array.from(unassigneds.values()).sort((a, b) => a - b)[0]]; - this.callbacks.reportMessage(LdmlCompilerMessages.Warn_UnassignedCharacters({ count, lowestCh })) + this.callbacks.reportMessage(LdmlCompilerMessages.Warn_UnassignedCharacters({ count, lowestCh: util.describeCodepoint(lowestCh) }, + findContextForString(String.fromCodePoint(lowestCh)))); } if (illegals) { // do this last, because we will return false. const [count, lowestCh] = [illegals.size, Array.from(illegals.values()).sort((a, b) => a - b)[0]]; - this.callbacks.reportMessage(LdmlCompilerMessages.Error_IllegalCharacters({ count, lowestCh })) + this.callbacks.reportMessage(LdmlCompilerMessages.Error_IllegalCharacters({ count, lowestCh: util.describeCodepoint(lowestCh) }, + findContextForString(String.fromCodePoint(lowestCh)))); return false; } } diff --git a/developer/src/kmc-ldml/src/compiler/keys.ts b/developer/src/kmc-ldml/src/compiler/keys.ts index 5ca61ceb48..3c890a5067 100644 --- a/developer/src/kmc-ldml/src/compiler/keys.ts +++ b/developer/src/kmc-ldml/src/compiler/keys.ts @@ -37,7 +37,7 @@ export class KeysCompiler extends SectionCompiler { for (const keyId of usedKeys.values()) { const key = keyBag.get(keyId); if (!key) continue; // key not found is handled elsewhere. - st.addStringAndMarkerSubstitution(SubstitutionUse.emit, key.output); + st.addStringAndMarkerSubstitution(SubstitutionUse.emit, key.output, key); } return true; } @@ -324,15 +324,16 @@ export class KeysCompiler extends SectionCompiler { // allocate the in-memory const flicks: KeysFlicks = new KeysFlicks( - sections.strs.allocString(flickId) + sections.strs.allocString(flickId, { x: flick }) ); // add data from each segment - for (const { keyId, directions } of flick.flickSegment) { - const keyIdStr = sections.strs.allocString(keyId); + for (const flickSegment of flick.flickSegment) { + const { keyId, directions } = flickSegment; + const keyIdStr = sections.strs.allocString(keyId, { x: flickSegment }); const directionsList: ListItem = sections.list.allocListFromSpaces( directions, - { }, + { x: flickSegment }, sections); flicks.flicks.push({ directions: directionsList, @@ -369,20 +370,20 @@ export class KeysCompiler extends SectionCompiler { if (!!gap) { flags |= constants.keys_key_flags_gap; } - const id = sections.strs.allocString(key.id); + const id = sections.strs.allocString(key.id, { x: key }); const longPress: ListItem = sections.list.allocListFromSpaces( - longPressKeyIds, {}, + longPressKeyIds, { x: key }, sections); const longPressDefault = sections.strs.allocString(longPressDefaultKeyId, - {}, + { x: key }, sections); const multiTap: ListItem = sections.list.allocListFromSpaces( multiTapKeyIds, - {}, + { x: key }, sections); - const keySwitch = sections.strs.allocString(layerId); // 'switch' is a reserved word + const keySwitch = sections.strs.allocString(layerId, { x: key }); // 'switch' is a reserved word const toRaw = output; @@ -395,6 +396,7 @@ export class KeysCompiler extends SectionCompiler { unescape: true, singleOk: true, nfd: true, + x: key, }, sections); if (!to.isOneChar) { @@ -461,7 +463,7 @@ export class KeysCompiler extends SectionCompiler { const { modifiers } = layer; if (!validModifier(modifiers)) { this.callbacks.reportMessage( - LdmlCompilerMessages.Error_InvalidModifier({ modifiers, id: layer.id }, layer) + LdmlCompilerMessages.Error_InvalidModifier({ modifiers }, layer) ); valid = false; } @@ -482,7 +484,7 @@ export class KeysCompiler extends SectionCompiler { LdmlCompilerMessages.Error_RowOnHardwareLayerHasTooManyKeys({ row: y + 1, hardware: layers.formId, - modifiers, + modifiers: modifiers || 'none', }, row) ); valid = false; diff --git a/developer/src/kmc-ldml/src/compiler/layr.ts b/developer/src/kmc-ldml/src/compiler/layr.ts index ec96a6bddd..9dff22a5bf 100644 --- a/developer/src/kmc-ldml/src/compiler/layr.ts +++ b/developer/src/kmc-ldml/src/compiler/layr.ts @@ -52,7 +52,7 @@ export class LayrCompiler extends SectionCompiler { const { modifiers } = layer; totalLayerCount++; if (!validModifier(modifiers)) { - this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidModifier({ modifiers, id: layer.id }, layer)); + this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidModifier({ modifiers }, layer)); valid = false; } }); @@ -69,13 +69,13 @@ export class LayrCompiler extends SectionCompiler { const sect = new Layr(); sect.lists = this.keyboard3.layers.map((layers) => { - const hardware = sections.strs.allocString(layers.formId); + const hardware = sections.strs.allocString(layers.formId, {x:layers}); // Already validated in validate const layerEntries = []; for (const layer of layers.layer) { const rows = layer.row.map((row) => { const erow: LayrRow = { - keys: row.keys.trim().split(/[ \t]+/).map((id) => sections.strs.allocString(id)), + keys: row.keys.trim().split(/[ \t]+/).map((id) => sections.strs.allocString(id, { x: row })), }; // include linenumber info for row return SectionCompiler.copySymbols(erow, row); @@ -84,7 +84,7 @@ export class LayrCompiler extends SectionCompiler { // push a layer entry for each modifier set for (const mod of mods) { layerEntries.push({ - id: sections.strs.allocString(layer.id), + id: sections.strs.allocString(layer.id, {x:layer}), mod, rows, }); diff --git a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts index 2955e1a95e..f76e0d3b3a 100644 --- a/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts +++ b/developer/src/kmc-ldml/src/compiler/ldml-compiler-messages.ts @@ -1,5 +1,4 @@ -import { util } from "@keymanapp/common-types"; -import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_FILENAME_SYMBOL, CompilerEvent, KeymanXMLReader, ObjectWithMetadata } from '@keymanapp/developer-utils'; +import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageObjectSpec as mx, CompilerMessageSpec as m, CompilerMessageDef as def, ObjectWithMetadata } from '@keymanapp/developer-utils'; // const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler; const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.LdmlKeyboardCompiler; @@ -9,42 +8,6 @@ const SevError = CompilerErrorSeverity.Error | CompilerErrorNamespace.LdmlKeyboa // sub-numberspace for transform errors const SevErrorTransform = SevError | 0xF00; -/** - * Convenience function for constructing CompilerEvents with line numbers. - * Use it as below: (abbreviated as mx()) - * - * ```js - * // Note: Indentation makes "InvalidScanCode" line up thrice - * static ERROR_InvalidScanCode = SevError | 0x0009; - * // Note: - * // 1. All parameters are passed in 'o', the context object is only used for context even if - * // it contains redundant info. - * // 2. No code execution within the arrow function other than the 'mx' call, string interpolation, - * // with `${def(o.property)}` as the max complexity of interpolation. - * static Error_InvalidScanCode = (o:{id: string, invalidCodeList: string}, x: ObjectWithMetadata) => mx( - * this.ERROR_InvalidScanCode, x, - * `Form '${def(o.id)}' has invalid/unknown scancodes '${def(o.codes)}'`, - * // Note: If detail is omitted, leave the trailing comma on the prior line to leave room for it - * `…additional markdown detail…` - * ); - * ``` - * - * @param code Unique numeric value of the event - * @param message A short description of the error presented to the user - * @param context Object to be used as a source for line number information - * @param detail Detailed Markdown-formatted description of the error - * including references to documentation, remediation options. - * @see CompilerMessageSpec - * @returns - */ -function CompilerMessageObjectSpec(code: number, context: ObjectWithMetadata, message: string, detail?: string): CompilerEvent { - let evt = m(code, message, detail); // constructs raw message - evt = LdmlCompilerMessages.offset(evt, context); // updates with offset from context - return evt; -}; - -const mx = CompilerMessageObjectSpec; - /** * @internal */ @@ -70,7 +33,7 @@ export class LdmlCompilerMessages { static ERROR_RowOnHardwareLayerHasTooManyKeys = SevError | 0x0004; static Error_RowOnHardwareLayerHasTooManyKeys = (o: { row: number, hardware: string, modifiers: string }, x?: ObjectWithMetadata) => mx( this.ERROR_RowOnHardwareLayerHasTooManyKeys, x, - `Row #${def(o.row)} on 'hardware' ${def(o.hardware)} layer for modifier ${o.modifiers || 'none'} has too many keys`, + `Row #${def(o.row)} on 'hardware' ${def(o.hardware)} layer for modifier ${def(o.modifiers)} has too many keys`, ); static ERROR_KeyNotFoundInKeyBag = SevError | 0x0005; @@ -85,6 +48,7 @@ export class LdmlCompilerMessages { `After minimization, one or more locales is repeated and has been removed`, ); + // This is the only allowed use of m() vs mx() in this file, all the others take context. static ERROR_InvalidFile = SevError | 0x0007; static Error_InvalidFile = (o:{errorText: string}) => m(this.ERROR_InvalidFile, `The source file has an invalid structure: ${def(o.errorText)}`); @@ -108,15 +72,16 @@ export class LdmlCompilerMessages { ); static ERROR_GestureKeyNotFoundInKeyBag = SevError | 0x000B; - static Error_GestureKeyNotFoundInKeyBag = (o:{keyId: string, parentKeyId: string, attribute: string}, x?: ObjectWithMetadata) => - mx( + static Error_GestureKeyNotFoundInKeyBag = (o:{keyId: string, parentKeyId: string, attribute: string}, x?: ObjectWithMetadata) => mx( this.ERROR_GestureKeyNotFoundInKeyBag, x, `Key '${def(o.keyId)}' not found in key bag, referenced from other '${def(o.parentKeyId)}' in ${def(o.attribute)}`, ); static HINT_NoDisplayForMarker = SevHint | 0x000C; - static Hint_NoDisplayForMarker = (o: { id: string }) => - m(this.HINT_NoDisplayForMarker, `Key element with id "${def(o.id)}" has only marker output, but there is no matching display element by output or keyId. Keycap may be blank.`); + static Hint_NoDisplayForMarker = (o: { id: string }, x?: ObjectWithMetadata) => mx( + this.HINT_NoDisplayForMarker, x, + `Key element with id "${def(o.id)}" has only marker output, but there is no matching display element by output or keyId. Keycap may be blank.`, + ); static ERROR_InvalidVersion = SevError | 0x000D; static Error_InvalidVersion = (o: { version: string; }, x?: ObjectWithMetadata) => mx( @@ -139,26 +104,15 @@ export class LdmlCompilerMessages { ); static HINT_NoDisplayForSwitch = SevHint | 0x000F; - static Hint_NoDisplayForSwitch = (o: { id: string }) => - m(this.HINT_NoDisplayForSwitch, `Key element with id "${def(o.id)}" is a layer switch key, but there is no matching display element by keyId. Keycap may be blank.`); - - /** annotate the to= or id= entry */ - private static outputOrKeyId(o:{output?: string, keyId?: string}) { - if (o.output && o.keyId) { - return `output='${o.output}' keyId='${o.keyId}'`; - } else if(o.keyId) { - return `keyId='${o.keyId}'`; - } else if (o.output) { - return `output='${o.output}'`; - } else { - return ''; - } - } + static Hint_NoDisplayForSwitch = (o: { id: string }, x?: ObjectWithMetadata) => mx( + this.HINT_NoDisplayForSwitch, x, + `Key element with id "${def(o.id)}" is a layer switch key, but there is no matching display element by keyId. Keycap may be blank.`, + ); static ERROR_DisplayIsRepeated = SevError | 0x0010; - static Error_DisplayIsRepeated = (o:{output?: string, keyId?: string}, x?: ObjectWithMetadata) => mx( + static Error_DisplayIsRepeated = (o:{display?: string}, x?: ObjectWithMetadata) => mx( this.ERROR_DisplayIsRepeated, x, - `display ${LdmlCompilerMessages.outputOrKeyId(o)} has more than one display entry.`, + `display display='${def(o.display)}' refers to the same keyId or output as another entry.`, ); static ERROR_KeyMissingToGapOrSwitch = SevError | 0x0011; @@ -179,18 +133,10 @@ export class LdmlCompilerMessages { `layers has invalid value formId=${def(o.formId)}`, ); - private static layerIdOrEmpty(layer : string) { - if (layer) { - return ` on layer id=${def(layer)}`; - } else { - return ''; - } - } - static ERROR_InvalidModifier = SevError | 0x0014; - static Error_InvalidModifier = (o:{modifiers: string, id: string}, x?: ObjectWithMetadata) => mx( + static Error_InvalidModifier = (o:{modifiers: string}, x?: ObjectWithMetadata) => mx( this.ERROR_InvalidModifier, x, - `layer has invalid modifiers='${def(o.modifiers)}'` + LdmlCompilerMessages.layerIdOrEmpty(o.id), + `layer has invalid modifiers='${def(o.modifiers)}'`, ); static ERROR_MissingFlicks = SevError | 0x0015; @@ -237,12 +183,16 @@ export class LdmlCompilerMessages { ); static ERROR_MissingSetVariable = SevError | 0x001D; - static Error_MissingSetVariable = (o:{id: string}) => - m(this.ERROR_MissingSetVariable, `Reference to undefined set variable: \$[${def(o.id)}]`); + static Error_MissingSetVariable = (o:{id: string}, x?: ObjectWithMetadata) => mx( + this.ERROR_MissingSetVariable, x, + `Reference to undefined set variable: \$[${def(o.id)}]`, + ); static ERROR_MissingUnicodeSetVariable = SevError | 0x001E; - static Error_MissingUnicodeSetVariable = (o:{id: string}) => - m(this.ERROR_MissingUnicodeSetVariable, `Reference to undefined UnicodeSet variable: \$[${def(o.id)}]`); + static Error_MissingUnicodeSetVariable = (o:{id: string}, x?: ObjectWithMetadata) => mx( + this.ERROR_MissingUnicodeSetVariable, x, + `Reference to undefined UnicodeSet variable: \$[${def(o.id)}]`, + ); static ERROR_NeedSpacesBetweenSetVariables = SevError | 0x001F; static Error_NeedSpacesBetweenSetVariables = (o:{item: string}, x?: ObjectWithMetadata) => mx( @@ -257,79 +207,88 @@ export class LdmlCompilerMessages { ); static ERROR_MissingMarkers = SevError | 0x0021; - static Error_MissingMarkers = (o: { ids: string[] }) => - m(this.ERROR_MissingMarkers, `Markers used for matching but not defined: ${def(o.ids?.join(','))}`); + static Error_MissingMarkers = (o: { ids: string }, x?: ObjectWithMetadata) => mx( + this.ERROR_MissingMarkers, x, + `Markers used for matching but not defined: ${def(o.ids)}` + ); static ERROR_DisplayNeedsToOrId = SevError | 0x0022; - static Error_DisplayNeedsToOrId = (o:{output?: string, keyId?: string}, x?: ObjectWithMetadata) => mx( + static Error_DisplayNeedsToOrId = (o:{display?: string}, x?: ObjectWithMetadata) => mx( this.ERROR_DisplayNeedsToOrId, x, - `display ${LdmlCompilerMessages.outputOrKeyId(o)} needs output= or keyId=, but not both`, + `display display='${def(o.display)}' needs output= or keyId=, but not both`, ); static HINT_PUACharacters = SevHint | 0x0023; - static Hint_PUACharacters = (o: { count: number, lowestCh: number }) => - m(this.HINT_PUACharacters, `File contains ${def(o.count)} PUA character(s), including ${util.describeCodepoint(o.lowestCh)}`); + static Hint_PUACharacters = (o: { count: number, lowestCh: string }, x?: ObjectWithMetadata) => mx( + this.HINT_PUACharacters, x, + `File contains ${def(o.count)} PUA character(s), including ${def(o.lowestCh)}`, + ); static WARN_UnassignedCharacters = SevWarn | 0x0024; - static Warn_UnassignedCharacters = (o: { count: number, lowestCh: number }) => - m(this.WARN_UnassignedCharacters, `File contains ${def(o.count)} unassigned character(s), including ${util.describeCodepoint(o.lowestCh)}`); + static Warn_UnassignedCharacters = (o: { count: number, lowestCh: string }, x?: ObjectWithMetadata) => mx( + this.WARN_UnassignedCharacters, x, + `File contains ${def(o.count)} unassigned character(s), including ${def(o.lowestCh)}`, + ); static ERROR_IllegalCharacters = SevError | 0x0025; - static Error_IllegalCharacters = (o: { count: number, lowestCh: number }) => - m(this.ERROR_IllegalCharacters, `File contains ${def(o.count)} illegal character(s), including ${util.describeCodepoint(o.lowestCh)}`); + static Error_IllegalCharacters = (o: { count: number, lowestCh: string }, x?: ObjectWithMetadata) => mx( + this.ERROR_IllegalCharacters, x, + `File contains ${def(o.count)} illegal character(s), including ${def(o.lowestCh) }`, + ); static HINT_CharClassImplicitDenorm = SevHint | 0x0026; static Hint_CharClassImplicitDenorm = (o: { lowestCh: number }, x?: ObjectWithMetadata) => mx( this.HINT_CharClassImplicitDenorm, x, - `File has character classes which span non-NFD character(s), including ${util.describeCodepoint(o.lowestCh)}. These will not match any text.`, + `File has character classes which span non-NFD character(s), including ${def(o.lowestCh)}. These will not match any text.`, ); static WARN_CharClassExplicitDenorm = SevWarn | 0x0027; static Warn_CharClassExplicitDenorm = (o: { lowestCh: number }, x?: ObjectWithMetadata) => mx( this.WARN_CharClassExplicitDenorm, x, - `File has character classes which include non-NFD characters(s), including ${util.describeCodepoint(o.lowestCh)}. These will not match any text.`, + `File has character classes which include non-NFD characters(s), including ${def(o.lowestCh)}. These will not match any text.`, ); - static ERROR_UnparseableReorderSet = SevError | 0x0028; - static Error_UnparseableReorderSet = (o: { from: string, set: string }) => - m(this.ERROR_UnparseableReorderSet, `Illegal UnicodeSet "${def(o.set)}" in reorder "${def(o.from)}`); + // Available: 0x0028 static ERROR_InvalidVariableIdentifier = SevError | 0x0029; - static Error_InvalidVariableIdentifier = (o: { id: string }) => m( - this.ERROR_InvalidVariableIdentifier, + static Error_InvalidVariableIdentifier = (o: { id: string }, x?: ObjectWithMetadata) => mx( + this.ERROR_InvalidVariableIdentifier, x, `Invalid variable identifier "${def(o.id)}". Identifiers must be between 1 and 32 characters, and can use A-Z, a-z, 0-9, and _.`, ); static ERROR_InvalidMarkerIdentifier = SevError | 0x002A; - static Error_InvalidMarkerIdentifier = (o: { id: string }) => m( - this.ERROR_InvalidMarkerIdentifier, + static Error_InvalidMarkerIdentifier = (o: { id: string }, x?: ObjectWithMetadata) => mx( + this.ERROR_InvalidMarkerIdentifier, x, `Invalid marker identifier "\m{${def(o.id)}}". Identifiers must be between 1 and 32 characters, and can use A-Z, a-z, 0-9, and _.`, ); static WARN_StringDenorm = SevWarn | 0x002B; - static Warn_StringDenorm = (o: { s: string }) => - m(this.WARN_StringDenorm, `File contains string "${def(o.s)}" that is neither NFC nor NFD.`); + static Warn_StringDenorm = (o: { s: string }, x?: ObjectWithMetadata) => mx( + this.WARN_StringDenorm, x, + `File contains string "${def(o.s)}" that is neither NFC nor NFD.`, + ); static ERROR_DuplicateLayerWidth = SevError | 0x002C; static Error_DuplicateLayerWidth = (o: { minDeviceWidth: number }, x?: ObjectWithMetadata) => mx( this.ERROR_DuplicateLayerWidth, x, - `Two or more layers have minDeviceWidth=${def(o?.minDeviceWidth)}`, + `Two or more layers have minDeviceWidth=${def(o.minDeviceWidth)}`, `Touch layers must have distinct widths.` ); static ERROR_InvalidLayerWidth = SevError | 0x002D; static Error_InvalidLayerWidth = (o: { minDeviceWidth: number }, x?: ObjectWithMetadata) => mx( this.ERROR_InvalidLayerWidth, x, - `Invalid Layers minDeviceWidth=${def(o?.minDeviceWidth)}`, + `Invalid Layers minDeviceWidth=${def(o.minDeviceWidth)}`, `Width must be between 1-999 (millimeters), inclusive.` // sync with layr_max_minDeviceWidth / layr_max_maxDeviceWidth (from spec) ); // Available: 0x02E-0x2F static ERROR_InvalidQuadEscape = SevError | 0x0030; - static Error_InvalidQuadEscape = (o: { cp: number }, x?: ObjectWithMetadata) => mx( + static Error_InvalidQuadEscape = (o: { cp: string, recommended: string }, x?: ObjectWithMetadata) => mx( this.ERROR_InvalidQuadEscape, x, - `Invalid escape "\\u${util.hexQuad(o?.cp || 0)}". Hint: Use "\\u{${def(o?.cp?.toString(16))}}"`, + `Invalid escape "${def(o.cp)}"`, + `**Hint**: Use "${def(o.recommended)}"`, ); // @@ -337,16 +296,18 @@ export class LdmlCompilerMessages { // This is a bit of a catch-all and represents messages bubbling up from the underlying regex engine static ERROR_UnparseableTransformFrom = SevErrorTransform | 0x00; - static Error_UnparseableTransformFrom = (o: { from: string, message: string }) => - m(this.ERROR_UnparseableTransformFrom, `Invalid transform from="${def(o.from)}": "${def(o.message)}"`); + static Error_UnparseableTransformFrom = (o: { from: string, message: string }, x?: ObjectWithMetadata) => mx( + this.ERROR_UnparseableTransformFrom, x, + `Invalid transform from="${def(o.from)}": "${def(o.message)}"`, + ); //------------------------------------------------------------------------------| // max length of detail message lines (checked by verifyCompilerMessagesObject) | //------------------------------------------------------------------------------| static ERROR_IllegalTransformDollarsign = SevErrorTransform | 0x01; - static Error_IllegalTransformDollarsign = (o: { from: string }) => m( - this.ERROR_IllegalTransformDollarsign, + static Error_IllegalTransformDollarsign = (o: { from: string }, x?: ObjectWithMetadata) => mx( + this.ERROR_IllegalTransformDollarsign, x, `Invalid transform from="${def(o.from)}": Unescaped dollar-sign ($) is not valid transform syntax.`, ` **Hint**: Use \`\\$\` to match a literal dollar-sign. If this precedes a variable name, the variable name may not be valid (A-Z, a-z, 0-9, _, 32 @@ -360,51 +321,30 @@ export class LdmlCompilerMessages { ); static ERROR_IllegalTransformPlus = SevErrorTransform | 0x03; - static Error_IllegalTransformPlus = (o: { from: string }) => m( - this.ERROR_IllegalTransformPlus, + static Error_IllegalTransformPlus = (o: { from: string }, x?: ObjectWithMetadata) => mx( + this.ERROR_IllegalTransformPlus, x, `Invalid transform from="${def(o.from)}": Unescaped plus (+) is not valid transform syntax.`, ` **Hint**: Use \`\\+\` to match a literal plus. `); static ERROR_IllegalTransformAsterisk = SevErrorTransform | 0x04; - static Error_IllegalTransformAsterisk = (o: { from: string }) =>m( - this.ERROR_IllegalTransformAsterisk, + static Error_IllegalTransformAsterisk = (o: { from: string }, x?: ObjectWithMetadata) =>mx( + this.ERROR_IllegalTransformAsterisk, x, `Invalid transform from="${def(o.from)}": Unescaped asterisk (*) is not valid transform syntax.`, ` **Hint**: Use \`\\*\` to match a literal asterisk. `); static ERROR_IllegalTransformToUset = SevErrorTransform | 0x05; - static Error_IllegalTransformToUset = (o: { to: string }) => m( - this.ERROR_IllegalTransformToUset, + static Error_IllegalTransformToUset = (o: { to: string }, x?: ObjectWithMetadata) => mx( + this.ERROR_IllegalTransformToUset, x, `Invalid transform to="${def(o.to)}": Set variable (\\$[…]) cannot be used in 'to=' unless part of a map.`, ` **Hint**: If a map was meant, must use the form \`\`. `); static ERROR_UnparseableTransformTo = SevErrorTransform | 0x06; - static Error_UnparseableTransformTo = (o: {to: string, message: string}) => m( - this.ERROR_UnparseableTransformTo, + static Error_UnparseableTransformTo = (o: {to: string, message: string}, x?: ObjectWithMetadata) => mx( + this.ERROR_UnparseableTransformTo, x, `Invalid transform to="${def(o.to)}": "${def(o.message)}"`, ); - - /** - * Get an offset from o and set e's offset field - * @param event a compiler event, such as from functions in this class - * @param x any object parsed from XML or with the XML_META_DATA_SYMBOL symbol copied over - * @returns modified event object - */ - static offset(event: CompilerEvent, x?: any): CompilerEvent { - if(x) { - const metadata = KeymanXMLReader.getMetaData(x) || {}; - const offset = metadata?.startIndex; - if (offset) { - event.offset = offset; - } - const filename = event.filename || metadata[XML_FILENAME_SYMBOL]; - if (filename) { - event.filename = filename; - } - } - return event; - } } diff --git a/developer/src/kmc-ldml/src/compiler/linter-keycaps.ts b/developer/src/kmc-ldml/src/compiler/linter-keycaps.ts index 8f1006c54a..9612190fdf 100644 --- a/developer/src/kmc-ldml/src/compiler/linter-keycaps.ts +++ b/developer/src/kmc-ldml/src/compiler/linter-keycaps.ts @@ -35,13 +35,13 @@ export class LinterKeycaps extends Linter { const disp = this.findDisp(id.value, to.value); if (!disp) { - this.callbacks.reportMessage(LdmlCompilerMessages.Hint_NoDisplayForSwitch({ id: id.value })); + this.callbacks.reportMessage(LdmlCompilerMessages.Hint_NoDisplayForSwitch({ id: id.value }, key)); } } else if (to.value !== '' && nonMarkerOutput === '') { // has output, but only markers const disp = this.findDisp(id.value, to.value); if (!disp) { - this.callbacks.reportMessage(LdmlCompilerMessages.Hint_NoDisplayForMarker({ id: id.value })); + this.callbacks.reportMessage(LdmlCompilerMessages.Hint_NoDisplayForMarker({ id: id.value }, key)); } } } diff --git a/developer/src/kmc-ldml/src/compiler/loca.ts b/developer/src/kmc-ldml/src/compiler/loca.ts index 48be2d032f..3cc222e613 100644 --- a/developer/src/kmc-ldml/src/compiler/loca.ts +++ b/developer/src/kmc-ldml/src/compiler/loca.ts @@ -71,7 +71,7 @@ export class LocaCompiler extends SectionCompiler { // yet include `getCanonicalLocales` but node 16 does include it so we can // safely use it. Also well supported in modern browsers. const canonicalLocales = (Intl as any).getCanonicalLocales(locales) as string[]; - result.locales = canonicalLocales.map(locale => sections.strs.allocString(locale)); + result.locales = canonicalLocales.map(locale => sections.strs.allocString(locale, {x: this.contextForLocale(locale)})); if(result.locales.length < locales.length) { this.callbacks.reportMessage(LdmlCompilerMessages.Hint_OneOrMoreRepeatedLocales(this.keyboard3?.locales)); diff --git a/developer/src/kmc-ldml/src/compiler/meta.ts b/developer/src/kmc-ldml/src/compiler/meta.ts index 57f0cb36c0..e6cd6371ee 100644 --- a/developer/src/kmc-ldml/src/compiler/meta.ts +++ b/developer/src/kmc-ldml/src/compiler/meta.ts @@ -53,12 +53,18 @@ export class MetaCompiler extends SectionCompiler { public compile(sections: DependencySections): Meta { const result = new Meta(); - result.author = sections.strs.allocString(this.keyboard3.info?.author); - result.conform = sections.strs.allocString(this.keyboard3.conformsTo); - result.layout = sections.strs.allocString(this.keyboard3.info?.layout); - result.name = sections.strs.allocString(this.keyboard3.info?.name); - result.indicator = sections.strs.allocString(this.keyboard3.info?.indicator); - result.version = sections.strs.allocString(this.keyboard3.version?.number ?? "0.0.0"); + result.author = sections.strs.allocString(this.keyboard3.info?.author, + {x: this.keyboard3.info}); + result.conform = sections.strs.allocString(this.keyboard3.conformsTo, + {x: this.keyboard3}); + result.layout = sections.strs.allocString(this.keyboard3.info?.layout, + {x: this.keyboard3.info}); + result.name = sections.strs.allocString(this.keyboard3.info?.name, + {x: this.keyboard3.info}); + result.indicator = sections.strs.allocString(this.keyboard3.info?.indicator, + {x: this.keyboard3.info}); + result.version = sections.strs.allocString(this.keyboard3.version?.number ?? "0.0.0", + {x: this.keyboard3.version}); result.settings = (this.keyboard3.settings?.normalization == "disabled" ? KeyboardSettings.normalizationDisabled : 0); return result; diff --git a/developer/src/kmc-ldml/src/compiler/substitution-tracker.ts b/developer/src/kmc-ldml/src/compiler/substitution-tracker.ts index 6f07475490..61daefa0cf 100644 --- a/developer/src/kmc-ldml/src/compiler/substitution-tracker.ts +++ b/developer/src/kmc-ldml/src/compiler/substitution-tracker.ts @@ -1,4 +1,5 @@ import { LdmlKeyboardTypes } from "@keymanapp/common-types"; +import { ObjectWithMetadata } from "@keymanapp/developer-utils"; /** * Verb for SubstitutionTracker.add() @@ -14,7 +15,7 @@ export enum SubstitutionUse { variable, } -type SubstitutionSet = Set; +type SubstitutionSet = Map; /** Tracks usage of markers */ export class SubstitutionTracker { @@ -28,10 +29,10 @@ export class SubstitutionTracker { all: SubstitutionSet; constructor() { - this.emitted = new Set(); - this.consumed = new Set(); - this.matched = new Set(); - this.all = new Set(); + this.emitted = new Map(); + this.consumed = new Map(); + this.matched = new Map(); + this.all = new Map(); } /** @@ -39,32 +40,33 @@ export class SubstitutionTracker { * @param verb what kind of use we are adding * @param markers list of substitutions to add */ - add(verb: SubstitutionUse, markers: string[]) { + add(verb: SubstitutionUse, markers: string[], x?: ObjectWithMetadata) { if (!markers.length) { return; // skip if empty } + x = x || {}; // need at least an empty object if (verb == SubstitutionUse.emit) { markers.forEach((m) => { - this.emitted.add(m); - this.all.add(m); + this.emitted.set(m, x); + this.all.set(m,x); }); } else if (verb == SubstitutionUse.consume) { markers.forEach((m) => { - this.consumed.add(m); - this.all.add(m); + this.consumed.set(m,x); + this.all.set(m,x); }); } else if (verb == SubstitutionUse.match) { markers.forEach((m) => { - this.matched.add(m); - this.all.add(m); + this.matched.set(m,x); + this.all.set(m,x); }); } else if (verb == SubstitutionUse.variable) { markers.forEach((m) => { // we don't know, so add it to all three - this.matched.add(m); - this.emitted.add(m); - this.consumed.add(m); - this.all.add(m); + this.matched.set(m,x); + this.emitted.set(m,x); + this.consumed.set(m,x); + this.all.set(m,x); }); /* c8 skip next 3 */ } else { @@ -75,27 +77,27 @@ export class SubstitutionTracker { /** rollup of several substitution types */ export class Substitutions { - addSetAndStringSubtitution(verb: SubstitutionUse, str?: string) { - this.set.add(verb, LdmlKeyboardTypes.VariableParser.allSetReferences(str)); - this.addStringAndMarkerSubstitution(verb, str); + addSetAndStringSubtitution(verb: SubstitutionUse, str?: string, x?: ObjectWithMetadata) { + this.set.add(verb, LdmlKeyboardTypes.VariableParser.allSetReferences(str), x); + this.addStringAndMarkerSubstitution(verb, str, x); } /** add a string that can have string var substitutions or markers */ - addStringAndMarkerSubstitution(verb: SubstitutionUse, str?: string) { - this.addMarkers(verb, str); - this.addStringSubstitution(verb, str); + addStringAndMarkerSubstitution(verb: SubstitutionUse, str?: string, x?: ObjectWithMetadata) { + this.addMarkers(verb, str, x); + this.addStringSubstitution(verb, str, x); } - addStringSubstitution(verb: SubstitutionUse, str?: string) { - this.string.add(verb, LdmlKeyboardTypes.VariableParser.allStringReferences(str)); + addStringSubstitution(verb: SubstitutionUse, str?: string, x?: ObjectWithMetadata) { + this.string.add(verb, LdmlKeyboardTypes.VariableParser.allStringReferences(str), x); } /** add a string that's just markers */ - addMarkers(verb: SubstitutionUse, str?: string) { - this.markers.add(verb, LdmlKeyboardTypes.MarkerParser.allReferences(str)); - LdmlKeyboardTypes.MarkerParser.allBrokenReferences(str).forEach(m => this.badMarkers.add(m)); + addMarkers(verb: SubstitutionUse, str?: string, x?: ObjectWithMetadata) { + this.markers.add(verb, LdmlKeyboardTypes.MarkerParser.allReferences(str), x); + LdmlKeyboardTypes.MarkerParser.allBrokenReferences(str).forEach(m => this.badMarkers.set(m, x)); } // all valid markers markers: SubstitutionTracker; // all invalid markers - badMarkers: Set; + badMarkers: SubstitutionSet; // all valid set ids set: SubstitutionTracker; // all valid string ids @@ -108,6 +110,6 @@ export class Substitutions { this.set = new SubstitutionTracker(); this.string = new SubstitutionTracker(); this.uset = new SubstitutionTracker(); - this.badMarkers = new Set(); + this.badMarkers = new Map(); } } diff --git a/developer/src/kmc-ldml/src/compiler/tran.ts b/developer/src/kmc-ldml/src/compiler/tran.ts index 32ebbca3dd..1cc9bd0289 100644 --- a/developer/src/kmc-ldml/src/compiler/tran.ts +++ b/developer/src/kmc-ldml/src/compiler/tran.ts @@ -1,6 +1,6 @@ import { constants, SectionIdent } from "@keymanapp/ldml-keyboard-constants"; import { KMXPlus, LdmlKeyboardTypes, util } from '@keymanapp/common-types'; -import { CompilerCallbacks, LDMLKeyboard } from "@keymanapp/developer-utils"; +import { CompilerCallbacks, LDMLKeyboard, ObjectWithMetadata } from "@keymanapp/developer-utils"; import { SectionCompiler } from "./section-compiler.js"; import Bksp = KMXPlus.Bksp; @@ -18,6 +18,7 @@ import { verifyValidAndUnique } from "../util/util.js"; import { LdmlCompilerMessages } from "./ldml-compiler-messages.js"; import { Substitutions, SubstitutionUse } from "./substitution-tracker.js"; import { transform_from_parse, transform_to_parse } from "../util/abnf/abnf.js"; +import { StrsOptions } from "../../../../../common/web/types/src/kmx/kmx-plus/kmx-plus.js"; type TransformCompilerType = 'simple' | 'backspace'; @@ -26,22 +27,24 @@ export abstract class TransformCompiler transforms.transformGroup.forEach(transformGroup => { - transformGroup.transform?.forEach(({ to, from }) => { - st.addSetAndStringSubtitution(SubstitutionUse.consume, from); - st.addSetAndStringSubtitution(SubstitutionUse.emit, to); + transformGroup.transform?.forEach((transform) => { + const { to, from } = transform; + st.addSetAndStringSubtitution(SubstitutionUse.consume, from, transform); + st.addSetAndStringSubtitution(SubstitutionUse.emit, to, transform); const mapFrom = LdmlKeyboardTypes.VariableParser.CAPTURE_SET_REFERENCE.exec(from); const mapTo = LdmlKeyboardTypes.VariableParser.MAPPED_SET_REFERENCE.exec(to || ''); if (mapFrom) { // add the 'from' as a match - st.set.add(SubstitutionUse.consume, [mapFrom[1]]); + st.set.add(SubstitutionUse.consume, [mapFrom[1]], transform); } if (mapTo) { // add the 'from' as a match - st.set.add(SubstitutionUse.emit, [mapTo[1]]); + st.set.add(SubstitutionUse.emit, [mapTo[1]], transform); } }); - transformGroup.reorder?.forEach(({ before }) => { - st.addStringSubstitution(SubstitutionUse.consume, before); + transformGroup.reorder?.forEach((reorder) => { + const { before } = reorder; + st.addStringSubstitution(SubstitutionUse.consume, before, reorder); }); })); return true; @@ -138,6 +141,8 @@ export abstract class TransformCompiler DTD - this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidVariableIdentifier({id})); + this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidVariableIdentifier({id}, x)); return false; } return true; @@ -77,7 +77,7 @@ export class VarsCompiler extends SectionCompiler { // Strings for (const e of variables.string) { const { id, value } = e; - if(!this.validateIdentifier(id)) { + if(!this.validateIdentifier(id, e)) { valid = false; continue; } @@ -90,9 +90,9 @@ export class VarsCompiler extends SectionCompiler { allStrings.add(ref); // avoids multiple reports of same missing variable } } - st.string.add(SubstitutionUse.variable, stringrefs); + st.string.add(SubstitutionUse.variable, stringrefs, e); allStrings.add(id); - st.addMarkers(SubstitutionUse.variable, value); + st.addMarkers(SubstitutionUse.variable, value, e); } // Sets for (const e of variables.set) { @@ -105,7 +105,7 @@ export class VarsCompiler extends SectionCompiler { allSets.add(id); // check for illegal references, here. const stringrefs = LdmlKeyboardTypes.VariableParser.allStringReferences(value); - st.string.add(SubstitutionUse.variable, stringrefs); + st.string.add(SubstitutionUse.variable, stringrefs, e); // Now split into spaces. const items: string[] = LdmlKeyboardTypes.VariableParser.setSplitter(value); @@ -116,7 +116,7 @@ export class VarsCompiler extends SectionCompiler { valid = false; this.callbacks.reportMessage(LdmlCompilerMessages.Error_NeedSpacesBetweenSetVariables({ item }, e)); } else { - st.set.add(SubstitutionUse.variable, setrefs); // the reference to a 'map' + st.set.add(SubstitutionUse.variable, setrefs, e); // the reference to a 'map' } // TODO-LDML: Are there other illegal cases here? what about "x$[set]"? } @@ -131,7 +131,7 @@ export class VarsCompiler extends SectionCompiler { addId(id, e); allUnicodeSets.add(id); const stringrefs = LdmlKeyboardTypes.VariableParser.allStringReferences(value); - st.string.add(SubstitutionUse.variable, stringrefs); + st.string.add(SubstitutionUse.variable, stringrefs, e); const setrefs = LdmlKeyboardTypes.VariableParser.allSetReferences(value); for (const id2 of setrefs) { if (!allUnicodeSets.has(id2)) { @@ -140,7 +140,7 @@ export class VarsCompiler extends SectionCompiler { // $[set] in a UnicodeSet must be another UnicodeSet. this.callbacks.reportMessage(LdmlCompilerMessages.Error_CantReferenceSetFromUnicodeSet({ id: id2 }, e)); } else { - st.uset.add(SubstitutionUse.variable, [id2]); + st.uset.add(SubstitutionUse.variable, [id2], e); } } } @@ -157,24 +157,24 @@ export class VarsCompiler extends SectionCompiler { } // check for any missing vars - for (const id of st.set.all) { + for (const [id,x] of st.set.all.entries()) { // note: we check the uset list also. we don't know until later which was // intended. collisions are handled separately. if (!allSets.has(id) && !allUnicodeSets.has(id)) { valid = false; - this.callbacks.reportMessage(LdmlCompilerMessages.Error_MissingSetVariable({ id })); + this.callbacks.reportMessage(LdmlCompilerMessages.Error_MissingSetVariable({ id }, x)); } } - for (const id of st.string.all) { + for (const [id,x] of st.string.all.entries()) { if (!allStrings.has(id)) { valid = false; - this.callbacks.reportMessage(LdmlCompilerMessages.Error_MissingStringVariable({ id })); + this.callbacks.reportMessage(LdmlCompilerMessages.Error_MissingStringVariable({ id }, x)); } } - for (const id of st.uset.all) { + for (const [id,x] of st.uset.all.entries()) { if (!allUnicodeSets.has(id)) { valid = false; - this.callbacks.reportMessage(LdmlCompilerMessages.Error_MissingUnicodeSetVariable({ id })); + this.callbacks.reportMessage(LdmlCompilerMessages.Error_MissingUnicodeSetVariable({ id }, x)); } } @@ -198,13 +198,13 @@ export class VarsCompiler extends SectionCompiler { // see if there are any matched-but-not-emitted markers const matchedNotEmitted : Set = new Set(); const mt = st.markers; - for (const m of mt.matched.values()) { + for (const m of mt.matched.keys()) { if (m === LdmlKeyboardTypes.MarkerParser.ANY_MARKER_ID) continue; // match-all marker if (!mt.emitted.has(m)) { matchedNotEmitted.add(m); } } - for (const m of mt.consumed.values()) { + for (const m of mt.consumed.keys()) { if (m === LdmlKeyboardTypes.MarkerParser.ANY_MARKER_ID) continue; // match-all marker if (!mt.emitted.has(m)) { matchedNotEmitted.add(m); @@ -213,7 +213,9 @@ export class VarsCompiler extends SectionCompiler { // report once if (matchedNotEmitted.size > 0) { - this.callbacks.reportMessage(LdmlCompilerMessages.Error_MissingMarkers({ ids: Array.from(matchedNotEmitted.values()).sort() })); + const ids = Array.from(matchedNotEmitted.values()).sort(); + const x = mt.all.get(ids[0]); // get the FIRST object for context + this.callbacks.reportMessage(LdmlCompilerMessages.Error_MissingMarkers({ ids: ids.join(',')}, x)); valid = false; } @@ -222,19 +224,24 @@ export class VarsCompiler extends SectionCompiler { if (!!st.badMarkers.size) { valid = false; - st.badMarkers.forEach(id => - this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidMarkerIdentifier({ id }))); + for (const [id, x] of st.badMarkers.entries()) { + this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidMarkerIdentifier({ id }, x)); + } } return valid; } validateSubstitutions(keyboard: LDMLKeyboard.LKKeyboard, st : Substitutions) : boolean { - keyboard?.variables?.string?.forEach(({value}) => - st.markers.add(SubstitutionUse.variable, LdmlKeyboardTypes.MarkerParser.allReferences(value))); + keyboard?.variables?.string?.forEach((string) => { + const {value} = string; + st.markers.add(SubstitutionUse.variable, LdmlKeyboardTypes.MarkerParser.allReferences(value), string); + }); // get markers mentioned in a set - keyboard?.variables?.set?.forEach(({ value }) => - LdmlKeyboardTypes.VariableParser.setSplitter(value).forEach(v => st.markers.add(SubstitutionUse.match, LdmlKeyboardTypes.MarkerParser.allReferences(v)))); + keyboard?.variables?.set?.forEach((set) => { + const { value } = set; + LdmlKeyboardTypes.VariableParser.setSplitter(value).forEach(v => st.markers.add(SubstitutionUse.match, LdmlKeyboardTypes.MarkerParser.allReferences(v), set)); + }); return true; } @@ -260,8 +267,16 @@ export class VarsCompiler extends SectionCompiler { const mt = st.markers; // collect all markers, excluding the match-all - const allMarkers : string[] = Array.from(mt.all).filter(m => m !== LdmlKeyboardTypes.MarkerParser.ANY_MARKER_ID).sort(); - result.markers = sections.list.allocList(allMarkers, {}, sections); + const allMarkers : string[] = Array.from(mt.all.keys()).filter(m => m !== LdmlKeyboardTypes.MarkerParser.ANY_MARKER_ID).sort(); + result.markers = sections.list.allocList(allMarkers, { + // pass the first object in the listr + x: mt.all.get(allMarkers[0]||'') + // however, this is the string with the marker *id*. So, from the StrsCompiler + // point of view, the value of this context would be if the marker ID had, say, + // invalid Unicode in it for ERROR_IllegalCharacters - but, that wouldn't be a + // valid Marker ID either. + // So it's not likely that this context will be used, but it's available. + }, sections); // sets need to be added late, because they can refer to markers variables?.set?.forEach((e) => diff --git a/developer/src/kmc-ldml/test/compiler-e2e.tests.ts b/developer/src/kmc-ldml/test/compiler-e2e.tests.ts index e7756226bd..7c05d39d22 100644 --- a/developer/src/kmc-ldml/test/compiler-e2e.tests.ts +++ b/developer/src/kmc-ldml/test/compiler-e2e.tests.ts @@ -1,12 +1,13 @@ import 'mocha'; import {assert} from 'chai'; import hextobin from '@keymanapp/hextobin'; -import {compileKeyboard, compilerTestCallbacks, compilerTestOptions, makePathToFixture} from './helpers/index.js'; +import {compileKeyboard, compilerTestCallbacks, compilerTestOptions, makePathToFixture, scrubContextFromMessages} from './helpers/index.js'; import { compareXml } from './helpers/compareXml.js'; import { LdmlKeyboardCompiler } from '../src/compiler/compiler.js'; import { kmxToXml } from '../src/util/serialize.js'; import { writeFileSync } from 'node:fs'; import { LdmlCompilerMessages } from '../src/main.js'; +import { util } from '@keymanapp/common-types'; /** Overall compiler tests */ describe('compiler-tests', function() { @@ -53,11 +54,11 @@ describe('compiler-tests', function() { const runOutput = await k.run(inputFilename, "invalid-illegal.kmx"); // need the exact name passed to build-fixtures assert.isNull(runOutput, "Expect invalid-illegal to fail to run()"); - assert.sameDeepMembers(compilerTestCallbacks.messages, [ + assert.sameDeepMembers( scrubContextFromMessages(compilerTestCallbacks.messages), [ // copied from strs.tests.ts // validation messages - LdmlCompilerMessages.Error_IllegalCharacters({ count: 5, lowestCh: 0xFDD0 }), - LdmlCompilerMessages.Hint_PUACharacters({ count: 2, lowestCh: 0xE010 }), + LdmlCompilerMessages.Error_IllegalCharacters({ count: 5, lowestCh: util.describeCodepoint(0xFDD0) }), + LdmlCompilerMessages.Hint_PUACharacters({ count: 2, lowestCh: util.describeCodepoint(0xE010) }), ]); }); diff --git a/developer/src/kmc-ldml/test/disp.tests.ts b/developer/src/kmc-ldml/test/disp.tests.ts index 032e4829bf..3bad95e339 100644 --- a/developer/src/kmc-ldml/test/disp.tests.ts +++ b/developer/src/kmc-ldml/test/disp.tests.ts @@ -67,25 +67,25 @@ describe('disp', function () { const disp = await loadSectionFixture(DispCompiler, 'sections/disp/invalid-dupto.xml', compilerTestCallbacks) as Disp; assert.isNull(disp); assert.equal(compilerTestCallbacks.messages.length, 1); - assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_DisplayIsRepeated({ output: 'e' }, withOffset(330))); + assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_DisplayIsRepeated({ display: '(e)' }, withOffset(330))); }); it('should reject duplicate ids', async function() { const disp = await loadSectionFixture(DispCompiler, 'sections/disp/invalid-dupid.xml', compilerTestCallbacks) as Disp;1 assert.isNull(disp); assert.equal(compilerTestCallbacks.messages.length, 1); - assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_DisplayIsRepeated({ keyId: 'e' }, withOffset(329))); + assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_DisplayIsRepeated({ display: '(e)' }, withOffset(329))); }); it('should reject if neither to nor id', async function() { const disp = await loadSectionFixture(DispCompiler, 'sections/disp/invalid-none.xml', compilerTestCallbacks) as Disp; assert.isNull(disp); assert.equal(compilerTestCallbacks.messages.length, 1); - assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_DisplayNeedsToOrId({}, withOffset(182))); + assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_DisplayNeedsToOrId({ display: '(f)'}, withOffset(182))); }); it('should reject if both to and id', async function() { const disp = await loadSectionFixture(DispCompiler, 'sections/disp/invalid-both.xml', compilerTestCallbacks) as Disp; assert.isNull(disp); assert.equal(compilerTestCallbacks.messages.length, 1); - assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_DisplayNeedsToOrId({ output: 'e', keyId: 'e' }, withOffset(182))); + assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_DisplayNeedsToOrId({ display: '(e)' }, withOffset(182))); }); testCompilationCases(DispCompiler, [ { diff --git a/developer/src/kmc-ldml/test/helpers/index.ts b/developer/src/kmc-ldml/test/helpers/index.ts index 7d714bb2b6..967287cf87 100644 --- a/developer/src/kmc-ldml/test/helpers/index.ts +++ b/developer/src/kmc-ldml/test/helpers/index.ts @@ -144,7 +144,7 @@ export async function compileKeyboard(inputFilename: string, options: LdmlCompil const valid = await k.validate(source); if (validateMessages) { - assert.sameDeepMembers(compilerTestCallbacks.messages, validateMessages, "validation messages mismatch"); + assert.sameDeepMembers(scrubContextFromMessages(compilerTestCallbacks.messages), validateMessages, "validation messages mismatch"); assert.notEqual(valid, expectFailValidate, 'validation failure'); } else { checkMessages(); @@ -155,7 +155,7 @@ export async function compileKeyboard(inputFilename: string, options: LdmlCompil const kmx = await k.compile(source); if (compileMessages) { - assert.sameDeepMembers(compilerTestCallbacks.messages, compileMessages, "compiler messages mismatch"); + assert.sameDeepMembers(scrubContextFromMessages(compilerTestCallbacks.messages), compileMessages, "compiler messages mismatch"); } else { checkMessages(); } @@ -259,6 +259,26 @@ export interface CompilationCase { retainOffsetInMessages?: boolean; } +/** + * Scrub 'context' from messages. to simplify unit tests + * @param messages input array of messages + * @returns copy of messages + */ +export function scrubContextFromMessages(messages: CompilerEvent[]): CompilerEvent[] { + return messages.map(m => { + const scrubbed = Object.assign({}, m); + // Turn this on once all messages have offsets, see messages.tests.ts + // if (!scrubbed.offset) { + // throw Error(`Error, no offset detected in message ${CompilerError.formatEvent(m)}`); + // } + delete scrubbed.offset; + delete scrubbed.line; + delete scrubbed.filename; + delete scrubbed.column; + return scrubbed; + }); +} + /** * Run a bunch of cases * @param cases cases to run @@ -283,15 +303,7 @@ export function testCompilationCases(compiler: SectionCompilerNew, cases : Compi let messagesToCheck = callbacks.messages; // scrub offsets from messages to reduce churn in the test casws if (!testcase.retainOffsetInMessages && callbacks.messages) { - messagesToCheck = callbacks.messages.map(m => { - const scrubbed = Object.assign({}, m); - // Turn this on once all messages have offsets, see messages.tests.ts - // if (!scrubbed.offset) { - // throw Error(`Error, no offset detected in message ${CompilerError.formatEvent(m)}`); - // } - delete scrubbed.offset; - return scrubbed; - }); + messagesToCheck = scrubContextFromMessages(callbacks.messages); } const testcaseErrors = matchCompilerEventsOrBoolean(messagesToCheck, testcase.errors); const testcaseWarnings = matchCompilerEvents(messagesToCheck, testcase.warnings); diff --git a/developer/src/kmc-ldml/test/keys.tests.ts b/developer/src/kmc-ldml/test/keys.tests.ts index 75b3fabd3e..3c3c842810 100644 --- a/developer/src/kmc-ldml/test/keys.tests.ts +++ b/developer/src/kmc-ldml/test/keys.tests.ts @@ -312,7 +312,7 @@ describe('keys.kmap', function () { { subpath: 'sections/keys/invalid-bad-modifier.xml', errors: [ - LdmlCompilerMessages.Error_InvalidModifier({id:'base',modifiers:'altR-shift'}), + LdmlCompilerMessages.Error_InvalidModifier({ modifiers: 'altR-shift' }), ] }, { diff --git a/developer/src/kmc-ldml/test/layr.tests.ts b/developer/src/kmc-ldml/test/layr.tests.ts index 610efff212..dc0334c0b1 100644 --- a/developer/src/kmc-ldml/test/layr.tests.ts +++ b/developer/src/kmc-ldml/test/layr.tests.ts @@ -96,8 +96,7 @@ describe('layr', function () { subpath: 'sections/keys/invalid-bad-modifier.xml', errors: [ LdmlCompilerMessages.Error_InvalidModifier({ - id: 'base', - modifiers: 'altR-shift' + modifiers: 'altR-shift', }), ], }, @@ -139,7 +138,7 @@ describe('layr', function () { { subpath: 'sections/layr/error-bogus-modifiers.xml', errors: [ - LdmlCompilerMessages.Error_InvalidModifier({ id: '', modifiers: 'caps bogus'}), + LdmlCompilerMessages.Error_InvalidModifier({ modifiers: 'caps bogus' }), ] }, { diff --git a/developer/src/kmc-ldml/test/messages.tests.ts b/developer/src/kmc-ldml/test/messages.tests.ts index a29644936e..a7d06006cb 100644 --- a/developer/src/kmc-ldml/test/messages.tests.ts +++ b/developer/src/kmc-ldml/test/messages.tests.ts @@ -15,18 +15,20 @@ describe('LdmlCompilerMessages', function () { /** all fns */ let total = 0; /** does not take line numbers */ - let noLines = 0; + const noLines = new Set(); /** takes line numbers */ let lines = 0; const fakeOffsetNumber = 1234; const fakeOffsetObject = withOffset(fakeOffsetNumber); for(const key of keys) { - if(typeof m[key] == 'function') { + // exclude this one, does not need line numbers + if (key == 'Error_InvalidFile') continue; + if (typeof m[key] == 'function') { total++; const f = m[key] as Function; // console.log(`${f.name}: ${f.length}`); if (f.length === 0) { // Error_foo() - noLines++; + noLines.add(key); continue; } // now try to call it @@ -42,16 +44,30 @@ describe('LdmlCompilerMessages', function () { expect(resp.offset).to.equal(fakeOffsetNumber, `Offset number round trip for error ${f.name} did not work, check the message function`); } else { // did not get a column number back - noLines++; + noLines.add(key); } } } expect(lines).to.not.be.equal(0, `None of ${total} messages had offset reporting.`); - if (noLines > 0) { - // Once this goes to zero, make it an error if it goes up! - // Oh, and while you're here, once this is zero, uncomment the code in testCompilationCases - // that asserts that all messages are actually generated with an offset. - console.warn(`TODO-LDML (#10622) ${noLines}/${total} ${Number((noLines/total)*1000).toFixed(0)}‰ of message(s) did not have detectable offset (line number) reporting.`); + expect(Array.from(noLines.values())).to.deep.equal([], `${noLines.size}/${total} ${Number((noLines.size/total)*100).toFixed(0)}% of message(s) did not have detectable offset (line number) reporting`); + }); + it('Should only have simple ${def(o...)} interpolation', () => { + const m = LdmlCompilerMessages as Record; + const keys = Object.keys(LdmlCompilerMessages); + const complexInterpolation : string[] = []; + for(const key of keys) { + // exclude this one, does not need line numbers + if (key == 'Error_InvalidFile') continue; + if (typeof m[key] == 'function') { + const f = m[key] as Function; + const s = f.toString(); + // scan for interpolation + // Interpolation.. A non-escaped ${ that's NOT followed by `def(o.` triggers this error. + if (/.*`.*(? Wed, 04 Jun 2025 19:14:08 +0200 + keyman (18.0.236-1) unstable; urgency=medium * New upstream release. diff --git a/linux/docs/help/common/index.md b/linux/docs/help/common/index.md index a7f9e786bf..6b80489211 100644 --- a/linux/docs/help/common/index.md +++ b/linux/docs/help/common/index.md @@ -15,12 +15,14 @@ sudo apt-get upgrade sudo apt-get install keyman onboard-keyman ``` -An older version of Keyman for Linux packages is also available in any current Debian at: +A possibly older version of Keyman for Linux packages is also available in +any current Debian at: [https://packages.debian.org/keyman](https://packages.debian.org/keyman) -**Note** in order to fully work you'll need a patched version of ibus -which is available in the above launchpad repo. Patched versions of ibus -will have the string `sil` in the package version number. +**Note** Versions before Keyman 18 required a patched version of ibus in order +to fully work. Patched versions of ibus were available in the above mentioned +launchpad repository and had the string `sil` in the package version number. +Keyman 18+ no longer requires a patched version of ibus. ## Q. How do I install a Keyman keyboard? diff --git a/linux/keyman-config/run-tests.sh b/linux/keyman-config/run-tests.sh index 3fa49f081d..57b6b47aa7 100755 --- a/linux/keyman-config/run-tests.sh +++ b/linux/keyman-config/run-tests.sh @@ -1,5 +1,5 @@ #!/bin/bash -PYTHONPATH=.:${PYTHONPATH} +export PYTHONPATH=.:${PYTHONPATH} XDG_CONFIG_HOME=$(mktemp --directory) export XDG_CONFIG_HOME @@ -13,6 +13,8 @@ if [[ "$1" == "--coverage" ]]; then fi if [[ -n "${TEAMCITY_GIT_PATH}" ]]; then + PYTHONPATH=$(dirname "$0")/../tools:${PYTHONPATH} + if ! pip3 list --format=columns | grep -q teamcity-messages; then if [[ -n "${TEAMCITY_PLATFORM}" ]] || [[ -n "${DOCKER_RUNNING}" ]]; then # Ubuntu 24.04+ prevents mixing pip and system packages and wants us @@ -29,7 +31,9 @@ if [[ -n "${TEAMCITY_GIT_PATH}" ]]; then # shellcheck disable=SC2086 pip3 install --user ${PIP_ARGS:-} teamcity-messages fi - test_module=teamcity.unittestpy + test_module=teamcity_pytestrunner.unittestpy + echo "##teamcity[testStarted name='|[keyman-config|] Running unit tests']" + echo "##teamcity[flowStarted flowId='unit_tests']" else test_module=unittest extra_opts=-v @@ -38,4 +42,8 @@ fi # shellcheck disable=SC2086 python3 ${coverage:-} -m "${test_module:-}" discover ${extra_opts:-} -s tests/ -p "*_tests.py" +if [[ -n "${TEAMCITY_GIT_PATH}" ]]; then + echo "##teamcity[flowFinished flowId='unit_tests']" + echo "##teamcity[testFinished name='|[keyman-config|] Finished running unit tests']" +fi rm -rf "${XDG_CONFIG_HOME}" diff --git a/linux/tools/teamcity_pytestrunner/README.md b/linux/tools/teamcity_pytestrunner/README.md new file mode 100644 index 0000000000..00122b59ec --- /dev/null +++ b/linux/tools/teamcity_pytestrunner/README.md @@ -0,0 +1,12 @@ +# Teamcity Python Test Runner + +This implements an improved version of JetBrains' python unittest runner for +Teamcity. It makes the output of the tests indented and collapsible under +the test block by specifying a parent flow. + +Requires the `teamcity-messages` package. + +## References + +- +- diff --git a/linux/tools/teamcity_pytestrunner/unittestpy.py b/linux/tools/teamcity_pytestrunner/unittestpy.py new file mode 100644 index 0000000000..543277cce1 --- /dev/null +++ b/linux/tools/teamcity_pytestrunner/unittestpy.py @@ -0,0 +1,61 @@ +import sys +from teamcity.messages import TeamcityServiceMessages +from teamcity.unittestpy import TeamcityTestResult, TeamcityTestRunner +from unittest import main + + +class KeymanTeamcityServiceMessages(TeamcityServiceMessages): + def __init__(self, parent_flow='unit_tests'): + self.parent_flow = parent_flow + super().__init__() + + def testStarted(self, testName, captureStandardOutput=None, flowId=None, metainfo=None): + self.message('testStarted', name=testName, metainfo=metainfo) + self.message('flowStarted', flowId=flowId, parent=self.parent_flow) + + def testFinished(self, testName, testDuration=None, flowId=None): + self.message('flowFinished', flowId=flowId, parent=self.parent_flow) + if testDuration is not None: + duration_ms = testDuration.days * 86400000 + \ + testDuration.seconds * 1000 + \ + int(testDuration.microseconds / 1000) + self.message('testFinished', name=testName, duration=str(duration_ms)) + else: + self.message('testFinished', name=testName) + + def testIgnored(self, testName, message='', flowId=None): + self.message('flowFinished', flowId=flowId, parent=self.parent_flow) + self.message('testIgnored', name=testName, message=message) + + def testFailed(self, testName, message='', details='', flowId=None, comparison_failure=None): + self.message('flowFinished', flowId=flowId, parent=self.parent_flow) + if not comparison_failure: + self.message('testFailed', name=testName, message=message, details=details) + else: + diff_message = u"\n{0} != {1}\n".format(comparison_failure.actual, comparison_failure.expected) + self.message( + 'testFailed', + name=testName, + message=str(message) + diff_message, + details=details, + type="comparisonFailure", + actual=comparison_failure.actual, + expected=comparison_failure.expected, + ) + + +class KeymanTestResult(TeamcityTestResult): + def __init__(self, stream=sys.stdout, descriptions=None, verbosity=None): + super().__init__(stream, descriptions, verbosity) + self.messages = KeymanTeamcityServiceMessages() + + +class KeymanTestRunner(TeamcityTestRunner): + resultclass = KeymanTestResult + + def run(self, test): + return super().run(test) + + +if __name__ == '__main__': + main(module=None, testRunner=KeymanTestRunner()) diff --git a/mac/build.sh b/mac/build.sh index db61eed053..ea9822f909 100755 --- a/mac/build.sh +++ b/mac/build.sh @@ -69,16 +69,12 @@ PREPRELEASE=false UPDATE_VERSION_IN_PLIST=true DO_CODESIGN=true CODESIGNING_SUPPRESSION="CODE_SIGN_IDENTITY=\"\" CODE_SIGNING_REQUIRED=NO" - -QUIET=true +QUIET=false if builder_verbose; then BUILD_OPTIONS="" - QUIET_FLAG= - QUIET=false else BUILD_OPTIONS="-quiet" - QUIET_FLAG="-quiet" fi BUILD_ACTIONS="build" @@ -290,7 +286,7 @@ do_publish() { builder_heading "Preparing files for release deployment..." ./setup/build.sh - "$KM4MIM_BASE_PATH/make-km-dmg.sh" $QUIET_FLAG + "$KM4MIM_BASE_PATH/make-km-dmg.sh" # Create download info "$KM4MIM_BASE_PATH/write-download_info.sh" diff --git a/package-lock.json b/package-lock.json index 268c33bf87..5262663a45 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1153,7 +1153,7 @@ "@sentry/node": "^7.57.0", "chalk": "^4.1.2", "express": "^4.20.0", - "multer": "^2.0.0", + "multer": "^2.0.1", "ngrok": "^5.0.0-beta.2", "open": "^8.4.0", "restructure": "^3.0.1", @@ -5748,20 +5748,34 @@ "license": "MIT" }, "node_modules/concat-stream": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", - "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", + "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", "engines": [ - "node >= 0.8" + "node >= 6.0" ], "license": "MIT", "dependencies": { "buffer-from": "^1.0.0", "inherits": "^2.0.3", - "readable-stream": "^2.2.2", + "readable-stream": "^3.0.2", "typedarray": "^0.0.6" } }, + "node_modules/concat-stream/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/content-disposition": { "version": "0.5.4", "license": "MIT", @@ -10479,18 +10493,18 @@ "license": "MIT" }, "node_modules/multer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/multer/-/multer-2.0.0.tgz", - "integrity": "sha512-bS8rPZurbAuHGAnApbM9d4h1wSoYqrOqkE+6a64KLMK9yWU7gJXBDDVklKQ3TPi9DRb85cRs6yXaC0+cjxRtRg==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/multer/-/multer-2.0.1.tgz", + "integrity": "sha512-Ug8bXeTIUlxurg8xLTEskKShvcKDZALo1THEX5E41pYCD2sCVub5/kIRIGqWNoqV6szyLyQKV6mD4QUrWE5GCQ==", "license": "MIT", "dependencies": { "append-field": "^1.0.0", - "busboy": "^1.0.0", - "concat-stream": "^1.5.2", - "mkdirp": "^0.5.4", + "busboy": "^1.6.0", + "concat-stream": "^2.0.0", + "mkdirp": "^0.5.6", "object-assign": "^4.1.1", - "type-is": "^1.6.4", - "xtend": "^4.0.0" + "type-is": "^1.6.18", + "xtend": "^4.0.2" }, "engines": { "node": ">= 10.16.0" @@ -12738,9 +12752,9 @@ } }, "node_modules/tar-fs": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.8.tgz", - "integrity": "sha512-ZoROL70jptorGAlgAYiLoBLItEKw/fUxg9BSYK/dF/GAGYFJOJJJMvjPAKDJraCXFwadD456FCuvLWgfhMsPwg==", + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.9.tgz", + "integrity": "sha512-XF4w9Xp+ZQgifKakjZYmFdkLoSWd34VGKcsTCwlNWM7QG3ZbaxnTsaBwnjFZqHRf/rROxaR8rXnbtwdvaDI+lA==", "dev": true, "license": "MIT", "dependencies": { diff --git a/resources/build/run-required-test-builds.sh b/resources/build/run-required-test-builds.sh index e5ff54ba7b..c0b3ce8680 100755 --- a/resources/build/run-required-test-builds.sh +++ b/resources/build/run-required-test-builds.sh @@ -166,9 +166,10 @@ while IFS= read -r line; do # Which platform are we watching? eval watch='$'watch_$platform # Add common patterns to the watch list - watch="^(${platform}|(oem/[^/]+/${platform})|resources/((?!teamcity)|teamcity/(${platform}|includes))|$watch)" - # Since bash doesn't support negative look-aheads we use grep to test - if echo "${line}" | grep --quiet --perl-regexp "${watch}"; then + watch="^(${platform}|(oem/[^/]+/${platform})|resources/((?!teamcity)|teamcity/(${platform}|includes))|${watch})" + # Since bash doesn't support negative look-aheads we use perl to test + # (grep has a --perl-regexp option, but not the version on macOS) + if perl -e 'exit($ARGV[0] =~ /$ARGV[1]/ ? 0 : 1)' "${line}" "${watch}"; then build_platforms+=($platform) fi fi diff --git a/resources/build/zip.inc.sh b/resources/build/zip.inc.sh index 3dc6a390c2..1f80a71edb 100644 --- a/resources/build/zip.inc.sh +++ b/resources/build/zip.inc.sh @@ -35,6 +35,12 @@ function add_zip_files() { SEVENZ_FLAGS+=($1) shift ;; + -x@*) + # Filename for a file containing list of files to exclude from the archive - Identical flag to zip and 7z + ZIP_FLAGS+=($1) + SEVENZ_FLAGS+=($1) + shift + ;; # Zip flags that have a corresponding 7z flag -q) @@ -50,7 +56,7 @@ function add_zip_files() { # -1 indicates low compression (fastest) # -9 indicates ultra compression (slowest) ZIP_FLAGS+=($1) - if [[ $1 =~ -([0-9]) ]] then + if [[ $1 =~ -([0-9]) ]]; then SEVENZ_FLAGS+=("-mx${BASH_REMATCH[1]}") fi shift; diff --git a/resources/builder.inc.sh b/resources/builder.inc.sh index 24683543b8..25c9b5994c 100755 --- a/resources/builder.inc.sh +++ b/resources/builder.inc.sh @@ -181,28 +181,30 @@ function builder_heading() { builder_echo() { - local color=white message= mark= block= action= do_output=true + local color=white message= mark= block= action= do_output=true test= local echo_target=echo if [[ $# -gt 1 ]]; then if [[ $1 =~ ^(white|grey|green|success|blue|heading|yellow|warning|red|error|purple|brightwhite|teal|debug|setmark)$ ]]; then color="$1" shift - elif [[ $1 == "start" ]]; then + elif [[ $1 == "start" ]] || [[ $1 == "startTest" ]]; then # builder_echo start block message - action="$1" + test="$1" block="$2" shift 2 + action="start" color="heading" if ! builder_is_running_on_teamcity && builder_is_child_build; then do_output=${_builder_debug_internal:-false} fi - elif [[ $1 == "end" ]]; then + elif [[ $1 == "end" ]] || [[ $1 == "endTest" ]]; then # builder_echo end block status message - action="$1" + test="$1" block="$2" color="$3" shift 3 + action="end" if [[ "${color}" != "error" ]] && ! builder_is_running_on_teamcity && builder_is_child_build; then do_output=${_builder_debug_internal:-false} fi @@ -211,7 +213,11 @@ builder_echo() { message="$*" if [[ "${action}" == "start" ]] && builder_is_running_on_teamcity; then - $echo_target -e "##teamcity[blockOpened name='|[${THIS_SCRIPT_IDENTIFIER}|] ${block}']" + if [[ "${test}" == "startTest" ]]; then + $echo_target -e "##teamcity[testSuiteStarted name='|[${THIS_SCRIPT_IDENTIFIER}|] ${block}']" + else + $echo_target -e "##teamcity[blockOpened name='|[${THIS_SCRIPT_IDENTIFIER}|] ${block}']" + fi fi if ${do_output}; then @@ -241,7 +247,11 @@ builder_echo() { fi if [[ "${action}" == "end" ]] && builder_is_running_on_teamcity; then - $echo_target -e "##teamcity[blockClosed name='|[${THIS_SCRIPT_IDENTIFIER}|] ${block}']" + if [[ "${test}" == "endTest" ]]; then + $echo_target -e "##teamcity[testSuiteFinished name='|[${THIS_SCRIPT_IDENTIFIER}|] ${block}']" + else + $echo_target -e "##teamcity[blockClosed name='|[${THIS_SCRIPT_IDENTIFIER}|] ${block}']" + fi fi } diff --git a/resources/docker-images/README.md b/resources/docker-images/README.md index 6f82b84319..c203d09948 100644 --- a/resources/docker-images/README.md +++ b/resources/docker-images/README.md @@ -22,11 +22,11 @@ resources/docker-images/build.sh ``` By default this will create 64-bit images for building -Keyman Core, Keyman for Android, Keyman for Linux and -Keyman for Web. These images are based on the Ubuntu 24.04 -with Node 20 and Emscripten 3.1.58 (for the exact versions, -see [`minimum-versions.inc.sh`](../build/minimum-versions.inc.sh)) -and are named e.g. `keyman-core-ci:default`. +Keyman Core, Keyman for Android, Keyman for Linux, Keyman for Web +and (cross-platform parts of) Keyman Developer. These images are based +on Ubuntu 24.04 with Node 20 and Emscripten 3.1.58 (for the exact versions, +see [`minimum-versions.inc.sh`](../build/minimum-versions.inc.sh)). +The images are named e.g. `keymanapp/keyman-core-ci:default`. The versions can be changed, e.g. @@ -34,7 +34,8 @@ The versions can be changed, e.g. resources/docker-images/build.sh --distro-version jammy --node 20 ``` -This will create an image named e.g. `keyman-core-ci:jammy-node20`. +This will create an image named e.g. +`keymanapp/keyman-core-ci:ubuntu-jammy-java21-node20.16.0-emsdk3.1.58`. Once the image is built, it may be used to build parts of Keyman. @@ -44,7 +45,7 @@ It is possible to build locally with these images: ```shell # build 'Keyman Core' in docker -resources/docker-images/run.sh core -- core/build.sh --debug build +resources/docker-images/run.sh :core -- core/build.sh --debug build ``` Note: For Core and Linux we put the generated binaries in a @@ -61,5 +62,5 @@ To run the tests locally, use the `run.sh` script: ```shell # Run common/web tests -resources/docker-images/run.sh web -- common/web/build.sh --debug test +resources/docker-images/run.sh :web -- common/web/build.sh --debug test ``` diff --git a/resources/docker-images/android/Dockerfile b/resources/docker-images/android/Dockerfile index 2b289ce077..489284b08c 100644 --- a/resources/docker-images/android/Dockerfile +++ b/resources/docker-images/android/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1.4 # Keyman is copyright (C) SIL Global. MIT License. ARG BASE_VERSION=default diff --git a/resources/docker-images/base/Dockerfile b/resources/docker-images/base/Dockerfile index 3c254edfad..de5a82fcf0 100644 --- a/resources/docker-images/base/Dockerfile +++ b/resources/docker-images/base/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1.4 # Keyman is copyright (C) SIL Global. MIT License. ARG DISTRO=ubuntu diff --git a/resources/docker-images/build.sh b/resources/docker-images/build.sh index ffe891525a..b650b9ce55 100755 --- a/resources/docker-images/build.sh +++ b/resources/docker-images/build.sh @@ -17,6 +17,7 @@ builder_describe \ ":android" \ ":base" \ ":core" \ + ":developer" \ ":linux" \ ":web" \ "--distro=DISTRO The distribution to use for the base image (debian or ubuntu, default: ubuntu)" \ @@ -45,6 +46,7 @@ build_action() { # shellcheck disable=SC2164 cd "${platform}" + export DOCKER_BUILDKIT=1 # shellcheck disable=SC2248,SC2086 docker build ${OPTION_NO_CACHE:-} -t "keymanapp/keyman-${platform}-ci:${build_version}" "${build_args[@]}" . # If the user didn't specify particular versions we will additionaly create an image @@ -63,18 +65,20 @@ test_action() { builder_echo debug "Testing image for ${platform}" ./run.sh --distro "${DISTRO}" --distro-version "${DISTRO_VERSION}" \ - "${platform}" -- ./build.sh configure,build,test:"${platform}" + ":${platform}" -- ./build.sh configure,build,test:"${platform}" } +check_buildx_available check_for_default_values convert_parameters_to_args if builder_has_action build; then build_action base - builder_run_action build:android build_action android - builder_run_action build:core build_action core - builder_run_action build:linux build_action linux - builder_run_action build:web build_action web + builder_run_action build:android build_action android + builder_run_action build:core build_action core + builder_run_action build:linux build_action linux + builder_run_action build:web build_action web + builder_run_action build:developer build_action developer fi builder_run_action test:core test_action core @@ -82,3 +86,4 @@ builder_run_action test:linux test_action linux builder_run_action test:web test_action web # Android uses artifacts from web, so it has to come after web builder_run_action test:android test_action android +builder_run_action test:developer test_action developer diff --git a/resources/docker-images/core/Dockerfile b/resources/docker-images/core/Dockerfile index a29cac8e04..8b3e2b4c7a 100644 --- a/resources/docker-images/core/Dockerfile +++ b/resources/docker-images/core/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1.4 # Keyman is copyright (C) SIL Global. MIT License. # # ARGS used in this file: diff --git a/resources/docker-images/developer/Dockerfile b/resources/docker-images/developer/Dockerfile new file mode 100644 index 0000000000..aaec7b55cf --- /dev/null +++ b/resources/docker-images/developer/Dockerfile @@ -0,0 +1,55 @@ +# syntax=docker/dockerfile:1.4 +# Keyman is copyright (C) SIL Global. MIT License. +# +# ARGS used in this file: +# - ARG BASE_VERSION=default +# - ARG REQUIRED_EMSCRIPTEN_VERSION=unset + +ARG BASE_VERSION=default +FROM keymanapp/keyman-base-ci:${BASE_VERSION} + +LABEL org.opencontainers.image.authors="SIL Global." +LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git" +LABEL org.opencontainers.image.title="Keyman Developer Build Image" + +USER root +RUN apt-get update && \ + apt-get install -qy devscripts jq meson + +# Pre-install emscripten +USER build +ARG REQUIRED_EMSCRIPTEN_VERSION=unset +RUN echo "Installing emscripten version ${REQUIRED_EMSCRIPTEN_VERSION}" && \ + export EMSDK_KEEP_DOWNLOADS=1 && \ + cd /home/build/ && \ + git clone https://github.com/emscripten-core/emsdk.git && \ + cd emsdk && \ + ./emsdk install ${REQUIRED_EMSCRIPTEN_VERSION} && \ + ./emsdk activate ${REQUIRED_EMSCRIPTEN_VERSION} +USER root +RUN echo "export EMSCRIPTEN_BASE=/home/build/emsdk/upstream/emscripten" >> /usr/bin/bashwrapper && \ + echo "export KEYMAN_USE_EMSDK=1" >> /usr/bin/bashwrapper + +# Finish bashwrapper script and adjust permissions +RUN <> /usr/bin/bashwrapper + +if [[ "\$@" =~ test ]] && [ -f /usr/bin/run-tests.sh ]; then + /usr/bin/run-tests.sh "\${@:-bash}" +else + "\${@:-bash}" +fi +EOF + +# now, switch to build user +USER build + +# Pre-install node +RUN export NVM_DIR="/home/build/.nvm" && \ + . /home/build/.nvm/nvm.sh && \ + cd /home/build/emsdk/upstream/emscripten && \ + npm install + +VOLUME /home/build/build +WORKDIR /home/build/build + +ENTRYPOINT [ "/usr/bin/bashwrapper" ] diff --git a/resources/docker-images/docker-build.inc.sh b/resources/docker-images/docker-build.inc.sh index 4bd9275d75..855ba6e641 100644 --- a/resources/docker-images/docker-build.inc.sh +++ b/resources/docker-images/docker-build.inc.sh @@ -59,3 +59,15 @@ check_for_default_values() { is_default_values() { ${is_default_values} } + +# Check that `docker buildx` is available +check_buildx_available() { + if ! docker --help | grep -q buildx; then + if [[ "${OSTYPE:-}" == "linux-gnu" ]]; then + builder_echo error "Docker buildx is not available. Please install Docker buildx to use this script. e.g. 'sudo apt install docker-buildx'" + else + builder_echo error "Docker buildx is not available. Please install Docker buildx to use this script." + fi + exit 1 + fi +} diff --git a/resources/docker-images/linux/Dockerfile b/resources/docker-images/linux/Dockerfile index 061e9f619b..da6bf59906 100644 --- a/resources/docker-images/linux/Dockerfile +++ b/resources/docker-images/linux/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1.4 # Keyman is copyright (C) SIL Global. MIT License. ARG BASE_VERSION=default diff --git a/resources/docker-images/run.sh b/resources/docker-images/run.sh index 05124768e9..2c8cd3e51d 100755 --- a/resources/docker-images/run.sh +++ b/resources/docker-images/run.sh @@ -14,10 +14,12 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" builder_describe \ "Run build.sh script inside of a docker image. Pass the build script and parameters after --." \ - "android" \ - "core" \ - "linux" \ - "web" \ + ":android" \ + ":core" \ + ":developer" \ + ":linux" \ + ":web" \ + "run+ Run command in docker image" \ "--distro=DISTRO The distribution (debian or ubuntu, default: ubuntu)" \ "--distro-version=DISTRO_VERSION The Ubuntu/Debian version (default: ${KEYMAN_DEFAULT_VERSION_UBUNTU_CONTAINER})" @@ -48,6 +50,13 @@ run_core() { "${builder_extra_params[@]}" } +run_developer() { + docker run -it --rm -v "${KEYMAN_ROOT}":/home/build/build \ + -v "${KEYMAN_ROOT}/core/build/docker-core/${build_dir}":/home/build/build/core/build \ + "keymanapp/keyman-developer-ci:${image_version}" \ + "${builder_extra_params[@]}" +} + run_linux() { mkdir -p "${KEYMAN_ROOT}/linux/build/docker-linux/${build_dir}" mkdir -p "${KEYMAN_ROOT}/linux/keyman-system-service/build/docker-linux/${build_dir}" @@ -69,7 +78,8 @@ run_web() { mkdir -p "${KEYMAN_ROOT}/core/build/docker-core/${build_dir}" -builder_run_action android run_android -builder_run_action core run_core -builder_run_action linux run_linux -builder_run_action web run_web +builder_run_action run:android run_android +builder_run_action run:core run_core +builder_run_action run:developer run_developer +builder_run_action run:linux run_linux +builder_run_action run:web run_web diff --git a/resources/docker-images/web/Dockerfile b/resources/docker-images/web/Dockerfile index 1f08daf92c..655a2d8592 100644 --- a/resources/docker-images/web/Dockerfile +++ b/resources/docker-images/web/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1.4 # Keyman is copyright (C) SIL Global. MIT License. # # ARGS used in this file: diff --git a/resources/teamcity/README.md b/resources/teamcity/README.md index 8a53868ca6..c6db3af4d9 100644 --- a/resources/teamcity/README.md +++ b/resources/teamcity/README.md @@ -20,19 +20,15 @@ docker run -v $(pwd):/Develop -it ubuntu:24.04 /bin/bash Inside of the container, run ```bash -apt update && apt install sudo +apt update && apt install -y sudo export DOCKER_RUNNING=1 +echo "ubuntu ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers +su ubuntu ``` and then run the build, e.g. ```bash +cd /Develop resources/teamcity/linux/keyman-linux-test.sh configure,build,test ``` - -NOTE: by default this will run the build as `root` in the container, -so you will end up with files owned by `root` in your tree. -Either create a user with the same user id as your local user in the -container, or use one of the images created by `resources/docker-images` -(though you won't be able to test if all the dependencies get installed -if using the docker-images containers). diff --git a/resources/teamcity/developer/developer-actions.inc.sh b/resources/teamcity/developer/developer-actions.inc.sh new file mode 100644 index 0000000000..45d3792dbe --- /dev/null +++ b/resources/teamcity/developer/developer-actions.inc.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Keyman is copyright (C) SIL Global. MIT License. + +developer_install_dependencies_on_linux_action() { + if ! is_ubuntu; then + return 0 + fi + + builder_echo start "install dependencies" "Installing dependencies" + + linux_check_and_install_packages devscripts jq meson + install_nvm + install_emscripten + + builder_echo end "install dependencies" success "Finished installing dependencies" +} diff --git a/resources/teamcity/developer/download-symbol-server-index.ps1 b/resources/teamcity/developer/download-symbol-server-index.ps1 new file mode 100644 index 0000000000..99c2c5a7fd --- /dev/null +++ b/resources/teamcity/developer/download-symbol-server-index.ps1 @@ -0,0 +1,60 @@ +# Keyman is copyright (C) SIL Global. MIT License. +# +# This script should be the identical for nightly/beta/stable for a given platform +# + +$ErrorActionPreference = "Stop" + +$RSYNC_PATH = $env:RSYNC_PATH +$RSYNC_HOME = $env:RSYNC_HOME +$USERPROFILE = $env:USERPROFILE + +New-Item -Force -ItemType Directory symbols\000admin +cd symbols\000admin + +# +# Download the symbol server index files with rsync from downloads.keyman.com +# (rsync requires that we are in the symbols folder to get folders in +# sync correctly; it is possible to resolve this but easier to just cd.) +# + +$rsync_args = @( + '-vrzltp', # verbose, recurse, zip, copy symlinks, preserve times, permissions + '--chmod=Dug=rwx,Do=rx,Fug=rw,Fo=r', # map Windows security to host security + '--stats', # show statistics for log + '--rsync-path="$RSYNC_PATH"', # path on remote server + "--rsh=$RSYNC_HOME\ssh -i $USERPROFILE\.ssh\id_rsa -o UserKnownHostsFile=$USERPROFILE\.ssh\known_hosts", # use ssh + "root@sysops.downloads.keyman.com:/var/www/virtual/downloads.keyman.com/htdocs/windows/symbols/000admin/lastid.txt", # target server + path + "." # download the whole symbols 000Admin folder +) + +& $RSYNC_HOME\rsync.exe $rsync_args +if ($LASTEXITCODE -ne 0) { throw "Exit code is $LASTEXITCODE" } + +$rsync_args = @( + '-vrzltp', # verbose, recurse, zip, copy symlinks, preserve times, permissions + '--chmod=Dug=rwx,Do=rx,Fug=rw,Fo=r', # map Windows security to host security + '--stats', # show statistics for log + '--rsync-path="$RSYNC_PATH"', # path on remote server + "--rsh=$RSYNC_HOME\ssh -i $USERPROFILE\.ssh\id_rsa -o UserKnownHostsFile=$USERPROFILE\.ssh\known_hosts", # use ssh + "root@sysops.downloads.keyman.com:/var/www/virtual/downloads.keyman.com/htdocs/windows/symbols/000admin/history.txt", # target server + path + "." # download the whole symbols 000Admin folder +) + +& $RSYNC_HOME\rsync.exe $rsync_args +if ($LASTEXITCODE -ne 0) { throw "Exit code is $LASTEXITCODE" } + +$rsync_args = @( + '-vrzltp', # verbose, recurse, zip, copy symlinks, preserve times, permissions + '--chmod=Dug=rwx,Do=rx,Fug=rw,Fo=r', # map Windows security to host security + '--stats', # show statistics for log + '--rsync-path="$RSYNC_PATH"', # path on remote server + "--rsh=$RSYNC_HOME\ssh -i $USERPROFILE\.ssh\id_rsa -o UserKnownHostsFile=$USERPROFILE\.ssh\known_hosts", # use ssh + "root@sysops.downloads.keyman.com:/var/www/virtual/downloads.keyman.com/htdocs/windows/symbols/000admin/server.txt", # target server + path + "." # download the whole symbols 000Admin folder +) + +& $RSYNC_HOME\rsync.exe $rsync_args +if ($LASTEXITCODE -ne 0) { throw "Exit code is $LASTEXITCODE" } + +# EOF diff --git a/resources/teamcity/developer/keyman-developer-release.sh b/resources/teamcity/developer/keyman-developer-release.sh new file mode 100755 index 0000000000..5aa8608a23 --- /dev/null +++ b/resources/teamcity/developer/keyman-developer-release.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# Keyman is copyright (C) SIL Global. MIT License. +# +# TC build script to build release of Keyman Developer + +# shellcheck disable=SC2164 +# shellcheck disable=SC1091 + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" +. "${THIS_SCRIPT%/*}/../../../resources/build/builder.inc.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +# shellcheck disable=SC2154 +. "${KEYMAN_ROOT}/resources/teamcity/includes/tc-helpers.inc.sh" + +################################ Main script ################################ + +builder_describe \ + "Build Keyman Developer on Windows" \ + "all run all actions" \ + "build build Keyman Developer and test keyboards" \ + "publish publish release of Keyman Developer" \ + "--rsync-path=RSYNC_PATH rsync path on remote server" \ + "--rsync-user=RSYNC_USER rsync user on remote server" \ + "--rsync-host=RSYNC_HOST rsync host on remote server" \ + "--rsync-root=RSYNC_ROOT rsync root on remote server" \ + "--help.keyman.com=HELP_KEYMAN_COM path to help.keyman.com repository" + +builder_parse "$@" + +# shellcheck disable=SC2154 +cd "${KEYMAN_ROOT}/developer/src" + +function _build_developer() { + builder_echo start "build developer" "Building Keyman Developer" + + ./build.sh configure build test api publish --npm-publish + + builder_echo end "build developer" success "Finished building Keyman Developer" +} + +function _build_testkeyboards() { + builder_echo start "build testkeyboards" "Building test keyboards" + + "${KEYMAN_ROOT}/common/test/keyboards/build.sh" + + builder_echo end "build testkeyboards" success "Finished building test keyboards" +} + +function _publish_sentry() { + builder_echo start "publish sentry" "Publishing debug information files to Sentry" + + "${KEYMAN_ROOT}/developer/src/tools/sentry-upload-difs.sh" + + builder_echo end "publish sentry" success "Finished publishing debug information files to Sentry" +} + +function _download_symbol_server_index() { + # Download symbol server index from symbol server + builder_echo start "download symbol server index" "Downloading symbol server index" + + cd "${KEYMAN_ROOT}/.." + # shellcheck disable=SC2154 + powershell -NonInteractive -ExecutionPolicy Bypass -File "${THIS_SCRIPT_PATH}/download-symbol-server-index.ps1" + cd "${KEYMAN_ROOT}/developer/src" + + builder_echo end "download symbol server index" success "Finished downloading symbol server index" +} + +function _publish_new_symbols() { + # Publish new symbols to symbol server + builder_echo start "publish new symbols" "Publishing new symbols to symbol server" + + cd "${KEYMAN_ROOT}/../symbols" + # shellcheck disable=SC2154 + powershell -NonInteractive -ExecutionPolicy Bypass -File "${THIS_SCRIPT_PATH}/publish-new-symbols.ps1" + cd "${KEYMAN_ROOT}/developer/src" + + builder_echo end "publish new symbols" success "Finished publishing new symbols to symbol server" +} + +function _publish_to_downloads_keyman_com() { + # Publish to downloads.keyman.com + builder_echo start "publish to downloads.keyman.com" "Publishing release to downloads.keyman.com" + + cd "${KEYMAN_ROOT}/developer" + # shellcheck disable=SC2154 + powershell -NonInteractive -ExecutionPolicy Bypass -File "${THIS_SCRIPT_PATH}/publish-to-downloads-keyman-com.ps1" + cd "${KEYMAN_ROOT}/developer/src" + + builder_echo end "publish to downloads.keyman.com" success "Finished publishing release to downloads.keyman.com" +} + +function _publish_api_documentation() { + # Upload new Keyman Developer API documentation to help.keyman.com + builder_echo start "publish api documentation" "Uploading new Keyman Developer API documentation to help.keyman.com" + + export HELP_KEYMAN_COM="${HELP_KEYMAN_COM:-${KEYMAN_ROOT}/../help.keyman.com}" + "${KEYMAN_ROOT}/resources/build/help-keyman-com.sh" developer + + builder_echo end "publish api documentation" success "Finished uploading new Keyman Developer API documentation to help.keyman.com" +} + +function build_developer_action() { + _build_developer + _build_testkeyboards +} + +function publish_action() { + if ! is_windows; then + # requires Powershell, so currently only supported on Windows + builder_echo error "This script is intended to be run on Windows only." + return 1 + fi + + export RSYNC_PATH + export RSYNC_USER + export RSYNC_HOST + export RSYNC_ROOT + + _publish_sentry + _download_symbol_server_index + _publish_new_symbols + _publish_to_downloads_keyman_com + _publish_api_documentation +} + +if builder_has_action all; then + build_developer_action + publish_action +else + builder_run_action build build_developer_action + builder_run_action publish publish_action +fi diff --git a/resources/teamcity/developer/keyman-developer-test-linux-macos.sh b/resources/teamcity/developer/keyman-developer-test-linux-macos.sh new file mode 100755 index 0000000000..c273873fe0 --- /dev/null +++ b/resources/teamcity/developer/keyman-developer-test-linux-macos.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Keyman is copyright (C) SIL Global. MIT License. +# +# TC build script for Keyman Developer on Linux and macOS + +# shellcheck disable=SC2164 +# shellcheck disable=SC1091 + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" +. "${THIS_SCRIPT%/*}/../../../resources/build/builder.inc.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +# shellcheck disable=SC2154 +. "${KEYMAN_ROOT}/resources/teamcity/developer/developer-actions.inc.sh" +. "${KEYMAN_ROOT}/resources/teamcity/includes/tc-actions.inc.sh" +. "${KEYMAN_ROOT}/resources/teamcity/includes/tc-helpers.inc.sh" +. "${KEYMAN_ROOT}/resources/teamcity/includes/tc-linux.inc.sh" + +################################ Main script ################################ + +builder_describe \ + "Build Keyman Developer on Linux and macOS" \ + "all run all actions" \ + "configure install dependencies" \ + "build build" + +builder_parse "$@" + +cd "${KEYMAN_ROOT}/developer/src" + +function build_developer_action() { + builder_echo start "build developer" "Building Keyman Developer" + + "${KEYMAN_ROOT}/developer/src/build.sh" configure build test + + builder_echo end "build developer" success "Finished building Keyman Developer" +} + +if is_windows; then + builder_echo error "This script is intended to be run on Linux or macOS only." + exit 1 +fi + +if builder_has_action all; then + developer_install_dependencies_on_linux_action + + set_variables_for_nvm + set_variables_for_emscripten + + build_developer_action +else + builder_run_action configure developer_install_dependencies_on_linux_action + + set_variables_for_nvm + set_variables_for_emscripten + + builder_run_action build build_developer_action +fi diff --git a/resources/teamcity/developer/keyman-developer-test-windows.sh b/resources/teamcity/developer/keyman-developer-test-windows.sh new file mode 100755 index 0000000000..75bb60b4ac --- /dev/null +++ b/resources/teamcity/developer/keyman-developer-test-windows.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Keyman is copyright (C) SIL Global. MIT License. +# +# TC build script for Keyman Developer on Windows + +# shellcheck disable=SC2164 +# shellcheck disable=SC1091 + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" +. "${THIS_SCRIPT%/*}/../../../resources/build/builder.inc.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +# shellcheck disable=SC2154 +. "${KEYMAN_ROOT}/resources/teamcity/includes/tc-helpers.inc.sh" + +################################ Main script ################################ + +builder_describe \ + "Build Keyman Developer on Windows" \ + "all run all actions" \ + "build build Keyman Developer and test keyboards" \ + "publish publish debug information files to sentry" + +builder_parse "$@" + +# shellcheck disable=SC2154 +cd "${KEYMAN_ROOT}/developer/src" + +function build_developer_action() { + _build_developer + _build_testkeyboards +} + +function _build_developer() { + builder_echo start "build developer" "Building Keyman Developer" + + ./build.sh configure build test api publish --dry-run + + builder_echo end "build developer" success "Finished building Keyman Developer" +} + +function _build_testkeyboards() { + builder_echo start "build testkeyboards" "Building test keyboards" + + "${KEYMAN_ROOT}/common/test/keyboards/build.sh" --zip-source --index + + builder_echo end "build testkeyboards" success "Finished building test keyboards" +} + +function publish_sentry_action() { + builder_echo start "publish sentry" "Publishing debug information files to Sentry" + + "${KEYMAN_ROOT}/developer/src/tools/sentry-upload-difs.sh" + + builder_echo end "publish sentry" success "Finished publishing debug information files to Sentry" +} + +if ! is_windows; then + builder_echo error "This script is intended to be run on Windows only." + exit 1 +fi + +if builder_has_action all; then + build_developer_action + publish_sentry_action +else + builder_run_action build build_developer_action + builder_run_action publish publish_sentry_action +fi diff --git a/resources/teamcity/developer/publish-new-symbols.ps1 b/resources/teamcity/developer/publish-new-symbols.ps1 new file mode 100644 index 0000000000..bf7d090b93 --- /dev/null +++ b/resources/teamcity/developer/publish-new-symbols.ps1 @@ -0,0 +1,39 @@ +# Keyman is copyright (C) SIL Global. MIT License. +# +# This script should be the identical for nightly/beta/stable for a given platform +# + +$ErrorActionPreference = "Stop" + +$RSYNC_HOME = $env:RSYNC_HOME +$RSYNC_PATH = $env:RSYNC_PATH +$RSYNC_USER = $env:RSYNC_USER +$RSYNC_HOST = $env:RSYNC_HOST +$RSYNC_ROOT = $env:RSYNC_ROOT +$USERPROFILE = $env:USERPROFILE + +# Rename 000Admin to 000admin + +ren 000Admin 000admin_ +ren 000admin_ 000admin + +# +# Upload with rsync to downloads.keyman.com +# (rsync requires that we are in the symbols folder to get folders in +# sync correctly; it is possible to resolve this but easier to just cd.) +# + +$rsync_args = @( + '-vrzltp', # verbose, recurse, zip, copy symlinks, preserve times, permissions + '--chmod=Dug=rwx,Do=rx,Fug=rw,Fo=r', # map Windows security to host security + '--stats', # show statistics for log + '--rsync-path="$RSYNC_PATH"', # path on remote server + "--rsh=$RSYNC_HOME\ssh -i $USERPROFILE\.ssh\id_rsa -o UserKnownHostsFile=$USERPROFILE\.ssh\known_hosts", # use ssh + ".", # upload the whole symbols folder + "$RSYNC_USER@$RSYNC_HOST:$RSYNC_ROOT/windows/symbols/" # target server + path +) + +& $RSYNC_HOME\rsync.exe $rsync_args +if ($LASTEXITCODE -ne 0) { throw "Exit code is $LASTEXITCODE" } + +# EOF diff --git a/resources/teamcity/developer/publish-to-downloads-keyman-com.ps1 b/resources/teamcity/developer/publish-to-downloads-keyman-com.ps1 new file mode 100644 index 0000000000..1899e694cb --- /dev/null +++ b/resources/teamcity/developer/publish-to-downloads-keyman-com.ps1 @@ -0,0 +1,223 @@ +# Keyman is copyright (C) SIL Global. MIT License. + +$ErrorActionPreference = "Stop" + +$tier = Get-Content ..\TIER.md +$build_number = Get-Content ..\VERSION.md +$build_counter = $build_number -replace "^\d+\.\d+\.(\d+)$", '$1' +$msi_major_version = $build_number -replace "^(\d+)\.(\d+)\.(\d+)$", '$1$2' + +if ( $tier -eq "alpha" -or $tier -eq "beta") { + $version_tag = $tier + $version_with_tag = $build_number # we may go with $build_number + "-" + $version_tag in the future one day +} else { + $version_tag = "" + $version_with_tag = $build_number +} + +$upload_path = "upload\$build_number" +$keyboards_path = "$upload_path\keyboards" + +# Keyman Developer installers +$kmcomp_zip = "kmcomp-$version_with_tag.zip" +$keyman_developer_exe = "keymandeveloper-$version_with_tag.exe" + +$setup_exe = "setup.exe" + +# Debug files +$debug_zip = "debug-$build_number.zip" + +$7Z_HOME = $env:7Z_HOME +$RSYNC_HOME = $env:RSYNC_HOME +$RSYNC_PATH = $env:RSYNC_PATH +$RSYNC_USER = $env:RSYNC_USER +$RSYNC_HOST = $env:RSYNC_HOST +$RSYNC_ROOT = $env:RSYNC_ROOT +$USERPROFILE = $env:USERPROFILE + +########################################################################################### +## Developer upload +########################################################################################### + +# +# Preparation +# + +if((Test-Path $upload_path) -ne 0) { + Remove-Item -Path $upload_path -Recurse +} + +mkdir $upload_path + +# +# Build Keyman Compiler WINE archive +# + +if((Test-Path ..\release\$kmcomp_zip) -ne 0) { + copy ..\release\$kmcomp_zip ..\$upload_path\$kmcomp_zip +} else { + cd bin + if((Test-Path ..\..\common\schemas\keyboard_info\keyboard_info.source.json) -ne 0) { + # Keyman versions through -16.0 + copy ..\..\common\schemas\keyboard_info\keyboard_info.source.json . + copy ..\..\common\schemas\keyboard_info\keyboard_info.distribution.json . + & "$7Z_HOME\7z.exe" a -bd -bb0 ..\$upload_path\$kmcomp_zip kmcomp.exe kmcmpdll.dll kmcomp.x64.exe kmcmpdll.x64.dll kmconvert.exe keyboard_info.source.json keyboard_info.distribution.json xml\layoutbuilder\*.keyman-touch-layout projects\ + + # Add Keyman Developer Server to the archive (15.0 late alpha - after 171) + if((Test-Path ..\src\server\build) -ne 0) { + copy ..\src\server\build\ server\ -Recurse + & "$7Z_HOME\7z.exe" a -bd -bb0 ..\$upload_path\$kmcomp_zip server\ + } + } else { + # Keyman versions 17.0+; note, use npm install for most modules + copy ..\..\common\schemas\keyboard_info\keyboard_info.schema.json . + & "$7Z_HOME\7z.exe" a -bd -bb0 ..\$upload_path\$kmcomp_zip kmconvert.exe keyboard_info.schema.json xml\layoutbuilder\*.keyman-touch-layout projects\ server\ + } + cd .. +} + +# +# Copy source files +# + +copy release\$build_number\$keyman_developer_exe $upload_path\$keyman_developer_exe + +# +# Construct keyman_developer.download_info +# + +$hash = get-filehash $upload_path\$keyman_developer_exe -Algorithm MD5 + +$keyman_developer_download_info = @" +{ + "name": "Keyman Developer", + "version": "$version_with_tag", + "date": "$([DateTime]::Now.ToString("yyyy-MM-dd"))", + "platform": "win", + "stability": "$tier", + "file": "keymandeveloper-$version_with_tag.exe", + "md5": "$($hash.Hash)", + "type": "exe", + "build": "$build_counter" +} +"@ + +[System.IO.File]::WriteAllLines("$pwd\$upload_path\$keyman_developer_exe.download_info", $keyman_developer_download_info) + +# +# Construct kmcomp.download_info +# + +$hash = get-filehash $upload_path\$kmcomp_zip -Algorithm MD5 + +$kmcomp_download_info = @" +{ + "name": "Keyman Developer Command-Line Compiler", + "version": "$version_with_tag", + "date": "$([DateTime]::Now.ToString("yyyy-MM-dd"))", + "platform": "win", + "stability": "$tier", + "file": "kmcomp-$version_with_tag.zip", + "md5": "$($hash.Hash)", + "type": "zip", + "build": "$build_counter" +} +"@ + +[System.IO.File]::WriteAllLines("$pwd\$upload_path\$kmcomp_zip.download_info", $kmcomp_download_info) + +# +# Copy common/test/keyboards/*/build/*.kmp to keyboards/ +# +if((Test-Path ..\common\test\keyboards) -ne 0) { + mkdir $keyboards_path + Copy-Item ..\common\test\keyboards\*\build\*.kmp $keyboards_path\ +} + +# +# Upload with rsync to downloads.keyman.com +# (rsync requires that we are in the upload folder to get folders in +# sync correctly; it is possible to resolve this but easier to just cd.) +# + +$rsync_args = @( + '-vrzltp', # verbose, recurse, zip, copy symlinks, preserve times, permissions + '--chmod=Dug=rwx,Do=rx,Fug=rw,Fo=r', # map Windows security to host security + '--stats', # show statistics for log + '--rsync-path="$RSYNC_PATH"', # path on remote server + "--rsh=$RSYNC_HOME\ssh -i $USERPROFILE\.ssh\id_rsa -o UserKnownHostsFile=$USERPROFILE\.ssh\known_hosts", # use ssh + "$build_number", # upload the whole build folder + "$RSYNC_USER@$RSYNC_HOST:$RSYNC_ROOT/developer/$tier/" # target server + path +) + +# Write-Output "rsync parameters:" $rsync_args + +cd upload +& $RSYNC_HOME\rsync.exe $rsync_args +if( $LASTEXITCODE -ne 0 ) { + Write-Output "rsync failed." + exit 99 +} +cd .. + +################# +# DEBUG.ZIP +################# + +if((Test-Path release\$build_number\$debug_zip) -ne 0) { + # + # Copy source files + # + + copy release\$build_number\$debug_zip $upload_path\$debug_zip + + # + # Construct debug-.download_info + # + + $hash = get-filehash $upload_path\$debug_zip -Algorithm MD5 + + $debug_zip_download_info = @" +{ + "name": "Keyman Developer debug files", + "version": "$version_with_tag", + "date": "$([DateTime]::Now.ToString("yyyy-MM-dd"))", + "platform": "win", + "stability": "$tier", + "file": "debug-$build_number.zip", + "md5": "$($hash.Hash)", + "type": "zip", + "build": "$build_counter" +} +"@ + + [System.IO.File]::WriteAllLines("$pwd\$upload_path\$debug_zip.download_info", $debug_zip_download_info) +} + +# +# Upload with rsync to downloads.keyman.com +# (rsync requires that we are in the upload folder to get folders in +# sync correctly; it is possible to resolve this but easier to just cd.) +# + +$rsync_args = @( + '-vrzltp', # verbose, recurse, zip, copy symlinks, preserve times, permissions + '--chmod=Dug=rwx,Do=rx,Fug=rw,Fo=r', # map Windows security to host security + '--stats', # show statistics for log + '--rsync-path="$RSYNC_PATH"', # path on remote server + "--rsh=$RSYNC_HOME\ssh -i $USERPROFILE\.ssh\id_rsa -o UserKnownHostsFile=$USERPROFILE\.ssh\known_hosts", # use ssh + "$build_number", # upload the whole build folder + "$RSYNC_USER@$RSYNC_HOST:$RSYNC_ROOT/developer/$tier/" # target server + path +) + +# Write-Output "rsync parameters:" $rsync_args + +cd upload +& $RSYNC_HOME\rsync.exe $rsync_args +if( $LASTEXITCODE -ne 0 ) { + Write-Output "rsync failed." + exit 99 +} +cd .. + +# EOF diff --git a/resources/teamcity/includes/tc-actions.inc.sh b/resources/teamcity/includes/tc-actions.inc.sh index 4b817a6f21..8f671e14b2 100644 --- a/resources/teamcity/includes/tc-actions.inc.sh +++ b/resources/teamcity/includes/tc-actions.inc.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Keyman is copyright (C) SIL Global. MIT License. # Run the clean action in the `linux` directory.` linux_clean_action() { @@ -25,7 +26,7 @@ linux_additional_test_dependencies_action() { fi # shellcheck disable=SC2086 - check_and_install_packages ${TOINSTALL} + linux_check_and_install_packages ${TOINSTALL} if ! is_os_version_or_higher 24.04; then builder_heading "Installing python3-coverage from pip" @@ -47,7 +48,8 @@ linux_build_action() { # Run unit tests for Keyman for Linux. linux_unit_tests_action() { - builder_echo start unit_tests "Running unit tests" + builder_echo startTest unit_tests "Running unit tests" + rm -f /tmp/ibus-engine-keyman.log rm -f /tmp/ibus-daemon.log # symlink might point to wrong location, so delete it - will be re-created during tests @@ -56,6 +58,5 @@ linux_unit_tests_action() { export NO_AT_BRIDGE=1 # shellcheck disable=SC2068 "${KEYMAN_ROOT}/linux/build.sh" test $@ - builder_echo end unit_tests success "Finished running unit tests" + builder_echo endTest unit_tests success "Finished running unit tests" } - diff --git a/resources/teamcity/includes/tc-helpers.inc.sh b/resources/teamcity/includes/tc-helpers.inc.sh new file mode 100644 index 0000000000..5170f3c25e --- /dev/null +++ b/resources/teamcity/includes/tc-helpers.inc.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Keyman is copyright (C) SIL Global. MIT License. + +# Returns 0 if we're running on Ubuntu. +is_ubuntu() { + if [[ "${OSTYPE:-}" == "linux-gnu" ]]; then + return 0 + else + return 1 + fi +} + +# Returns 0 if we're running on Windows, i.e. if the environment variable +# `OSTYPE` is set to "msys" or "cygwin". +is_windows() { + if [[ "${OSTYPE:-}" == "msys" ]] || [[ "${OSTYPE:-}" == "cygwin" ]]; then + return 0 + else + return 1 + fi +} + +# Returns 0 if we're running on macOS. +is_macos() { + if [[ "${OSTYPE:-}" == "darwin" ]]; then + return 0 + else + return 1 + fi +} + +install_nvm() { + if ! is_ubuntu; then + # on Windows and macOS build agents are configured manually + return 0 + fi + linux_install_nvm +} + +# Set the environment variables required to use node/nvm and set the +# `KEYMAN_USE_NVM` variable so that the build can automatically install +# the required node version. +set_variables_for_nvm() { + if [[ -f "${HOME}/.nvm/nvm.sh" ]] && [[ -d "${HOME}/.keyman/node" ]]; then + # nvm.sh uses some variables that might not be initialized, so we + # disable the "unbound variable" check temporarily + set -u + export NVM_DIR="${HOME}/.nvm" + # shellcheck disable=SC1091 + . "${NVM_DIR}/nvm.sh" + set +u + export KEYMAN_USE_NVM=1 + PATH=${HOME}/.keyman/node:${PATH} + fi +} + +install_emscripten() { + if ! is_ubuntu; then + # on Windows and macOS build agents are configured manually + return 0 + fi + linux_install_emscripten +} + +set_variables_for_emscripten() { + export EMSCRIPTEN_BASE="${EMSCRIPTEN_BASE:-${HOME}/emsdk/upstream/emscripten}" + export KEYMAN_USE_EMSDK=1 +} diff --git a/resources/teamcity/includes/tc-linux.inc.sh b/resources/teamcity/includes/tc-linux.inc.sh index 32d311cad3..cf1964de21 100644 --- a/resources/teamcity/includes/tc-linux.inc.sh +++ b/resources/teamcity/includes/tc-linux.inc.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Keyman is copyright (C) SIL Global. MIT License. # Returns 0 if the OS version is greater than or equal to the specified version. # Parameter: @@ -23,7 +24,12 @@ is_package_installed() { # Check if the specified packages are installed and install them if not. # Parameters: # $* - List of package names to check and install (e.g., "lcov jq") -check_and_install_packages() { +linux_check_and_install_packages() { + if ! is_ubuntu; then + return 0 + fi + + builder_echo start "check and install packages" "Checking and installing packages" local PACKAGES=$* local TOINSTALL="" @@ -39,21 +45,7 @@ check_and_install_packages() { # shellcheck disable=SC2086 sudo DEBIAN_FRONTEND="noninteractive" apt-get install -qy ${TOINSTALL} fi -} - -# Set the environment variables required to use node/nvm and set the -# `KEYMAN_USE_NVM` variable so that the build can automatically install -# the required node version. -set_variables_for_nvm() { - # nvm.sh uses some variables that might not be initialized, so we - # disable the "unbound variable" check temporarily - set -u - export NVM_DIR="${HOME}/.nvm" - # shellcheck disable=SC1091 - . "${NVM_DIR}/nvm.sh" - set +u - export KEYMAN_USE_NVM=1 - PATH=${HOME}/.keyman/node:${PATH} + builder_echo end "check and install packages" success "Finished checking and installing packages" } # Install nvm if it is not already installed and install the latest LTS @@ -74,12 +66,60 @@ linux_install_nvm() { builder_echo end install_nvm success "Finished checking and installing nvm" } +linux_install_emscripten() { + builder_echo start "install emscripten" "Checking and installing emscripten" + if { [[ ! -z "${EMSCRIPTEN_BASE:-}" ]] && [[ -f "${EMSCRIPTEN_BASE}/emcc" ]] ; } || + [[ -f "${HOME}/emsdk/upstream/emscripten/emcc" ]]; then + builder_echo "Emscripten is already installed at ${EMSCRIPTEN_BASE:-${HOME}/emsdk/upstream/emscripten}" + else + # shellcheck disable=SC2154 + . "${KEYMAN_ROOT}/resources/build/minimum-versions.inc.sh" + + builder_echo "Installing emscripten version ${KEYMAN_MIN_VERSION_EMSCRIPTEN}" + export EMSDK_KEEP_DOWNLOADS=1 + # shellcheck disable=SC2164 + cd "${HOME}" + git clone https://github.com/emscripten-core/emsdk.git + # shellcheck disable=SC2164 + cd emsdk + ./emsdk install "${KEYMAN_MIN_VERSION_EMSCRIPTEN}" + ./emsdk activate "${KEYMAN_MIN_VERSION_EMSCRIPTEN}" + fi + set_variables_for_emscripten + builder_echo end "install emscripten" success "Finished checking and installing emscripten" +} + # Install additional dependencies required for running integration tests. linux_install_dependencies_for_tests() { builder_echo start install_dependencies_for_tests "Installing dependencies for tests" - check_and_install_packages xvfb xserver-xephyr metacity mutter dbus-x11 weston xwayland + linux_check_and_install_packages xvfb xserver-xephyr metacity mutter dbus-x11 weston xwayland builder_echo end install_dependencies_for_tests success "Finished installing dependencies for tests" } +linux_start_xvfb() { + # On Linux start Xvfb etc + local PID_FILE=/tmp/keymanweb-pids + builder_echo "Starting Xvfb..." + Xvfb -screen 0 1024x768x24 :33 &> /dev/null & + echo "kill -9 $! || true" > "${PID_FILE}" + sleep 1 + builder_echo "Starting Xephyr..." + DISPLAY=:33 Xephyr :32 -screen 1024x768 &> /dev/null & + echo "kill -9 $! || true" >> "${PID_FILE}" + sleep 1 + builder_echo "Starting metacity" + metacity --display=:32 &> /dev/null & + echo "kill -9 $! || true" >> "${PID_FILE}" + export DISPLAY=:32 +} + +linux_stop_xvfb() { + # On Linux stop Xvfb etc + local PID_FILE=/tmp/keymanweb-pids + if [[ -f "${PID_FILE}" ]]; then + bash "${PID_FILE}" + rm -f "${PID_FILE}" + fi +} diff --git a/resources/teamcity/linux/keyman-linux-release.sh b/resources/teamcity/linux/keyman-linux-release.sh index 38849fa9ba..14457a54fb 100755 --- a/resources/teamcity/linux/keyman-linux-release.sh +++ b/resources/teamcity/linux/keyman-linux-release.sh @@ -11,6 +11,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" # shellcheck disable=SC2154 . "${KEYMAN_ROOT}/resources/shellHelperFunctions.sh" . "${KEYMAN_ROOT}/resources/teamcity/includes/tc-actions.inc.sh" +. "${KEYMAN_ROOT}/resources/teamcity/includes/tc-helpers.inc.sh" . "${KEYMAN_ROOT}/resources/teamcity/includes/tc-linux.inc.sh" ################################ Main script ################################ diff --git a/resources/teamcity/linux/keyman-linux-test-integration.sh b/resources/teamcity/linux/keyman-linux-test-integration.sh index edb287f696..0868cbb835 100755 --- a/resources/teamcity/linux/keyman-linux-test-integration.sh +++ b/resources/teamcity/linux/keyman-linux-test-integration.sh @@ -12,6 +12,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" # shellcheck disable=SC2154 . "${KEYMAN_ROOT}/resources/shellHelperFunctions.sh" . "${KEYMAN_ROOT}/resources/teamcity/includes/tc-actions.inc.sh" +. "${KEYMAN_ROOT}/resources/teamcity/includes/tc-helpers.inc.sh" . "${KEYMAN_ROOT}/resources/teamcity/includes/tc-linux.inc.sh" ################################ Main script ################################ diff --git a/resources/teamcity/linux/keyman-linux-test.sh b/resources/teamcity/linux/keyman-linux-test.sh index 0d9505d351..7c8387d153 100755 --- a/resources/teamcity/linux/keyman-linux-test.sh +++ b/resources/teamcity/linux/keyman-linux-test.sh @@ -12,6 +12,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" # shellcheck disable=SC2154 . "${KEYMAN_ROOT}/resources/shellHelperFunctions.sh" . "${KEYMAN_ROOT}/resources/teamcity/includes/tc-actions.inc.sh" +. "${KEYMAN_ROOT}/resources/teamcity/includes/tc-helpers.inc.sh" . "${KEYMAN_ROOT}/resources/teamcity/includes/tc-linux.inc.sh" ################################ Main script ################################ diff --git a/resources/teamcity/web/keyman-web-release.sh b/resources/teamcity/web/keyman-web-release.sh new file mode 100755 index 0000000000..634485da3e --- /dev/null +++ b/resources/teamcity/web/keyman-web-release.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# Keyman is copyright (C) SIL Global. MIT License. +# +# TC build script to build release of KeymanWeb. + +# shellcheck disable=SC2164 +# shellcheck disable=SC1091 + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" +. "${THIS_SCRIPT%/*}/../../../resources/build/builder.inc.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +# shellcheck disable=SC2154 +. "${KEYMAN_ROOT}/resources/shellHelperFunctions.sh" +. "${KEYMAN_ROOT}/resources/teamcity/web/web-actions.inc.sh" +. "${KEYMAN_ROOT}/resources/teamcity/includes/tc-helpers.inc.sh" +. "${KEYMAN_ROOT}/resources/teamcity/includes/tc-linux.inc.sh" + +################################ Main script ################################ + +builder_describe \ + "Run tests for native KeymanWeb" \ + "all run all actions" \ + "configure install dependencies" \ + "build build Web + embedded" \ + "publish publish release" \ + "--s.keyman.com=S_KEYMAN_COM path to s.keyman.com repository" \ + "--help.keyman.com=HELP_KEYMAN_COM path to help.keyman.com repository" + +builder_parse "$@" + +cd "${KEYMAN_ROOT}/web" + +function _push_release_to_skeymancom() { + # Push release to s.keyman.com/kmw/engine (do this before updating + # downloads.keyman.com so we can ensure files are available) + builder_echo start publish "Publishing release to s.keyman.com" + + cd "${S_KEYMAN_COM:=${KEYMAN_ROOT}/../s.keyman.com}" + git pull https://github.com/keymanapp/s.keyman.com.git master + cd "${KEYMAN_ROOT}/web" + "${KEYMAN_ROOT}/web/ci.sh" prepare:s.keyman.com --s.keyman.com "${S_KEYMAN_COM}" + + builder_echo end publish success "Finished publishing release to s.keyman.com" +} + +function _zip_and_upload_artifacts() { + builder_echo start "zip and upload artifacts" "Zipping and uploading artifacts" + + # shellcheck disable=SC2154 + powershell -NonInteractive -ExecutionPolicy Bypass -File "${THIS_SCRIPT_PATH}/zip-and-upload-artifacts.ps1" + + builder_echo end "zip and upload artifacts" success "Finished zipping and uploading artifacts" +} + +function _upload_help() { + builder_echo start "upload help" "Uploading new Keyman for Web help to help.keyman.com" + + export HELP_KEYMAN_COM="${HELP_KEYMAN_COM:-${KEYMAN_ROOT}/../help.keyman.com}" + cd "${KEYMAN_ROOT}/resources/build" + "${KEYMAN_ROOT}/resources/build/help-keyman-com.sh" web + cd "${KEYMAN_ROOT}/web" + + builder_echo end "upload help" success "Finished uploading new Keyman for Web help to help.keyman.com" +} + +function publish_web_action() { + builder_echo start publish "Publishing KeymanWeb release" + + # TODO: refactor to allow to run on Linux/macOS as well + if ! is_windows; then + builder_echo end publish error "Publishing KeymanWeb is only supported on Windows" + return 1 + fi + + # Push release to s.keyman.com/kmw/engine (do this before updating + # downloads.keyman.com so we can ensure files are available) + _push_release_to_skeymancom + + _zip_and_upload_artifacts + _upload_help + + builder_echo end publish success "Finished publishing KeymanWeb release" +} + +if builder_has_action all; then + web_install_dependencies_on_linux_action + + set_variables_for_nvm + + web_build_action + publish_web_action +else + builder_run_action configure web_install_dependencies_on_linux_action + + set_variables_for_nvm + + builder_run_action build web_build_action + builder_run_action publish publish_web_action +fi diff --git a/resources/teamcity/web/keyman-web-test.sh b/resources/teamcity/web/keyman-web-test.sh new file mode 100755 index 0000000000..0996ab7ec3 --- /dev/null +++ b/resources/teamcity/web/keyman-web-test.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Keyman is copyright (C) SIL Global. MIT License. +# +# TC build script for Keyman Web/Test + +# shellcheck disable=SC2164 +# shellcheck disable=SC1091 + +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" +. "${THIS_SCRIPT%/*}/../../../resources/build/builder.inc.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +# shellcheck disable=SC2154 +. "${KEYMAN_ROOT}/resources/shellHelperFunctions.sh" +. "${KEYMAN_ROOT}/resources/teamcity/web/web-actions.inc.sh" +. "${KEYMAN_ROOT}/resources/teamcity/includes/tc-helpers.inc.sh" +. "${KEYMAN_ROOT}/resources/teamcity/includes/tc-linux.inc.sh" + +################################ Main script ################################ + +builder_describe \ + "Run tests for native KeymanWeb" \ + "all run all actions" \ + "configure install dependencies" \ + "build build Web + embedded" \ + "test run native KeymanWeb tests and check build size" + +builder_parse "$@" + +cd "${KEYMAN_ROOT}/web" + +function check_build_size_action() { + builder_echo start "check build size" "Check build size" + "${KEYMAN_ROOT}/web/ci.sh" validate-size + builder_echo end "check build size" success "Finished checking build size" +} + +if builder_has_action all; then + web_install_dependencies_on_linux_action + + set_variables_for_nvm + + web_build_action + web_test_action + check_build_size_action +else + builder_run_action configure web_install_dependencies_on_linux_action + + set_variables_for_nvm + + builder_run_action build web_build_action + builder_run_action test web_test_action + builder_run_action test check_build_size_action +fi diff --git a/resources/teamcity/web/web-actions.inc.sh b/resources/teamcity/web/web-actions.inc.sh new file mode 100644 index 0000000000..187eef8bb6 --- /dev/null +++ b/resources/teamcity/web/web-actions.inc.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Keyman is copyright (C) SIL Global. MIT License. + +web_install_dependencies_on_linux_action() { + if ! is_ubuntu; then + return 0 + fi + + builder_echo start "install dependencies" "Install dependencies" + + # shellcheck disable=SC2086 + linux_check_and_install_packages devscripts jq + + linux_install_nvm + _install_playwright_dependencies + + builder_echo end "install dependencies" success "Finished installing dependencies" +} + +_install_playwright_dependencies() { + if ! is_ubuntu || ! is_os_version_or_higher 24.04; then + return 0 + fi + + # shellcheck disable=SC2086 + linux_check_and_install_packages ibevent-2.1-7t64 libxslt1.1 libwoff1 \ + libvpx9 libgstreamer-plugins-bad1.0-0 libwebpdemux2 libharfbuzz-icu0 \ + libenchant-2-2 libsecret-1-0 libhyphen0 libmanette-0.2-0 libflite1 \ + gstreamer1.0-libav libnss3 libnspr4 libatk1.0-0t64 libatk-bridge2.0-0t64 \ + libcups2t64 libatspi2.0-0t64 libxcomposite1 libxdamage1 libxrandr2 \ + libxcursor1 libgtk-3-0t64 libgles2 +} + +web_build_action() { + builder_echo start web_build "Building web" + # shellcheck disable=SC2154 + "${KEYMAN_ROOT}/web/ci.sh" build + builder_echo end web_build success "Finished building web" +} + +web_test_action() { + builder_echo start web_test "Running tests for native KeymanWeb" + if is_ubuntu; then + linux_start_xvfb + trap "linux_stop_xvfb" ERR + fi + + "${KEYMAN_ROOT}/web/ci.sh" test + + if is_ubuntu; then + linux_stop_xvfb + trap ERR + fi + + "${KEYMAN_ROOT}/web/build.sh" coverage + + builder_echo end web_test success "Finished running tests for native KeymanWeb" +} + diff --git a/resources/teamcity/web/zip-and-upload-artifacts.ps1 b/resources/teamcity/web/zip-and-upload-artifacts.ps1 new file mode 100755 index 0000000000..6b29bf7612 --- /dev/null +++ b/resources/teamcity/web/zip-and-upload-artifacts.ps1 @@ -0,0 +1,75 @@ +# Keyman is copyright (C) SIL Global. MIT License. +# +# This script should be the identical for nightly/beta/stable for a given platform +# +# Working directory should be ${KEYMAN_ROOT}/web +# +$ErrorActionPreference = "Stop"; + +$tier = Get-Content ..\TIER.md +$build_number = Get-Content ..\VERSION.md +$build_counter = $build_number -replace "^\d+\.\d+\.(\d+)$", '$1' + +$upload_path = "build\upload\$build_number" +$zip = "$upload_path\keymanweb-$build_number.zip" + +$7Z_HOME = $env:7Z_HOME +$RSYNC_HOME = $env:RSYNC_HOME +$USERPROFILE = $env:USERPROFILE + +# Since shell-scripting doesn't like number-initial variables, we convert it to a friendlier name. +$env:SEVEN_Z_HOME=$7Z_HOME + +# PowerShell fun: `--` parameters will break a PowerShell command when not escaped in some form. +# We can build the command in string form first and then execute the string, fortunately. +$cmd = '"C:\Program Files\Git\bin\bash.exe" --init-file "c:\Program Files\Git\etc\profile" -l ./ci.sh prepare:downloads.keyman.com' +cmd /c $cmd + +$hash = get-filehash $zip -Algorithm MD5 + +# +# Construct .build_info +# + +$download_info = @" +{ + "name": "KeymanWeb", + "version": "$build_number", + "date": "$([DateTime]::Now.ToString("yyyy-MM-dd"))", + "platform": "web", + "stability": "$tier", + "file": "keymanweb-$build_number.zip", + "md5": "$($hash.Hash)", + "type": "zip", + "build": "$build_counter" +} +"@ + +# WriteAllLines is needed to avoid BOM +[System.IO.File]::WriteAllLines("$pwd\$upload_path\keymanweb-$build_number.zip.download_info", $download_info) +# $download_info | Out-File $upload_path\keymanweb-$build_number.zip.download_info -Encoding utf8 + +# +# Upload with rsync to downloads.keyman.com +# (rsync requires that we are in the upload folder to get folders in +# sync correctly; it is possible to resolve this but easier to just cd.) +# + +$rsync_args = @( + '-vrzltp', # verbose, recurse, zip, copy symlinks, preserve times, permissions + '--chmod=Dug=rwx,Do=rx,Fug=rw,Fo=r', # map Windows security to host security + '--stats', # show statistics for log + '--rsync-path="%downloads_rsync_path%"', # path on remote server + "--rsh=$RSYNC_HOME\ssh -i $USERPROFILE\.ssh\id_rsa -o UserKnownHostsFile=$USERPROFILE\.ssh\known_hosts", # use ssh + "$build_number", # upload the whole build folder + "%downloads_rsync_user%@%downloads_rsync_host%:%downloads_rsync_root%/web/$tier/" # target server + path +) + +# Write-Output "rsync parameters:" $rsync_args + +cd "$upload_path\.." +& $RSYNC_HOME\rsync.exe $rsync_args +if ($LASTEXITCODE -ne 0) { throw "Exit code is $LASTEXITCODE" } +cd .. + +# EOF diff --git a/web/src/engine/sentry-manager/src/index.ts b/web/src/engine/sentry-manager/src/index.ts index b8129766d1..92085e7869 100644 --- a/web/src/engine/sentry-manager/src/index.ts +++ b/web/src/engine/sentry-manager/src/index.ts @@ -12,8 +12,8 @@ type Options = { }; export class KeymanSentryManager { - keymanPlatform: string; - _enabled: boolean = true; + private keymanPlatform: string; + private _enabled: boolean = true; static STANDARD_ALIASABLE_FILES = { 'keymanweb.js': 'keymanweb.js', @@ -34,7 +34,7 @@ export class KeymanSentryManager { } // If we've recognized one of our source files, - aliasFilename(filename: string): string|null { + private aliasFilename(filename: string): string|null { if(!this.mayAlias(filename)) { return null; } @@ -49,14 +49,14 @@ export class KeymanSentryManager { } } - mayAlias(filename: string): boolean { + private mayAlias(filename: string): boolean { return !!KeymanSentryManager.STANDARD_ALIASABLE_FILES[filename]; } // Filters all expected but unnecessary path prefixes, affixes, and suffixes reported by Sentry from our products. // This allows us to mask all different sorts of installations with a single uploaded path. // Modifies original object. - pathFilter(event: any) { + private pathFilter(event: any) { // Get the underlying JS error. let exception = event.exception; if(!exception) { @@ -90,7 +90,7 @@ export class KeymanSentryManager { } // Attaches some useful debugging information to the specified object, pass-by-reference style. - attachEventMetadata(event: any) { + private attachEventMetadata(event: any) { // Ensure that the 'extra' object exists. (May not exist for synthetic/custom Errors.) event.extra = event.extra || {}; //@ts-ignore @@ -100,7 +100,7 @@ export class KeymanSentryManager { // Sanitizes the event object (in-place) to remove sensitive information // from the breadcrumbs and url (for embedded KeymanWeb) - sanitizeEvent(event: any) { + private sanitizeEvent(event: any) { if (event && event.breadcrumbs) { event.breadcrumbs.forEach((b: any) => { if (b.category == 'navigation') { @@ -123,7 +123,7 @@ export class KeymanSentryManager { * Also will sanitize the Sentry event. * @param event A Sentry-generated event */ - prepareEvent(event: any): boolean { + private prepareEvent(event: any): boolean { this.pathFilter(event); this.attachEventMetadata(event); this.sanitizeEvent(event); @@ -148,7 +148,7 @@ export class KeymanSentryManager { * if we want to prevent the event from being sent to the server. * @param event */ - prepareEventDebugWrapper(event: any) { + public prepareEventDebugWrapper(event: any) { if(DEBUG) { try { if(this.prepareEvent(event)) { @@ -178,7 +178,7 @@ export class KeymanSentryManager { * * https://stackoverflow.com/a/53214615/1836776 */ - initConsole() { + private initConsole() { // creating function declarations for better stacktraces (otherwise they'd be anonymous function expressions) let oldConsoleError = console.error; let _this = this; @@ -223,7 +223,7 @@ export class KeymanSentryManager { } } - init() { + public init() { // Do the actual Sentry initialization. //@ts-ignore Sentry.init({ @@ -238,11 +238,11 @@ export class KeymanSentryManager { this.initConsole(); } - get enabled(): boolean { + public get enabled(): boolean { return this._enabled; } - set enabled(value: boolean) { + public set enabled(value: boolean) { this._enabled = value; } } diff --git a/windows/src/desktop/kmshell/main/Keyman.System.UpdateStateMachine.pas b/windows/src/desktop/kmshell/main/Keyman.System.UpdateStateMachine.pas index b883a9ba30..4e2afbe89f 100644 --- a/windows/src/desktop/kmshell/main/Keyman.System.UpdateStateMachine.pas +++ b/windows/src/desktop/kmshell/main/Keyman.System.UpdateStateMachine.pas @@ -782,6 +782,7 @@ begin // Return to Idle state bucStateContext.RemoveCachedFiles; ChangeState(IdleState); + Exit; end; DownloadResult := DownloadUpdatesBackground; @@ -920,7 +921,7 @@ begin CheckForUpdates.Free; end; { Response OK and go back to update available so files can be downloaded } - // TODO: This acually needs to check if the updates available are newer then the already downloaded updates + // TODO: This actually needs to check if the updates available are newer then the already downloaded updates if Result = wucUpdateAvailable then begin @@ -980,10 +981,11 @@ begin executeResult := WaitForElevatedConfiguration(0, '-ikp'); if (executeResult <> 0) then begin - TKeymanSentryClient.Client.MessageEvent - (Sentry.Client.SENTRY_LEVEL_ERROR, - 'Executing kmshell process to install keyboard packages failed:"' + - IntToStr(Ord(executeResult)) + '"'); + TKeymanSentryClient.Breadcrumb('error', + 'Executing kmshell process to install keyboard packages failed"' + + IntToStr(Ord(executeResult)) + '"', 'update'); + KL.Log('InstallingState.LaunchInstallPackageProcess failed executing kmshell ' + + 'process to install keyboard packages: "' + IntToStr(Ord(executeResult)) + '"'); ChangeState(IdleState); end; end @@ -1035,9 +1037,10 @@ begin if not FResult then begin bucStateContext.RemoveCachedFiles; - TKeymanSentryClient.Client.MessageEvent(Sentry.Client.SENTRY_LEVEL_ERROR, - 'Executing kmshell process to install failed:"' + - IntToStr(Ord(FResult)) + '"'); + TKeymanSentryClient.Breadcrumb('error', + 'InstallingState.DoInstallKeyman: failed executing kmshell or file not found', 'update'); + KL.Log('InstallingState.DoInstallKeyman failed executing kmshell ' + + 'or the update was not found in the cache'); ChangeState(IdleState); end;