erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-09-04 22:57:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 17

Merge pull request #9420 from keymanapp/chore/merge-master-into-feature-kmc-kmw-a17s18

Browse source 
chore: merge master into feature-kmc-kmw (A17S18) 🗜
This commit is contained in:
Marc Durdin 2023-08-04 18:56:45 +10:00 • committed by GitHub
commit c92f28e5f7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
105 changed files with 2233 additions and 1017 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.github/workflows/crowdin.yml vendored
View file

@ -2,8 +2,8 @@ name: Upload translation sources to Crowdin translate.keyman.com
on:
schedule:
# At 06:00 every two weeks
- cron: '0 6 1,15 * *'
# At 06:00 every day. https://crontab.cronhub.io/
- cron: '0 6 * * *'
jobs:
upload-sources-to-crowdin:

9
.github/workflows/deb-packaging.yml vendored
View file

@ -1,5 +1,5 @@
name: "Ubuntu packaging"
run-name: "Ubuntu packaging - ${{ github.ref_name }} by @${{ github.actor }}"
run-name: "Ubuntu packaging - ${{ github.event.client_payload.branch }} (branch ${{ github.head_ref }}), by @${{ github.actor }}"
on:
repository_dispatch:
types: ['deb-release-packaging:*', 'deb-pr-packaging:*']
@ -18,6 +18,8 @@ jobs:
VERSION: ${{ steps.version_step.outputs.VERSION }}
PRERELEASE_TAG: ${{ steps.prerelease_tag.outputs.PRERELEASE_TAG }}
GIT_SHA: ${{ steps.set_status.outputs.GIT_SHA }}
GHA_TEST_BUILD: ${{ github.event.client_payload.isTestBuild }}
GHA_BRANCH: ${{ github.event.client_payload.branch }}
steps:
- name: Checkout
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c #v3.3.0
@ -103,7 +105,10 @@ jobs:
strategy:
fail-fast: true
matrix:
dist: [focal, jammy, kinetic, lunar]
# Currently not building mantic until ibus version on mantic stabilizied
# and we can provide a patched version
# dist: [focal, jammy, lunar, mantic]
dist: [focal, jammy, lunar]
arch: [amd64]
runs-on: ubuntu-latest

55
HISTORY.md
View file

@ -1,5 +1,60 @@
# Keyman Version History
## 17.0.153 alpha 2023-08-03
* docs(windows): Update OS requirement to Windows 10 (#9381)
* fix(web): maintenance of focus when changing keyboard via Toolbar UI (#9397)
* chore(linux): Remove Kinetic from GHA (#9399)
* chore(linux): Properly treat test builds with packaging GHA (#9400)
## 17.0.152 alpha 2023-08-02
* fix(developer): more wasm uset fixes (#9382)
* docs(windows): corrected nmake cmd for certificates (#9376)
* chore: add run-name to deb-packaging (#9386)
* chore: try another variable for reporting (#9388)
* chore(linux): Remove package build on Jenkins for Keyman 17 (#9380)
* docs(linux): Add build doc for Keyman Web and Android (#9383)
## 17.0.151 alpha 2023-08-01
* feat(developer) marker steps (#9364)
* feat(common): marker processing (#9365)
* chore(linux): Don't fail on parallel builds (#9368)
* fix(developer): fix breakage from emscripten 3.1.44 (#9375)
* docs(core): Document how to build Core on Linux (#9328)
## 17.0.150 alpha 2023-07-31
* chore(linux): Update debian changelog (#9358)
* chore(linux): Fix creation of PRs after uploading to Debian (#9360)
## 17.0.149 alpha 2023-07-30
* fix(core): Better range check for Uni_IsValid() (#9346)
* chore(core): update documentation in transform logic and processor (#9352)
## 17.0.148 alpha 2023-07-27
* feat(core): merge transform/reorder processing w/ u32 (#9293)
* chore(developer): make unknown vkey a hint, not error (#9344)
* chore(linux): Update supported Ubuntu versions (#9341)
## 17.0.147 alpha 2023-07-25
* chore(linux): Update debian changelog (#9327)
## 17.0.146 alpha 2023-07-24
* chore(deps-dev): bump word-wrap from 1.2.3 to 1.2.4 (#9314)
## 17.0.145 alpha 2023-07-21
* fix(linux): Fix logging (#9310)
* fix(windows): open pdf in an external browser (#9295)
* fix(linux): Fix installation of keyboards with lang tag `mul` (#9027)
* fix(web): allows registering precached keyboards (#9304)
## 17.0.144 alpha 2023-07-20
* refactor(linux): Use better way to get username (#9313)

2
VERSION.md
View file

@ -1 +1 @@
17.0.145
17.0.154

1
common/web/lm-worker/build-wrap-and-minify.js
View file

@ -35,6 +35,7 @@ if(MINIFY) {
sourcesContent: DEBUG,
minify: true,
keepNames: true,
target: 'es5',
outfile: `build/lib/worker-main.polyfilled.min.js`
});
}

3
common/web/types/src/kmx/kmx-plus-builder/build-list.ts
View file

@ -86,6 +86,9 @@ export function build_list(source_list: List, sect_strs: BUILDER_STRS): BUILDER_
* @returns
*/
export function build_list_index(sect_list: BUILDER_LIST, value: ListItem) : BUILDER_LIST_REF {
if (!value) {
return 0; // empty list
}
if(!(value instanceof ListItem)) {
throw new Error('unexpected value '+ value);
}

6
common/web/types/src/kmx/kmx-plus-builder/build-vars.ts
View file

@ -2,7 +2,7 @@ import { constants } from "@keymanapp/ldml-keyboard-constants";
import { KMXPlusData } from "../kmx-plus.js";
import { build_strs_index, BUILDER_STR_REF, BUILDER_STRS } from "./build-strs.js";
import { BUILDER_SECTION } from "./builder-section.js";
import { BUILDER_LIST_REF } from "./build-list.js";
import { build_list_index, BUILDER_LIST, BUILDER_LIST_REF } from "./build-list.js";
import { build_elem_index, BUILDER_ELEM, BUILDER_ELEM_REF } from "./build-elem.js";
@ -22,7 +22,7 @@ export interface BUILDER_VARS extends BUILDER_SECTION {
/**
* Builder for the 'vars' section
*/
export function build_vars(kmxplus: KMXPlusData, sect_strs: BUILDER_STRS, sect_elem: BUILDER_ELEM) : BUILDER_VARS {
export function build_vars(kmxplus: KMXPlusData, sect_strs: BUILDER_STRS, sect_elem: BUILDER_ELEM, sect_list: BUILDER_LIST) : BUILDER_VARS {
if(!kmxplus.vars) {
return null;
}
@ -49,7 +49,7 @@ export function build_vars(kmxplus: KMXPlusData, sect_strs: BUILDER_STRS, sect_e
size: constants.length_vars +
(constants.length_vars_item * kmxplus.vars.totalCount()),
_offset: 0,
markers: 0,
markers: build_list_index(sect_list, kmxplus.vars.markers),
varCount: kmxplus.vars.totalCount(),
varEntries: [
...stringVars,

2
common/web/types/src/kmx/kmx-plus-builder/kmx-plus-builder.ts
View file

@ -99,7 +99,7 @@ export default class KMXPlusBuilder {
this.sect.name = build_name(this.file.kmxplus, this.sect.strs);
this.sect.tran = build_tran(this.file.kmxplus.tran, this.sect.strs, this.sect.elem);
this.sect.uset = build_uset(this.file.kmxplus, this.sect.strs);
this.sect.vars = build_vars(this.file.kmxplus, this.sect.strs, this.sect.elem);
this.sect.vars = build_vars(this.file.kmxplus, this.sect.strs, this.sect.elem, this.sect.list);
this.sect.vkey = build_vkey(this.file.kmxplus);
// Finalize the sect (index) section

4
common/web/types/src/kmx/kmx-plus.ts
View file

@ -6,6 +6,7 @@ import { isOneChar, toOneChar, unescapeString } from '../util/util.js';
import { KMXFile } from './kmx.js';
import { UnicodeSetParser, UnicodeSet } from '@keymanapp/common-types';
import { VariableParser } from '../ldml-keyboard/pattern-parser.js';
import { MarkerParser } from '../ldml-keyboard/pattern-parser.js';
// Implementation of file structures from /core/src/ldml/C7043_ldml.md
// Writer in kmx-builder.ts
@ -292,6 +293,9 @@ export class Vars extends Section {
return v[0];
}
}
substituteMarkerString(s : string) : string {
return MarkerParser.toSentinelString(s, this.markers);
}
};
/**

13
common/web/types/src/kmx/string-list.ts
View file

@ -1,3 +1,4 @@
import { OrderedStringList } from 'src/ldml-keyboard/pattern-parser.js';
import { Strs, StrsItem } from './kmx-plus.js';
/**
@ -22,7 +23,7 @@ export class ListIndex {
* A string list in memory. This will be replaced with an index
* into the string table at finalization.
*/
export class ListItem extends Array<ListIndex> {
export class ListItem extends Array<ListIndex> implements OrderedStringList {
/**
* Construct a new list from an array of strings.
* Use List. This is meant to be called by the List.allocString*() functions.
@ -41,6 +42,9 @@ export class ListItem extends Array<ListIndex> {
this.push(index);
}
}
getItemOrder(item: string): number {
return this.findIndex(({value}) => value.value === item);
}
isEqual(a: ListItem | string[]): boolean {
if (a.length != this.length) {
return false;
@ -68,7 +72,12 @@ export class ListItem extends Array<ListIndex> {
return 0;
}
}
/** for debugging, print as single string */
toString(): string {
return this.map(v => v.value.value).join(' ');
return this.toStringArray().join(' ');
}
/** for debugging, map to string array */
toStringArray(): string[] {
return this.map(v => v.value.value);
}
};

58
common/web/types/src/ldml-keyboard/pattern-parser.ts
View file

@ -2,6 +2,7 @@
* Utilities for transform and marker processing
*/
import { constants } from "@keymanapp/ldml-keyboard-constants";
import { MATCH_QUAD_ESCAPE, isOneChar, unescapeOneQuadString, unescapeString } from "../util/util.js";
@ -21,6 +22,12 @@ function matchArray(str: string, match: RegExp) : string[] {
*/
const COMMON_ID = /^[0-9A-Za-z_]{1,32}$/;
/** for use with markers, means an ordering can be determined */
export interface OrderedStringList {
/** @returns the ordering of an item (0..), or -1 if not found */
getItemOrder(item : string) : number;
}
/**
* Class for helping with markers
*/
@ -40,6 +47,25 @@ export class MarkerParser {
*/
public static readonly ANY_MARKER_ID = '.';
/**
* Marker sentinel as a string - U+FFFF
*/
public static readonly SENTINEL = String.fromCodePoint(constants.marker_sentinel);
/**
* Matches all markers.
*/
public static readonly SENTINEL_ALL_MARKERS = this.SENTINEL + this.SENTINEL;
/** Minimum ID (trailing code unit) */
public static readonly MIN_MARKER_INDEX = constants.marker_min_index;
/** Index meaning 'any marker' == `\m{.}` */
public static readonly ANY_MARKER_INDEX = constants.marker_any_index;
/** Maximum usable marker index */
public static readonly MAX_MARKER_INDEX = constants.marker_max_index;
/** Max count of markers */
public static readonly MAX_MARKER_COUNT = constants.marker_max_count;
/**
* Pattern for matching a marker reference, OR the special marker \m{.}
*/
@ -51,8 +77,40 @@ export class MarkerParser {
* @returns `[]` or an array of all markers referenced
*/
public static allReferences(str: string): string[] {
if (!str) {
return [];
}
return matchArray(str, this.REFERENCE);
}
/** @returns string for marker #n */
public static markerOutput(n: number): string {
if (n < MarkerParser.MIN_MARKER_INDEX || n > MarkerParser.ANY_MARKER_INDEX) {
throw RangeError(`Internal Error: marker index out of range ${n}`);
}
return this.SENTINEL + String.fromCharCode(n);
}
/** @returns all marker strings as sentinel values */
public static toSentinelString(s: string, markers?: OrderedStringList) : string {
if (!s) return s;
return s.replaceAll(this.REFERENCE, (sub, arg) => {
if (arg === MarkerParser.ANY_MARKER_ID) {
return MarkerParser.SENTINEL_ALL_MARKERS;
}
if (!markers) {
throw RangeError(`Internal Error: Could not find marker \\m{${arg}} (no markers defined)`);
}
const order = markers.getItemOrder(arg);
if (order === -1) {
throw RangeError(`Internal Error: Could not find marker \\m{${arg}}`);
} else if(order >= MarkerParser.MAX_MARKER_INDEX) {
throw RangeError(`Internal Error: marker \\m{${arg}} has out of range index ${order}`);
} else {
return MarkerParser.markerOutput(order+1);
}
});
}
}
/**

53
common/web/types/test/ldml-keyboard/test-pattern-parser.ts
View file

@ -1,6 +1,6 @@
import 'mocha';
import { assert } from 'chai';
import { ElementParser, ElementSegment, ElementType, MarkerParser, VariableParser } from '../../src/ldml-keyboard/pattern-parser.js';
import { ElementParser, ElementSegment, ElementType, MarkerParser, OrderedStringList, VariableParser } from '../../src/ldml-keyboard/pattern-parser.js';
describe('Test of Pattern Parsers', () => {
describe('should test MarkerParser', () => {
@ -51,6 +51,57 @@ describe('Test of Pattern Parsers', () => {
assert.deepEqual(MarkerParser.allReferences(str), [], `expected no markers: ${str}`);
}
});
it('should be able to emit sentinel values', () => {
assert.equal(MarkerParser.markerOutput(295), '\uFFFF\u0127', 'Wrong sentinel value emitted');
assert.equal(MarkerParser.markerOutput(MarkerParser.ANY_MARKER_INDEX), MarkerParser.SENTINEL_ALL_MARKERS, 'Wrong sentinel value emitted for ffff');
assert.throws(() => MarkerParser.markerOutput(0)); // below MIN
assert.throws(() => MarkerParser.markerOutput(0x10000)); // above MAX
});
it('should be able to output sentinel strings', () => {
// with nothing (no markers)
assert.equal(
MarkerParser.toSentinelString(`No markers here!`),
`No markers here!`
);
assert.throws(() =>
MarkerParser.toSentinelString(`Marker \\m{sorryNoMarkers}`)
);
// with a custom class
class MyMarkers implements OrderedStringList {
getItemOrder(item: string): number {
const m : any = {
'a': 0,
'b': 1,
'c': 2,
'zzz': 0x2FFFFF,
};
const o = m[item];
if (o === undefined) return -1;
return o;
}
};
const markers = new MyMarkers();
assert.equal(MarkerParser.toSentinelString(
`No markers here!`, markers),
`No markers here!`
);
assert.equal(MarkerParser.toSentinelString(
`Give me \\m{a} and \\m{c}, or \\m{.}.`, markers),
`Give me \uFFFF\u0001 and \uFFFF\u0003, or \uFFFF\uFFFF.`
);
assert.throws(() =>
MarkerParser.toSentinelString(
`Want to see something funny? \\m{zzz}`, // out of range
markers
)
);
assert.throws(() =>
MarkerParser.toSentinelString(
`Want to see something sad? \\m{nothing}`, // non existent
markers
)
);
});
});
describe('should test VariableParser', () => {
// same test as for markers

5
core/include/ldml/keyboardprocessor_ldml.h
View file

@ -93,6 +93,11 @@
#define LDML_LENGTH_VARS_ITEM 0x10
#define LDML_LENGTH_VKEY 0xC
#define LDML_LENGTH_VKEY_ITEM 0x8
#define LDML_MARKER_ANY_INDEX 0xFFFF
#define LDML_MARKER_MAX_COUNT 0xFFFD
#define LDML_MARKER_MAX_INDEX 0xFFFE
#define LDML_MARKER_MIN_INDEX 0x1
#define LDML_MARKER_SENTINEL 0xFFFF
#define LDML_META_SETTINGS_FALLBACK_OMIT 0x1
#define LDML_META_SETTINGS_TRANSFORMFAILURE_OMIT 0x2
#define LDML_META_SETTINGS_TRANSFORMPARTIAL_HIDE 0x4

13
core/include/ldml/keyboardprocessor_ldml.ts
View file

@ -613,6 +613,19 @@ class Constants {
}
return chars.join('');
}
// ---- marker stuff ----
/** sentinel value indicating a marker follows */
readonly marker_sentinel = 0xFFFF;
/** minimum usable marker index */
readonly marker_min_index = 0x0001;
/** index value referring to the 'any' marker match */
readonly marker_any_index = 0xFFFF;
/** maximum marker index prior to the 'any' value */
readonly marker_max_index = this.marker_any_index - 1;
/** maximum count of markers (not including 'any') */
readonly marker_max_count = this.marker_max_index - this.marker_min_index;
};
export const constants = new Constants();

101
core/src/kmx/kmx_plus.cpp
View file

@ -18,6 +18,20 @@ namespace km {
namespace kbp {
namespace kmx {
/**
* \def KMXPLUS_DEBUG_LOAD set to 1 to print messages on KMXPLUS loading.
* Off by default.
*/
#ifndef KMXPLUS_DEBUG_LOAD
#define KMXPLUS_DEBUG_LOAD 0
#endif
#if KMXPLUS_DEBUG_LOAD
#define DebugLoad(msg,...) DebugLog(msg, __VA_ARGS__)
#else
#define DebugLoad(msg,...)
#endif
// double check these modifier mappings
static_assert(LCTRLFLAG == LDML_KEYS_MOD_CTRLL, "LDML modifier bitfield vs. kmx_file.h #define mismatch");
static_assert(RCTRLFLAG == LDML_KEYS_MOD_CTRLR, "LDML modifier bitfield vs. kmx_file.h #define mismatch");
@ -231,7 +245,7 @@ COMP_KMXPLUS_DISP::valid(KMX_DWORD _kmn_unused(length)) const {
DebugLog("disp: baseCharacter str#0x%X", baseCharacter);
}
for (KMX_DWORD i=0; i<count; i++) {
DebugLog("disp#%d: to: str0x%X -> str0x%X", i, entries[i].to, entries[i].display);
DebugLoad("disp#%d: to: str0x%X -> str0x%X", i, entries[i].to, entries[i].display);
if (entries[i].to == 0 || entries[i].display == 0) {
DebugLog("disp to: or display: has a zero string");
assert(false);
@ -265,7 +279,7 @@ COMP_KMXPLUS_STRS::valid(KMX_DWORD _kmn_unused(length)) const {
return false;
}
// TODO-LDML: validate valid UTF-16LE?
DebugLog("strs #0x%X: '%s'", i, Debug_UnicodeString(start));
DebugLoad("strs #0x%X: '%s'", i, Debug_UnicodeString(start));
}
return true;
}
@ -740,7 +754,7 @@ COMP_KMXPLUS_KEYS_Helper::setKeys(const COMP_KMXPLUS_KEYS *newKeys) {
for(KMX_DWORD i = 0; is_valid && i < key2->keyCount; i++) {
const auto &key = keys[i];
// is the count off the end?
DebugLog( "<key #%d> id=0x%X, to=0x%X, flicks=%d", i, key.id, key.to, key.flicks); // TODO-LDML: could dump more fields here
DebugLoad( "<key #%d> id=0x%X, to=0x%X, flicks=%d", i, key.id, key.to, key.flicks); // TODO-LDML: could dump more fields here
if (key.flicks >0 && key.flicks >= key2->flicksCount) {
DebugLog("key[%d] has invalid flicks index %d", i, key.flicks);
is_valid = false;
@ -750,7 +764,7 @@ COMP_KMXPLUS_KEYS_Helper::setKeys(const COMP_KMXPLUS_KEYS *newKeys) {
for(KMX_DWORD i = 0; is_valid && i < key2->flicksCount; i++) {
const auto &e = flickLists[i];
// is the count off the end?
DebugLog("<flicks> %d: index %d, count %d", i, e.flick, e.count);
DebugLoad("<flicks> %d: index %d, count %d", i, e.flick, e.count);
if (i == 0) {
if (e.flick != 0 || e.count != 0) {
DebugLog("Error: Invalid Flick #0");
@ -765,18 +779,22 @@ COMP_KMXPLUS_KEYS_Helper::setKeys(const COMP_KMXPLUS_KEYS *newKeys) {
}
for(KMX_DWORD i = 0; is_valid && i < key2->flickCount; i++) {
const auto &e = flickElements[i];
// is the count off the end?
DebugLog("<flick> %d: to=0x%X, directions=0x%X, flags=0x%X", i, e.to, e.directions, e.flags);
// validate to is present
if (e.to == 0 || e.directions == 0) {
DebugLog("flickElement[%d] has empty to=%0x%X or directions=%0x%X", i, e.to, e.directions);
is_valid = false;
assert(is_valid);
}
DebugLoad("<flick> %d: to=0x%X, directions=0x%X, flags=0x%X", i, e.to, e.directions, e.flags);
}
// now the kmap
DebugLog(" kmap count: #0x%X", key2->kmapCount);
DebugLoad(" kmap count: #0x%X", key2->kmapCount);
for (KMX_DWORD i = 0; i < key2->kmapCount; i++) {
// These are pretty noisy, drop them from the log
// DebugLog(" #0x%d\n", i);
DebugLoad(" #0x%d\n", i);
auto &entry = kmap[i];
// DebugLog(" vkey\t0x%X", entry.vkey);
// DebugLog(" mod\t0x%X", entry.mod);
// DebugLog(" key\t#0x%X", entry.key);
DebugLoad(" vkey\t0x%X", entry.vkey);
DebugLoad(" mod\t0x%X", entry.mod);
DebugLoad(" key\t#0x%X", entry.key);
if (!LDML_IS_VALID_MODIFIER_BITS(entry.mod)) {
DebugLog("Invalid modifier value");
assert(false);
@ -928,10 +946,12 @@ COMP_KMXPLUS_LIST_Helper::setList(const COMP_KMXPLUS_LIST *newList) {
assert(is_valid);
}
}
#if KMXPLUS_DEBUG_LOAD
for (KMX_DWORD i = 0; is_valid && i < list->indexCount; i++) {
const auto &e = indices[i];
DebugLog(" index %d: str 0x%X", i, e);
DebugLoad(" index %d: str 0x%X", i, e);
}
#endif
}
// Return results
DebugLog("COMP_KMXPLUS_LIST_Helper.setList(): %s", is_valid ? "valid" : "invalid");
@ -969,7 +989,13 @@ COMP_KMXPLUS_USET::valid(KMX_DWORD _kmn_unused(length)) const {
assert(false);
return false;
}
return true;
return true; // see helper
}
COMP_KMXPLUS_USET_RANGE::COMP_KMXPLUS_USET_RANGE(KMX_DWORD s, KMX_DWORD e) : start(s), end(e) {
}
COMP_KMXPLUS_USET_RANGE::COMP_KMXPLUS_USET_RANGE(const COMP_KMXPLUS_USET_RANGE &other) : start(other.start), end(other.end) {
}
COMP_KMXPLUS_USET_Helper::COMP_KMXPLUS_USET_Helper() : uset(nullptr), is_valid(false), usets(nullptr), ranges(nullptr) {
@ -977,7 +1003,7 @@ COMP_KMXPLUS_USET_Helper::COMP_KMXPLUS_USET_Helper() : uset(nullptr), is_valid(f
bool
COMP_KMXPLUS_USET_Helper::setUset(const COMP_KMXPLUS_USET *newUset) {
DebugLog("validating newUset=%p", newUset);
DebugLoad("validating newUset=%p", newUset);
is_valid = true;
if (newUset == nullptr) {
// Note: kmx_plus::kmx_plus has already called section_from_bytes()
@ -1017,9 +1043,13 @@ COMP_KMXPLUS_USET_Helper::setUset(const COMP_KMXPLUS_USET *newUset) {
} else {
/** last lastEnd value */
KMX_DWORD lastEnd = 0x0;
for (KMX_DWORD r = 0; r < e.count; r++) {
for (KMX_DWORD r = 0; is_valid && r < e.count; r++) {
const auto &range = ranges[e.range + r]; // already range-checked 'r' above
if (range.end < range.start) {
if (!Uni_IsValid(range.start, range.end)) {
DebugLog("uset[%d][%d] not valid: [U+%04X-U+%04X]", i, r, range.start, range.end);
is_valid = false;
assert(is_valid);
} else if (range.end < range.start) {
// range swapped
DebugLog("uset[%d]: range[%d+%d] end 0x%X<start 0x%X", i, e.range, r, range.end, range.start);
is_valid = false;
@ -1042,15 +1072,17 @@ COMP_KMXPLUS_USET_Helper::setUset(const COMP_KMXPLUS_USET *newUset) {
return is_valid;
}
USet::USet(const COMP_KMXPLUS_USET_RANGE *newRange, size_t newCount) : ranges(newRange), count(newCount) {
USet::USet(const COMP_KMXPLUS_USET_RANGE *newRange, size_t newCount) {
for (size_t i = 0; i < newCount; i++) {
ranges.emplace_back(newRange[i].start, newRange[i].end);
}
}
USet::USet() : ranges(nullptr), count(0) {
USet::USet() {
}
bool USet::contains(km_kbp_usv ch) const {
for (size_t i = 0; i < count; i++) {
const auto &range = ranges[i];
for (const auto &range : ranges) {
if (range.start <= ch && range.end >= ch) {
return true;
}
@ -1058,6 +1090,30 @@ bool USet::contains(km_kbp_usv ch) const {
return false;
}
bool
USet::valid() const {
// double check
for (const auto &range : ranges) {
if (!Uni_IsValid(range.start, range.end)) {
DebugLog("Invalid UnicodeSet (contains noncharacters): [U+%04X,U+%04X]", (int)range.start, (int)range.end);
return false;
}
}
return true;
}
void
USet::dump() const {
DebugLog(" - USet size=%d", ranges.size());
for (const auto &range : ranges) {
if (range.start == range.end) {
DebugLog(" - [U+%04X]", (uint32_t)range.start);
} else {
DebugLog(" - [U+%04X-U+%04X]", (uint32_t)range.start, (uint32_t)range.end);
}
}
}
USet
COMP_KMXPLUS_USET_Helper::getUset(KMXPLUS_USET i) const {
if (!valid() || i >= uset->usetCount) {
@ -1083,6 +1139,9 @@ kmx_plus::kmx_plus(const COMP_KEYBOARD *keyboard, size_t length)
: bksp(nullptr), disp(nullptr), elem(nullptr), key2(nullptr), layr(nullptr), list(nullptr), loca(nullptr), meta(nullptr),
sect(nullptr), strs(nullptr), tran(nullptr), vars(nullptr), vkey(nullptr), valid(false) {
DebugLog("kmx_plus: Got a COMP_KEYBOARD at %p\n", keyboard);
#if !KMXPLUS_DEBUG_LOAD
DebugLog("Note: define KMXPLUS_DEBUG_LOAD=1 at compile time for more verbosity in loading");
#endif
if (!(keyboard->dwFlags & KF_KMXPLUS)) {
DebugLog("Err: flags COMP_KEYBOARD.dwFlags did not have KF_KMXPLUS set");
valid = false;

14
core/src/kmx/kmx_plus.h
View file

@ -11,6 +11,7 @@
#include <kmx/kmx_base.h>
#include <kmx_file.h>
#include <ldml/keyboardprocessor_ldml.h>
#include <list>
namespace km {
namespace kbp {
@ -683,24 +684,27 @@ struct COMP_KMXPLUS_USET_USET {
struct COMP_KMXPLUS_USET_RANGE {
km_kbp_usv start;
km_kbp_usv end;
public:
COMP_KMXPLUS_USET_RANGE(const COMP_KMXPLUS_USET_RANGE& other);
COMP_KMXPLUS_USET_RANGE(KMX_DWORD start, KMX_DWORD end);
};
/**
* represents one of the uset elements
* Aliases, does not copy memory.
* The original KMX+ memory must stay around while this object is held.
*/
class USet {
public:
/** construct a set over the specified range. */
/** construct a set over the specified range. Data is copied. */
USet(const COMP_KMXPLUS_USET_RANGE* newStart, size_t newCount);
/** empty set */
USet();
/** true if the uset contains this char */
bool contains(km_kbp_usv ch) const;
/** debugging */
void dump() const;
bool valid() const;
private:
const COMP_KMXPLUS_USET_RANGE *ranges;
size_t count;
std::list<COMP_KMXPLUS_USET_RANGE> ranges;
};
class COMP_KMXPLUS_USET_Helper {

91
core/src/kmx/kmx_xstring.h
View file

@ -6,22 +6,43 @@ namespace km {
namespace kbp {
namespace kmx {
const char16_t Uni_LEAD_SURROGATE_START = 0xD800;
const char16_t Uni_LEAD_SURROGATE_END = 0xDBFF;
const char16_t Uni_TRAIL_SURROGATE_START = 0xDC00;
const char16_t Uni_TRAIL_SURROGATE_END = 0xDFFF;
const char16_t Uni_SURROGATE_START = Uni_LEAD_SURROGATE_START;
const char16_t Uni_SURROGATE_END = Uni_TRAIL_SURROGATE_END;
const char16_t Uni_FD_NONCHARACTER_START = 0xFDD0;
const char16_t Uni_FD_NONCHARACTER_END = 0xFDEF;
const char16_t Uni_FFFE_NONCHARACTER = 0xFFFE;
const char16_t Uni_FFFF_NONCHARACTER = 0xFFFF;
const char16_t Uni_BMP_END = 0xFFFF;
const km_kbp_usv Uni_SMP_START = 0x010000;
const km_kbp_usv Uni_PLANE_MASK = 0x1F0000;
const km_kbp_usv Uni_MAX_CODEPOINT = 0x10FFFF;
/**
* @brief True if a lead surrogate
* \def Uni_IsSurrogate1
*/
#define Uni_IsSurrogate1(ch) ((ch) >= 0xD800 && (ch) <= 0xDBFF)
#define Uni_IsSurrogate1(ch) ((ch) >= km::kbp::kmx::Uni_LEAD_SURROGATE_START && (ch) <= km::kbp::kmx::Uni_LEAD_SURROGATE_END)
/**
* @brief True if a trail surrogate
* \def Uni_IsSurrogate2
*/
#define Uni_IsSurrogate2(ch) ((ch) >= 0xDC00 && (ch) <= 0xDFFF)
#define Uni_IsSurrogate2(ch) ((ch) >= km::kbp::kmx::Uni_TRAIL_SURROGATE_START && (ch) <= km::kbp::kmx::Uni_TRAIL_SURROGATE_END)
/**
* @brief True if any surrogate
* \def UniIsSurrogate
*/
#define Uni_IsSurrogate(ch) (Uni_IsSurrogate1(ch) || Uni_IsSurrogate2(ch))
/**
* @brief Returns true if BMP (Plane 0)
* \def Uni_IsBMP
*/
#define Uni_IsBMP(ch) ((ch) < 0x10000)
#define Uni_IsBMP(ch) ((ch) <= km::kbp::kmx::Uni_BMP_END)
/**
* @brief Convert two UTF-16 surrogates into one UTF-32 codepoint
@ -29,17 +50,35 @@ namespace kmx {
* @param cl trail surrogate - Uni_IsSurrogate2(cl) must == true
* \def Uni_SurrogateToUTF
*/
#define Uni_SurrogateToUTF32(ch, cl) (((ch) - 0xD800) * 0x400 + ((cl) - 0xDC00) + 0x10000)
#define Uni_SurrogateToUTF32(ch, cl) (((ch) - km::kbp::kmx::Uni_LEAD_SURROGATE_START) * 0x400 + ((cl) - km::kbp::kmx::Uni_TRAIL_SURROGATE_START) + km::kbp::kmx::Uni_SMP_START)
/**
* @brief Convert UTF-32 BMP to UTF-16 BMP
* @param ch codepoint - Uni_IsBMP(ch) must == true
* \def Uni_UTF32BMPToUTF16
*/
#define Uni_UTF32BMPToUTF16(ch) (ch & 0xFFFF)
#define Uni_UTF32BMPToUTF16(ch) ((ch) & Uni_FFFF_NONCHARACTER)
#define Uni_UTF32ToSurrogate1(ch) (char16_t)(((ch) - 0x10000) / 0x400 + 0xD800)
#define Uni_UTF32ToSurrogate2(ch) (char16_t)(((ch) - 0x10000) % 0x400 + 0xDC00)
#define Uni_UTF32ToSurrogate1(ch) (char16_t)(((ch) - km::kbp::kmx::Uni_SMP_START) / 0x400 + km::kbp::kmx::Uni_LEAD_SURROGATE_START)
#define Uni_UTF32ToSurrogate2(ch) (char16_t)(((ch) - km::kbp::kmx::Uni_SMP_START) % 0x400 + km::kbp::kmx::Uni_TRAIL_SURROGATE_START)
/**
* @returns true if the character is a noncharacter
*/
bool Uni_IsNonCharacter(km_kbp_usv ch);
/**
* @returns true if the character is a valid Unicode code point.
* Surrogates belong to UTF-16 and are invalid.
*/
bool Uni_IsValid(km_kbp_usv ch);
/**
* @returns true if the character is a valid Unicode code point range, that is, [start-end] are all
* valid.
* Surrogates belong to UTF-16 and are invalid.
*/
bool Uni_IsValid(km_kbp_usv start, km_kbp_usv range);
/**
* char16_t array big enough to hold a single Unicode codepoint,
@ -133,6 +172,44 @@ u16string_to_u32string(const std::u16string &source) {
return out;
}
inline bool Uni_IsEndOfPlaneNonCharacter(km_kbp_usv ch) {
return (((ch) & Uni_FFFE_NONCHARACTER) == Uni_FFFE_NONCHARACTER); // matches FFFF or FFFE
}
inline bool Uni_IsNoncharacter(km_kbp_usv ch) {
return (((ch) >= Uni_FD_NONCHARACTER_START && (ch) <= Uni_FD_NONCHARACTER_END) || Uni_IsEndOfPlaneNonCharacter(ch));
}
inline bool Uni_InCodespace(km_kbp_usv ch) {
return ((ch) <= Uni_MAX_CODEPOINT);
};
inline bool Uni_IsValid(km_kbp_usv ch) {
return (Uni_InCodespace(ch) && !Uni_IsSurrogate(ch) && !Uni_IsNoncharacter(ch));
}
inline bool Uni_IsValid(km_kbp_usv start, km_kbp_usv end) {
if (!Uni_IsValid(end) || !Uni_IsValid(start) || (end < start)) {
// start or end out of range, or inverted range
return false;
} else if ((start <= Uni_SURROGATE_END) && (end >= Uni_SURROGATE_START)) {
// contains some of the surrogate range
return false;
} else if ((start <= Uni_FD_NONCHARACTER_END) && (end >= Uni_FD_NONCHARACTER_START)) {
// contains some of the noncharacter range
return false;
} else if ((start & Uni_PLANE_MASK) != (end & Uni_PLANE_MASK)) {
// start and end are on different planes, meaning that the U+__FFFE/U+__FFFF noncharacters
// are contained.
// As a reminder, we already checked that start/end are themselves valid,
// so we know that 'end' is not on a noncharacter at end of plane.
return false;
} else {
return true;
}
}
} // namespace kmx
} // namespace kbp
} // namespace km

1
core/src/ldml/C9134_ldml_markers.md
View file

@ -27,7 +27,6 @@ Markers can appear in both 'emitting' and 'matching-only' areas:
#### Match only
- `transform from=` to match markers
- `transform after=` to match markers
- `display to=` for matching keys which contain markers
## Theory / Encoding

38
core/src/ldml/ldml_processor.cpp
View file

@ -240,51 +240,50 @@ ldml_processor::process_event(
// Construct a context buffer of all the KM_KBP_BT_CHAR items
// Extract the context into 'ctxt' for transforms to process
if (!!transforms) {
// if no transforms, no reason to do this extraction
// if no transforms, no reason to do this extraction (ctxt will remain empty)
auto &cp = state->context();
// We're only interested in as much of the context as is a KM_KBP_BT_CHAR.
uint8_t last_type = KM_KBP_BT_UNKNOWN;
for (auto c = cp.rbegin(); c != cp.rend(); c++) {
last_type = c->type;
if (last_type != KM_KBP_BT_CHAR) {
// not a char, get out
// not a char, stop here
// TODO-LDML: markers?
break;
}
ctxt.emplace_front(1, c->character);
// extract UTF-32 to 1 or 2 UTF-16 chars in a string
km::kbp::kmx::char16_single buf;
const int len = km::kbp::kmx::Utf32CharToUtf16(c->character, buf);
const std::u16string str(buf.ch, len);
ctxt.push_front(str); // prepend to string
}
}
// Look up the key
const std::u16string str = keys.lookup(vk, modifier_state);
if (str.empty()) {
// not found
// not found, so pass the keystroke on to the Engine
state->actions().push_invalidate_context();
state->actions().push_emit_keystroke();
break; // ----- commit and exit
}
// found the correct string - push it into the context and actions
const std::u32string str32 = kmx::u16string_to_u32string(str);
for(size_t i=0; i<str32.length(); i++) {
state->context().push_character(str32[i]);
state->actions().push_character(str32[i]);
for (const auto &ch : str32) {
state->context().push_character(ch);
state->actions().push_character(ch);
}
// Now process transforms
// Process the transforms
if (!!transforms) {
// add the newly added char to ctxt
ctxt.push_back(str);
ctxt.push_back(str32);
std::u32string outputString;
std::u16string outputString;
// TODO-LDML: unroll ctxt into a str. Would be better to have transforms be able to process a vector
std::u16string ctxtstr;
for (size_t i = 0; i < ctxt.size(); i++) {
ctxtstr.append(ctxt[i]);
std::u32string ctxtstr;
for (const auto &ch : ctxt) {
ctxtstr.append(ch);
}
// check if the context matched, and if so how much (at the end)
const size_t matchedContext = transforms->apply(ctxtstr, outputString);
if (matchedContext > 0) {
@ -296,10 +295,9 @@ ldml_processor::process_event(
state->actions().push_backspace(KM_KBP_BT_CHAR, deletedChar); // Cause prior char to be removed
}
// Now, add in the updated text
const std::u32string outstr32 = kmx::u16string_to_u32string(outputString);
for (size_t i = 0; i < outstr32.length(); i++) {
state->context().push_character(outstr32[i]);
state->actions().push_character(outstr32[i]);
for (const auto &ch : outputString) {
state->context().push_character(ch);
state->actions().push_character(ch);
}
}
}

289
core/src/ldml/ldml_transforms.cpp
View file

@ -9,6 +9,8 @@
#include "debuglog.h"
#include <algorithm>
#include <string>
#include "kmx/kmx_xstring.h"
#ifndef assert
#define assert(x) // TODO-LDML
@ -18,6 +20,22 @@ namespace km {
namespace kbp {
namespace ldml {
/**
* \def KMXPLUS_DEBUG_TRANSFORM
* define KMXPLUS_DEBUG_TRANSFORM=1 to enable verbose processing of transforms/reorders
* The default is 0, which only notes initialization and exceptional cases
*/
#ifndef KMXPLUS_DEBUG_TRANSFORM
#define KMXPLUS_DEBUG_TRANSFORM 0
#endif
#if KMXPLUS_DEBUG_TRANSFORM
#define DebugTran(msg, ...) DebugLog(msg, ##__VA_ARGS__)
#else
#define DebugTran(msg, ...)
#endif
element::element(const USet &new_u, KMX_DWORD new_flags)
: chr(), uset(new_u), flags((new_flags & ~LDML_ELEM_FLAGS_TYPE) | LDML_ELEM_FLAGS_TYPE_USET) {
}
@ -67,6 +85,16 @@ element::matches(km_kbp_usv ch) const {
}
}
void
element::dump() const {
if (is_uset()) {
DebugLog("element order=%d USET", (int)get_order());
uset.dump();
} else {
DebugLog("element order=%d U+%04X", (int)get_order(), (int)chr);
}
}
int
reorder_sort_key::compare(const reorder_sort_key &other) const {
int primaryResult = (int)primary - (int)other.primary;
@ -83,7 +111,10 @@ reorder_sort_key::compare(const reorder_sort_key &other) const {
} else if (quaternaryResult) {
return quaternaryResult;
} else {
assert(quaternaryResult); // quaternary is a string index, should always be !=
// We don't expect to get here. quaternaryResult is the string index, which
// should be unequal.
assert(quaternaryResult);
// We have the underlying character, so use the binary order as a tiebreaker.
int identityResult = (int)ch - (int)other.ch; // tie breaker
return identityResult;
}
@ -94,6 +125,11 @@ reorder_sort_key::operator<(const reorder_sort_key &other) const {
return (compare(other) < 0);
}
bool
reorder_sort_key::operator>(const reorder_sort_key &other) const {
return (compare(other) > 0);
}
std::deque<reorder_sort_key>
reorder_sort_key::from(const std::u32string &str) {
// construct a 'baseline' sort key, that is, in the absence of
@ -102,6 +138,10 @@ reorder_sort_key::from(const std::u32string &str) {
auto s = str.begin(); // str iterator
size_t c = 0; // str index
for (auto e = str.begin(); e < str.end(); e++, s++, c++) {
// primary weight: 0
// seconary weight: c (the string index)
// tertiary weight: 0
// quaternary weight: c (the index again)
keylist.emplace_back(reorder_sort_key{*s, 0, c, 0, c});
}
return keylist;
@ -116,9 +156,13 @@ reorder_sort_key::dump() const {
size_t
element_list::match_end(const std::u32string &str) const {
if (str.size() < size()) {
return 0; // input string too short, can't possibly match
// input string too short, can't possibly match.
// This assumes each element is a single char, no string elements.
return 0;
}
// s: iterate from end to front of string
// For example, if str = 'abcd', we try to match 'd', then 'c', then 'b', then 'a'
// starting with the end of the element list.
auto s = str.rbegin();
// e: end to front on elements.
// we know the # of elements is <= length of string,
@ -137,8 +181,8 @@ element_list::match_end(const std::u32string &str) const {
bool
element_list::load(const kmx::kmx_plus &kplus, kmx::KMXPLUS_ELEM id) {
KMX_DWORD elementsLength;
auto elements = kplus.elem->getElementList(id, elementsLength);
assert((elementsLength == 0) || (elements != nullptr));
auto elements = kplus.elem->getElementList(id, elementsLength); // pointer to beginning of element list
assert((elementsLength == 0) || (elements != nullptr)); // it could be a 0-length list
for (size_t i = 0; i<elementsLength; i++) {
auto e = elements[i];
KMX_DWORD flags = e.flags;
@ -147,34 +191,63 @@ element_list::load(const kmx::kmx_plus &kplus, kmx::KMXPLUS_ELEM id) {
km_kbp_usv ch = e.element;
emplace_back(ch, flags); // char
} else if (type == LDML_ELEM_FLAGS_TYPE_USET) {
// need to load a USet
auto u = kplus.usetHelper.getUset(e.element);
if (!u.valid()) {
DebugLog("Error, invalid UnicodeSet at element %d", (int)i);
u.dump();
assert(u.valid());
return false;
}
emplace_back(u, flags);
} else {
// not handled
// reorders don't use 'string' element types, so we don't expect them here.
assert((type != LDML_ELEM_FLAGS_TYPE_USET) && (type != LDML_ELEM_FLAGS_TYPE_CHAR));
return false;
}
}
#if KMXPLUS_DEBUG_TRANSFORM
DebugTran("Loaded:");
dump();
#endif
return true;
}
std::deque<reorder_sort_key> &
element_list::update_sort_key(size_t offset, std::deque<reorder_sort_key> &key) const {
/** string index */
size_t c = 0;
for (auto e = begin(); e < end(); e++) {
for (auto e = begin(); e < end(); e++, c++) {
/** update this key */
auto &k = key.at(offset + c);
// we double check that the character matches. otherwise something
// has really gone awry, because we shouldn't be here if this element list doesn't apply.
if (!e->matches(k.ch)) {
DebugLog("!! updateSortKey(%d+%d): element did not re-match the sortkey", offset, c);
DebugLog("!! Internal Error: updateSortKey(%d+%d): element did not re-match the sortkey", offset, c);
k.dump();
// TODO-LDML: assertion follows
assert(e->matches(k.ch)); // double check that this element matches
}
assert(e->matches(k.ch)); // double check that this element matches
// we only update primary and tertiary weights
k.primary = e->get_order();
k.tertiary = e->get_tertiary(); // TODO-LDML: need more detailed tertiary work
c++;
// TODO-LDML: need more detailed tertiary work
k.tertiary = e->get_tertiary();
#if KMXPLUS_DEBUG_TRANSFORM
DebugTran("Updating at +%d", c);
k.dump();
#endif
}
return key;
}
void
element_list::dump() const {
DebugLog("element_list[%d]", size());
for (const auto &e : *this) {
e.dump();
}
}
reorder_entry::reorder_entry(const element_list &new_elements) : elements(new_elements), before() {
}
reorder_entry::reorder_entry(const element_list &new_elements, const element_list &new_before) : elements(new_elements), before(new_before) {
@ -183,11 +256,14 @@ reorder_entry::reorder_entry(const element_list &new_elements, const element_lis
size_t
reorder_entry::match_end(std::u32string &str, size_t offset, size_t len) const {
auto substr = str.substr(offset, len);
// first, see if the elements match. If not, this entry doesn't apply
size_t match_len = elements.match_end(substr);
if (match_len == 0) {
return 0;
}
// Now we need to check if there is a "before=" element string that
// is also a precondition.
if (!before.empty()) {
// does not match before offset
std::u32string prefix = substr.substr(0, substr.size() - match_len);
@ -209,95 +285,138 @@ reorder_group::apply(std::u32string &str) const {
// get a baseline sort key
auto sort_keys = reorder_sort_key::from(str);
// DebugLog("Baseline Keys:");
// for (auto e = sort_keys.begin(); e < sort_keys.end(); e++) {
// e->dump();
// }
// apply ALL reorders in the group.
// size_t c = 0;
for (auto r = list.begin(); r < list.end(); r++) {
for (const auto &r : list) {
// work backward from end of string forward
// That is, see if "abc" matches "abc" or "ab" or "a"
for (size_t s = str.size(); s > 0; s--) {
size_t submatch = r->match_end(str, 0, s);
size_t submatch = r.match_end(str, 0, s);
if (submatch != 0) {
#if KMXPLUS_DEBUG_TRANSFORM
DebugTran("Matched: %S (off=%d, len=%d)", str.c_str(), 0, s);
r.elements.dump();
#endif
// update the sort key
size_t sub_match_start = s - submatch;
r->elements.update_sort_key(sub_match_start, sort_keys);
some_match = true;
r.elements.update_sort_key(sub_match_start, sort_keys);
some_match = true; // record that there was a match
}
}
// c++;
}
if (!some_match) {
// DebugLog("Skip: No reorder elements matched.");
// get out if nothing matched.
// the sortkey won't be "interesting", and the sort
// will be a no-op.
DebugTran("Skip: No reorder elements matched.");
return false; // nothing matched, so no work.
}
size_t match_len = str.size(); // TODO-LDML: for now, assume entire match
// DebugLog("Updated Keys:");
// for (auto e = sort_keys.begin(); e < sort_keys.end(); e++) {
// e->dump();
// }
std::u32string prefix = str;
prefix.resize(str.size() - match_len); // just the part before the matched part.
// just the suffix (the matched part)
std::u32string suffix = str.substr(prefix.size(), match_len);
// sort it! Here's where the reorder happens
// TODO: need to sort only between primary bases…
std::sort(sort_keys.begin(), sort_keys.end());
#if 0
// TODO-LDML :need to sort sub-runs
for(auto e = sort_keys.end(); !applied && e > sort_keys.begin(); e--) {
if (e->primary == 0) {
// Got it.
std::sort(e, sort_keys.end());
// DebugLog("… sorting at q=%d", (int)e->quaternary);
}
#if KMXPLUS_DEBUG_TRANSFORM
DebugTran("Updated sortkey");
for (const auto &r : sort_keys) {
r.dump();
}
#endif
// recombine into a str
// TODO-LDML: for now, assume matches entire string.
// A needed optimization here would be to detect a common substring
// at the end of the old and new strings, and keep the match_len
// minimal. This reduces thrash in core's context.
size_t match_len = str.size();
// 'prefix' is the unmatched string before the match
// TODO-LDML: right now, this is empty.
std::u32string prefix = str;
prefix.resize(str.size() - match_len); // just the part before the matched part.
// Now, we need to actually do the sorting, but we must only sort
// 'runs' beginning with 0-weight keys.
// Consider the 'roast' example in the spec, you might end up with the following:
// codepoint (pri, sec, ter, quat)
// U+1A21 (0, 0, 0, 0)
// U+1A60 (127, 1, 0, 1)
// U+1A45 (0, 2, 0, 2)
// U+1A6B (42, 3, 0, 3)
// U+1A76 (55, 4, 0, 4)
// This example happens to be in order, but must be sorted in two diferent ranges,
// with secondary (index) values of [0,1] and [2,4]
//
// Another example might look like the following:
// U+1A21 (0, 0, 0, 0)
// U+1A6B (42, 1, 0, 1)
// U+1A76 (55, 2, 0, 2)
// U+1A60 (10, 3, 0, 3)
// U+1A45 (10, 4, 0, 3)
// Here there is only a single range to sort [0,4]
/** pointer to the beginning of the current run. */
std::deque<reorder_sort_key>::iterator run_start = sort_keys.begin();
for(auto e = run_start; e != sort_keys.end(); e++) {
if ((e->primary == 0) && (e != run_start)) { // it's a base
auto run_end = e - 1;
DebugTran("Sorting subrange quaternary=[%d..]", run_start->quaternary);
std::sort(run_start, run_end); // reversed because it's a reverse iterator…?
// move the start
run_start = e; // next run starts here
}
}
// sort the last run in the string as well.
if (run_start != sort_keys.end()) { // TODO-LDML: skip if a single-char run
DebugTran("Sorting final subrange quaternary=[%d..]", run_start->quaternary);
std::sort(run_start, sort_keys.end()); // reversed because it's a reverse iterator…?
}
// recombine into a string by pulling out the 'ch' value
// that's in each sortkey element.
std::u32string newSuffix;
size_t q = sort_keys.begin()->quaternary; //
size_t q = sort_keys.begin()->quaternary; // start with the first quaternary
for (auto e = sort_keys.begin(); e < sort_keys.end(); e++, q++) {
if (q != e->quaternary) { // something rearranged in this subrange
if (q != e->quaternary) {
// something rearranged in this subrange, because the quaternary values are out of order.
applied = true;
}
// collect the characters
newSuffix.append(1, e->ch);
}
if (applied) {
// DebugLog("Final Sort");
// for (auto e = sort_keys.begin(); e < sort_keys.end(); e++) {
// e->dump();
// }
str.resize(prefix.size());
str.append(newSuffix);
} else {
// DebugLog("Skip: no reordering change detected");
DebugTran("Skip: sorting caused no reordering");
}
#if KMXPLUS_DEBUG_TRANSFORM
DebugTran("Sorted sortkey");
for (const auto &r : sort_keys) {
r.dump();
}
#endif
return applied;
}
transform_entry::transform_entry(const std::u16string &from, const std::u16string &to) : fFrom(from), fTo(to) {
transform_entry::transform_entry(const std::u32string &from, const std::u32string &to) : fFrom(from), fTo(to) {
}
size_t
transform_entry::match(const std::u16string &input) const {
transform_entry::match(const std::u32string &input) const {
if (input.length() < fFrom.length()) {
// TODO-LDML: regex
// Too small, can't match.
return 0;
}
// string at end
auto substr = input.substr(input.length() - fFrom.length(), fFrom.length());
if (substr != fFrom) {
// end of string doesn't match
return 0;
}
// match length == fFrom.length
return substr.length();
}
std::u16string
transform_entry::apply(const std::u16string & /*input*/, size_t /*matchLen*/) const {
std::u32string
transform_entry::apply(const std::u32string & /*input*/, size_t /*matchLen*/) const {
// TODO-LDML: regex
// For now, we just return the 'to' string literally.
return fTo;
}
@ -326,7 +445,7 @@ transform_group::transform_group() {
* return the first transform match in this group
*/
const transform_entry *
transform_group::match(const std::u16string &input, size_t &subMatched) const {
transform_group::match(const std::u32string &input, size_t &subMatched) const {
for (auto transform = begin(); (subMatched == 0) && (transform < end()); transform++) {
// TODO-LDML: non regex implementation
// is the match area too short?
@ -346,7 +465,7 @@ transform_group::match(const std::u16string &input, size_t &subMatched) const {
* @return match length: number of chars at end of input string to modify. 0 if no match.
*/
size_t
transforms::apply(const std::u16string &input, std::u16string &output) {
transforms::apply(const std::u32string &input, std::u32string &output) {
/**
* Example:
* Group0: za -> c, a -> bb
@ -381,7 +500,7 @@ transforms::apply(const std::u16string &input, std::u16string &output) {
*/
size_t matched = 0;
/** modified copy of input */
std::u16string updatedInput = input;
std::u32string updatedInput = input;
for (auto group = transform_groups.begin(); group < transform_groups.end(); group++) {
// for each transform group
// break out once there's a match
@ -393,13 +512,14 @@ transforms::apply(const std::u16string &input, std::u16string &output) {
// find the first match in this group (if present)
// TODO-LDML: check if reorder
if (group->type == any_group_type::transform) {
auto transform = group->transform.match(updatedInput, subMatched);
auto entry = group->transform.match(updatedInput, subMatched);
if (transform != nullptr) {
if (entry != nullptr) {
// now apply the found transform
// update subOutput (string) and subMatched
std::u16string subOutput = transform->apply(updatedInput, subMatched);
// the returned string must replace the last "subMatched" chars of the string.
std::u32string subOutput = entry->apply(updatedInput, subMatched);
// remove the matched part of the updatedInput
updatedInput.resize(updatedInput.length() - subMatched); // chop of the subMatched part at end
@ -421,7 +541,17 @@ transforms::apply(const std::u16string &input, std::u16string &output) {
}
}
} else if (group->type == any_group_type::reorder) {
// TODO-LDML reorder
// TODO-LDML: cheesy solution. We should be finding a smaller
// common match here.
std::u32string str2 = updatedInput;
if (group->reorder.apply(str2)) {
// pretend the whole thing matched
output.resize(0);
output.append(str2);
updatedInput.resize(0);
updatedInput.append(str2);
matched = output.length();
}
}
// else: continue to next group
}
@ -441,10 +571,10 @@ transforms::apply(const std::u16string &input, std::u16string &output) {
return matched;
}
// simple impl
bool
transforms::apply(std::u16string &str) {
std::u16string output;
transforms::apply(std::u32string &str) {
// simple implementation for tests
std::u32string output;
size_t matchLength = apply(str, output);
if (matchLength == 0) {
return false;
@ -454,23 +584,6 @@ transforms::apply(std::u16string &str) {
return true;
}
bool
transforms::apply(std::u32string &str) {
bool rc = false;
// TODO-LDML: PoC implementation for now, need to refactor into fcns
// ONLY reorder
for (auto group = transform_groups.begin(); group < transform_groups.end(); group++) {
assert(group->type == reorder); // TODO-LDML
auto rgroup = group->reorder;
if (rgroup.apply(str)) {
rc = true;
}
}
return rc;
}
// Loader
transforms *
transforms::load(
const kmx::kmx_plus &kplus,
@ -516,17 +629,18 @@ transforms::load(
for (KMX_DWORD itemNumber = 0; itemNumber < group->count; itemNumber++) {
const kmx::COMP_KMXPLUS_TRAN_TRANSFORM *element = tranHelper.getTransform(group->index + itemNumber);
const std::u16string fromStr = kplus.strs->get(element->from);
const std::u16string toStr = kplus.strs->get(element->to);
const std::u32string fromStr = kmx::u16string_to_u32string(kplus.strs->get(element->from));
const std::u32string toStr = kmx::u16string_to_u32string(kplus.strs->get(element->to));
std::u16string mapFrom, mapTo;
if (element->mapFrom && element->mapTo) {
// strings: variable name
// strings: variable name of from/to
// TODO-LDML: not implemented
mapFrom = kplus.strs->get(element->mapFrom);
mapTo = kplus.strs->get(element->mapTo);
}
newGroup.emplace_back(fromStr, toStr); // creating a transform_entry
newGroup.emplace_back(fromStr, toStr /* ,mapFrom, mapTo */); // creating a transform_entry
}
transforms->addGroup(newGroup);
} else if (group->type == LDML_TRAN_GROUP_TYPE_REORDER) {
@ -544,6 +658,7 @@ transforms::load(
if (load_ok) {
newGroup.list.emplace_back(elements, before);
} else {
DebugLog("reorder elements(%d+%d) failed to load", group->index, itemNumber);
return nullptr;
}
}

61
core/src/ldml/ldml_transforms.hpp
View file

@ -24,7 +24,7 @@ using km::kbp::kmx::USet;
* Type of a group
*/
enum any_group_type {
transform = LDML_TRAN_GROUP_TYPE_REORDER,
transform = LDML_TRAN_GROUP_TYPE_TRANSFORM,
reorder = LDML_TRAN_GROUP_TYPE_REORDER,
};
@ -33,24 +33,30 @@ enum any_group_type {
*/
class element {
public:
/** from a USet */
/** construct from a USet */
element(const USet &u, KMX_DWORD flags);
/** from a single char */
/** construct from a single char */
element(km_kbp_usv ch, KMX_DWORD flags);
/** @returns true if a USet type */
bool is_uset() const;
/** @returns true if prebase bit set*/
bool is_prebase() const;
/** @returns true if tertiary base bit set */
bool is_tertiary_base() const;
signed char get_tertiary() const;
/** @returns the primary order */
signed char get_order() const;
/** @returns the tertiary order */
signed char get_tertiary() const;
/** @returns raw elem flags */
KMX_DWORD get_flags() const;
/** @returns true if matches this character*/
bool matches(km_kbp_usv ch) const;
/** debugging: dump this element via DebugLog() */
void dump() const;
private:
// TODO-LDML: support multi-char strings
// TODO-LDML: support multi-char strings?
const km_kbp_usv chr;
const USet uset;
const KMX_DWORD flags;
@ -62,30 +68,30 @@ private:
class transform_entry {
public:
transform_entry(
const std::u16string &from,
const std::u16string &to
const std::u32string &from,
const std::u32string &to
/*TODO-LDML: mapFrom, mapTo*/
);
/**
* @returns length if it's a match
*/
size_t match(const std::u16string &input) const;
size_t match(const std::u32string &input) const;
/**
* @returns output string
*/
std::u16string apply(const std::u16string &input, size_t matchLen) const;
std::u32string apply(const std::u32string &input, size_t matchLen) const;
private:
const std::u16string fFrom; // TODO-LDML: regex
const std::u16string fTo;
const std::u32string fFrom; // TODO-LDML: regex
const std::u32string fTo;
};
/**
* An ordered list of strings.
*/
typedef std::deque<std::u16string> string_list;
typedef std::deque<std::u32string> string_list;
/**
* a group of <transform> entries - a <transformGroup>
@ -100,7 +106,7 @@ public:
* @param subMatched on output, the matched length
* @returns alias to transform_entry or nullptr
*/
const transform_entry *match(const std::u16string &input, size_t &subMatched) const;
const transform_entry *match(const std::u32string &input, size_t &subMatched) const;
};
/** a single char, categorized according to reorder rules*/
@ -111,13 +117,12 @@ struct reorder_sort_key {
signed char tertiary; // tertiary value, defaults to 0
size_t quaternary; // index again
/**
* Return -1, 0, 1 depending on order
*/
/** @returns -1, 0, 1 depending on ordering */
int compare(const reorder_sort_key &other) const;
bool operator<(const reorder_sort_key &other) const;
bool operator>(const reorder_sort_key &other) const;
/** create a 'baseline' sort key, all 0 primary weights */
/** create a 'baseline' sort key, with each character having primary weight 0 */
static std::deque<reorder_sort_key> from(const std::u32string &str);
/** TODO-LDML: for debugging. */
@ -136,7 +141,7 @@ public:
* Update the deque (see reorder_sort_key::from()) with the weights from this element list
* starting at the beginning of this element list
* @param offset start at this offset in the deque. Still starts at the first element
* @param the key deque to update
* @param key key deque to update
* @returns the key parameter
*/
std::deque<reorder_sort_key> &update_sort_key(size_t offset, std::deque<reorder_sort_key> &key) const;
@ -144,6 +149,9 @@ public:
/** construct from KMX+ elem id*/
bool
load(const kmx::kmx_plus& kplus, kmx::KMXPLUS_ELEM id);
/** TODO-LDML: for debugging */
void dump() const;
};
class reorder_entry {
@ -217,13 +225,7 @@ public:
* @param output if matched, contains the replacement output text
* @return length in chars of the input (counting from the end) which matched context
*/
size_t apply(const std::u16string &input, std::u16string &output);
/**
* For tests
* @return true if str was altered
*/
bool apply(std::u16string &str);
size_t apply(const std::u32string &input, std::u32string &output);
/**
* For tests - TODO-LDML only supports reorder
@ -232,12 +234,13 @@ public:
bool apply(std::u32string &str);
public:
/** load from a kmx_plus data section, either tran or bksp */
static transforms *
load(const kmx::kmx_plus &kplus, const kbp::kmx::COMP_KMXPLUS_TRAN *tran, const kbp::kmx::COMP_KMXPLUS_TRAN_Helper &tranHelper);
load(const kmx::kmx_plus &kplus,
const kbp::kmx::COMP_KMXPLUS_TRAN *tran,
const kbp::kmx::COMP_KMXPLUS_TRAN_Helper &tranHelper);
};
/**
* Loader for transform groups (from tran or bksp)
*/
} // namespace ldml
} // namespace kbp
} // namespace km

53
core/tests/unit/kmnkbd/test_kmx_xstring.cpp
View file

@ -1238,6 +1238,7 @@ test_xstrlen_ignoreifopt() {
void
test_utf32() {
std::cout << "== " << __FUNCTION__ << std::endl;
const KMX_DWORD u295 = 0x0127; // ħ
assert(Uni_IsBMP(u295));
@ -1270,6 +1271,7 @@ test_utf32() {
void
test_u16string_to_u32string() {
std::cout << "== " << __FUNCTION__ << std::endl;
// normal cases
{
const std::u32string str = u16string_to_u32string(u"");
@ -1327,6 +1329,56 @@ test_u16string_to_u32string() {
}
}
void test_is_valid() {
std::cout << "== " << __FUNCTION__ << std::endl;
// valid
assert_equal(Uni_IsValid(0x0000), true);
assert_equal(Uni_IsValid(0x0127), true);
assert_equal(Uni_IsValid(U'🙀'), true);
// invalid
assert_equal(Uni_IsValid(0xDECAFBAD), false); // out of range
assert_equal(Uni_IsValid(0x566D4128), false);
assert_equal(Uni_IsValid(0xFFFF), false); // nonchar
assert_equal(Uni_IsValid(0xFFFE), false); // nonchar
assert_equal(Uni_IsValid(0x10FFFF), false); // nonchar
assert_equal(Uni_IsValid(0x10FFFE), false); // nonchar
assert_equal(Uni_IsValid(0x01FFFF), false); // nonchar
assert_equal(Uni_IsValid(0x01FFFE), false); // nonchar
assert_equal(Uni_IsValid(0x02FFFF), false); // nonchar
assert_equal(Uni_IsValid(0x02FFFE), false); // nonchar
assert_equal(Uni_IsValid(0xFDD1), false); // nonchar
assert_equal(Uni_IsValid(0xFDD0), false); // nonchar
// positive range test
assert_equal(Uni_IsValid(0x100000, 0x10FFFD), true);
assert_equal(Uni_IsValid(0x10, 0x20), true);
assert_equal(Uni_IsValid(0x100000, 0x10FFFD), true);
// all valid ranges in BMP
assert_equal(Uni_IsValid(0x0000, 0xD7FF), true);
assert_equal(Uni_IsValid(0xD800, 0xDFFF), false);
assert_equal(Uni_IsValid(0xE000, 0xFDCF), true);
assert_equal(Uni_IsValid(0xFDD0, 0xFDEF), false);
assert_equal(Uni_IsValid(0xFDF0, 0xFDFF), true);
assert_equal(Uni_IsValid(0xFDF0, 0xFFFD), true);
// negative range test
assert_equal(Uni_IsValid(0, 0x10FFFF), false); // ends with nonchar
assert_equal(Uni_IsValid(0, 0x10FFFD), false); // contains lots o' nonchars
assert_equal(Uni_IsValid(0x20, 0x10), false); // swapped
assert_equal(Uni_IsValid(0xFDEF, 0xFDF0), false); // just outside range
assert_equal(Uni_IsValid(0x0000, 0x010000), false); // crosses noncharacter plane boundary and other stuff
assert_equal(Uni_IsValid(0x010000, 0x020000), false); // crosses noncharacter plane boundary
assert_equal(Uni_IsValid(0x0000, 0xFFFF), false); // crosses other BMP prohibited and plane boundary
assert_equal(Uni_IsValid(0x0000, 0xFFFD), false); // crosses other BMP prohibited
assert_equal(Uni_IsValid(0x0000, 0xE000), false); // crosses surrogate space
assert_equal(Uni_IsValid(0x0000, 0x20FFFF), false); // out of bounds
assert_equal(Uni_IsValid(0x10FFFD, 0x20FFFF), false); // out of bounds
}
constexpr const auto help_str = u"\
test_kmx_xstring [--color]\n\
\n\
@ -1349,6 +1401,7 @@ int main(int argc, char *argv []) {
test_xstrlen_ignoreifopt();
test_utf32();
test_u16string_to_u32string();
test_is_valid();
return 0;
}

5
core/tests/unit/ldml/keyboards/fr-t-k0-azerty-test.xml → core/tests/unit/ldml/keyboards/k_020_fr-test.xml
View file

@ -1,10 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- Note: similar to, but slightly ahead of, fr-t-k0-azerty -->
<!DOCTYPE keyboardTest SYSTEM "../../../../../resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboardTest.dtd">
<keyboardTest conformsTo="techpreview">
<info keyboard="fr-t-k0-azerty.xml" author="Team Keyboard" name="fr-test" />
<info keyboard="k_020_fr.xml" author="Team Keyboard" name="fr-test-updated" />
<repertoire name="simple-repertoire" chars="[a b c d e \u0022]" type="simple" /> <!-- verify that these outputs are all available from simple keys on any layer, for all form factors -->
<repertoire name="chars-repertoire" chars="[á é ó]" type="gesture" /> <!-- verify that these outputs are all available from simple or gesture keys on any layer, for touch -->
<tests name="key-tests">
<tests name="key-tests-updated">
<test name="key-test">
<startContext to="abc\u0022..."/>
<!-- tests by pressing key ids -->

13
core/tests/unit/ldml/keyboards/fr-t-k0-azerty.xml → core/tests/unit/ldml/keyboards/k_020_fr.xml
View file

@ -1,4 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- Note: similar to, but slightly ahead of, fr-t-k0-azerty -->
<!DOCTYPE keyboard SYSTEM "../../../../../resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard.dtd">
<keyboard locale="fr-t-k0-azerty" conformsTo="techpreview">
<locales>
@ -7,11 +8,11 @@
<!-- 'conformsTo' gives CLDR spec conformance. Distinguishes from prior
keyboard formats -->
<!-- 'version' element is now optional -->
<version number="1.0.0" />
<version number="1.0.1" />
<info author="Team Keyboard" normalization="NFC" layout="AZERTY" indicator="FR" />
<names>
<name value="French Test" />
<name value="French Test AZERTY" />
<name value="French Test Updated" />
<name value="French Test AZERTY Updated" />
</names>
<settings fallback="omit" /> <!-- "hide" makes this act like a dead keyTBD -->
@ -22,15 +23,15 @@
<vkey from="A" to="Q" />
<vkey from="Z" to="W" />
<vkey from="GRAVE" to="QUOTE" /> <!-- TODO-LDML: GRAVE=BKQUOTE in ldml? -->
<!-- <vkey from="MINUS" to="LBRACKET" /> --> <!-- TODO-LDML: missing MINUS, LBRKT=LBRACKET in ldml? -->
<vkey from="MINUS" to="LBRACKET" /> <!-- TODO-LDML: missing MINUS, LBRKT=LBRACKET in ldml? -->
<!-- i.e., OEM_3 >>> OEM_7 -->
<vkey from="LBRACKET" to="RBRACKET" /> <!-- TODO-LDML: RBRKT=RBRACKET in ldml? -->
<!-- <vkey from="COLON" to="M" /> --> <!-- TODO-LDML: missing COLON-->
<vkey from="COLON" to="M" /> <!-- TODO-LDML: missing COLON-->
<vkey from="QUOTE" to="GRAVE" /> <!-- TODO-LDML: GRAVE=BKQUOTE in ldml? -->
<vkey from="M" to="COMMA" />
<vkey from="COMMA" to="PERIOD" />
<vkey from="PERIOD" to="SLASH" />
<!-- <vkey from="SLASH" to="EXCLAMATION" /> --> <!-- TODO-LDML: missing: EXCLAMATION -->
<vkey from="SLASH" to="EXCLAMATION" /> <!-- TODO-LDML: missing: EXCLAMATION -->
<!-- i.e., OEM_2 >>> OEM_8 -->
</vkeys>

20
core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana-test.xml
View file

@ -3,16 +3,30 @@
<keyboardTest conformsTo="techpreview">
<info keyboard="k_200_reorder_nod_Lana.xml" author="Team Keyboard" name="nod-Lana" />
<tests name="reorder-tests">
<test name="roast0-orig">
<test name="roast00-orig-emit">
<startContext to="" />
<emit to="\u1A21\u1A60\u1A45\u1A6B\u1A76" />
<check result="\u1A21\u1A60\u1A45\u1A6B\u1A76" />
</test>
<test name="roast01-orig-key">
<startContext to="" />
<keystroke key="kha" />
<check result="\u1A21" />
<keystroke key="sakot" />
<check result="\u1A21\u1A60" />
<keystroke key="wa" />
<check result="\u1A21\u1A60\u1A45" />
<keystroke key="o" />
<check result="\u1A21\u1A60\u1A45\u1A6B" />
<keystroke key="t2" />
<check result="\u1A21\u1A60\u1A45\u1A6B\u1A76" />
</test>
<!-- <test name="roast1-upper-first">
<test name="roast02-orig-1key">
<startContext to="" />
<keystroke key="roast" />
<check result="\u1A21\u1A60\u1A45\u1A6B\u1A76" />
</test>
<test name="roast1-upper-first">
<startContext to="" />
<keystroke key="kha" />
<keystroke key="o" />
@ -38,6 +52,6 @@
<keystroke key="wa" />
<keystroke key="t2" />
<check result="\u1A21\u1A60\u1A45\u1A6B\u1A76" />
</test> -->
</test>
</tests>
</keyboardTest>

6
core/tests/unit/ldml/keyboards/k_200_reorder_nod_Lana.xml
View file

@ -13,16 +13,20 @@
</names>
<keys>
<!-- keys in order for 'roast' -->
<key id="kha" to="\u{1A21}" />
<key id="sakot" to="\u{1A60}" />
<key id="wa" to="\u{1A45}" />
<key id="o" to="\u{1A6B}" />
<key id="t2" to="\u{1A76}" />
<!-- and a shortcut key for this common word -->
<key id="roast" to="\u{1A21}\u{1A60}\u{1A45}\u{1A6B}\u{1A76}" />
</keys>
<layers form="us">
<layer modifier="none" id="base">
<row keys="gap" />
<row keys="roast" />
<row keys="gap wa gap gap t2 gap gap gap o gap" />
<row keys="gap sakot gap gap gap gap gap kha gap" />
<row keys="gap gap gap gap gap gap gap" />

6
core/tests/unit/ldml/keyboards/meson.build
View file

@ -9,8 +9,8 @@
# tests in resources/standards-data/ldml-keyboards/techpreview/test/
tests_from_cldr = [
'ja-Latn',
'pt-k0-abnt2',
# 'fr-t-k0-azerty', # vkey issues
# 'pt-k0-abnt2', #TODO-LDML: marker syntax fail!
'fr-t-k0-azerty',
]
tests_without_testdata = [
@ -32,7 +32,7 @@ tests_without_testdata = [
# These tests have a k_001_tiny-test.xml file as well.
tests_with_testdata = [
'k_001_tiny',
'fr-t-k0-azerty', # TODO-LDML: move to cldr above (fix vkey)
'k_020_fr', # TODO-LDML: move to cldr above (fix vkey)
'k_200_reorder_nod_Lana',
]

88
core/tests/unit/ldml/test_transforms.cpp
View file

@ -40,7 +40,7 @@ test_transforms() {
std::cout << __FILE__ << ":" << __LINE__ << " - basic " << std::endl;
{
// start with one
transform_entry te(std::u16string(u"e^"), std::u16string(u"E")); // keep it simple
transform_entry te(std::u32string(U"e^"), std::u32string(U"E")); // keep it simple
// OK now make a group do it
transforms tr;
transform_group st;
@ -51,17 +51,17 @@ test_transforms() {
// see if we can match the same
{
std::u16string src(u"barQ^");
std::u32string src(U"barQ^");
bool res = tr.apply(src);
zassert_equal(res, false);
zassert_string_equal(src, std::u16string(u"barQ^")); // no change
zassert_string_equal(src, std::u32string(U"barQ^")); // no change
}
{
std::u16string src(u"fooe^");
std::u32string src(U"fooe^");
bool res = tr.apply(src);
zassert_equal(res, true);
zassert_string_equal(src, std::u16string(u"fooE"));
zassert_string_equal(src, std::u32string(U"fooE"));
}
}
@ -73,23 +73,23 @@ test_transforms() {
// setup
{
transform_group st;
st.emplace_back(std::u16string(u"za"), std::u16string(u"c"));
st.emplace_back(std::u16string(u"a"), std::u16string(u"bb"));
st.emplace_back(std::u32string(U"za"), std::u32string(U"c"));
st.emplace_back(std::u32string(U"a"), std::u32string(U"bb"));
tr.addGroup(st);
}
{
transform_group st;
st.emplace_back(std::u16string(u"bb"), std::u16string(u"ccc"));
st.emplace_back(std::u32string(U"bb"), std::u32string(U"ccc"));
tr.addGroup(st);
}
{
transform_group st;
st.emplace_back(std::u16string(u"cc"), std::u16string(u"d"));
st.emplace_back(std::u32string(U"cc"), std::u32string(U"d"));
tr.addGroup(st);
}
{
transform_group st;
st.emplace_back(std::u16string(u"tcd"), std::u16string(u"e"));
st.emplace_back(std::u32string(U"tcd"), std::u32string(U"e"));
tr.addGroup(st);
}
@ -97,31 +97,31 @@ test_transforms() {
// see if we can match the same
{
std::u16string src(u"ta");
std::u32string src(U"ta");
bool res = tr.apply(src);
// pipe (|) symbol shows where the 'output' is delineated
// t|a --> t|bb --> t|ccc --> t|cd --> |e
zassert_string_equal(src, std::u16string(u"e"));
zassert_string_equal(src, std::u32string(U"e"));
zassert_equal(res, true);
}
{
std::u16string src(u"qza");
std::u32string src(U"qza");
bool res = tr.apply(src);
// pipe (|) symbol shows where the 'output' is delineated
// q|za -> q|c
zassert_string_equal(src, std::u16string(u"qc"));
zassert_string_equal(src, std::u32string(U"qc"));
zassert_equal(res, true);
}
{
std::u16string src(u"qa");
std::u32string src(U"qa");
bool res = tr.apply(src);
zassert_string_equal(src, std::u16string(u"qcd"));
zassert_string_equal(src, std::u32string(U"qcd"));
zassert_equal(res, true);
}
{
std::u16string src(u"tb");
std::u32string src(U"tb");
bool res = tr.apply(src);
zassert_string_equal(src, std::u16string(u"tb"));
zassert_string_equal(src, std::u32string(U"tb"));
zassert_equal(res, false);
}
}
@ -132,13 +132,13 @@ test_transforms() {
transforms tr;
{
transform_group st;
st.emplace_back(std::u16string(u"िह"), std::u16string(u"हि"));
st.emplace_back(std::u32string(U"िह"), std::u32string(U"हि"));
tr.addGroup(st);
}
{
std::u16string src(u"िह");
std::u32string src(U"िह");
bool res = tr.apply(src);
zassert_string_equal(src, std::u16string(u"हि"));
zassert_string_equal(src, std::u32string(U"हि"));
zassert_equal(res, true);
}
}
@ -163,7 +163,7 @@ test_reorder_standalone() {
const std::u32string expect = roasts[0];
// now setup the rules
const COMP_KMXPLUS_USET_RANGE ranges[] = {// 0
{0x1A75, 0x1A79}};
COMP_KMXPLUS_USET_RANGE(0x1A75, 0x1A79)};
const COMP_KMXPLUS_USET_USET usets[] = {{0, 1, 0xFFFFFFFF}};
const COMP_KMXPLUS_USET_USET &toneMarksUset = usets[0];
const USet toneMarks(&ranges[toneMarksUset.range], toneMarksUset.count);
@ -288,7 +288,7 @@ test_reorder_standalone() {
// <reorder from="\u1A60" order="127" />
element_list e0;
e0.emplace_back(U'\u1A6B', 127 << LDML_ELEM_FLAGS_ORDER_BITSHIFT);
e0.emplace_back(U'\u1A60', 127 << LDML_ELEM_FLAGS_ORDER_BITSHIFT);
rg.list.emplace_back(e0);
// <reorder from="\u1A6B" order="42" />
@ -334,19 +334,41 @@ test_reorder_standalone() {
std::cout << __FILE__ << ":" << __LINE__ << " - back to nod-Lana " << std::endl;
// TODO-LDML: move this into test code perhaps
for (size_t r = 0; r < sizeof(roasts) / sizeof(roasts[0]); r++) {
std::cout << __FILE__ << ":" << __LINE__ << " - trying roast #" << r << std::endl;
const auto &roast = roasts[r];
std::cout << __FILE__ << ":" << __LINE__ << " - trying roast #" << r << "=" << roast << std::endl;
// try apply with string
{
std::cout << "- try apply(text, output)" << std::endl;
std::u32string text = roast;
std::u32string output;
size_t len = tr.apply(text, output);
if (len == 0) {
std::cout << " (did not apply)" << std::endl;
} else {
std::cout << " applied, matchLen= " << len << std::endl;
text.resize(text.size()-len); // shrink
text.append(output);
std::cout << " = " << text << std::endl;
}
zassert_string_equal(text, expect);
}
// try all-at-once
{
std::cout << "- try apply(text)" << std::endl;
std::u32string text = roast;
if (!tr.apply(text)) {
std::cout << " (did not apply)" << std::endl;
} else if (text == roast) {
std::cout << " (suboptimal: apply returned true but made no change)" << std::endl;
} else {
std::cout << " changed to " << text;
}
zassert_string_equal(text, expect);
std::cout << " matched (converting all at once)!" << std::endl;
}
// simulate typing this one char at a time;
{
std::cout << "- try key-at-a-time" << std::endl;
std::u32string text;
for (auto ch = roast.begin(); ch < roast.end(); ch++) {
// append the string
@ -362,6 +384,24 @@ test_reorder_standalone() {
std::cout << std::endl;
}
}
// special test
{
std::cout << __FILE__ << ":" << __LINE__ << " - special test " << std::endl;
const std::u32string expect = U"\u1A21\u1A60\u1A45"; // this string shouldn't mutate at all.
{
std::u32string text = expect;
tr.apply(text);
zassert_string_equal(text, expect);
}
{
// try submatch
std::u32string text = expect;
std::u32string output;
size_t len = tr.apply(text, output);
zassert_string_equal(output, U"");
assert_equal(len, 0);
}
}
}
return EXIT_SUCCESS;
}

27
developer/src/kmc-kmn/src/compiler/compiler.ts
View file

@ -89,12 +89,18 @@ let callbackProcIdentifier = 0;
const
callbackPrefix = 'kmnCompilerCallbacks_';
interface MallocAndFree {
malloc(sz: number) : number;
free(p: number) : null;
};
let
Module: any;
export class KmnCompiler implements UnicodeSetParser {
callbackID: string; // a unique numeric id added to globals with prefixed names
callbacks: CompilerCallbacks;
wasmExports: MallocAndFree;
constructor() {
this.callbackID = callbackPrefix + callbackProcIdentifier.toString();
@ -112,7 +118,7 @@ export class KmnCompiler implements UnicodeSetParser {
return false;
}
}
this.wasmExports = (Module.wasmExports ?? Module.asm);
return this.verifyInitialized();
}
@ -427,25 +433,23 @@ export class KmnCompiler implements UnicodeSetParser {
return null;
}
const buf = Module.asm.malloc(rangeCount * 2 * Module.HEAPU32.BYTES_PER_ELEMENT);
// TODO-LDML: Catch OOM
/** return code, if positive: range count */
const buf = this.wasmExports.malloc(rangeCount * 2 * Module.HEAPU32.BYTES_PER_ELEMENT);
/** If <= 0: return code. If positive: range count */
const rc = Module.kmcmp_parseUnicodeSet(pattern, buf, rangeCount * 2);
if (rc >= 0) {
const ranges = [];
const startu = (buf / Module.HEAPU32.BYTES_PER_ELEMENT);
for (let i = 0; i < rc; i++) {
const low = Module.HEAPU32[startu + (i * 2) + 0];
const high = Module.HEAPU32[startu + (i * 2) + 1];
ranges.push([low, high]);
const start = Module.HEAPU32[startu + (i * 2) + 0];
const end = Module.HEAPU32[startu + (i * 2) + 1];
ranges.push([start, end]);
}
// TODO-LDML: no free??
// Module.asm.free(buf);
this.wasmExports.free(buf);
return new UnicodeSet(pattern, ranges);
} else {
// translate error
// TODO-LDML: no free??
// Module.asm.free(buf);
this.wasmExports.free(buf);
// translate error code into callback
this.callbacks.reportMessage(getUnicodeSetError(rc));
return null;
}
@ -455,6 +459,7 @@ export class KmnCompiler implements UnicodeSetParser {
/* c8 ignore next 2 */
return null;
}
// call with rangeCount = 0 to invoke in 'preflight' mode.
const rc = Module.kmcmp_parseUnicodeSet(pattern, 0, 0);
if (rc >= 0) {
return rc;

25
developer/src/kmc-kmn/test/test-wasm-uset.ts
View file

@ -65,12 +65,25 @@ describe('Compiler UnicodeSet function', function() {
'[[]': CompilerMessages.ERROR_UnicodeSetSyntaxError,
};
for(const [pat, expected] of Object.entries(failures)) {
callbacks.clear();
assert.notOk(compiler.parseUnicodeSet(pat, 1));
assert.equal(callbacks.messages.length, 1);
const firstMessage = callbacks.messages[0];
const code = firstMessage.code;
assert.equal(code, expected, `${compilerErrorFormatCode(code)}≠${compilerErrorFormatCode(expected)} got ${firstMessage.message} for ${pat}`);
{
// verify fails parse
callbacks.clear();
assert.notOk(compiler.parseUnicodeSet(pat, 1));
assert.equal(callbacks.messages.length, 1);
const firstMessage = callbacks.messages[0];
const code = firstMessage.code;
assert.equal(code, expected, `${compilerErrorFormatCode(code)}≠${compilerErrorFormatCode(expected)} got ${firstMessage.message} for parsing ${pat}`);
}
// skip 'out of range' because that one won't fail during sizing.
if (expected !== CompilerMessages.FATAL_UnicodeSetOutOfRange) {
// verify fails size
callbacks.clear();
assert.equal(compiler.sizeUnicodeSet(pat), -1, `sizing ${pat}`);
assert.equal(callbacks.messages.length, 1);
const firstMessage = callbacks.messages[0];
const code = firstMessage.code;
assert.equal(code, expected, `${compilerErrorFormatCode(code)}≠${compilerErrorFormatCode(expected)} got ${firstMessage.message} for sizing ${pat}`);
}
}
});
});

12
developer/src/kmc-ldml/src/compiler/disp.ts
View file

@ -1,5 +1,5 @@
import { constants } from "@keymanapp/ldml-keyboard-constants";
import { KMXPlus } from '@keymanapp/common-types';
import { KMXPlus, LDMLKeyboard, MarkerParser } from '@keymanapp/common-types';
import { CompilerMessages } from "./messages.js";
import { SectionCompiler } from "./section-compiler.js";
@ -7,8 +7,14 @@ import { SectionCompiler } from "./section-compiler.js";
import DependencySections = KMXPlus.DependencySections;
import Disp = KMXPlus.Disp;
import DispItem = KMXPlus.DispItem;
import { MarkerTracker, MarkerUse } from "./marker-tracker.js";
export class DispCompiler extends SectionCompiler {
static validateMarkers(keyboard: LDMLKeyboard.LKKeyboard, mt : MarkerTracker): boolean {
keyboard.displays?.display?.forEach(({ to }) =>
mt.add(MarkerUse.match, MarkerParser.allReferences(to)));
return true;
}
public get id() {
return constants.section.disp;
@ -38,9 +44,11 @@ export class DispCompiler extends SectionCompiler {
// displayOptions
result.baseCharacter = sections.strs.allocAndUnescapeString(this.keyboard.displays?.displayOptions?.baseCharacter);
// TODO-LDML: substitute variables!
// displays
result.disps = this.keyboard.displays?.display.map(display => ({
to: sections.strs.allocAndUnescapeString(display.to),
to: sections.strs.allocAndUnescapeString(sections.vars.substituteMarkerString(display.to)),
display: sections.strs.allocAndUnescapeString(display.display),
})) || []; // TODO-LDML: need coverage for the []

5
developer/src/kmc-ldml/src/compiler/empty-compiler.ts
View file

@ -1,6 +1,7 @@
import { SectionIdent, constants } from '@keymanapp/ldml-keyboard-constants';
import { SectionCompiler } from "./section-compiler.js";
import { LDMLKeyboard, KMXPlus, CompilerCallbacks } from "@keymanapp/common-types";
import { VarsCompiler } from './vars.js';
/**
* Compiler for typrs that don't actually consume input XML
@ -69,6 +70,6 @@ export class UsetCompiler extends EmptyCompiler {
}
/**
* For test use. The top three compilers.
* For test use. The top compilers.
*/
export const BASIC_DEPENDENCIES = [ StrsCompiler, ListCompiler, ElemCompiler ];
export const BASIC_DEPENDENCIES = [ StrsCompiler, ListCompiler, ElemCompiler, VarsCompiler ];

120
developer/src/kmc-ldml/src/compiler/keys.ts
View file

@ -1,5 +1,5 @@
import { constants } from '@keymanapp/ldml-keyboard-constants';
import { LDMLKeyboard, KMXPlus, Constants } from '@keymanapp/common-types';
import { LDMLKeyboard, KMXPlus, Constants, MarkerParser } from '@keymanapp/common-types';
import { CompilerMessages } from './messages.js';
import { SectionCompiler } from "./section-compiler.js";
@ -8,8 +8,18 @@ import Keys = KMXPlus.Keys;
import ListItem = KMXPlus.ListItem;
import KeysFlicks = KMXPlus.KeysFlicks;
import { allUsedKeyIdsInLayers, calculateUniqueKeys, translateLayerAttrToModifier, validModifier } from '../util/util.js';
import { MarkerTracker, MarkerUse } from './marker-tracker.js';
export class KeysCompiler extends SectionCompiler {
static validateMarkers(
keyboard: LDMLKeyboard.LKKeyboard,
mt: MarkerTracker
): boolean {
keyboard.keys?.key?.forEach(({ to }) =>
mt.add(MarkerUse.emit, MarkerParser.allReferences(to))
);
return true;
}
public get id() {
return constants.section.keys;
@ -20,7 +30,7 @@ export class KeysCompiler extends SectionCompiler {
* @returns just the non-touch layers.
*/
public hardwareLayers() {
return this.keyboard.layers?.filter(({form}) => form !== 'touch');
return this.keyboard.layers?.filter(({ form }) => form !== "touch");
}
public validate() {
@ -30,7 +40,7 @@ export class KeysCompiler extends SectionCompiler {
const usedKeys = allUsedKeyIdsInLayers(this.keyboard?.layers);
const uniqueKeys = calculateUniqueKeys([...this.keyboard.keys?.key]);
for (let key of uniqueKeys) {
const {id, flicks} = key;
const { id, flicks } = key;
if (!usedKeys.has(id)) {
continue; // unused key, ignore
}
@ -38,10 +48,14 @@ export class KeysCompiler extends SectionCompiler {
if (!flicks) {
continue; // no flicks
}
const flickEntry = this.keyboard.keys?.flicks?.find(x => x.id === flicks);
if (!flickEntry ) {
const flickEntry = this.keyboard.keys?.flicks?.find(
(x) => x.id === flicks
);
if (!flickEntry) {
valid = false;
this.callbacks.reportMessage(CompilerMessages.Error_MissingFlicks({flicks, id}));
this.callbacks.reportMessage(
CompilerMessages.Error_MissingFlicks({ flicks, id })
);
}
}
@ -53,8 +67,9 @@ export class KeysCompiler extends SectionCompiler {
if (hardwareLayers.length >= 1) {
// validate all errors
for (let layers of hardwareLayers) {
for(let layer of layers.layer) {
valid = this.validateHardwareLayerForKmap(layers.form, layer) && valid; // note: always validate even if previously invalid results found
for (let layer of layers.layer) {
valid =
this.validateHardwareLayerForKmap(layers.form, layer) && valid; // note: always validate even if previously invalid results found
}
}
// TODO-LDML: } else { touch?
@ -84,11 +99,13 @@ export class KeysCompiler extends SectionCompiler {
/* c8 ignore next 3 */
if (hardwareLayers.length > 1) {
// validation should have already caught this
throw Error(`Internal error: Expected 0 or 1 hardware layer, not ${hardwareLayers.length}`);
throw Error(
`Internal error: Expected 0 or 1 hardware layer, not ${hardwareLayers.length}`
);
} else if (hardwareLayers.length === 1) {
const theLayers = hardwareLayers[0];
const { form } = theLayers;
for(let layer of theLayers.layer) {
for (let layer of theLayers.layer) {
this.compileHardwareLayerToKmap(sections, layer, sect, form);
}
} // else: TODO-LDML do nothing if only touch layers
@ -98,7 +115,9 @@ export class KeysCompiler extends SectionCompiler {
public loadFlicks(sections: DependencySections, sect: Keys) {
for (let lkflicks of this.keyboard.keys.flicks) {
let flicks: KeysFlicks = new KeysFlicks(sections.strs.allocString(lkflicks.id));
let flicks: KeysFlicks = new KeysFlicks(
sections.strs.allocString(lkflicks.id)
);
for (let lkflick of lkflicks.flick) {
let flags = 0;
@ -106,10 +125,14 @@ export class KeysCompiler extends SectionCompiler {
if (!to.isOneChar) {
flags |= constants.keys_flick_flags_extend;
}
let directions: ListItem = sections.list.allocListFromSpaces(sections.strs, lkflick.directions);
let directions: ListItem = sections.list.allocListFromSpaces(
sections.strs,
lkflick.directions
);
flicks.flicks.push({
directions,
flags,
// TODO-LDML: markers,variables
to,
});
}
@ -132,19 +155,33 @@ export class KeysCompiler extends SectionCompiler {
if (!!key.gap) {
flags |= constants.keys_key_flags_gap;
}
if (key.transform === 'no') {
if (key.transform === "no") {
flags |= constants.keys_key_flags_notransform;
}
const id = sections.strs.allocString(key.id);
const longPress: ListItem = sections.list.allocListFromEscapedSpaces(sections.strs, key.longPress);
const longPressDefault = sections.strs.allocAndUnescapeString(key.longPressDefault);
const multiTap: ListItem = sections.list.allocListFromEscapedSpaces(sections.strs, key.multiTap);
const longPress: ListItem = sections.list.allocListFromEscapedSpaces(
sections.strs,
// TODO-LDML: markers,variables
key.longPress
);
const longPressDefault = sections.strs.allocAndUnescapeString(
// TODO-LDML: markers,variables
key.longPressDefault
);
const multiTap: ListItem = sections.list.allocListFromEscapedSpaces(
sections.strs,
// TODO-LDML: markers,variables
key.multiTap
);
const keySwitch = sections.strs.allocString(key.switch); // 'switch' is a reserved word
const to = sections.strs.allocAndUnescapeString(key.to, true);
const toRaw = key.to;
// TODO-LDML: variables
let toCooked = sections.vars.substituteMarkerString(toRaw);
const to = sections.strs.allocAndUnescapeString(toCooked, true);
if (!to.isOneChar) {
flags |= constants.keys_key_flags_extend;
}
const width = Math.ceil((key.width || 1) * 10.0); // default, width=1
const width = Math.ceil((key.width || 1) * 10.0); // default, width=1
sect.keys.push({
flags,
flicks,
@ -166,12 +203,17 @@ export class KeysCompiler extends SectionCompiler {
* @param layer
* @returns
*/
private validateHardwareLayerForKmap(hardware: string, layer: LDMLKeyboard.LKLayer) {
private validateHardwareLayerForKmap(
hardware: string,
layer: LDMLKeyboard.LKLayer
) {
let valid = true;
const { modifier } = layer;
if (!validModifier(modifier)) {
this.callbacks.reportMessage(CompilerMessages.Error_InvalidModifier({ modifier, layer: layer.id }));
this.callbacks.reportMessage(
CompilerMessages.Error_InvalidModifier({ modifier, layer: layer.id })
);
valid = false;
}
@ -179,21 +221,31 @@ export class KeysCompiler extends SectionCompiler {
/* c8 ignore next 5 */
if (!keymap) {
// not reached due to XML validation
this.callbacks.reportMessage(CompilerMessages.Error_InvalidHardware({ form: hardware }));
this.callbacks.reportMessage(
CompilerMessages.Error_InvalidHardware({ form: hardware })
);
valid = false;
}
const uniqueKeys = calculateUniqueKeys([...this.keyboard.keys?.key]);
if (layer.row.length > keymap.length) {
this.callbacks.reportMessage(CompilerMessages.Error_HardwareLayerHasTooManyRows());
this.callbacks.reportMessage(
CompilerMessages.Error_HardwareLayerHasTooManyRows()
);
valid = false;
}
for (let y = 0; y < layer.row.length && y < keymap.length; y++) {
const keys = layer.row[y].keys.split(' ');
const keys = layer.row[y].keys.split(" ");
if (keys.length > keymap[y].length) {
this.callbacks.reportMessage(CompilerMessages.Error_RowOnHardwareLayerHasTooManyKeys({ row: y + 1, hardware, modifier }));
this.callbacks.reportMessage(
CompilerMessages.Error_RowOnHardwareLayerHasTooManyKeys({
row: y + 1,
hardware,
modifier,
})
);
valid = false;
}
@ -201,14 +253,24 @@ export class KeysCompiler extends SectionCompiler {
for (let key of keys) {
x++;
let keydef = uniqueKeys.find(x => x.id == key);
let keydef = uniqueKeys.find((x) => x.id == key);
if (!keydef) {
this.callbacks.reportMessage(CompilerMessages.Error_KeyNotFoundInKeyBag({ keyId: key, col: x + 1, row: y + 1, layer: layer.id, form: 'hardware' }));
this.callbacks.reportMessage(
CompilerMessages.Error_KeyNotFoundInKeyBag({
keyId: key,
col: x + 1,
row: y + 1,
layer: layer.id,
form: "hardware",
})
);
valid = false;
continue;
}
if (!keydef.to && !keydef.gap && !keydef.switch) {
this.callbacks.reportMessage(CompilerMessages.Error_KeyMissingToGapOrSwitch({ keyId: key }));
this.callbacks.reportMessage(
CompilerMessages.Error_KeyMissingToGapOrSwitch({ keyId: key })
);
valid = false;
continue;
}
@ -222,7 +284,7 @@ export class KeysCompiler extends SectionCompiler {
sections: DependencySections,
layer: LDMLKeyboard.LKLayer,
sect: Keys,
hardware: string,
hardware: string
): Keys {
const mod = translateLayerAttrToModifier(layer);
const keymap = Constants.HardwareToKeymap.get(hardware);
@ -231,7 +293,7 @@ export class KeysCompiler extends SectionCompiler {
for (let row of layer.row) {
y++;
const keys = row.keys.split(' ');
const keys = row.keys.split(" ");
let x = -1;
for (let key of keys) {
x++;

72
developer/src/kmc-ldml/src/compiler/marker-tracker.ts Normal file
View file

@ -0,0 +1,72 @@
/**
* Verb for MarkerTracker.add()
*/
export enum MarkerUse {
/** outputs this marker into context (e.g. transform to= or key to=) */
emit,
/** consumes this marker out of the context (e.g. transform from=) */
consume,
/** matches the marker, but doesn't consume (e.g. display to=) */
match,
/** variable definition: might consume, emit, or match. */
variable,
}
type MarkerSet = Set<string>;
/** Tracks usage of markers */
export class MarkerTracker {
/** markers that were emitted */
emitted: MarkerSet;
/** markers that were consumed and removed from the context */
consumed: MarkerSet;
/** markers that were matched, but not necessarily consumed */
matched: MarkerSet;
/** all markers */
all: MarkerSet;
constructor() {
this.emitted = new Set<string>();
this.consumed = new Set<string>();
this.matched = new Set<string>();
this.all = new Set<string>();
}
/**
*
* @param verb what kind of use we are adding
* @param markers list of markers to add
*/
add(verb: MarkerUse, markers: string[]) {
if (!markers.length) {
return; // skip if empty
}
if (verb == MarkerUse.emit) {
markers.forEach((m) => {
this.emitted.add(m);
this.all.add(m);
});
} else if (verb == MarkerUse.consume) {
markers.forEach((m) => {
this.consumed.add(m);
this.all.add(m);
});
} else if (verb == MarkerUse.match) {
markers.forEach((m) => {
this.matched.add(m);
this.all.add(m);
});
} else if (verb == MarkerUse.variable) {
markers.forEach((m) => {
// we don't know, so add it to all three
this.matched.add(m);
this.emitted.add(m);
this.consumed.add(m);
this.all.add(m);
});
/* c8 skip next 3 */
} else {
throw Error(`Internal error: unsupported verb ${verb} for match`);
}
}
}

12
developer/src/kmc-ldml/src/compiler/messages.ts
View file

@ -2,7 +2,7 @@ import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m
const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler;
const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler;
// const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.KeyboardCompiler;
// const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.LdmlKeyboardCompiler;
const SevError = CompilerErrorSeverity.Error | CompilerErrorNamespace.LdmlKeyboardCompiler;
const SevFatal = CompilerErrorSeverity.Fatal | CompilerErrorNamespace.LdmlKeyboardCompiler;
@ -35,9 +35,9 @@ export class CompilerMessages {
m(this.HINT_LocaleIsNotMinimalAndClean, `Locale '${o.sourceLocale}' is not minimal or correctly formatted and should be '${o.locale}'`);
static HINT_LocaleIsNotMinimalAndClean = SevHint | 0x0008;
static Error_VkeyIsNotValid = (o:{vkey: string}) =>
m(this.ERROR_VkeyIsNotValid, `Virtual key '${o.vkey}' is not found in the CLDR VKey Enum table.`);
static ERROR_VkeyIsNotValid = SevError | 0x0009;
static Hint_VkeyIsNotValid = (o:{vkey: string}) =>
m(this.HINT_VkeyIsNotValid, `Virtual key '${o.vkey}' is not found in the CLDR VKey Enum table.`);
static HINT_VkeyIsNotValid = SevHint | 0x0009;
static Hint_VkeyIsRedundant = (o:{vkey: string}) =>
m(this.HINT_VkeyIsRedundant, `Virtual key '${o.vkey}' is mapped to itself, which is redundant.`);
@ -130,5 +130,9 @@ export class CompilerMessages {
static Error_CantReferenceSetFromUnicodeSet = (o:{id: string}) =>
m(this.ERROR_CantReferenceSetFromUnicodeSet, `Illegal use of set variable from within UnicodeSet: \$[${o.id}]`);
static ERROR_CantReferenceSetFromUnicodeSet = SevError | 0x0020;
static Error_MissingMarkers = (o: { ids: string[] }) =>
m(this.ERROR_MissingMarkers, `Markers used for matching but not defined: ${o.ids?.join(',')}`);
static ERROR_MissingMarkers = SevError | 0x0021;
}

3
developer/src/kmc-ldml/src/compiler/section-compiler.ts
View file

@ -32,7 +32,8 @@ export class SectionCompiler {
const defaults = new Set(<SectionIdent[]>[
constants.section.strs,
constants.section.list,
constants.section.elem
constants.section.elem,
constants.section.vars,
]);
return defaults;
}

19
developer/src/kmc-ldml/src/compiler/tran.ts
View file

@ -1,5 +1,5 @@
import { constants, SectionIdent } from "@keymanapp/ldml-keyboard-constants";
import { KMXPlus, LDMLKeyboard, CompilerCallbacks, VariableParser } from '@keymanapp/common-types';
import { KMXPlus, LDMLKeyboard, CompilerCallbacks, VariableParser, MarkerParser } from '@keymanapp/common-types';
import { SectionCompiler } from "./section-compiler.js";
import Bksp = KMXPlus.Bksp;
@ -15,10 +15,21 @@ import LKTransform = LDMLKeyboard.LKTransform;
import LKTransforms = LDMLKeyboard.LKTransforms;
import { verifyValidAndUnique } from "../util/util.js";
import { CompilerMessages } from "./messages.js";
import { MarkerTracker, MarkerUse } from "./marker-tracker.js";
type TransformCompilerType = 'simple' | 'backspace';
class TransformCompiler<T extends TransformCompilerType, TranBase extends Tran> extends SectionCompiler {
export class TransformCompiler<T extends TransformCompilerType, TranBase extends Tran> extends SectionCompiler {
static validateMarkers(keyboard: LDMLKeyboard.LKKeyboard, mt : MarkerTracker): boolean {
keyboard?.transforms?.forEach(transforms =>
transforms.transformGroup.forEach(transformGroup => {
transformGroup.transform?.forEach(({ to, from }) => {
mt.add(MarkerUse.emit, MarkerParser.allReferences(to));
mt.add(MarkerUse.consume, MarkerParser.allReferences(from));
})}));
return true;
}
protected type: T;
@ -130,6 +141,10 @@ class TransformCompiler<T extends TransformCompilerType, TranBase extends Tran>
cookedTo = sections.vars.substituteStrings(cookedTo, sections);
}
// add in markers. idempotent if no markers.
cookedFrom = sections.vars.substituteMarkerString(cookedFrom); // TODO-LDML: need to support \m{.} here, maybe other edge cases
cookedTo = sections.vars.substituteMarkerString(cookedTo);
result.from = sections.strs.allocAndUnescapeString(cookedFrom); // TODO-LDML: not unescaped here, done previously
result.to = sections.strs.allocAndUnescapeString(cookedTo); // TODO-LDML: not unescaped here, done previously
return result;

67
developer/src/kmc-ldml/src/compiler/vars.ts
View file

@ -1,5 +1,5 @@
import { SectionIdent, constants } from "@keymanapp/ldml-keyboard-constants";
import { KMXPlus, LDMLKeyboard, CompilerCallbacks } from '@keymanapp/common-types';
import { KMXPlus, LDMLKeyboard, CompilerCallbacks, MarkerParser } from '@keymanapp/common-types';
import { VariableParser } from '@keymanapp/common-types';
import { SectionCompiler } from "./section-compiler.js";
import Vars = KMXPlus.Vars;
@ -9,6 +9,10 @@ import UnicodeSetItem = KMXPlus.UnicodeSetItem;
import DependencySections = KMXPlus.DependencySections;
import LDMLKeyboardXMLSourceFile = LDMLKeyboard.LDMLKeyboardXMLSourceFile;
import { CompilerMessages } from "./messages.js";
import { KeysCompiler } from "./keys.js";
import { TransformCompiler } from "./tran.js";
import { DispCompiler } from "./disp.js";
import { MarkerTracker, MarkerUse } from "./marker-tracker.js";
export class VarsCompiler extends SectionCompiler {
public get id() {
return constants.section.vars;
@ -17,7 +21,8 @@ export class VarsCompiler extends SectionCompiler {
public get dependencies(): Set<SectionIdent> {
const defaults = new Set(<SectionIdent[]>[
constants.section.strs,
constants.section.elem
constants.section.elem,
constants.section.list,
]);
defaults.delete(this.id);
return defaults;
@ -29,7 +34,6 @@ export class VarsCompiler extends SectionCompiler {
public validate(): boolean {
let valid = true;
// TODO-LDML scan for markers?
// Check for duplicate ids
const allIds = new Set();
@ -130,14 +134,61 @@ export class VarsCompiler extends SectionCompiler {
}));
valid = false;
}
valid = this.validateMarkers() && valid; // accumulate validity
return valid;
}
private collectMarkers(mt : MarkerTracker) : boolean {
let valid = true;
// call our friends to validate
valid = this.validateVarsMarkers(this.keyboard, mt) && valid; // accumulate validity
valid = KeysCompiler.validateMarkers(this.keyboard, mt) && valid; // accumulate validity
valid = TransformCompiler.validateMarkers(this.keyboard, mt) && valid; // accumulate validity
valid = DispCompiler.validateMarkers(this.keyboard, mt) && valid; // accumulate validity
return valid;
}
private validateMarkers(): boolean {
const mt = new MarkerTracker();
let valid = this.collectMarkers(mt);
// see if there are any matched-but-not-emitted
const matchedNotEmitted : Set<string> = new Set<string>();
for (const m of mt.matched.values()) {
if (m === MarkerParser.ANY_MARKER_ID) continue; // match-all marker
if (!mt.emitted.has(m)) {
matchedNotEmitted.add(m);
}
}
for (const m of mt.consumed.values()) {
if (m === MarkerParser.ANY_MARKER_ID) continue; // match-all marker
if (!mt.emitted.has(m)) {
matchedNotEmitted.add(m);
}
}
// report once
if (matchedNotEmitted.size > 0) {
this.callbacks.reportMessage(CompilerMessages.Error_MissingMarkers({ ids: Array.from(matchedNotEmitted.values()).sort() }));
valid = false;
}
return valid;
}
validateVarsMarkers(keyboard: LDMLKeyboard.LKKeyboard, mt : MarkerTracker) : boolean {
keyboard?.variables?.string?.forEach(({value}) =>
mt.add(MarkerUse.variable, MarkerParser.allReferences(value)));
return true;
}
public compile(sections: DependencySections): Vars {
const result = new Vars();
const variables = this.keyboard?.variables;
// we always have vars, it's depended on by other sections
if (!variables) return result; // Empty vars, to simplify other sections
// we already know the variables do not conflict with each other
@ -153,6 +204,14 @@ export class VarsCompiler extends SectionCompiler {
variables?.unicodeSet?.forEach((e) =>
this.addUnicodeSet(result, e, sections));
// reload markers - TODO-LDML: double work!
const mt = new MarkerTracker();
this.collectMarkers(mt);
// collect all markers, excluding the match-all
const allMarkers : string[] = Array.from(mt.all).filter(m => m !== MarkerParser.ANY_MARKER_ID).sort();
result.markers = sections.list.allocList(sections.strs, allMarkers);
return result.valid() ? result : null;
}

4
developer/src/kmc-ldml/src/compiler/visual-keyboard-compiler.ts
View file

@ -40,6 +40,10 @@ export class LdmlKeyboardVisualKeyboardCompiler {
let keydef = source.keyboard.keys?.key?.find(x => x.id == key);
if (!keydef) {
throw Error(`Internal Error: could not find key id="${key}" in layer "${layer.id || '<none>'}", row "${y}"`);
}
vk.keys.push({
flags: VisualKeyboard.VisualKeyboardKeyFlags.kvkkUnicode,
shift: shift,

10
developer/src/kmc-ldml/src/compiler/vkey.ts
View file

@ -19,13 +19,15 @@ export class VkeyCompiler extends SectionCompiler {
this.keyboard.vkeys.vkey.forEach(vk => {
if(LdmlVkeyNames[vk.from] === undefined) {
this.callbacks.reportMessage(CompilerMessages.Error_VkeyIsNotValid({vkey: vk.from}));
valid = false;
// TODO-LDML: When we do #7135 this may need to change back to an error.
this.callbacks.reportMessage(CompilerMessages.Hint_VkeyIsNotValid({vkey: vk.from}));
return;
}
if(LdmlVkeyNames[vk.to] === undefined) {
this.callbacks.reportMessage(CompilerMessages.Error_VkeyIsNotValid({vkey: vk.to}));
valid = false;
// TODO-LDML: When we do #7135 this may need to change back to an error.
this.callbacks.reportMessage(CompilerMessages.Hint_VkeyIsNotValid({vkey: vk.to}));
return;
}
if(vk.from == vk.to) {

52
developer/src/kmc-ldml/test/fixtures/basic.txt vendored
View file

@ -257,7 +257,7 @@ block(keys) # struct COMP_KMXPLUS_KEYS {
index(strNull,strHmaqtugha,2) # KMXPLUS_STR 'hmaqtugha'
00 00 00 00 # KMXPLUS_STR switch
0A 00 00 00 # KMX_DWORD width*10
01 00 00 00 # TODO: index(listNull,indexAe,4) # LIST longPress 'a e'
02 00 00 00 # TODO: index(listNull,indexAe,4) # LIST longPress 'a e'
00 00 00 00 # STR longPressDefault
00 00 00 00 # TODO: index(listNull,listNull,4) # LIST multiTap
00 00 00 00 # flicks 0
@ -324,21 +324,25 @@ block(layr) # struct COMP_KMXPLUS_LAYR {
block(list) # struct COMP_KMXPLUS_LAYR_LIST {
6c 69 73 74 # KMX_DWORD header.ident; // 0000 Section name - list
diff(list,endList) # KMX_DWORD header.size; // 0004 Section length
02 00 00 00 # KMX_DWORD listCount (should be 2)
02 00 00 00 # KMX_DWORD indexCount (should be 2)
03 00 00 00 # KMX_DWORD listCount (should be 2)
03 00 00 00 # KMX_DWORD indexCount (should be 2)
# list #0 the null list
block(listNull)
00 00 00 00 #index(indexNull,indexNull,2) # KMX_DWORD list index (0)
00 00 00 00 # KMX_DWORD lists[0].count
# list #1 the ae list
block(listA)
00 00 00 00 # first index
01 00 00 00 #count
block(listAe)
00 00 00 00 # index(indexAe,indexNull,2) # KMX_DWORD list index (also 0)
01 00 00 00 # index(indexAe,indexNull,2) # KMX_DWORD list index (also 0)
02 00 00 00 # KMX_DWORD count
block(endLists)
# indices
#block(indexNull)
# No null index
# index(strNull,strNull,2) # KMXPLUS_STR string index
block(indexA)
index(strNull,strA,2) # a
block(indexAe)
index(strNull,strA,2) # KMXPLUS_STR a
index(strNull,strElemBkspFrom2,2) # KMXPLUS_STR e
@ -401,6 +405,7 @@ block(strs) # struct COMP_KMXPLUS_STRS {
diff(strs,strName) sizeof(strName,2)
diff(strs,strFromSet) sizeof(strFromSet,2)
diff(strs,strUSet) sizeof(strUSet,2)
diff(strs,strAmarker) sizeof(strAmarker,2)
diff(strs,strElemTranFrom1) sizeof(strElemTranFrom1,2)
diff(strs,strElemTranFrom1a) sizeof(strElemTranFrom1a,2)
diff(strs,strElemTranFrom1b) sizeof(strElemTranFrom1b,2)
@ -421,6 +426,7 @@ block(strs) # struct COMP_KMXPLUS_STRS {
diff(strs,strTranTo) sizeof(strTranTo,2)
diff(strs,strKeys) sizeof(strKeys,2)
diff(strs,strIndicator) sizeof(strIndicator,2)
diff(strs,strSentinel0001) sizeof(strSentinel0001,2)
# String table -- block(x) is used to store the null u16char at end of each string
@ -433,6 +439,7 @@ block(strs) # struct COMP_KMXPLUS_STRS {
block(strName) 54 00 65 00 73 00 74 00 4b 00 62 00 64 00 block(x) 00 00 # 'TestKbd'
block(strFromSet) 5B 00 5C 00 75 00 31 00 41 00 37 00 35 00 2D 00 5C 00 75 00 31 00 41 00 37 00 39 00 5D 00 block(x) 00 00 # [\u1a75-\u1a79]
block(strUSet) 5b 00 61 00 62 00 63 00 5d 00 block(x) 00 00 # '[abc]'
block(strAmarker) 5C 00 6D 00 7B 00 61 00 7D 00 block(x) 00 00 # '\m{a}'
block(strElemTranFrom1) 5E 00 block(x) 00 00 # '^'
block(strElemTranFrom1a) 5E 00 61 00 block(x) 00 00 # '^a'
block(strElemTranFrom1b) 5E 00 65 00 block(x) 00 00 # '^e'
@ -458,7 +465,7 @@ block(strs) # struct COMP_KMXPLUS_STRS {
block(strKeys) 90 17 b6 17 block(x) 00 00 # 'ថា'
# <reorder before="ᩫ" from="᩠᩵ᩅ" order="10 55 10" />
block(strIndicator) 3d d8 40 de block(x) 00 00 # '🙀'
block(strSentinel0001) FF FF 01 00 block(x) 00 00 # U+FFFF U+0001
@ -478,10 +485,15 @@ block(tran) # struct COMP_KMXPLUS_TRAN {
block(tranGroupStart) # COMP_KMXPLUS_TRAN_GROUP
# group 0
00 00 00 00 # KMX_DWORD type = transform
01 00 00 00 # KMX_DWORD count
02 00 00 00 # KMX_DWORD count
diff(tranTransformStart,tranTransform0,16) # KMX_DWORD index
# group 1
00 00 00 00 # KMX_DWORD type = transform
01 00 00 00 # KMX_DWORD count
diff(tranTransformStart,tranTransform2,16) # KMX_DWORD index
# group 2
01 00 00 00 # KMX_DWORD type = reorder
01 00 00 00 # KMX_DWORD count
diff(tranReorderStart,tranReorder0,8) # KMX_DWORD index
@ -494,6 +506,18 @@ block(tran) # struct COMP_KMXPLUS_TRAN {
index(strNull,strNull,2) # mapFrom
index(strNull,strNull,2) # mapTo
block(tranTransform1)
index(strNull,strA,2) # KMXPLUS_STR from; 'a'
index(strNull,strSentinel0001,2) # KMXPLUS_STR to; \m{a}
index(strNull,strNull,2) # mapFrom
index(strNull,strNull,2) # mapTo
block(tranTransform2) # Next group
index(strNull,strSentinel0001,2) # KMXPLUS_STR from; (\m{a})
index(strNull,strNull,2) # KMXPLUS_STR to; (none)
index(strNull,strNull,2) # mapFrom
index(strNull,strNull,2) # mapTo
# reorders
block(tranReorderStart) # COMP_KMXPLUS_TRAN_REORDER
block(tranReorder0)
@ -520,23 +544,29 @@ block(uset)
block(vars) # struct COMP_KMXPLUS_VARS {
76 61 72 73 # KMX_DWORD header.ident; // 0000 Section name - vars
diff(vars,varsEnd) # KMX_DWORD header.size; // 0004 Section length
00 00 00 00 # KMX_DWORD markers - list
01 00 00 00 # KMX_DWORD markers - list 1 ['a']
diff(varsBegin,varsEnd,16) # KMX_DWORD varCount
# var 0
block(varsBegin)
# var 0
00 00 00 00 # KMX_DWORD type = str
index(strNull,strA,2) # KMXPLUS_STR id 'a'
index(strNull,strAmarker,2) # KMXPLUS_STR value '\m{a}'
00 00 00 00 # KMXPLUS_ELEM
# var 1
01 00 00 00 # KMX_DWORD type = set
index(strNull,strVse,2) # KMXPLUS_STR id 'vse'
index(strNull,strSet,2) # KMXPLUS_STR value 'a b c'
01 00 00 00 # KMXPLUS_ELEM elem 'a b c' see 'elemSet'
# var 1
# var 2
00 00 00 00 # KMX_DWORD type = string
index(strNull,strVst,2) # KMXPLUS_STR id 'vst'
index(strNull,strSet2,2) # KMXPLUS_STR value 'abc'
00 00 00 00 # KMXPLUS_ELEM elem
# var 2
# var 3
02 00 00 00 # KMX_DWORD type = string
index(strNull,strVus,2) # KMXPLUS_STR id 'vus'
index(strNull,strUSet,2) # KMXPLUS_STR value '[abc]'

6
developer/src/kmc-ldml/test/fixtures/basic.xml vendored
View file

@ -37,6 +37,7 @@
</layers>
<variables>
<string id="a" value="\m{a}"/>
<string id="vst" value="abc"/>
<set id="vse" value="a b c"/>
<unicodeSet id="vus" value="[abc]"/>
@ -45,6 +46,11 @@
<transforms type="simple">
<transformGroup>
<transform from="^a" to="â" />
<transform from="a" to="\m{a}" />
</transformGroup>
<transformGroup>
<!-- delete that marker -->
<transform from="\m{a}" />
</transformGroup>
<transformGroup>

43
developer/src/kmc-ldml/test/fixtures/sections/vars/fail-markers-badref-0.xml vendored Normal file
View file

@ -0,0 +1,43 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
This will fail because the two markers given don't exist anywhere.
-->
<!DOCTYPE keyboard SYSTEM "../../../../../../../resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard.dtd">
<keyboard locale="mt" conformsTo="techpreview">
<names>
<name value="vars-markers-badref-0" />
</names>
<displays>
<!-- Not ok, nobody emits this, so this display doesn't make sense -->
<display to="\m{doesnt_exist_1}" display="¬" />
<!-- OK - it is in the strings list. Could warn that it matches no keys -->
<display to="\m{maybe_emitted}" display="‽" />
</displays>
<variables>
<!-- This may be emitted, so at present it does not cause an error. -->
<!-- A linter could, however, hint on an unused variable. -->
<!-- Removing this variable would cause \m{maybe-emitted} to error in the transform group -->
<string id="unused_variable" value="\m{maybe_emitted}" />
</variables>
<transforms type="simple">
<transformGroup>
<!-- not OK, nobody emits this. -->
<transform from="\m{doesnt_exist_3}" to="3" />
</transformGroup>
</transforms>
<transforms type="backspace">
<transformGroup>
<!-- not OK, nobody emits this. -->
<transform from="\m{doesnt_exist_2}" />
<!-- OK - it's in the variable list -->
<transform from="\m{maybe_emitted}" />
<!-- OK - matches all markers -->
<transform from="\m{.}" />
</transformGroup>
</transforms>
</keyboard>

44
developer/src/kmc-ldml/test/fixtures/sections/vars/markers-maximal.xml vendored Normal file
View file

@ -0,0 +1,44 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
This keyboard won't make a lot of sense, it's mostly
a test of validating markers.
When we get further on in implementation, we'll shape this up
into something testable for implementation.
-->
<!DOCTYPE keyboard SYSTEM "../../../../../../../resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard.dtd">
<keyboard locale="mt" conformsTo="techpreview">
<names>
<name value="vars-markers-maximal" />
</names>
<displays>
<display to="\m{m}" display="Ⓜ️" />
</displays>
<keys>
<key id="m" to="\m{m}" />
</keys>
<!-- from spec -->
<variables>
<string id="x" value="\m{x}" />
</variables>
<transforms type="simple">
<transformGroup>
<transform from="m" to="\m{m}"/>
<transform from="x" to="\m{x}"/>
</transformGroup>
<transformGroup>
<transform from="e\m{x}" to="é" />
</transformGroup>
</transforms>
<transforms type="backspace">
<transformGroup>
<transform from="A\m{.}B" />
</transformGroup>
</transforms>
</keyboard>

3
developer/src/kmc-ldml/test/helpers/index.ts
View file

@ -19,6 +19,7 @@ import DependencySections = KMXPlus.DependencySections;
import Section = KMXPlus.Section;
import { ElemCompiler, ListCompiler, StrsCompiler } from '../../src/compiler/empty-compiler.js';
import { KmnCompiler } from '@keymanapp/kmc-kmn';
import { VarsCompiler } from '../../src/compiler/vars.js';
// import Vars = KMXPlus.Vars;
/**
@ -92,7 +93,7 @@ async function loadDepsFor(sections: DependencySections, parentCompiler: Section
const parentId = parentCompiler.id;
if (!dependencies) {
// default dependencies
dependencies = [ StrsCompiler, ListCompiler, ElemCompiler ];
dependencies = [ StrsCompiler, ListCompiler, ElemCompiler, VarsCompiler ];
}
for (const dep of dependencies) {
const compiler = new dep(source, callbacks);

3
developer/src/kmc-ldml/test/test-tran.ts
View file

@ -1,7 +1,6 @@
import 'mocha';
import { assert } from 'chai';
import { TranCompiler, BkspCompiler } from '../src/compiler/tran.js';
import { VarsCompiler } from '../src/compiler/vars.js';
import { BASIC_DEPENDENCIES, UsetCompiler } from '../src/compiler/empty-compiler.js';
import { CompilerMessages } from '../src/compiler/messages.js';
import { compilerTestCallbacks, testCompilationCases } from './helpers/index.js';
@ -10,7 +9,7 @@ import { KMXPlus } from '@keymanapp/common-types';
import Tran = KMXPlus.Tran;// for tests…
import Bksp = KMXPlus.Bksp;// for tests…
import { constants } from '@keymanapp/ldml-keyboard-constants';
const tranDependencies = [ ...BASIC_DEPENDENCIES, UsetCompiler, VarsCompiler ];
const tranDependencies = [ ...BASIC_DEPENDENCIES, UsetCompiler ];
const bkspDependencies = tranDependencies;
describe('tran', function () {

32
developer/src/kmc-ldml/test/test-vars.ts
View file

@ -5,6 +5,9 @@ import { CompilerMessages } from '../src/compiler/messages.js';
import { CompilerMessages as KmnCompilerMessages } from '@keymanapp/kmc-kmn';
import { testCompilationCases } from './helpers/index.js';
import { KMXPlus } from '@keymanapp/common-types';
import { BASIC_DEPENDENCIES } from '../src/compiler/empty-compiler.js';
// now that 'everything' depends on vars, we need an explicit dependency here
const varsDependencies = BASIC_DEPENDENCIES.filter(c => c !== VarsCompiler);
import Vars = KMXPlus.Vars;
@ -182,5 +185,32 @@ describe('vars', function () {
CompilerMessages.Error_MissingStringVariable({id: 'missingStringInSet'})
],
},
]);
], varsDependencies);
describe('markers', function () {
this.slow(500); // 0.5 sec -- json schema validation takes a while
testCompilationCases(VarsCompiler, [
{
subpath: 'sections/vars/markers-maximal.xml',
callback(sect) {
const vars = <Vars> sect;
assert.ok(vars.markers);
assert.sameDeepOrderedMembers(vars.markers.toStringArray(),
['m','x']);
},
},
{
subpath: 'sections/vars/fail-markers-badref-0.xml',
errors: [
CompilerMessages.Error_MissingMarkers({
ids: [
'doesnt_exist_1',
'doesnt_exist_2',
'doesnt_exist_3',
]
}),
],
},
], varsDependencies);
});
});

14
developer/src/kmc-ldml/test/test-vkey.ts
View file

@ -37,19 +37,19 @@ describe('vkey compiler', function () {
assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Info_MultipleVkeysHaveSameTarget({vkey: 'Q'}));
});
it('should error on invalid "from" vkey', async function() {
it('should hint on invalid "from" vkey', async function() {
let vkey = await loadSectionFixture(VkeyCompiler, 'sections/vkey/invalid-from-vkey.xml', compilerTestCallbacks) as Vkey;
assert.isNull(vkey);
assert.isNotNull(vkey);
assert.equal(compilerTestCallbacks.messages.length, 2);
assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Error_VkeyIsNotValid({vkey: 'q'}));
assert.deepEqual(compilerTestCallbacks.messages[1], CompilerMessages.Error_VkeyIsNotValid({vkey: 'HYFEN'}));
assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Hint_VkeyIsNotValid({vkey: 'q'}));
assert.deepEqual(compilerTestCallbacks.messages[1], CompilerMessages.Hint_VkeyIsNotValid({vkey: 'HYFEN'}));
});
it('should error on invalid "to" vkey', async function() {
it('should hint on invalid "to" vkey', async function() {
let vkey = await loadSectionFixture(VkeyCompiler, 'sections/vkey/invalid-to-vkey.xml', compilerTestCallbacks) as Vkey;
assert.isNull(vkey);
assert.isNotNull(vkey);
assert.equal(compilerTestCallbacks.messages.length, 1);
assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Error_VkeyIsNotValid({vkey: 'A-ACUTE'}));
assert.deepEqual(compilerTestCallbacks.messages[0], CompilerMessages.Hint_VkeyIsNotValid({vkey: 'A-ACUTE'}));
});
it('should error on repeated vkeys', async function() {

11
developer/src/kmcmplib/src/meson.build
View file

@ -38,13 +38,22 @@ if cpp_compiler.get_id() == 'emscripten'
links += [
# Forcing inclusion of debug symbols
'-g', '-Wlimited-postlink-optimizations',
'-fwasm-exceptions', '--bind', '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\']'] + sanitize
'-fwasm-exceptions', '--bind'] + sanitize
# Alternative to sanitize is adding '-sASSERTIONS', '-sSAFE_HEAP' to lib_links:
# lib_links += ['-sASSERTIONS', '-sSAFE_HEAP']
# For profiling, add:
# lib_links += ['--profiling-funcs', '-sDEMANGLE_SUPPORT=1']
if cpp_compiler.version().version_compare('>=3.1.44')
# emscripten 3.1.44 removes .asm object and so we need to export `wasmExports`
# #9375; https://github.com/emscripten-core/emscripten/blob/main/ChangeLog.md#3144---072523
links += ['-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\',\'wasmExports\']']
else
# emscripten < 3.1.44 does not include `wasmExports`
links += ['-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\']']
endif
endif
icu = subproject('icu-for-uset', default_options: [ 'default_library=static', 'cpp_std=c++17', 'warning_level=0', 'werror=false'])

234
docs/build/linux-ubuntu.md vendored
View file

@ -4,28 +4,30 @@
On Linux, you can build the following projects:
* [Keyman for Linux](#keyman-for-linux)
* [Keyman Core](#keyman-core) (Linux only) (aka core)
* [Keyman for Android](#keyman-for-android)
<!-- TODO: document how to build for Web, Core-Wasm and Common/Web on Linux. See TC build agent for details. -->
* Keyman Core (wasm targets)
* Common/Web
* KeymanWeb
- [Keyman Core](#keyman-core) (aka core)
- [Keyman for Linux](#keyman-for-linux)
- [Keyman Web](#keyman-web)
- [Keyman for Android](#keyman-for-android)
<!-- TODO: document how to build for Common/Web on Linux.
See TC build agent for details. -->
- Common/Web
The following projects **cannot** be built on Linux:
* Keyman for Windows
* Keyman Developer
* Keyman for macOS
* Keyman for iOS
- Keyman for Windows
- Keyman Developer
- Keyman for macOS
- Keyman for iOS
## System Requirements
## Requirements
* Minimum Ubuntu version: Ubuntu 20.04
### System Requirements
- Minimum Ubuntu version: Ubuntu 20.04
Other Linux distributions will also work if appropriate dependencies are installed.
## Repository Paths
### Repository Paths
Recommended filesystem layout:
@ -38,7 +40,7 @@ $HOME/keyman/
...
```
## Prerequisites
### Prerequisites
The current list of dependencies can be found in the `Build-Depends` section of `linux/debian/control`.
They are most easily installed with the `mk-build-deps` tool:
@ -49,101 +51,139 @@ sudo apt install devscripts equivs
sudo mk-build-deps --install linux/debian/control
```
### Node.js
#### Node.js
Node.js v18 is required for Core build, Web tests, and Developer command line tools.
Node.js v18 is required for Core builds, Web builds, and Developer command line tool builds and usage.
## Keyman for Linux
You can install it with:
All dependencies are already installed if you followed the instructions under [Prerequisites](#Prerequisites).
```shell
curl -sL https://deb.nodesource.com/setup_18.x | bash
apt-get -q -y install nodejs
```
Building:
#### Emscripten
* [Building Keyman for Linux](../../linux/README.md)
You'll also have to install `emscripten` (version 3.1.44 is known to work):
```shell
git clone https://github.com/emscripten-core/emsdk.git
cd emsdk
./emsdk install latest
./emsdk activate latest
export EMSCRIPTEN_BASE=$(pwd)/upstream/emscripten
```
**NOTE:** Don't put EMSDK on the path, i.e. don't source `emsdk_env.sh`.
## Keyman Core
All dependencies are already installed if you followed the instructions under [Prerequisites](#Prerequisites).
All dependencies are already installed if you followed the instructions under
[Prerequisites](#prerequisites).
Building:
### Building Keyman Core
* [Building Keyman Core](../../core/doc/BUILDING.md)
Keyman Core can be built with the `core/build.sh` script.
## Docker Builder
- [Building Keyman Core](../../core/doc/BUILDING.md)
The Docker builder allows you to perform a linux build from anywhere Docker is supported.
To build the docker image:
## Keyman for Linux
```shell
cd linux
docker pull ubuntu:latest
docker build . -t keymanapp/keyman-linux-builder:latest
All dependencies are already installed if you followed the instructions
under [Prerequisites](#prerequisites).
### Building Keyman for Linux
Keyman for Linux can be built with the `linux/build.sh` script.
- [Building Keyman for Linux](../../linux/README.md)
## Keyman Web
Most dependencies are already installed if you followed the instructions under
[Prerequisites](#prerequisites). You'll still have to install Chrome:
```bash
wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
sudo apt install ./google-chrome-stable_current_amd64.deb
```
Once the image is built, it may be used to build parts of Keyman.
And add the `CHROME_BIN` environment variable to `.bashrc:
- core
```shell
# build 'core' in docker
cd ../core
# keep linux build artifacts separate
mkdir -p build/linux
docker run -it --rm -v $(pwd)/..:/home/build -v $(pwd)/build/linux:/home/build/core/build keymanapp/keyman-linux-builder:latest bash -c 'core/build.sh --debug'
```bash
export CHROME_BIN=/opt/google/chrome/chrome
```
- linux
### Environment variables for Keyman Web
```shell
# build 'linux' installation in docker
cd keymanapp/keyman
docker run -it --rm -v $(pwd):/home/build/src/keyman -w /home/build/src/keyman keymanapp/keyman-linux-builder:latest bash -c "DESTDIR=. linux/build.sh --debug build install"
```
`CHROME_BIN` pointing to the Google Chrome binary.
### Building Keyman Web
Keyman Web can be built with the `web/build.sh` script.
- [Building Keyman Web](../../web/README.md)
## Keyman for Android
**Dependencies:**
* [Base](#base-dependencies)
* [Web](./windows#web-dependencies)
Most dependencies are already installed if you followed the instructions
under [Prerequisites](#prerequisites).
**Additional requirements:**
* Android SDK
* [Android Studio](https://developer.android.com/studio/install#linux)
* Gradle
* Maven
* OpenJDK 11 (for Keyman 17.0+)
* pandoc
- [Android Studio](https://developer.android.com/studio/install#linux)
or sdkmanager
- Maven
- pandoc
- Android SDK
- Gradle
- jq
If you only use the command line you don't need Android Studio, however
to do development it's recommended to install it.
Run Android Studio once after installation to install additional components
such as emulator images and SDK updates.
Maven, jq and pandoc can be installed with:
```shell
sudo apt update
sudo apt install maven pandoc jq
```
If necessary, Android SDK and Gradle will be installed by the build script.
In order for that to work, run the following command once. You won't need
this if you install Android SDK through Android Studio.
```shell
sudo apt install sdkmanager
sudo sdkmanager platform-tools
sudo chown -R $USER:$USER /opt/android-sdk/
sdkmanager --licenses
```
### Environment variables for Keyman for Android
**Required environment variable:**
* `ANDROID_HOME` pointing to Android SDK (`$HOME/Android/Sdk`)
- `ANDROID_HOME` pointing to Android SDK (`$HOME/Android/Sdk`)
**Recommended environment variable:**
* [`JAVA_HOME`](#java_home)
- [`JAVA_HOME`](#java_home)
Building:
### Building Keyman for Android
* [Building Keyman for Android](../../android/README.md)
Keyman for Android can be built with the `android/build.sh` script.
## Prerequisites
- [Building Keyman for Android](../../android/README.md)
Many dependencies are only required for specific projects.
### Notes on Environment Variables
### Base Dependencies
**Environment variables:**
* --
## Notes on Environment Variables
### JAVA_HOME
#### JAVA_HOME
This environment variable tells Gradle what version of Java to use for building
Keyman for Android. OpenJDK 11 is used for master.
@ -166,3 +206,59 @@ older versions, you can set `JAVA_HOME_11` to the OpenJDK 11 path and
from command line. But note that you do need to update your `JAVA_HOME` env
var to the associated version before opening Android Studio and loading any
Android projects. `JAVA_HOME_11` is mostly used by CI.
## Docker Builder
The Docker builder allows you to perform a build from anywhere Docker is supported.
To build the docker image:
```shell
cd linux
docker pull ubuntu:latest
docker build . -t keymanapp/keyman-linux-builder:latest
```
Once the image is built, it may be used to build parts of Keyman.
**Note** that it's not yet possible to run tests in the Docker container.
- core
```shell
# build 'Keyman Core' in docker
# keep linux build artifacts separate
mkdir -p $(git rev-parse --show-toplevel)/core/build/linux
docker run -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \
-v $(git rev-parse --show-toplevel)/core/build/linux:/home/build/build/core/build \
keymanapp/keyman-linux-builder:latest \
core/build.sh --debug
```
- linux
```shell
# build 'Keyman for Linux' installation in docker
docker run -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \
--entrypoint /bin/bash keymanapp/keyman-linux-builder:latest \
-c 'DESTDIR=/home/build /usr/bin/bashwrapper linux/build.sh --debug build install'
```
- Keyman Web
```shell
# build 'Keyman Web' in docker
docker run --privileged -it --rm \
-v $(git rev-parse --show-toplevel):/home/build/build \
keymanapp/keyman-linux-builder:latest \
web/build.sh --debug
```
- Keyman for Android
```shell
# build 'Keyman for Android' in docker
docker run -it --rm -v $(git rev-parse --show-toplevel):/home/build/build \
keymanapp/keyman-linux-builder:latest \
android/build.sh --debug
```

1
docs/build/windows.md vendored
View file

@ -153,6 +153,7 @@ choco install git jq python ninja pandoc
refreshenv
# choco meson (0.55) is too old, 1.0 required:
python -m pip install meson
```
**Environment variables**:
* [`KEYMAN_ROOT`](#keyman_root)

32
docs/linux/README.md
View file

@ -2,27 +2,18 @@
## Projects
- [keyman-config](../../linux/keyman-config) - km-config and some other tools to install, uninstall
and view information about Keyman keyboard packages.
- [keyman-config](../../linux/keyman-config) - `km-config` and some other tools
to install, uninstall and view information about Keyman keyboard packages.
- [ibus-keyman](../../linux/ibus-keyman) - IBUS integration to use .kmp Keyman keyboards
- [keyman-system-service](../../linux/keyman-system-service) - A DBus system service
that allows to perform keyboard related actions when running under Wayland.
- [core](../../core) - common keyboardprocessor library
See [license information](../../linux/LICENSE.md) about licensing.
## Linux Requirements/Setup
- It is helpful to be using the [packages.sil.org](http://packages.sil.org) repo
- Install packages required for building and developing Keyman for Linux.
The list of required packages can be seen in `linux/debian/control`.
It is easiest to use the `mk-build-deps` tool to install the
dependencies:
```bash
sudo apt update
sudo apt install devscripts equivs
sudo mk-build-deps --install linux/debian/control
```
See [document in ../build](../build/linux-ubuntu.md).
## Compiling from Command Line
@ -61,15 +52,18 @@ for details on building Linux packages for Keyman.
## Testing
### keyman-config
The unit tests can be run with the following command:
The tests can be run with the following command:
```bash
cd linux/keyman-config
./run-tests.sh
linux/build.sh test
```
To just run the unit tests without integration tests, add the
`--no-integration` parameter.
It's also possible to only run the tests for one of the subprojects. You
can use `build.sh` in the subdirectory for that.
### ibus-keyman
If you want to run the ibus-keyman tests with Wayland, you'll have to

210
docs/linux/packaging.md
View file

@ -10,7 +10,7 @@ We use different channels to build and distribute the Linux packages:
[alpha](https://launchpad.net/~keymanapp/+archive/ubuntu/keyman-alpha) versions
- [pso](http://packages.sil.org/) and [llso](http://linux.lsdev.sil.org/ubuntu/)
for stable, beta, and alpha versions
- artifacts on [Jenkins](https://jenkins.lsdev.sil.org/view/Keyman/view/Pipeline/job/pipeline-keyman-packaging/view/change-requests/)
- artifacts on [GitHub](https://github.com/keymanapp/keyman/actions/workflows/deb-packaging.yml)
for pull requests
Packages on [llso](http://linux.lsdev.sil.org/ubuntu/) are uploaded automatically and are
@ -21,191 +21,63 @@ pso enabled.
## Package builds
Package builds happen on [Launchpad](#package-builds-on-launchpad) and
[Jenkins](#package-builds-on-jenkins). Package builds for the official Ubuntu/Debian
[GitHub](#github-actions-package-builds). Package builds for the official Ubuntu/Debian
repos happen outside of our control. However, we
[upload source packages](#uploading-debian-source-packages) to the Debian community.
## Package builds on Jenkins
## GitHub Actions package builds
### Build jobs
The definition of the packaging jobs, the triggering of the jobs and the necessary build scripts
are scattered over several source repos:
The [Keyman GitHub repo](https://github.com/keymanapp/keyman) contains various
scripts that are used to trigger a build and as part of the package build,
and of course the source code for the packages:
- [ci-builder-scripts](https://github.com/sillsdev/ci-builder-scripts) contains the definition of
a meta job (multi-branch pipeline job) that gets triggered when a change gets pushed to the
[Keyman GitHub repo](https://github.com/keymanapp/keyman). The meta job creates a new build
configuration/job for each branch/pull request on GitHub. The new job gets triggered to initialize
itself, but then exits immediately. We use the Jenkins
[Job DSL plugin](https://github.com/jenkinsci/job-dsl-plugin/wiki) to define the meta job.
- [.github/workflows/deb-packaging.yml](https://github.com/keymanapp/keyman/blob/master/.github/workflows/deb-packaging.yml)
contains the definition of the packaging GHA
- [resources/build/run-required-test-builds.sh](https://github.com/keymanapp/keyman/blob/master/resources/build/run-required-test-builds.sh)
runs on [TeamCity](https://build.palaso.org/buildConfiguration/Keyman_Test?)
to trigger the builds for the various platforms, among them the GHA package build.
- [resources/build/increment-version.sh](https://github.com/keymanapp/keyman/blob/master/resources/build/increment-version.sh)
runs on [TeamCity](https://build.palaso.org/buildConfiguration/Keyman_TriggerReleaseBuildsMaster?)
and increments the version number before triggering the builds for the
various platforms.
- The [linux/scripts](https://github.com/keymanapp/keyman/tree/master/linux/scripts)
subdirectory contains `bash` scripts that are used during the package build.
Some are only needed for Launchpad builds.
ci-builder-scripts also contains several generic scripts to set up a package build environment
(using `sbuilder`) and for building source and binary packages. These scripts are shared with
other projects.
- [deb-packaging.sh](https://github.com/keymanapp/keyman/blob/master/linux/scripts/deb-packaging.sh)
gets called by the packaging GHA to install dependencies, create the source
package and to verify the API.
The Keyman GitHub repo defines a webhook that triggers the meta job on Jenkins.
Changes to ci-builder-scripts go through [Gerrit](https://gerrit.lsdev.sil.org). See
[CONTRIBUTING.md](https://github.com/sillsdev/ci-builder-scripts/blob/master/CONTRIBUTING.md)
for details.
File structure:
- [groovy/KeymanPackagingJobs.groovy](https://github.com/sillsdev/ci-builder-scripts/blob/master/groovy/KeymanPackagingJobs.groovy)
contains the meta job definition
- The [bash/](https://github.com/sillsdev/ci-builder-scripts/tree/master/bash) subdirectory
contains `bash` scripts:
- [setup.sh](https://github.com/sillsdev/ci-builder-scripts/blob/master/bash/setup.sh) -
setup sbuild chroot environment
- [update](https://github.com/sillsdev/ci-builder-scripts/blob/master/bash/update) -
update the sbuild chroot environment
- [build-package](https://github.com/sillsdev/ci-builder-scripts/blob/master/bash/build-package) -
create a binary package
- [lsdev-pipeline-library](https://github.com/sillsdev/lsdev-pipeline-library) contains a reusable
Jenkins pipeline library. The
[vars/keymanPackaging.groovy](https://github.com/sillsdev/lsdev-pipeline-library/blob/master/vars/keymanPackaging.groovy)
file contains the bulk of the logic of the Keyman packaging job.
- The [Keyman GitHub repo](https://github.com/keymanapp/keyman) contains various scripts that are
used to trigger a build and as part of the package build, and of course the source code for the
packages:
- [resources/build/run-required-test-builds.sh](https://github.com/keymanapp/keyman/blob/master/resources/build/run-required-test-builds.sh)
runs on [TeamCity](https://build.palaso.org/buildConfiguration/Keyman_Test?) to trigger the
builds for the various platforms, among them the Jenkins package build.
- [resources/build/increment-version.sh](https://github.com/keymanapp/keyman/blob/master/resources/build/increment-version.sh)
runs on [TeamCity](https://build.palaso.org/buildConfiguration/Keyman_TriggerReleaseBuildsMaster?)
and increments the version number before triggering the builds for the various platforms.
- [linux/Jenkinsfile](https://github.com/keymanapp/keyman/blob/master/linux/Jenkinsfile) is a flag
for the meta job. If the meta job finds this file, it will create a new build configuration. This
file simply calls the packaging functionality defined in `lsdev-pipeline-library` and passes the
distributions and architectures to build as parameters.
- [linux/build/agent/install-deps](https://github.com/keymanapp/keyman/blob/master/linux/build/agent/install-deps)
installs dependencies on the current build agent.
- The [linux/scripts](https://github.com/keymanapp/keyman/tree/master/linux/scripts) subdirectory
contains `bash` scripts that are used during the package build. Some are only needed for
Launchpad builds.
- [jenkins.sh](https://github.com/keymanapp/keyman/blob/master/linux/scripts/jenkins.sh)
gets called from `lsdev-pipeline-library` to create a source package.
- [linux/debian](https://github.com/keymanapp/keyman/tree/master/linux/debian) - this is the `debian`
subdirectory for Keyman for Linux with the meta data for the Linux package.
See [Debian New Maintainers' Guide](https://www.debian.org/doc/manuals/maint-guide/) for
details to the various files.
- [linux/debian](https://github.com/keymanapp/keyman/tree/master/linux/debian) -
this is the `debian` subdirectory for Keyman for Linux with the meta data
for the Linux package.
See [Debian New Maintainers' Guide](https://www.debian.org/doc/manuals/maint-guide/)
for details to the various files in the `debian` directory.
### Flow of a Linux package build
- TeamCity jobs [Keyman_Test](https://build.palaso.org/buildConfiguration/Keyman_Test) or
[Keyman_TriggerReleaseBuilds*](https://build.palaso.org/buildConfiguration/Keyman_TriggerReleaseBuildsBeta)
trigger a build on [Jenkins](https://jenkins.lsdev.sil.org/view/Keyman/view/Pipeline/job/pipeline-keyman-packaging/)
- Jenkins verifies the build parameters and starts the matching build configuration for the PR or
branch
- The [build job](https://github.com/sillsdev/lsdev-pipeline-library/blob/master/vars/keymanPackaging.groovy) runs several checks:
- it exits immediately if the build is not manually triggered and no parameters are passed in
(i.e. it got triggered by the GitHub webhook)
- it doesn't build if this is a PR, didn't get triggered manually and the PR is not from a trusted
user
- it doesn't build if no Linux-relevant files changed unless the parameter `force` was passed
- manually triggered builds will always build
- build job installs
[dependencies](https://github.com/keymanapp/keyman/blob/master/linux/build/agent/install-deps)
on the current build agent
- build job creates a source package for the linux packages (keyman, kmflcomp,
libkmfl, and ibus-kmfl). This is done by calling
[scripts/jenkins.sh](https://github.com/keymanapp/keyman/blob/master/linux/scripts/jenkins.sh).
- build job creates the binary package for each linux package on each distribution (currently
bionic, focal, and groovy) and each architecture (amd64, i386 only for bionic)
- at the end of the build if it is not a build of a PR, the `.deb` file gets uploaded to llso
(alpha packages to e.g. `bionic-experimental`, beta packages to `bionic-proposed` and
packages build from the stable branch to the main section `bionic`)
- TeamCity jobs [Keyman_Test](https://build.palaso.org/buildConfiguration/Keyman_Test)
or [Keyman_TriggerReleaseBuilds*](https://build.palaso.org/buildConfiguration/Keyman_TriggerReleaseBuildsBeta)
trigger a packaging GHA build
- packaging GHA calls [deb-packaging.sh](https://github.com/keymanapp/keyman/blob/master/linux/scripts/deb-packaging.sh)
which installs dependencies and creates the source package
- packaging GHA creates the binary package for each linux package on each
distribution
- packaging GHA verifies that the API didn't change with the help of
[deb-packaging.sh](https://github.com/keymanapp/keyman/blob/master/linux/scripts/deb-packaging.sh)
- at the end of the build if it is not a build of a PR, the `.deb` files get
uploaded to llso (alpha packages to e.g. `jammy-experimental`, beta
packages to `jammy-proposed` and packages build from the stable branch
to the main section `jammy`)
- if the build is successful the job archives the artifacts
The Jenkins build progress is visible in two ways:
- [traditional view](https://jenkins.lsdev.sil.org/view/Keyman/view/Pipeline/job/pipeline-keyman-packaging/)
- [blue ocean view](https://jenkins.lsdev.sil.org/blue/organizations/jenkins/pipeline-keyman-packaging/activity)
**Note:** TC release builds pass the git tag to build to the Jenkins job. The same tag
gets passed twice as parameters `tag` and `tag2`. The first parameter gets persisted between
builds, allowing to retrigger a tag-build. The second parameter is necessary to distinguish
if this is a retriggered build of a tag-build.
### Local package builds
It is possible to use the usual Debian/Ubuntu tools to create the package locally. For someone who
only occasionally deals with packaging it might be easier to use the scripts that Jenkins runs:
#### Prerequisites for local package builds
Install `sbuild` (and probably some other packages that I forgot).
You’ll need a chroot image before you can use sbuild. The scripts in
[ci-builder-scripts](https://github.com/sillsdev/ci-builder-scripts) will help
with that. [`setup.sh`](https://github.com/sillsdev/ci-builder-scripts/blob/master/bash/setup.sh)
can setup such chroots:
```bash
bash/setup.sh --dists "focal bionic" --arches "amd64 i386"
```
[`update`](https://github.com/sillsdev/ci-builder-scripts/blob/master/bash/update) is used to
later update those chroots:
```bash
bash/update --dists "focal bionic" --arches "amd64 i386"
```
Set the `DEBSIGNKEY` environment variable to your public GPG key that will be used to sign
the packages.
#### Building packages
Building packages happen in the [Keyman source tree](https://github.com/keymanapp/keyman).
The Keyman
[`linux/scripts/jenkins.sh`](https://github.com/keymanapp/keyman/blob/master/linux/scripts/jenkins.sh)
script can be used to create a source package.
```bash
cd linux
./scripts/jenkins.sh keyman ${DEBSIGNKEY}
```
This creates a source package (`keyman_<version>-1.dsc`) and some `*.tar.?z`
files in the source root directory for `keyman`.
ci-builder-script's [`build-package`](https://github.com/sillsdev/ci-builder-scripts/blob/master/bash/build-package)
script creates the binary packages:
```bash
cd $KEYMAN_ROOT
~/ci-builder-scripts/bash/build-package \
--dists "focal bionic" --arches "amd64 i386" \
--debkeyid ${DEBSIGNKEY} --build-in-place --no-upload
```
This will create the binary package `keyman_<version>-1+<dist>1_<arch>.deb`.
To speed up package building you might want to limit the build to a single dist
(e.g. `--dists "bionic"`) and arch (e.g. `--arches "amd64"`).
After building packages it might be a good idea to clean up the source tree
before doing further work:
```bash
git clean -dxf
```
### Local package builds (Docker)
### Local package builds with Docker
It is possible to use the usual Debian/Ubuntu tools to create the package locally.
For someone who only occasionally deals with packaging it might be easier to use
the scripts that run on GitHub actions:
Docker and the scripts that run on GitHub actions:
#### Prerequisites for local package builds with Docker

2
linux/.pbuilderrc
View file

@ -14,7 +14,7 @@ DEBIAN_SUITES=($UNSTABLE_CODENAME $TESTING_CODENAME $STABLE_CODENAME $STABLE_BAC
"experimental" "unstable" "testing" "stable")
# List of Ubuntu suites. Update these when needed.
UBUNTU_SUITES=("lunar" "kinetic" "jammy" "focal")
UBUNTU_SUITES=("mantic" "lunar" "jammy" "focal")
# Mirrors to use. Update these to your preferred mirror.
DEBIAN_MIRROR="deb.debian.org"

71
linux/Dockerfile
View file

@ -1,4 +1,4 @@
# Copyright (c) 2022 SIL International. All rights reserved.
# Copyright (c) 2022-2023 SIL International. All rights reserved.
#
# builder image for a linux build
# see ../docs/build/linux-ubuntu.md
@ -7,27 +7,80 @@ FROM --platform=amd64 ubuntu:latest
LABEL org.opencontainers.image.authors="SIL International."
LABEL org.opencontainers.image.url="https://github.com/keymanapp/keyman.git"
LABEL org.opencontainers.image.title="Keyman Linux Build Image"
# We will switch to a build user after some installation
USER root
RUN useradd -c "Build user" -d $HOME -m build
ENV HOME /home/build
VOLUME /home/build
WORKDIR /home/build
RUN useradd -c "Build user" --home-dir $HOME --create-home --shell /usr/bin/bashwrapper build
VOLUME /home/build/build
WORKDIR /home/build/build
ENV DEBIAN_FRONTEND noninteractive
ENV DEBIAN_PRIORITY critical
ENV DEBCONF_NOWARNINGS yes
# Update to the latest
RUN apt-get -q -y update && \
apt-get -q -y install devscripts equivs meson python3 python3-setuptools software-properties-common && \
apt-get -q -y install devscripts equivs meson python3 python3-setuptools software-properties-common curl && \
add-apt-repository ppa:keymanapp/keyman && \
add-apt-repository ppa:keymanapp/keyman-alpha && \
add-apt-repository ppa:keymanapp/keyman-alpha
RUN apt-get -q -y update && \
apt-get -q -y upgrade
# Install dependencies
ADD debian/control /tmp/control
# Answer 'yes' to install questions
RUN (yes | mk-build-deps --install /tmp/control) || true
RUN curl -sL https://deb.nodesource.com/setup_18.x | bash
RUN apt-get -q -y install nodejs
RUN (yes | mk-build-deps --install /tmp/control) || true && \
rm /tmp/control
# Install Node
RUN curl -sL https://deb.nodesource.com/setup_18.x | bash && \
apt-get -q -y install nodejs
# Install emscripten
RUN cd /usr/share && \
git clone https://github.com/emscripten-core/emsdk.git && \
cd emsdk && \
./emsdk install latest && \
./emsdk activate latest && \
echo "#!/bin/bash" > /usr/bin/bashwrapper && \
echo "export EMSCRIPTEN_BASE=/usr/share/emsdk/upstream/emscripten" >> /usr/bin/bashwrapper
# Keyman Web
RUN curl --output google-chrome-stable_current_amd64.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \
apt-get -q -y install ./google-chrome-stable_current_amd64.deb && \
rm google-chrome-stable_current_amd64.deb && \
echo "export CHROME_BIN=/opt/google/chrome/chrome" >> /usr/bin/bashwrapper
# Keyman for Android
RUN apt-get -q -y install gradle maven pandoc sdkmanager jq && \
sdkmanager platform-tools && \
yes | sdkmanager --licenses && \
chown -R build:build /opt/android-sdk/ && \
echo "export ANDROID_HOME=/opt/android-sdk" >> /usr/bin/bashwrapper && \
echo "export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64" >> /usr/bin/bashwrapper
# Finish bashwrapper script and adjust permissions
RUN echo "\${@:-bash}" >> /usr/bin/bashwrapper && \
chmod +x /usr/bin/bashwrapper && \
chown -R build:build $HOME
# now, switch to build user
USER build
# Pre-install gradle. This will put files in ~/.gradle which will speed up builds.
RUN mkdir -p $HOME/tmp/gradle/wrapper && \
# KMEA uses gradle-7.5.1-bin
curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.jar https://raw.githubusercontent.com/keymanapp/keyman/master/android/KMEA/gradle/wrapper/gradle-wrapper.jar && \
curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.properties https://raw.githubusercontent.com/keymanapp/keyman/master/android/KMEA/gradle/wrapper/gradle-wrapper.properties && \
curl --location --output $HOME/tmp/gradlew https://raw.githubusercontent.com/keymanapp/keyman/master/android/KMEA/gradlew && \
chmod +x $HOME/tmp/gradlew && \
$HOME/tmp/gradlew --quiet && \
# Some projects use gradle-7.5.1-all, so we pre-install that as well
curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.jar https://raw.githubusercontent.com/keymanapp/keyman/master/android/Samples/KMSample1/gradle/wrapper/gradle-wrapper.jar && \
curl --location --output $HOME/tmp/gradle/wrapper/gradle-wrapper.properties https://raw.githubusercontent.com/keymanapp/keyman/master/android/Samples/KMSample1/gradle/wrapper/gradle-wrapper.properties && \
curl --location --output $HOME/tmp/gradlew https://raw.githubusercontent.com/keymanapp/keyman/master/android/Samples/KMSample1/gradlew && \
chmod +x $HOME/tmp/gradlew && \
$HOME/tmp/gradlew --quiet && \
rm -rf $HOME/tmp
ENTRYPOINT [ "/usr/bin/bashwrapper" ]

11
linux/Jenkinsfile vendored
View file

@ -1,11 +0,0 @@
#!groovy
// Copyright (c) 2019-2022 SIL International
// This software is licensed under the MIT license (http://opensource.org/licenses/MIT)
@Library('lsdev-pipeline-library') _
keymanPackaging {
distributionsToPackage = 'focal jammy kinetic lunar'
arches = 'amd64 i386'
packagesToBuild = ['keyman']
}

15
linux/debian/changelog
View file

@ -1,3 +1,18 @@
keyman (16.0.141-1) unstable; urgency=medium
* Work around mips64el build failure (#1041499)
* New upstream release.
* Re-release to Debian
-- Eberhard Beilharz <eb1@sil.org> Thu, 27 Jul 2023 16:30:04 +0200
keyman (16.0.140-1) unstable; urgency=medium
* New upstream release (closes: #1037707).
* Re-release to Debian
-- Eberhard Beilharz <eb1@sil.org> Mon, 24 Jul 2023 11:41:07 +0200
keyman (16.0.139-4) unstable; urgency=medium
* debian/tests: Revert previous change and ignore s390x from autopkgtests

2
linux/ibus-keyman/build.sh
View file

@ -61,7 +61,7 @@ fi
if builder_start_action test; then
cd "$THIS_SCRIPT_PATH/$MESON_PATH"
if builder_has_option --no-integration; then
meson test --print-errorlogs $builder_verbose keymanutil-tests print-kmpdetails-test print-kmp-test
meson test --print-errorlogs $builder_verbose setup-src-test keymanutil-tests print-kmpdetails-test print-kmp-test bcp47-util-tests teardown-src-test
else
meson test --print-errorlogs $builder_verbose
fi

1
linux/ibus-keyman/meson.build
View file

@ -13,6 +13,7 @@ ibus = dependency('ibus-1.0', version: '>= 1.2.0')
gtk = dependency('gtk+-3.0', version: '>= 2.4')
json_glib = dependency('json-glib-1.0', version: '>= 1.0')
systemd = dependency('libsystemd')
icu = dependency('icu-i18n')
core_dir = meson.current_source_dir() / '../../core'
common_dir = meson.current_source_dir() / '../../common'

99
linux/ibus-keyman/src/bcp47util.c Normal file
View file

@ -0,0 +1,99 @@
#include <gio/gio.h>
#include <unicode/locid.h>
#include <unicode/uloc.h>
/// Minimize the BCP-47 `tag` so that unnecessary parts get ommitted.
/// The result gets stored in `minimzedTag`.
///
/// @param tag The tag to process
/// @param minimizedTag Caller-provided character array for the
/// resulting minimized tag.
/// @param tagCapacity Array size of `minimizedTag`
/// @return The length of the minimized tag, or -1 in error case
int bcp47_minimize(const char* tag, char* minimizedTag, int tagCapacity) {
UErrorCode status = U_ZERO_ERROR;
if (!tag || strlen(tag) == 0) {
strncpy(minimizedTag, "", tagCapacity);
return -1;
}
// special treatment for `und-Latn` which is used by sil_ipa keyboard
if (strcmp(tag, "und-Latn") == 0) {
strncpy(minimizedTag, "und-Latn", tagCapacity);
return strlen(minimizedTag);
}
int capacity = 255;
char workingTag[capacity];
// special treatment for tags that start with `und`: replace `und` with `en`.
// ICU 70 doesn't properly treat `und`.
int isUnd = strncmp(tag, "und", 3) == 0;
if (isUnd) {
strcpy(workingTag, "en");
strncat(&workingTag[2], &tag[3], capacity - 3);
} else {
strncpy(workingTag, tag, capacity - 1);
}
workingTag[capacity - 1] = 0;
char localeId[capacity];
uloc_forLanguageTag(workingTag, localeId, capacity, NULL, &status);
if (U_FAILURE(status)) {
g_error("%s: uloc_forLanguageTag returned %0x", __FUNCTION__, status);
return -1;
}
char minimizedLocaleId[capacity];
uloc_minimizeSubtags(localeId, minimizedLocaleId, capacity, &status);
if (U_FAILURE(status)) {
g_error("%s: uloc_minimizeSubtags returned %0x", __FUNCTION__, status);
return -1;
}
int taglen = uloc_toLanguageTag(minimizedLocaleId, minimizedTag, tagCapacity, FALSE, &status);
if (U_FAILURE(status)) {
g_error("%s: uloc_toLanguageTag returned %0x", __FUNCTION__, status);
return -1;
}
if (isUnd) {
// Replace 'en' with 'und' again
strncpy(workingTag, &minimizedTag[2], capacity - 1);
workingTag[capacity - 1] = 0;
strcpy(minimizedTag, "und");
strncat(minimizedTag, workingTag, tagCapacity - 4);
minimizedTag[tagCapacity - 1] = 0;
taglen = strlen(minimizedTag);
}
return taglen;
}
/// Extract the language code from the BCP-47 `tag`
///
/// @param tag The BCP-47 tag
/// @param lang_code Caller-provided character array that will receive
/// the language code extracted from `tag`
/// @param capacity Array size of `lang_code`
/// @return TRUE if successful, otherwise FALSE
int bcp47_get_language_code(const char* tag, char* lang_code, int capacity) {
UErrorCode status = U_ZERO_ERROR;
if (!tag || strlen(tag) == 0) {
strncpy(lang_code, "", capacity);
return FALSE;
}
// ICU 70 doesn't properly treat `und`
if (strncmp(tag, "und", 3) == 0) {
strncpy(lang_code, "und", capacity);
return TRUE;
}
uloc_getLanguage(tag, lang_code, capacity, &status);
if (U_FAILURE(status)) {
g_error("%s: uloc_getLanguage returned %0x", __FUNCTION__, status);
return FALSE;
}
return TRUE;
}

7
linux/ibus-keyman/src/bcp47util.h Normal file
View file

@ -0,0 +1,7 @@
#ifndef __BCP47UTIL_H__
#define __BCP47UTIL_H__
int bcp47_minimize(const char* tag, char* minimizedTag, int capacity);
int bcp47_get_language_code(const char* tag, char* lang_code, int capacity);
#endif // __BCP47UTIL_H__

72
linux/ibus-keyman/src/keymanutil.c
View file

@ -57,6 +57,7 @@
#include <string.h>
#include <keyman/keyboardprocessor.h>
#include "bcp47util.h"
#include "keymanutil.h"
#include "kmpdetails.h"
#include "keyman-version.h"
@ -163,8 +164,6 @@ ibus_keyman_add_engines(GList * engines, GList * kmpdir_list)
get_kmp_details(kmp_dir, details);
for (k=details->keyboards; k != NULL; k = k->next) {
gchar *lang=NULL;
gchar *name_with_lang = NULL;
kmp_keyboard *keyboard = (kmp_keyboard *) k->data;
gboolean alreadyexists = FALSE;
@ -191,35 +190,50 @@ ibus_keyman_add_engines(GList * engines, GList * kmpdir_list)
for (l=keyboard->languages; l != NULL; l = l->next) {
kmp_language *language = (kmp_language *) l->data;
if (language->id != NULL) {
gchar **tagparts = g_strsplit(language->id, "-", 2);
lang = g_strdup(tagparts[0]);
g_strfreev(tagparts);
// If ibus doesn't know about the language then append the
// language name to the keyboard name
if (language->name != NULL) {
if (g_strcmp0(ibus_get_untranslated_language_name (lang), "Other") == 0) {
name_with_lang = g_strjoin(" - ", keyboard->name, language->name, NULL);
}
int capacity = 255;
gchar *name_with_lang = NULL;
gchar *minimized_tag = g_new0(gchar, capacity);
int result = bcp47_minimize(language->id, minimized_tag, capacity);
if (result < 0) {
g_strlcpy(minimized_tag, language->id, capacity);
}
gchar *lang_code = g_new0(gchar, capacity);
if (!bcp47_get_language_code(minimized_tag, lang_code, capacity)) {
g_strlcpy(lang_code, minimized_tag, capacity);
}
// If ibus doesn't know about the language then append the
// language name to the keyboard name
if (language->name != NULL) {
gchar *ibus_lang = ibus_get_untranslated_language_name(lang_code);
g_debug("%s: untranslated ibus language for %s: %s", __FUNCTION__, minimized_tag, ibus_lang);
if (g_strcmp0(ibus_lang, "Other") == 0) {
name_with_lang = g_strjoin(" - ", keyboard->name, language->name, NULL);
}
g_free(ibus_lang);
}
gchar *id_with_lang = g_strjoin(":", language->id, abs_kmx, NULL);
gchar *id_with_lang = g_strjoin(":", minimized_tag, abs_kmx, NULL);
g_message("adding engine %s", id_with_lang);
engines = g_list_append (engines,
ibus_keyman_engine_desc_new (id_with_lang, // lang:kmx full path
name_with_lang ? name_with_lang : keyboard->name, // longname
kbd_details->description, // description
details->info.copyright, // copyright if available
lang, // language, most are ignored by ibus except major languages
kbd_details->license, // license
details->info.author_desc, // author name only, not email
keyman_get_icon_file(abs_kmx), // icon full path
"us", // layout defaulting to us (en-US)
keyboard->version));
g_free(lang);
g_free(id_with_lang);
g_free(name_with_lang);
name_with_lang = NULL;
g_message("adding engine %s", id_with_lang);
engines = g_list_append(
engines,
ibus_keyman_engine_desc_new(
id_with_lang, // lang:kmx full path
name_with_lang ? name_with_lang : keyboard->name, // longname
kbd_details->description, // description
details->info.copyright, // copyright if available
lang_code, // language, most are ignored by ibus except major languages
kbd_details->license, // license
details->info.author_desc, // author name only, not email
keyman_get_icon_file(abs_kmx), // icon full path
"us", // layout defaulting to us (en-US)
keyboard->version));
g_free(lang_code);
g_free(minimized_tag);
g_free(id_with_lang);
g_free(name_with_lang);
}
}
}
@ -230,7 +244,7 @@ ibus_keyman_add_engines(GList * engines, GList * kmpdir_list)
keyboard->name, // longname
kbd_details->description, // description
details->info.copyright, // copyright if available
lang, // language, most are ignored by ibus except major languages
NULL, // language, most are ignored by ibus except major languages
kbd_details->license, // license
details->info.author_desc, // author name only, not email
keyman_get_icon_file(abs_kmx), // icon full path

3
linux/ibus-keyman/src/meson.build
View file

@ -1,6 +1,7 @@
util_files = files(
'keymanutil.c',
'kmpdetails.c',
'bcp47util.c',
)
engine_files = files(
@ -16,7 +17,7 @@ include_dirs = [
include_directories(meson.current_build_dir() / '..'),
]
deps = [ibus, gtk, json_glib, kmnkbp_lib, systemd]
deps = [gtk, ibus, icu, json_glib, kmnkbp_lib, systemd]
prefix = get_option('prefix')
cfg = configuration_data()

100
linux/ibus-keyman/src/test/bcp47util_tests.c Normal file
View file

@ -0,0 +1,100 @@
#include <glib-object.h>
#include <glib.h>
#include <gtk/gtk.h>
#include "bcp47util.h"
typedef struct {
} Bcp47UtilFixture;
typedef struct {
const char *tag;
const char *expected;
int expectedResult;
} TestData;
static void
test_bcp47_minimize(Bcp47UtilFixture *fixture, gconstpointer user_data) {
TestData *testData = (TestData *)user_data;
int capacity = 255;
char minimizedTag[capacity];
int result = bcp47_minimize(testData->tag, minimizedTag, capacity);
g_assert_cmpint(result, ==, testData->expectedResult);
g_assert_cmpstr(minimizedTag, ==, testData->expected);
}
static void
test_bcp47_get_language_code(Bcp47UtilFixture *fixture, gconstpointer user_data) {
TestData *testData = (TestData *)user_data;
int capacity = 255;
char lang_code[capacity];
int result = bcp47_get_language_code(testData->tag, lang_code, capacity);
g_assert_cmpint(result, ==, testData->expectedResult);
g_assert_cmpstr(lang_code, ==, testData->expected);
}
int
main(int argc, char *argv[]) {
gtk_init(&argc, &argv);
g_test_init(&argc, &argv, NULL);
g_test_set_nonfatal_assertions();
TestData testData1 = {NULL, "", -1};
g_test_add("/bcp47util/minimize/NULL", Bcp47UtilFixture, &testData1, NULL, test_bcp47_minimize, NULL);
TestData testData2 = {"", "", -1};
g_test_add("/bcp47util/minimize/EmptyString", Bcp47UtilFixture, &testData2, NULL, test_bcp47_minimize, NULL);
TestData testData3 = { "fuf", "fuf", 3 };
g_test_add("/bcp47util/minimize/fuf", Bcp47UtilFixture, &testData3, NULL, test_bcp47_minimize, NULL);
TestData testData4 = {"fuf-Latn", "fuf", 3};
g_test_add("/bcp47util/minimize/fuf-Latn", Bcp47UtilFixture, &testData4, NULL, test_bcp47_minimize, NULL);
TestData testData5 = {"fuf-Arab", "fuf-Arab", 8};
g_test_add("/bcp47util/minimize/fuf-Arab", Bcp47UtilFixture, &testData5, NULL, test_bcp47_minimize, NULL);
TestData testData6 = {"fuf-Adlm-ML", "fuf-Adlm-ML", 11};
g_test_add("/bcp47util/minimize/fuf-Adlm-ML", Bcp47UtilFixture, &testData6, NULL, test_bcp47_minimize, NULL);
TestData testData7 = {"und", "und", 3};
g_test_add("/bcp47util/minimize/und", Bcp47UtilFixture, &testData7, NULL, test_bcp47_minimize, NULL);
TestData testData8 = {"und-Latn", "und-Latn", 8};
g_test_add("/bcp47util/minimize/und-Latn", Bcp47UtilFixture, &testData8, NULL, test_bcp47_minimize, NULL);
TestData testData9 = {"und-fonipa", "und-fonipa", 10};
g_test_add("/bcp47util/minimize/und-fonipa", Bcp47UtilFixture, &testData9, NULL, test_bcp47_minimize, NULL);
TestData testData10 = {"und-Latn-fonipa", "und-fonipa", 10};
g_test_add("/bcp47util/minimize/und-Latn-fonipa", Bcp47UtilFixture, &testData10, NULL, test_bcp47_minimize, NULL);
TestData testData11 = {"mul", "mul", 3};
g_test_add("/bcp47util/minimize/mul", Bcp47UtilFixture, &testData11, NULL, test_bcp47_minimize, NULL);
// bcp47_get_language_code tests
TestData testData21 = {NULL, "", FALSE};
g_test_add("/bcp47util/langcode/NULL", Bcp47UtilFixture, &testData21, NULL, test_bcp47_get_language_code, NULL);
TestData testData22 = {"", "", FALSE};
g_test_add("/bcp47util/langcode/EmptyString", Bcp47UtilFixture, &testData22, NULL, test_bcp47_get_language_code, NULL);
TestData testData23 = {"fuf", "fuf", TRUE};
g_test_add("/bcp47util/langcode/fuf", Bcp47UtilFixture, &testData23, NULL, test_bcp47_get_language_code, NULL);
TestData testData24 = {"fuf-Latn", "fuf", TRUE};
g_test_add("/bcp47util/langcode/fuf-Latn", Bcp47UtilFixture, &testData24, NULL, test_bcp47_get_language_code, NULL);
TestData testData25 = {"fuf-Arab", "fuf", TRUE};
g_test_add("/bcp47util/langcode/fuf-Arab", Bcp47UtilFixture, &testData25, NULL, test_bcp47_get_language_code, NULL);
TestData testData26 = {"fuf-Adlm-ML", "fuf", TRUE};
g_test_add("/bcp47util/langcode/fuf-Adlm-ML", Bcp47UtilFixture, &testData26, NULL, test_bcp47_get_language_code, NULL);
TestData testData27 = {"und", "und", TRUE};
g_test_add("/bcp47util/langcode/und", Bcp47UtilFixture, &testData27, NULL, test_bcp47_get_language_code, NULL);
TestData testData28 = {"und-Latn", "und", TRUE};
g_test_add("/bcp47util/langcode/und-Latn", Bcp47UtilFixture, &testData28, NULL, test_bcp47_get_language_code, NULL);
return g_test_run();
}

101
linux/ibus-keyman/src/test/meson.build
View file

@ -3,11 +3,13 @@ keymanutil_sources = [
util_files,
]
keymanutil_deps = [ibus, gtk, json_glib, kmnkbp_lib]
keymanutil_deps = [gtk, ibus, icu, json_glib, kmnkbp_lib]
test_env = [
'G_TEST_SRCDIR=' + meson.current_source_dir(),
'G_TEST_BUILDDIR=' + meson.current_build_dir(),
'TOP_SRCDIR=' + meson.global_source_root(),
'TOP_BINDIR=' + meson.build_root(),
]
test_include_dirs = [
@ -16,47 +18,102 @@ test_include_dirs = [
include_directories(meson.current_build_dir() / '..'),
]
executable(
env_file = '/tmp/env-src-test.txt'
pid_file = '/tmp/ibus-keyman-src-test-pids'
setup_src_test_tests = find_program('setup-tests.sh', dirs: [meson.current_source_dir()])
teardown_tests = find_program('teardown-tests.sh', dirs: [meson.current_source_dir() / '../../tests/scripts'])
run_src_test = find_program('run-single-test.sh', dirs: [meson.current_source_dir()])
keymanutil_tests = executable(
'keymanutil-tests',
sources: keymanutil_sources,
dependencies: keymanutil_deps,
include_directories : test_include_dirs
)
print_kmpdetails_test = executable(
'print_kmpdetails',
sources: [
'print_kmpdetails.c',
'../kmpdetails.c'
],
dependencies: [ json_glib ],
include_directories: test_include_dirs
)
print_kmp_test = executable(
'print_kmp',
sources: [
'print_kmp.c',
],
dependencies: [ json_glib ],
include_directories: test_include_dirs
)
bcp47_util_tests = executable(
'bcp47-util-tests',
sources: [
'bcp47util_tests.c',
'../bcp47util.c'
],
dependencies: [ gtk, icu ],
include_directories: test_include_dirs
)
test(
'setup-src-test',
setup_src_test_tests,
args: ['--x11', env_file, pid_file],
env: test_env,
priority: -1,
is_parallel: false,
protocol: 'exitcode'
)
test(
'teardown-src-test',
teardown_tests,
args: [pid_file],
priority: -9,
is_parallel: false,
protocol: 'exitcode'
)
test(
'keymanutil-tests',
find_program('run-tests.sh'),
run_src_test,
args: [ '--tap', '-k', '--env', env_file, '--', keymanutil_tests],
env: test_env,
priority: -2,
is_parallel: false,
protocol: 'tap',
)
test(
'print-kmpdetails-test',
executable(
'print_kmpdetails',
sources: [
'print_kmpdetails.c',
'../kmpdetails.c'
],
dependencies: [ json_glib ],
include_directories: test_include_dirs
),
args: [ meson.current_source_dir() ],
run_src_test,
args: [ '--', print_kmpdetails_test, meson.current_source_dir() ],
env: test_env,
priority: -2,
protocol: 'exitcode',
)
test(
'print-kmp-test',
executable(
'print_kmp',
sources: [
'print_kmp.c',
],
dependencies: [ json_glib ],
include_directories: test_include_dirs
),
args: [ meson.current_source_dir() / 'kmp.json' ],
run_src_test,
args: [ '--', print_kmp_test, meson.current_source_dir() / 'kmp.json' ],
env: test_env,
priority: -2,
protocol: 'exitcode',
)
test(
'bcp47-util-tests',
run_src_test,
args: [ '--tap', '-k', '--env', env_file, '--', bcp47_util_tests ],
env: test_env,
priority: -2,
is_parallel: false,
protocol: 'tap',
)

48
linux/ibus-keyman/src/test/run-single-test.sh Executable file
View file

@ -0,0 +1,48 @@
#!/bin/bash
function help() {
echo "Usage:"
echo " $0 [--env <envfile>] [-k] [--tap] [--] TESTFILE [TESTARGS]"
echo
echo "Arguments:"
echo " --help, -h, -? Display this help"
echo " --verbose, -v Run tests verbosely"
echo " --debug debug test logging output"
echo " -k passed to GLib testing framework"
echo " --tap output in TAP format. Passed to GLib testing framework"
echo " --env <envfile> Name of the file containing environment variables to use"
exit 0
}
function run_tests() {
# Output these lines to stderr - the first line on stdout has to be the TAP version number
# which running ${TESTFILE} outputs
echo "# NOTE: When the tests fail check /tmp/ibus-engine-keyman.log and /tmp/ibus-daemon.log!" >&2
echo "" >&2
echo "# Starting tests..." >&2
# Note: -k and --tap are consumed by the GLib testing framework
# shellcheck disable=SC2086
"${TESTFILE}" ${ARG_K-} ${ARG_TAP-} ${ARG_VERBOSE-} ${ARG_DEBUG-} "$@"
echo "# Finished tests."
}
while (( $# )); do
case $1 in
--help|-h|-\?) help ;;
-k) ARG_K=$1 ;;
--tap) ARG_TAP=$1 ;;
--verbose|-v) ARG_VERBOSE=--verbose;;
--debug) ARG_DEBUG=--debug-log;;
--env) shift ; ARG_ENV=$1 ;;
--) shift ; TESTFILE=$1; shift ; break ;;
*) echo "Error: Unexpected argument \"$1\". Exiting." ; exit 4 ;;
esac
shift || (echo "Error: The last argument is missing a value. Exiting."; false) || exit 5
done
# shellcheck source=/dev/null
. "$ARG_ENV"
run_tests "$@"

6
linux/ibus-keyman/src/test/run-tests.sh
View file

@ -7,8 +7,8 @@ if [ -v KEYMAN_PKG_BUILD ]; then
# During package builds we skip these tests - they often fail, e.g.
# during Debian reproducibility testing with an error like
# "cannot open display: :32"
echo "1..1"
echo "ok 1 # SKIP on package build"
echo "TAP version 14"
echo "1..0 # SKIP on package build"
exit 0
fi
@ -58,4 +58,4 @@ glib-compile-schemas "$SCHEMA_DIR"
export GSETTINGS_BACKEND=memory
${G_TEST_BUILDDIR:-.}/keymanutil-tests "$@"
"${G_TEST_BUILDDIR:-.}/keymanutil-tests" "$@"

6
linux/ibus-keyman/src/test/setup-tests.sh Executable file
View file

@ -0,0 +1,6 @@
#!/usr/bin/env bash
set -eu
. "$(dirname "$0")/../../tests/scripts/test-helper.inc.sh"
setup_display_server_only "$1" "$2" "$3"

90
linux/ibus-keyman/tests/meson.build
View file

@ -10,7 +10,7 @@ kmnkbp_tests_lib = cc.find_library(
dirs: [ core_dir / 'build/arch' / get_option('buildtype') / 'tests/kmx_test_source' ]
)
test_deps = [ibus, gtk, json_glib, kmnkbp_lib, kmnkbp_tests_lib, systemd]
test_deps = [gtk, ibus, icu, json_glib, kmnkbp_lib, kmnkbp_tests_lib, systemd]
dbus_deps = [gtk, systemd]
test_env = [
@ -57,6 +57,10 @@ teardown_tests = find_program('teardown-tests.sh', dirs: [meson.current_source_d
run_test = find_program('run-single-test.sh', dirs: [meson.current_source_dir() / 'scripts'])
find_tests = find_program('find-tests.sh', dirs: [meson.current_source_dir() / 'scripts'])
# Mutter 40.x added the --headless option wich we need in order to be able to run the Wayland tests
mutter = find_program('mutter', required: false, version: '>=40')
can_build_wayland = mutter.found()
test(
'setup-x11',
setup_tests,
@ -67,16 +71,6 @@ test(
protocol: 'exitcode'
)
test(
'setup-wayland',
setup_tests,
args: ['--wayland', env_file, pid_file],
env: test_env,
priority: -20,
is_parallel: false,
protocol: 'exitcode'
)
test(
'teardown-x11',
teardown_tests,
@ -86,14 +80,26 @@ test(
protocol: 'exitcode'
)
test(
'teardown-wayland',
teardown_tests,
args: [pid_file],
priority: -29,
is_parallel: false,
protocol: 'exitcode'
)
if can_build_wayland
test(
'setup-wayland',
setup_tests,
args: ['--wayland', env_file, pid_file],
env: test_env,
priority: -20,
is_parallel: false,
protocol: 'exitcode'
)
test(
'teardown-wayland',
teardown_tests,
args: [pid_file],
priority: -29,
is_parallel: false,
protocol: 'exitcode'
)
endif
kmxtest_files = run_command(
find_tests,
@ -130,26 +136,28 @@ foreach kmx: kmxtest_files
timeout: 120,
protocol: 'tap',
)
test(
'Wayland-' + testname + '__surrounding-text',
run_test,
args: [ '--wayland', '--surrounding-text', test_args],
env: test_env,
depends: [test_exe],
priority: -21,
is_parallel: false,
timeout: 120,
protocol: 'tap',
)
test(
'Wayland-' + testname + '__no-surrounding-text',
run_test,
args: [ '--wayland', '--no-surrounding-text', test_args],
env: test_env,
depends: [test_exe],
priority: -22,
is_parallel: false,
timeout: 120,
protocol: 'tap',
)
if can_build_wayland
test(
'Wayland-' + testname + '__surrounding-text',
run_test,
args: [ '--wayland', '--surrounding-text', test_args],
env: test_env,
depends: [test_exe],
priority: -21,
is_parallel: false,
timeout: 120,
protocol: 'tap',
)
test(
'Wayland-' + testname + '__no-surrounding-text',
run_test,
args: [ '--wayland', '--no-surrounding-text', test_args],
env: test_env,
depends: [test_exe],
priority: -22,
is_parallel: false,
timeout: 120,
protocol: 'tap',
)
endif
endforeach

13
linux/ibus-keyman/tests/scripts/run-single-test.sh
View file

@ -9,8 +9,8 @@ if [ -v KEYMAN_PKG_BUILD ]; then
# ibus requires to find /var/lib/dbus/machine-id or /etc/machine-id, otherwise it fails with:
# "Bail out! IBUS-FATAL-WARNING: Unable to load /var/lib/dbus/machine-id: Failed to open file
# “/var/lib/dbus/machine-id”: No such file or directory"
echo "1..1"
echo "ok 1 - Integration tests # SKIP on package build"
echo "TAP version 14"
echo "1..0 # SKIP on package build"
exit 0
fi
@ -38,10 +38,13 @@ function help() {
}
function run_tests() {
echo "# NOTE: When the tests fail check /tmp/ibus-engine-keyman.log and /tmp/ibus-daemon.log!"
echo ""
# Output these lines to stderr - the first line on stdout has to be the TAP version number
# which running ${TESTFILE} outputs
echo "# NOTE: When the tests fail check /tmp/ibus-engine-keyman.log and /tmp/ibus-daemon.log!" >&2
echo "" >&2
echo "# Starting tests..." >&2
echo "# Starting tests..."
# Note: -k and --tap are consumed by the GLib testing framework
# shellcheck disable=SC2086
"${G_TEST_BUILDDIR:-.}"/ibus-keyman-tests ${ARG_K-} ${ARG_TAP-} \

3
linux/ibus-keyman/tests/scripts/run-tests.sh
View file

@ -18,8 +18,7 @@ if [ -v KEYMAN_PKG_BUILD ]; then
# ibus requires to find /var/lib/dbus/machine-id or /etc/machine-id, otherwise it fails with:
# "Bail out! IBUS-FATAL-WARNING: Unable to load /var/lib/dbus/machine-id: Failed to open file
# “/var/lib/dbus/machine-id”: No such file or directory"
echo "1..1"
echo "ok 1 - Integration tests # SKIP on package build"
echo "1..0 # SKIP on package build"
exit 0
fi

2
linux/ibus-keyman/tests/scripts/setup-tests.sh
View file

@ -3,4 +3,6 @@ set -eu
. "$(dirname "$0")/test-helper.inc.sh"
exit_on_package_build
setup "$1" "$2" "$3"

2
linux/ibus-keyman/tests/scripts/teardown-tests.sh
View file

@ -3,4 +3,6 @@ set -eu
. "$(dirname "$0")/test-helper.inc.sh"
exit_on_package_build
cleanup "$1"

106
linux/ibus-keyman/tests/scripts/test-helper.inc.sh
View file

@ -10,7 +10,7 @@ function can_run_wayland() {
fi
}
function generate_kmpjson() {
function _generate_kmpjson() {
local TESTDIR
TESTDIR="$1"
pushd "$TESTDIR" > /dev/null || exit
@ -78,7 +78,7 @@ EOF
popd > /dev/null || exit
}
function link_test_keyboards() {
function _link_test_keyboards() {
KMX_TEST_DIR=$1
TESTDIR=$2
TESTBASEDIR=$3
@ -93,22 +93,23 @@ function link_test_keyboards() {
fi
}
function setup() {
local DISPLAY_SERVER ENV_FILE PID_FILE TOP_SRCDIR TOP_BINDIR TESTBASEDIR TESTDIR
DISPLAY_SERVER=$1
ENV_FILE=$2
PID_FILE=$3
function _setup_init() {
local ENV_FILE PID_FILE
ENV_FILE=$1
PID_FILE=$2
TOP_SRCDIR=${G_TEST_SRCDIR:-$(realpath "$(dirname "$0")/..")}/..
TOP_BINDIR=${G_TEST_BUILDDIR:-$(realpath "$(dirname "$0/..")")}/..
TESTBASEDIR=${XDG_DATA_HOME:-$HOME/.local/share}/keyman
TESTDIR=${TESTBASEDIR}/test_kmx
if [ -z "${TOP_SRCDIR:-}" ]; then
TOP_SRCDIR=${G_TEST_SRCDIR:-$(realpath "$(dirname "$0")/..")}/..
fi
if [ -z "${TOP_BINDIR:-}" ]; then
TOP_BINDIR=${G_TEST_BUILDDIR:-$(realpath "$(dirname "$0/..")")}/..
fi
echo > "$ENV_FILE"
if [ -f "$PID_FILE" ]; then
# kill previous instances
"$(dirname "$0")"/teardown-tests.sh "$PID_FILE"
"$(dirname "$0")"/teardown-tests.sh "$PID_FILE" || true
fi
echo > "$PID_FILE"
@ -127,19 +128,34 @@ function setup() {
exit 2
fi
link_test_keyboards "${TOP_SRCDIR}/../../common/test/keyboards/baseline" "$TESTDIR" "$TESTBASEDIR"
export LD_LIBRARY_PATH=${COMMON_ARCH_DIR}/src:${LD_LIBRARY_PATH-}
echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> "$ENV_FILE"
}
generate_kmpjson "$TESTDIR"
function _setup_test_dbus_server() {
local ENV_FILE PID_FILE
ENV_FILE=$1
PID_FILE=$2
# Start test dbus server
# Start test dbus server. This will create `/tmp/km-test-server.env`.
"${TOP_BINDIR}/tests/km-dbus-test-server" &> /dev/null &
sleep 1
source /tmp/km-test-server.env
cat /tmp/km-test-server.env >> "$ENV_FILE"
cat /tmp/km-test-server.env >> "$PID_FILE"
echo "${TOP_BINDIR}/tests/stop-test-server" >> "$PID_FILE"
if [ "$DISPLAY_SERVER" == "wayland" ]; then
source /tmp/km-test-server.env
echo "# DBUS_SESSION_BUS_ADDRESS=$DBUS_SESSION_BUS_ADDRESS"
}
function _setup_display_server() {
local DISPLAY_SERVER ENV_FILE PID_FILE
ENV_FILE=$1
PID_FILE=$2
DISPLAY_SERVER=$3
if [ "$DISPLAY_SERVER" == "--wayland" ]; then
if ! can_run_wayland; then
# support for --headless got added in mutter 40.x
echo "ERROR: mutter doesn't support running headless. Can't run Wayland tests."
@ -173,6 +189,12 @@ function setup() {
export DISPLAY=:32
echo "export DISPLAY=\"$DISPLAY\"" >> "$ENV_FILE"
fi
}
function _setup_schema_and_gsettings() {
local ENV_FILE PID_FILE
ENV_FILE=$1
PID_FILE=$2
# Install schema to temporary directory. This removes the build dependency on the keyman package.
SCHEMA_DIR=$TEMP_DATA_DIR/glib-2.0/schemas
@ -186,9 +208,6 @@ function setup() {
cp "${TOP_SRCDIR}"/../keyman-config/resources/com.keyman.gschema.xml "$SCHEMA_DIR"/
glib-compile-schemas "$SCHEMA_DIR"
export LD_LIBRARY_PATH=${COMMON_ARCH_DIR}/src:${LD_LIBRARY_PATH-}
echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> "$ENV_FILE"
# Ubuntu 18.04 Bionic doesn't have ibus-memconf, and glib is not compiled with the keyfile
# backend enabled, so we just use the default backend. Otherwise we use the keyfile
# store which interferes less when running on a dev machine.
@ -197,6 +216,12 @@ function setup() {
echo "export GSETTINGS_BACKEND=\"$GSETTINGS_BACKEND\"" >> "$ENV_FILE"
IBUS_CONFIG=--config=/usr/libexec/ibus-memconf
fi
}
function _setup_ibus() {
local ENV_FILE PID_FILE
ENV_FILE=$1
PID_FILE=$2
#shellcheck disable=SC2086
ibus-daemon ${ARG_VERBOSE-} --daemonize --panel=disable --address=unix:abstract="${TEMP_DATA_DIR}/test-ibus" ${IBUS_CONFIG-} &> /tmp/ibus-daemon.log
@ -208,11 +233,42 @@ function setup() {
echo "export IBUS_ADDRESS=\"$IBUS_ADDRESS\"" >> "$ENV_FILE"
echo "# DBUS_SESSION_BUS_ADDRESS=$DBUS_SESSION_BUS_ADDRESS"
#shellcheck disable=SC2086
"${TOP_BINDIR}/src/ibus-engine-keyman" --testing ${ARG_VERBOSE-} &> /tmp/ibus-engine-keyman.log &
echo "kill -9 $! || true" >> "$PID_FILE"
sleep 1s
}
function setup() {
local DISPLAY_SERVER ENV_FILE PID_FILE TESTBASEDIR TESTDIR
DISPLAY_SERVER=$1
ENV_FILE=$2
PID_FILE=$3
_setup_init "${ENV_FILE}" "${PID_FILE}"
TESTBASEDIR=${XDG_DATA_HOME:-$HOME/.local/share}/keyman
TESTDIR=${TESTBASEDIR}/test_kmx
_link_test_keyboards "${TOP_SRCDIR}/../../common/test/keyboards/baseline" "$TESTDIR" "$TESTBASEDIR"
_generate_kmpjson "$TESTDIR"
_setup_test_dbus_server "${ENV_FILE}" "${PID_FILE}"
_setup_display_server "${ENV_FILE}" "${PID_FILE}" "${DISPLAY_SERVER}"
_setup_schema_and_gsettings "${ENV_FILE}" "${PID_FILE}"
_setup_ibus "${ENV_FILE}" "${PID_FILE}"
}
function setup_display_server_only() {
local DISPLAY_SERVER ENV_FILE PID_FILE TESTBASEDIR TESTDIR
DISPLAY_SERVER=$1
ENV_FILE=$2
PID_FILE=$3
_setup_init "${ENV_FILE}" "${PID_FILE}"
_setup_display_server "${ENV_FILE}" "${PID_FILE}" "${DISPLAY_SERVER}"
_setup_schema_and_gsettings "${ENV_FILE}" "${PID_FILE}"
}
function cleanup() {
@ -227,3 +283,11 @@ function cleanup() {
echo "# Finished shutdown of processes."
fi
}
function exit_on_package_build() {
if [ -v KEYMAN_PKG_BUILD ]; then
# Skip setup during package builds - can't run headless and we won't
# run the other tests anyway
exit 0
fi
}

4
linux/keyman-config/keyman_config/install_kmp.py
View file

@ -238,9 +238,9 @@ class InstallKmp():
if not language:
return language
language = CanonicalLanguageCodeUtils.findBestTag(language, False, True)
language = CanonicalLanguageCodeUtils.findBestTag(language, False, False)
for supportedLanguage in supportedLanguages:
tag = CanonicalLanguageCodeUtils.findBestTag(supportedLanguage['id'], False, True)
tag = CanonicalLanguageCodeUtils.findBestTag(supportedLanguage['id'], False, False)
if tag == language:
return tag
return None

13
linux/keyman-config/tests/test_install_kmp.py
View file

@ -199,19 +199,22 @@ class InstallKmpTests(unittest.TestCase):
languages = [
{'id': 'de'},
{'id': 'esi-Latn'},
{'id': 'dyo'}
{'id': 'dyo'},
{'id': 'fuh-Arab'}
]
for testcase in [
{'given': 'de', 'expected': 'de'},
{'given': 'esi', 'expected': 'esi-Latn'},
{'given': 'esi-Latn', 'expected': 'esi-Latn'},
{'given': 'esi', 'expected': 'esi'},
{'given': 'esi-Latn', 'expected': 'esi'},
{'given': 'es', 'expected': None},
{'given': 'en', 'expected': None},
{'given': None, 'expected': None},
# #3399
{'given': 'dyo-latn', 'expected': 'dyo-Latn'},
{'given': 'dyo', 'expected': 'dyo-Latn'},
{'given': 'dyo-latn', 'expected': 'dyo'},
{'given': 'dyo', 'expected': 'dyo'},
{'given': 'fuh-Arab', 'expected': 'fuh-Arab'},
{'given': 'fuh', 'expected': None},
]:
with self.subTest(data=testcase):
# Execute

6
linux/keyman-system-service/README.md
View file

@ -1,8 +1,8 @@
# keyman-system-service
A DBus system service that allows to access /dev/input/* devices
to toggle capslock and perform other keyboard related actions when
running under Wayland.
A DBus system service that allows to access `/dev/input/*` devices
to toggle capslock and perform other keyboard related actions. This is
required when running under Wayland, but also used with X11.
See <https://0pointer.net/blog/the-new-sd-bus-api-of-systemd.html>,
<https://stackoverflow.com/a/44281937>,

2
linux/scripts/cow.sh
View file

@ -3,7 +3,7 @@
# If needed set cowbuilder up for building Keyman Debian packages
# Then cowbuilder update
distributions='focal jammy kinetic lunar'
distributions='focal jammy lunar mantic'
if ! dpkg-query -l cowbuilder; then
echo "installing pbuilder and cowbuilder"

2
linux/scripts/deb.sh
View file

@ -11,7 +11,7 @@
set -e
all_distributions="focal jammy"
all_distributions="focal jammy lunar mantic"
distributions=""
echo "all_distributions: ${all_distributions}"

2
linux/scripts/dist.sh
View file

@ -49,7 +49,7 @@ dpkg-source --tar-ignore=*~ --tar-ignore=.git --tar-ignore=.gitattributes \
--tar-ignore=core/build \
--tar-ignore=developer --tar-ignore=docs --tar-ignore=ios \
--tar-ignore=linux/keyman-config/buildtools/build-langtags.py --tar-ignore=__pycache__ \
--tar-ignore=linux/help --tar-ignore=linux/Jenkinsfile \
--tar-ignore=linux/help \
--tar-ignore=mac --tar-ignore=node_modules --tar-ignore=oem \
--tar-ignore=linux/build \
--tar-ignore=linux/builddebs \

65
linux/scripts/jenkins.sh
View file

@ -1,65 +0,0 @@
#!/bin/bash
# $1 - project name with appended tier, e.g. keyman-alpha
# $2 - GPG key used for signing the source package
set -e
set -u
## START STANDARD BUILD SCRIPT INCLUDE
# adjust relative paths as necessary
THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")"
. "${THIS_SCRIPT%/*}/../../resources/build/build-utils.sh"
## END STANDARD BUILD SCRIPT INCLUDE
. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh"
. "$THIS_SCRIPT_PATH/package-build.inc.sh"
keyman_projects="keyman"
tier="stable"
if [[ "$1" =~ "-alpha" ]]; then
tier="alpha"
elif [[ "$1" =~ "-beta" ]]; then
tier="beta"
fi
proj="$1"
proj=${proj%"-alpha"}
proj=${proj%"-beta"}
fullsourcename="keyman"
sourcedir="$KEYMAN_ROOT"
sourcename=${fullsourcename%"-alpha"}
sourcename=${sourcename%"-beta"}
# set Debian/changelog environment
export DEBFULLNAME="${fullsourcename} Package Signing Key"
export DEBEMAIL='jenkins@sil.org'
checkAndInstallRequirements
# clean up prev deb builds
builder_heading "cleaning previous builds of $1"
rm -rf builddebs
rm -rf "$sourcedir/${1}"_*.{dsc,build,buildinfo,changes,tar.?z,log}
rm -rf "$sourcedir/../${1}"_*.{dsc,build,buildinfo,changes,tar.?z,log}
builder_heading "Make source package for $fullsourcename"
builder_heading "reconfigure"
TIER="$tier" ./scripts/reconf.sh
builder_heading "Make origdist"
./scripts/dist.sh origdist
builder_heading "Make deb source"
./scripts/deb.sh sourcepackage
#sign source package
for file in builddebs/*.dsc; do
builder_heading "Signing source package $file"
debsign -k"$2" "$file"
done
mv builddebs/* ..

2
linux/scripts/launchpad.sh
View file

@ -33,7 +33,7 @@ else
fi
echo "ppa: ${ppa}"
distributions="${DIST:-focal jammy kinetic lunar}"
distributions="${DIST:-focal jammy lunar mantic}"
packageversion="${PACKAGEVERSION:-1~sil1}"
BASEDIR=$(pwd)

14
linux/scripts/package-build.inc.sh
View file

@ -49,6 +49,14 @@ function downloadSource() {
sha256sum -c --ignore-missing SHA256SUMS |grep "${proj}"
}
function wait_for_apt_deb {
# from https://gist.github.com/hrpatel/117419dcc3a75e46f79a9f1dce99ef52
while sudo fuser /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock &>/dev/null 2>&1; do
echo "Waiting for apt/dpkg lock to release, sleeping 10s"
sleep 10
done
}
function checkAndInstallRequirements()
{
local TOINSTALL=""
@ -63,12 +71,12 @@ function checkAndInstallRequirements()
export DEBIAN_FRONTEND=noninteractive
if [ -n "$TOINSTALL" ]; then
sudo apt-get update
wait_for_apt_deb && sudo apt-get update
# shellcheck disable=SC2086
sudo apt-get -qy install $TOINSTALL
wait_for_apt_deb && sudo apt-get -qy install $TOINSTALL
fi
sudo mk-build-deps debian/control
sudo apt-get -qy --allow-downgrades install ./keyman-build-deps_*.deb
wait_for_apt_deb && sudo apt-get -qy --allow-downgrades install ./keyman-build-deps_*.deb
sudo rm -f keyman-buid-deps_*
}

4
linux/scripts/upload-to-debian.sh
View file

@ -105,7 +105,7 @@ git add debian/changelog
git commit -m "chore(linux): Update debian changelog"
if [ -n "$PUSH" ]; then
$NOOP git push --force-with-lease origin chore/linux/changelog
$NOOP gh pr create --draft --base "$stable_branch" --title "chore(linux): Update debian changelog" --body "@keymanapp-test-bot skip"
$NOOP gh pr create --draft --base "${stable_branch#origin/}" --title "chore(linux): Update debian changelog" --body "@keymanapp-test-bot skip"
fi
if $ISBETA; then
@ -118,7 +118,7 @@ git checkout -B chore/linux/cherry-pick/changelog ${CLBRANCH}
git cherry-pick -x chore/linux/changelog
if [ -n "$PUSH" ]; then
$NOOP git push --force-with-lease origin chore/linux/cherry-pick/changelog
$NOOP gh pr create --draft --base ${CLBRANCH} --title "chore(linux): Update debian changelog 🍒" --body "@keymanapp-test-bot skip"
$NOOP gh pr create --draft --base ${CLBRANCH#origin/} --title "chore(linux): Update debian changelog 🍒" --body "@keymanapp-test-bot skip"
fi
builder_heading "Finishing"

6
package-lock.json generated
View file

@ -11214,9 +11214,9 @@
}
},
"node_modules/word-wrap": {
"version": "1.2.3",
"resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",
"integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==",
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.4.tgz",
"integrity": "sha512-2V81OA4ugVo5pRo46hAoD2ivUJx8jXmWXfUkY4KFNw0hEptvN0QfH3K4nHiwzGeKl5rFKedV48QVoqYavy4YpA==",
"dev": true,
"engines": {
"node": ">=0.10.0"

33
resources/build/build-utils.sh
View file

@ -74,24 +74,31 @@ function findVersion() {
VERSION_TAG=
fi
if [ -z "${TEAMCITY_VERSION-}" -a -z "${JENKINS_HOME-}" ]; then
# Local dev machine, not TeamCity
if [ -z "${TEAMCITY_VERSION-}" ] && [ -z "${GITHUB_ACTIONS-}" ]; then
# Local dev machine, not TeamCity or GitHub Action
VERSION_TAG="$VERSION_TAG-local"
VERSION_ENVIRONMENT=local
else
elif [ -n "${TEAMCITY_PR_NUMBER-}" ]; then
# On TeamCity: are we running a pull request build or a master/beta/stable build?
if [ ! -z "${TEAMCITY_PR_NUMBER-}" ]; then
VERSION_ENVIRONMENT=test
# Note TEAMCITY_PR_NUMBER can also be 'master', 'beta', or 'stable-x.y'
# This indicates we are running a Test build.
if [[ $TEAMCITY_PR_NUMBER =~ ^(master|beta|stable(-[0-9]+\.[0-9]+)?)$ ]]; then
VERSION_TAG="$VERSION_TAG-test"
else
VERSION_TAG="$VERSION_TAG-test-$TEAMCITY_PR_NUMBER"
fi
VERSION_ENVIRONMENT="test"
# Note TEAMCITY_PR_NUMBER can also be 'master', 'beta', or 'stable-x.y'
# This indicates we are running a Test build.
if [[ $TEAMCITY_PR_NUMBER =~ ^(master|beta|stable(-[0-9]+\.[0-9]+)?)$ ]]; then
VERSION_TAG="$VERSION_TAG-test"
else
VERSION_ENVIRONMENT="$TIER"
VERSION_TAG="$VERSION_TAG-test-$TEAMCITY_PR_NUMBER"
fi
elif [ -n "${GITHUB_ACTIONS-}" ] && ${GHA_TEST_BUILD-}; then
VERSION_ENVIRONMENT="test"
# Note GHA_BRANCH can be 'master', 'beta', or 'stable-x.y'
# This indicates we are running a Test build.
if [[ ${GHA_BRANCH-} =~ ^(master|beta|stable(-[0-9]+\.[0-9]+)?)$ ]]; then
VERSION_TAG="${VERSION_TAG}-test"
else
VERSION_TAG="${VERSION_TAG}-test-${GHA_BRANCH-unset}"
fi
else
VERSION_ENVIRONMENT="$TIER"
fi
VERSION_WITH_TAG="$VERSION$VERSION_TAG"

2
resources/build/increment-version.sh
View file

@ -143,7 +143,7 @@ if [ "$action" == "commit" ]; then
popd > /dev/null
#
# Trigger builds for the previous version on TeamCity, Jenkins and GitHub
# Trigger builds for the previous version on TeamCity and GitHub
#
triggerBuilds

6
resources/build/run-required-test-builds.sh
View file

@ -54,11 +54,7 @@ function triggerTestBuilds() {
eval test_builds='(${'bc_test_$platform'[@]})'
for test_build in "${test_builds[@]}"; do
if [[ $test_build == "" ]]; then continue; fi
if [ "${test_build:(-8)}" == "_Jenkins" ]; then
local job=${test_build%_Jenkins}
echo " -- Triggering build configuration $job/$branch on Jenkins"
triggerJenkinsBuild "$job" "$branch" "$force"
elif [ "${test_build:(-7)}" == "_GitHub" ]; then
if [ "${test_build:(-7)}" == "_GitHub" ]; then
local job=${test_build%_GitHub}
echo " -- Triggering GitHub action build $job/$branch"
triggerGitHubActionsBuild true "$job" "$branch"

62
resources/build/trigger-builds.inc.sh
View file

@ -17,11 +17,7 @@ function triggerBuilds() {
eval builds='(${'bc_${bcbase}_${platform}'[@]})'
for build in "${builds[@]}"; do
if [[ $build == "" ]]; then continue; fi
if [ "${build:(-8)}" == "_Jenkins" ]; then
local job=${build%_Jenkins}
echo Triggering Jenkins build "$job" "$base" "true"
triggerJenkinsBuild "$job" "$base" "true"
elif [ "${build:(-7)}" == "_GitHub" ]; then
if [ "${build:(-7)}" == "_GitHub" ]; then
local job=${build%_GitHub}
echo Triggering GitHub action build "$job" "$base"
triggerGitHubActionsBuild false "$job" "$base"
@ -69,62 +65,6 @@ function triggerTeamCityBuild() {
-d "$command"
}
function triggerJenkinsBuild() {
local JENKINS_JOB="$1"
local JENKINS_BRANCH="${2:-master}"
local JENKINS_SERVER=https://jenkins.lsdev.sil.org
local FORCE=""
if [ "${3:-false}" == "true" ]; then
FORCE=", \"force\": true"
fi
local TAG=""
# This will only be true if we created and pushed a tag
if [ "${action:-""}" == "commit" ]; then
TAG=", \"tag\": \"$VERSION_GIT_TAG\", \"tag2\": \"$VERSION_GIT_TAG\""
fi
if [[ $JENKINS_BRANCH != stable-* ]] && [[ $JENKINS_BRANCH =~ [0-9]+ ]]; then
JENKINS_BRANCH="PR-${JENKINS_BRANCH}"
fi
local OUTPUT=$(curl --silent --write-out '\n' \
-X POST \
--header "token: $JENKINS_TOKEN" \
--header "Content-Type: application/json" \
$JENKINS_SERVER/generic-webhook-trigger/invoke \
--data "{ \"project\": \"$JENKINS_JOB/$JENKINS_BRANCH\", \"branch\": \"$JENKINS_BRANCH\" $TAG $FORCE }")
if echo "$OUTPUT" | grep -q "\"triggered\":true"; then
echo -n " job triggered: "
else
echo "##teamcity[buildProblem description='Triggering Jenkins build failed']"
echo -n " triggering failed: "
fi
# Strip {"jobs":{ from the beginning of OUTPUT
OUTPUT=${OUTPUT#\{\"jobs\":\{}
# Split json string to lines with one job each
local jobs count
count=0
IFS='|' jobs=(${OUTPUT//\},\"pipeline/\},|\"pipeline})
# Find job that actually got triggered (or that we should have triggered)
for line in "${jobs[@]}"; do
if [[ $line == \"$JENKINS_JOB/$JENKINS_BRANCH* ]]; then
echo "$line"
count=$((++count))
fi
done
if [[ $count < 1 ]]; then
# DEBUG
echo -n $OUTPUT
echo
fi
}
function triggerGitHubActionsBuild() {
local IS_TEST_BUILD="$1"
local GITHUB_ACTION="$2"

15
resources/build/trigger-definitions.inc.sh
View file

@ -34,9 +34,6 @@ watch_common_linux='common/linux|common/web'
# These bc_x_y variables ARE used in trigger-builds.inc.sh by pattern so the names are important,
# and you won't find them directly in a grep search.
#
# _Jenkins should be appended to any build configuration (pipeline) name that is from Jenkins,
# not TeamCity.
#
# _GitHub should be appended to any build configuration name that is from GitHub, not TeamCity.
# Test Build Configurations
@ -46,7 +43,7 @@ bc_test_all=()
bc_test_android=(KeymanAndroid_TestPullRequests KeymanAndroid_TestSamplesAndTestProjects)
bc_test_ios=(Keyman_iOS_TestPullRequests Keyman_iOS_TestSamplesAndTestProjects)
bc_test_linux=(KeymanLinux_TestPullRequests Keyman_Linux_Test_Integration Keyman_Common_KPAPI_TestPullRequests_Linux pipeline-keyman-packaging_Jenkins deb-pr-packaging_GitHub)
bc_test_linux=(KeymanLinux_TestPullRequests Keyman_Linux_Test_Integration Keyman_Common_KPAPI_TestPullRequests_Linux deb-pr-packaging_GitHub)
bc_test_mac=(Keyman_KeymanMac_PullRequests Keyman_Common_KPAPI_TestPullRequests_macOS)
bc_test_windows=(KeymanDesktop_TestPullRequests KeymanDesktop_TestPrRenderOnScreenKeyboards Keyman_Common_KPAPI_TestPullRequests_Windows)
bc_test_web=(Keymanweb_TestPullRequests Keyman_Common_LMLayer_TestPullRequests Keyman_Common_KPAPI_TestPullRequests_WASM)
@ -66,7 +63,7 @@ vcs_test=HttpsGithubComKeymanappKeymanPRs
bc_master_android=(KeymanAndroid_Build)
bc_master_ios=(Keyman_iOS_Master)
bc_master_linux=(KeymanLinux_Master pipeline-keyman-packaging_Jenkins deb-release-packaging_GitHub)
bc_master_linux=(KeymanLinux_Master deb-release-packaging_GitHub)
bc_master_mac=(KeymanMac_Master)
bc_master_windows=(Keyman_Build)
bc_master_web=(Keymanweb_Build)
@ -78,7 +75,7 @@ vcs_master=HttpsGithubComKeymanappKeyman
bc_beta_android=(KeymanAndroid_Build)
bc_beta_ios=(Keyman_iOS_Master)
bc_beta_linux=(KeymanLinux_Master pipeline-keyman-packaging_Jenkins deb-release-packaging_GitHub)
bc_beta_linux=(KeymanLinux_Master deb-release-packaging_GitHub)
bc_beta_mac=(KeymanMac_Master)
bc_beta_windows=(Keyman_Build)
bc_beta_web=(Keymanweb_Build)
@ -90,7 +87,7 @@ vcs_beta=HttpsGithubComKeymanappKeyman
bc_stable_14_0_android=(KeymanAndroid_Build)
bc_stable_14_0_ios=(Keyman_iOS_Master)
bc_stable_14_0_linux=(KeymanLinux_Master pipeline-keyman-packaging_Jenkins deb-release-packaging_GitHub)
bc_stable_14_0_linux=(KeymanLinux_Master deb-release-packaging_GitHub)
bc_stable_14_0_mac=(KeymanMac_Master)
bc_stable_14_0_windows=(Keyman_Build)
bc_stable_14_0_web=(Keymanweb_Build)
@ -105,7 +102,7 @@ vcs_stable_14_0=HttpsGithubComKeymanappKeyman
bc_stable_15_0_android=(KeymanAndroid_Build)
bc_stable_15_0_ios=(Keyman_iOS_Master)
bc_stable_15_0_linux=(KeymanLinux_Master pipeline-keyman-packaging_Jenkins deb-release-packaging_GitHub)
bc_stable_15_0_linux=(KeymanLinux_Master deb-release-packaging_GitHub)
bc_stable_15_0_mac=(KeymanMac_Master)
bc_stable_15_0_windows=(Keyman_Build)
bc_stable_15_0_web=(Keymanweb_Build)
@ -116,7 +113,7 @@ vcs_stable_15_0=HttpsGithubComKeymanappKeyman
bc_stable_16_0_android=(KeymanAndroid_Build)
bc_stable_16_0_ios=(Keyman_iOS_Master)
bc_stable_16_0_linux=(KeymanLinux_Master pipeline-keyman-packaging_Jenkins)
bc_stable_16_0_linux=(KeymanLinux_Master)
bc_stable_16_0_mac=(KeymanMac_Master)
bc_stable_16_0_windows=(Keyman_Build)
bc_stable_16_0_web=(Keymanweb_Build)

3
resources/standards-data/ldml-keyboards/techpreview/3.0/fr-t-k0-azerty.xml
View file

@ -63,12 +63,9 @@
<key id="symbol" switch="symbol" />
<key id="base" switch="base" />
<!--
TODO: need discussion
<key id="bksp" gap="true" />
<key id="extra" gap="true" />
<key id="enter" to="\u{000A}" />
-->
<!-- extra keys -->
<key id="u-grave" to="ü" />

6
web/src/app/browser/src/contextManager.ts
View file

@ -211,6 +211,12 @@ export default class ContextManager extends ContextManagerBase<BrowserConfigurat
const originalTarget = this.activeTarget; // may differ, depending on focus state.
if(target == originalTarget) {
// A focus state may have .currentTarget as null at this stage; if the func
// is being called with a non-null parameter, we want this SET. #9404
if(originalTarget) {
this.currentTarget = originalTarget;
}
/**
* If it's already active, we should cancel early.
*

20
web/src/engine/package-cache/src/cloud/queryEngine.ts
View file

@ -1,3 +1,5 @@
import EventEmitter from 'eventemitter3';
import { PathConfiguration } from 'keyman/engine/paths';
import { default as KeyboardStub, ErrorStub, KeyboardAPISpec, mergeAndResolveStubPromises } from '../keyboardStub.js';
@ -55,7 +57,11 @@ type CloudLanguagesQueryResult = {
export type CloudQueryResult = CloudKeyboardQueryResult | CloudLanguagesQueryResult;
export default class CloudQueryEngine {
interface EventMap {
'unboundregister': (registration: ReturnType<CloudQueryEngine['_registerCore']>) => void
}
export default class CloudQueryEngine extends EventEmitter<EventMap> {
private cloudResolutionPromises: Record<number, ManagedPromise<KeyboardStub[] | ManagedPromise<LanguageAPIPropertySpec[]>>> = {};
private _languageListPromise: ManagedPromise<LanguageAPIPropertySpec[]>;
@ -65,6 +71,8 @@ export default class CloudQueryEngine {
private pathConfig: PathConfiguration;
constructor(requestEngine: CloudRequesterInterface, pathConfig: PathConfiguration) {
super();
this.requestEngine = requestEngine;
this.pathConfig = pathConfig;
@ -138,10 +146,16 @@ export default class CloudQueryEngine {
result = new Error(CLOUD_REGISTRATION_ERR + err);
}
if(promiseid) {
if(!promiseid) {
this.emit('unboundregister', result);
return;
} else {
const promise: ManagedPromise<KeyboardStub[]> | ManagedPromise<LanguageAPIPropertySpec[]> = this.cloudResolutionPromises[promiseid];
if(promise) {
if(!promise) {
this.emit('unboundregister', result);
return;
} else {
try {
if(result instanceof Error) {
promise.reject(result as Error);

13
web/src/engine/package-cache/src/keyboardRequisitioner.ts
View file

@ -97,6 +97,19 @@ export default class KeyboardRequisitioner {
this.pathConfig = pathConfig;
this.cache = new StubAndKeyboardCache(keyboardLoader);
this.cloudQueryEngine = new CloudQueryEngine(keyboardRequester, this.pathConfig);
// Handles keymanweb.com's precached keyboard array. There is no associated promise,
// so there's nothing handling the `register` call's results otherwise.
this.cloudQueryEngine.on('unboundregister', (registration) => {
// Internal, undocumented use-case of `keyman.register`: precached keyboard loading
// Other uses may trigger errors, especially if there's a type-structure mismatch.
// Those errors should not be handled here; let them surface.
if(Array.isArray(registration)) {
registration.forEach((entry) => {
this.cache.addStub(entry);
});
}
});
}
addKeyboardArray(x: (string|RawKeyboardMetadata)[]): Promise<(KeyboardStub | ErrorStub)[]> {

4
windows/src/README.md
View file

@ -59,13 +59,13 @@ will need to uninstall and reinstall.
## Certificates
In order to create a release build, you will need a code signing certiicate.
In order to create a release build, you will need a code signing certificate.
You can use your own certificate, or you can use test certificates which are not
globally trusted. The environment variables `SC_PFX_SHA1` and `SC_PFX_SHA256`
can be set to custom certificate paths.
The Keyman repo no longer includes test certificates. To build your own, run
`nmake test-certificates` from **common/windows/delphi/tools/certificates** to
`nmake test-certificate` from **common/windows/delphi/tools/certificates** to
build and install your own local root CA "**KeymanTestCA**" certificates. If you
specify a password for the certificate, you'll need to set that in the
environment variable `SC_PWD`.

15
windows/src/desktop/help/about/requirements.md
View file

@ -4,20 +4,9 @@ title: System Requirements
## Supported Windows Operating Systems
Keyman fully supports 32-bit *and* 64-bit versions of the following
Windows operating systems:
- Windows 7
- Windows 8
- Windows 8.1
- Windows 10
- Windows 11
- Windows Server 2008 and 2008 R2
- Windows Server 2012 and 2012 R2
**Note:** Keyman works slightly differently in different versions of Windows. Older versions of Windows have more language limitations and need extra configuration.
* See [What operating systems does Keyman support?](../common/os) topic.
## Resource Requirements
Keyman has minimal resource requirements. Any computer that can run
Windows 7 should be able to run Keyman without trouble.
Windows 10 should be able to run Keyman without trouble.

8
windows/src/desktop/help/advanced/text_services_framework.md
View file

@ -10,14 +10,6 @@ supports features such as keyboard drivers, handwriting recognition,
speech recognition, as well as spell checking and other text processing
functions.
In Windows 7, the Language Bar is the core user interface for TSF. From
the Language Bar, you can select the input language, and control
keyboard input, handwriting recognition and speech recognition. In
Windows 8, the Language Bar is no longer used, and the interface is
tightly integrated into the taskbar.
![](../desktop_images/language-bar.png)
With Keyman Desktop 9 and later versions, all keyboards are registered
through the Windows interfaces, and the key advantage is that Keyman now
automatically detects applications that have support for TSF and

17
windows/src/desktop/help/basic/config/options.md
View file

@ -60,23 +60,6 @@ To open the Options tab of Keyman Configuration:
Click the Reset Hints button to switch all hint messages on again,
even those that you have switched off on a case-by-case basis.
- Select keyboard layout for all applications (Windows 7)
Unlike Windows 7 default behaviour, Keyman allows you to select one
Windows language and Keyman keyboard for all open applications and
text fields across your entire system. Tick this option to select
one Windows language and Keyman keyboard across your entire system.
Untick this option to select Windows language and Keyman keyboards
independently for different programs.
On Windows 8, 8.1, 10 and later versions, this checkbox is disabled
because Windows has this functionality built in. The setting can be
changed in Windows, with the following steps:
- Search for "Advanced Keyboard Settings"
- Check the \"Let me set a different input method for each app
window\" checkbox.
- Automatically report errors to keyman.com
If Keyman crashes, then it can automatically send a report to the

27
windows/src/desktop/help/basic/make-taskbar-icon-visible.md
View file

@ -37,30 +37,3 @@ Here's how to make the change with Windows Settings:
5. The Keyman icon will now always appear in the Windows
Taskbar near the clock, if Keyman is on.
6. Continue on to [Step 5](../start/tutorial#step-5-) of this guide.
- On Windows 8:
1. Right-click on the Windows Taskbar.
![](../desktop_images/win8-taskbar1.png)
2. Select 'Properties'.
3. Next to 'Notification area', click Customize….
![](../desktop_images/win8-taskbar2.png)
4. From the dropdown menu beside 'Keyman
Engine x86', select 'Show icon and notifications'.
![](../desktop_images/win8-taskbar3.png)
5. Click OK to apply changes. The Keyman icon will now always
appear in the Windows Taskbar near the clock, if Keyman is
on.
6. Continue on to [Step 5](../start/tutorial#step-5-) of this guide.
- On Windows 7:
1. Open the Windows Start menu.
2. In the search field, type and enter: Notification Area Icons
3. From the dropdown menu beside Keyman
Engine x86, select 'Show icon and notifications'.
![](../desktop_images/7-taskbar.png)
4. Click OK to apply changes. The Keyman icon will now always
appear in the Windows Taskbar near the clock, if Keyman is
on.
5. Continue on to [Step 5](../start/tutorial#step-5-) of this guide.

1
windows/src/desktop/help/basic/select-keyboard.md
View file

@ -120,4 +120,3 @@ reach the desired keyboard, then release <kbd>spacebar</kbd>.
- [Keyman Configuration - Keyboard Layouts Tab](config/keyboards)
- [How To - Download and Install a Keyman Keyboard](../start/download-and-install-keyboard)
- [Keyboard Task - Enable or Disable a Keyboard](enable-or-disable-keyboard)
- [How To - Fix A Problem with an Active Keyman Keyboard Not Typing](../troubleshooting/hidden)

Some files were not shown because too many files have changed in this diff Show more