From 860fc0cb624a65006aaec74df0ff58daf965a626 Mon Sep 17 00:00:00 2001 From: Sabine Date: Wed, 5 Aug 2026 10:45:54 +0200 Subject: [PATCH] feat(developer): add message for use of Unicode codepoints; edit regex, unescape_string; handle empty input --- .../keylayout-to-kmn-converter.ts | 2 +- .../src/keylayout-to-kmn/kmn-file-writer.ts | 28 +++++----- .../test/data/Test_messages.keylayout | 8 ++- .../test/keylayout-file-reader.tests.ts | 19 ------- .../test/keylayout-to-kmn-converter.tests.ts | 8 +-- .../kmc-convert/test/kmn-file-writer.tests.ts | 55 ++++++++++--------- 6 files changed, 54 insertions(+), 66 deletions(-) diff --git a/developer/src/kmc-convert/src/keylayout-to-kmn/keylayout-to-kmn-converter.ts b/developer/src/kmc-convert/src/keylayout-to-kmn/keylayout-to-kmn-converter.ts index c17d74537d..dbf9e593fe 100644 --- a/developer/src/kmc-convert/src/keylayout-to-kmn/keylayout-to-kmn-converter.ts +++ b/developer/src/kmc-convert/src/keylayout-to-kmn/keylayout-to-kmn-converter.ts @@ -140,7 +140,7 @@ export class KeylayoutToKmnConverter { const outputKmn = kmnFileWriter.write(processedData); const result: ConverterToKmnResult = { artifacts: { - kmn: { data: outputKmn, filename: processedData.kmnFilename } + kmn: { data: outputKmn, filename: processedData?.kmnFilename ?? '' } } }; return result; diff --git a/developer/src/kmc-convert/src/keylayout-to-kmn/kmn-file-writer.ts b/developer/src/kmc-convert/src/keylayout-to-kmn/kmn-file-writer.ts index 9f63c8fca0..faf83e1cf1 100644 --- a/developer/src/kmc-convert/src/keylayout-to-kmn/kmn-file-writer.ts +++ b/developer/src/kmc-convert/src/keylayout-to-kmn/kmn-file-writer.ts @@ -68,10 +68,7 @@ export class UnicodeCharacterConversion { public static re_dec = /^&#([0-9]{1,7});$/; // &#x followed by 1.-6. hex digits or &# followed by 1.-7. decimal digits - private static re_hexdec = /^&#x?([0-9a-f]{1,7};)/i; - - // & followed by gt, lt, quot, amp, apos and ; (>) - private static re_nam = /&(gt|lt|quot|amp|apos);/i; + private static re_hexdec = /^&#(?:x[0-9a-f]{1,6}|[0-9]{1,7});/i /** * @brief function to convert a (character or) numeric html character reference to a character @@ -127,9 +124,7 @@ export class UnicodeCharacterConversion { .replace(/'/g, "'") .replace(/&/g, "&"); - const m_nam = UnicodeCharacterConversion.re_nam.exec(unescaped); - - if (!m_nam) + if ((unescaped === inputString)) return unescaped; return this.unescape_string(unescaped); } @@ -154,12 +149,12 @@ export class UnicodeCharacterConversion { inputString.carryOver = ''; inputString.input = this.unescape_string(inputString.input) ?? ''; - const re_hexdec = UnicodeCharacterConversion.re_hexdec.exec(inputString.input); + const m_hexdec = UnicodeCharacterConversion.re_hexdec.exec(inputString.input); // if the (remaining) input string starts with a hex or dec html entity ( &#x...; or &#...;) we need to convert this part to a character - if (re_hexdec) { - returnChar = this.convert_htmlToCharacter(re_hexdec[0] as string); - const to_be_replaced = re_hexdec[0] as string; + if (m_hexdec) { + returnChar = this.convert_htmlToCharacter(m_hexdec[0] as string); + const to_be_replaced = m_hexdec[0] as string; const replace_len = [...to_be_replaced].length; // Use of a carry over happens for example when the input string is '&gt;' which should result in '>' and then '>' (without the carry over we would get 'gt;' and then 'gt' which is wrong) @@ -189,7 +184,9 @@ export class UnicodeCharacterConversion { const replace_len = [...to_be_replaced].length; inputString.rest_string = inputString.input.substring(replace_len); - inputString.replaced_string += to_be_replaced; + if (inputString.replaced_string !== undefined) { + inputString.replaced_string += to_be_replaced; + } // use the rest of the string after the replaced portion inputString.input = inputString.input.substring(replace_len); } @@ -244,7 +241,7 @@ export class KmnFileWriter { data += "c ..................................................................................................................\n"; data += "c ..................................................................................................................\n"; data += "c Keyman keyboard generated by kmn-convert version: " + KEYMAN_VERSION.VERSION + "\n"; - data += "c from Ukelele file: " + dataUkelele.keylayoutFilename + "\n"; + data += "c from Ukelele file: " + (dataUkelele?.keylayoutFilename ?? '') + "\n"; data += "c ..................................................................................................................\n"; data += "c ..................................................................................................................\n"; data += "\n"; @@ -273,7 +270,7 @@ export class KmnFileWriter { // during the process of creating Rule[], duplicate rules might occur // (e.g. when in a keylayout file the same modifiers occur in several behaviors thus producing the same rules). // This is to filter out those duplicate Rule objects - const uniqueDataRules: Rule[] = dataUkelele.rules.filter((curr) => { + const uniqueDataRules: Rule[] = (dataUkelele?.rules ?? []).filter((curr) => { return (!(curr.output === undefined) && (curr.key !== "") && ((curr.ruleType === "C0") @@ -1631,6 +1628,9 @@ export class KmnFileWriter { if ((out.character.indexOf('&') > -1) && (out.character.length > 1)) { msg_entity = msg_entity + "specified string might not be a valid html entity: "; } + if ((out.character.indexOf('U+') > -1) && (out.character.length > 2)) { + msg_entity = msg_entity + "(invalid) Unicode code point used: "; + } } // add a warning message diff --git a/developer/src/kmc-convert/test/data/Test_messages.keylayout b/developer/src/kmc-convert/test/data/Test_messages.keylayout index 85362cd262..87f7436b6d 100644 --- a/developer/src/kmc-convert/test/data/Test_messages.keylayout +++ b/developer/src/kmc-convert/test/data/Test_messages.keylayout @@ -43,7 +43,11 @@ - + + + + + @@ -52,7 +56,7 @@ - + diff --git a/developer/src/kmc-convert/test/keylayout-file-reader.tests.ts b/developer/src/kmc-convert/test/keylayout-file-reader.tests.ts index 888acb069a..a72c635bec 100644 --- a/developer/src/kmc-convert/test/keylayout-file-reader.tests.ts +++ b/developer/src/kmc-convert/test/keylayout-file-reader.tests.ts @@ -51,25 +51,6 @@ describe('KeylayoutFileReader', function () { const validated = sutR.validate(result as Keylayout.KeylayoutXMLSourceFile, inputFilename); assert.isFalse(validated); }); - it('validate() should return false on no entries in action-when', async function () { - const sutR = new KeylayoutFileReader(compilerTestCallbacks); - const inputFilename = makePathToFixture('../data/Test_noActionWhen.keylayout'); - const result: Keylayout.KeylayoutXMLSourceFile | null = sutR.read(compilerTestCallbacks.loadFile(inputFilename)); - const validated = sutR.validate(result as Keylayout.KeylayoutXMLSourceFile, inputFilename); - assert.isFalse(validated); - }); - it('validate() should return false on null as input', async function () { - const sutR = new KeylayoutFileReader(compilerTestCallbacks); - const inputFilename = makePathToFixture('../data/Test_noActionWhen.keylayout'); - const validated = sutR.validate(null, inputFilename); - assert.isFalse(validated); - }); - it('validate() should return false on undefined as input', async function () { - const sutR = new KeylayoutFileReader(compilerTestCallbacks); - const inputFilename = makePathToFixture('../data/Test_noActionWhen.keylayout'); - const validated = sutR.validate(undefined, inputFilename); - assert.isFalse(validated); - }); }); describe("read() ", function () { diff --git a/developer/src/kmc-convert/test/keylayout-to-kmn-converter.tests.ts b/developer/src/kmc-convert/test/keylayout-to-kmn-converter.tests.ts index dfb1321438..5aacad8e6d 100644 --- a/developer/src/kmc-convert/test/keylayout-to-kmn-converter.tests.ts +++ b/developer/src/kmc-convert/test/keylayout-to-kmn-converter.tests.ts @@ -31,7 +31,7 @@ describe('KeylayoutToKmnConverter', function () { [makePathToFixture('../data/Test_OtherOutputName.kmn')], ].forEach(function (files) { it(infile + " should run ", async function () { - await NodeAssert.doesNotReject(async () => sut.run(makePathToFixture(infile), files[0])); + await NodeAssert.doesNotReject(async () =>await sut.run(makePathToFixture(infile), files[0])); assert.equal(compilerTestCallbacks.messages.length, 0); }); }); @@ -51,7 +51,7 @@ describe('KeylayoutToKmnConverter', function () { [makePathToFixture('../data/Test_MissingAllERROR.keylayout')], ].forEach(function (files) { it(files + " should give an error ", async function () { - sut.run(files[0]); + await sut.run(files[0]); assert.isTrue(compilerTestCallbacks.messages.length > 0); }); }); @@ -96,7 +96,7 @@ describe('KeylayoutToKmnConverter', function () { ['../data/Test.keylayout'], ].forEach(function (files) { it(files + " should give no errors ", async function () { - sut.run(makePathToFixture(files[0])); + await sut.run(makePathToFixture(files[0])); assert.equal(compilerTestCallbacks.messages.length, 0); }); }); @@ -130,7 +130,7 @@ describe('KeylayoutToKmnConverter', function () { ['../data/Test_undefinedAction.keylayout'], ].forEach(function (files) { it(files + " should give Error: undefined action detected", async function () { - sut.run(makePathToFixture(files[0])); + await sut.run(makePathToFixture(files[0])); assert.equal(compilerTestCallbacks.messages.length, 1); assert.equal(compilerTestCallbacks.messages[0].code, 5292040); }); diff --git a/developer/src/kmc-convert/test/kmn-file-writer.tests.ts b/developer/src/kmc-convert/test/kmn-file-writer.tests.ts index f82e7390e0..2ae5e5a3ee 100644 --- a/developer/src/kmc-convert/test/kmn-file-writer.tests.ts +++ b/developer/src/kmc-convert/test/kmn-file-writer.tests.ts @@ -505,7 +505,7 @@ describe('KmnFileWriter', function () { ['typing a &gt', 'typing a >'], ].forEach(function (values) { - it(('readXmlOutput should convert "' + values[0] + '"').padEnd(30, " ") + 'to "' + values[1] + '"', async function () { + it(('readXmlOutput should convert ' + values[0] + "'").padEnd(30, " ") + " to '" + values[1] + "'", async function () { const out: ReplacedOutputString = { input: values[0] as string, replaced_character: '', @@ -521,39 +521,42 @@ describe('KmnFileWriter', function () { describe('unescape_string', function () { [ - ["", ''], + ['', ''], [undefined, undefined], [null, undefined], - ["&", '&'], - ["&amp;", '&'], - ["&amp;amp;", '&'], - ["&gt;", '>'], - ["'", "'"], - [">", '>'], - ["<", '<'], - ["&#x1234;", 'ሴ'], - ["&#x1234;&", 'ሴ&'], + ['&', '&'], + ['&amp;', '&'], + ['&amp;amp;', '&'], + ['&gt;', '>'], + [''', "'"], + ['>', '>'], + ['>', '>'], + ['≫', '≫'], + ['&gT;', '&gT;'], + ['<', '<'], + ['&#x1234;', 'ሴ'], + ['&#x1234;&', 'ሴ&'], ['aሴ😏bẘ>😆z😆z'], - ["a>b", 'a>b'], - ["a > b", 'a > b'], - ["typing a '>'", "typing a \'>\'"], - ["typing a >", 'typing a >'], - ["typing a &gt", 'typing a >'], - ["a&bcd", "a&bcd"], + ['a>', 'a>'], + ['a>b', 'a>b'], + ['a > b', 'a > b'], + ["typing a '>'", "typing a '>'"], + ['typing a >', 'typing a >'], + ['typing a &gt', 'typing a >'], + ['a&bcd', 'a&bcd'], ['&', '&'], - ["&#;", "&#;"], - ["&#x;", "&#x;"], - ['&##;', "&##;"], - ["�", '�'], - ["�", '�'], + ['&#;', '&#;'], + ['&#x;', '&#x;'], + ['&##;', '&##;'], + ['�', '�'], + ['�', '�'], ['Ӓ56', 'Ӓ56'], ['&#Ӓ56', '&#Ӓ56'], ['a&#Ӓ56', 'a&#Ӓ56'], - ["&gt;", '&gt;'], - ["&gt", "&gt"], + ['&gt;', '&gt;'], + ['&gt', '&gt'], ].forEach(function (values) { - it(('unescape_string should unescape "' + values[0] + '"').padEnd(30, " ") + 'to "' + values[1] + '"', async function () { + it(('unescape_string should unescape ' + values[0] + "'").padEnd(30, ' ') + " to '" + values[1] + "'", async function () { const result = UnicodeCharacterConversion.unescape_string(values[0] as string); assert.equal(result, values[1]); });