feat(developer): add message for use of Unicode codepoints; edit regex, unescape_string; handle empty input

developer/src/kmc-convert/src/keylayout-to-kmn/keylayout-to-kmn-converter.ts

@@ -140,7 +140,7 @@ export class KeylayoutToKmnConverter {
     const outputKmn = kmnFileWriter.write(processedData);
     const result: ConverterToKmnResult = {
       artifacts: {
-        kmn: { data: outputKmn, filename: processedData.kmnFilename }
+        kmn: { data: outputKmn, filename: processedData?.kmnFilename ?? '' }
       }
     };
     return result;

developer/src/kmc-convert/src/keylayout-to-kmn/kmn-file-writer.ts

@@ -68,10 +68,7 @@ export class UnicodeCharacterConversion {
   public static re_dec = /^&#([0-9]{1,7});$/;
 
   // &#x followed by 1.-6. hex digits or &# followed by 1.-7. decimal digits
-  private static re_hexdec = /^&#x?([0-9a-f]{1,7};)/i;
-
-  // & followed by gt, lt, quot, amp, apos and ; (>)
-  private static re_nam = /&(gt|lt|quot|amp|apos);/i;
+  private static re_hexdec = /^&#(?:x[0-9a-f]{1,6}|[0-9]{1,7});/i
 
   /**
     * @brief  function to convert a (character or) numeric html character reference to a character
@@ -127,9 +124,7 @@ export class UnicodeCharacterConversion {
       .replace(/'/g, "'")
       .replace(/&/g, "&");
 
-    const m_nam = UnicodeCharacterConversion.re_nam.exec(unescaped);
-
-    if (!m_nam)
+    if ((unescaped === inputString))
       return unescaped;
     return this.unescape_string(unescaped);
   }
@@ -154,12 +149,12 @@ export class UnicodeCharacterConversion {
     inputString.carryOver = '';
     inputString.input = this.unescape_string(inputString.input) ?? '';
 
-    const re_hexdec = UnicodeCharacterConversion.re_hexdec.exec(inputString.input);
+    const m_hexdec = UnicodeCharacterConversion.re_hexdec.exec(inputString.input);
 
     // if the (remaining) input string starts with a hex or dec html entity ( &#x...; or &#...;) we need to convert this part to a character
-    if (re_hexdec) {
-      returnChar = this.convert_htmlToCharacter(re_hexdec[0] as string);
-      const to_be_replaced = re_hexdec[0] as string;
+    if (m_hexdec) {
+      returnChar = this.convert_htmlToCharacter(m_hexdec[0] as string);
+      const to_be_replaced = m_hexdec[0] as string;
       const replace_len = [...to_be_replaced].length;
 
       // Use of a carry over happens for example when the input string is '>' which should result in '>' and then '>'  (without the carry over we would get 'gt;' and then 'gt' which is wrong)
@@ -189,7 +184,9 @@ export class UnicodeCharacterConversion {
       const replace_len = [...to_be_replaced].length;
 
       inputString.rest_string = inputString.input.substring(replace_len);
-      inputString.replaced_string += to_be_replaced;
+      if (inputString.replaced_string !== undefined) {
+        inputString.replaced_string += to_be_replaced;
+      }
       // use the rest of the string after the replaced portion
       inputString.input = inputString.input.substring(replace_len);
     }
@@ -244,7 +241,7 @@ export class KmnFileWriter {
     data += "c ..................................................................................................................\n";
     data += "c ..................................................................................................................\n";
     data += "c Keyman keyboard generated by kmn-convert version: " + KEYMAN_VERSION.VERSION + "\n";
-    data += "c from Ukelele file: " + dataUkelele.keylayoutFilename + "\n";
+    data += "c from Ukelele file: " + (dataUkelele?.keylayoutFilename ?? '') + "\n";
     data += "c ..................................................................................................................\n";
     data += "c ..................................................................................................................\n";
     data += "\n";
@@ -273,7 +270,7 @@ export class KmnFileWriter {
     // during the process of creating Rule[], duplicate rules might occur
     // (e.g. when in a keylayout file the same modifiers occur in several behaviors thus producing the same rules).
     // This is to filter out those duplicate Rule objects
-    const uniqueDataRules: Rule[] = dataUkelele.rules.filter((curr) => {
+    const uniqueDataRules: Rule[] = (dataUkelele?.rules ?? []).filter((curr) => {
       return (!(curr.output === undefined)
         && (curr.key !== "")
         && ((curr.ruleType === "C0")
@@ -1631,6 +1628,9 @@ export class KmnFileWriter {
       if ((out.character.indexOf('&') > -1) && (out.character.length > 1)) {
         msg_entity = msg_entity + "specified string might not be a valid html entity: ";
       }
+      if ((out.character.indexOf('U+') > -1) && (out.character.length > 2)) {
+        msg_entity = msg_entity + "(invalid) Unicode code point used: ";
+      }
     }
 
     // add a warning message

developer/src/kmc-convert/test/data/Test_messages.keylayout

@@ -43,7 +43,11 @@
     <keyMapSet id="ANSI">
         <keyMap index="0">
             <key code="5" output="G"/>
-            <key code="5" output="G"/>
+            <key code="9" output="U+0047"/>
+            <key code="11" output="&#x0026;gt"/>
+            <key code="12" output="&#0001;"/>
+            <key code="13" output=""/>
+            <key code="14" output="&#x0009;"/>
         </keyMap>
         <keyMap index="1">
             <key code="5" output="&#x0007;"/>
@@ -52,7 +56,7 @@
             <key code="25" action="a0"/>
             <key code="6" output="Z"/>
             <key code="7" output="X"/>
-            <key code="7" output="ẍ"/>
+            <key code="8" output="ẍ"/>
         </keyMap>
         <keyMap index="3">
             <key code="5" output="∞"/>

developer/src/kmc-convert/test/keylayout-file-reader.tests.ts

@@ -51,25 +51,6 @@ describe('KeylayoutFileReader', function () {
       const validated = sutR.validate(result as Keylayout.KeylayoutXMLSourceFile, inputFilename);
       assert.isFalse(validated);
     });
-    it('validate() should return false on no entries in action-when', async function () {
-      const sutR = new KeylayoutFileReader(compilerTestCallbacks);
-      const inputFilename = makePathToFixture('../data/Test_noActionWhen.keylayout');
-      const result: Keylayout.KeylayoutXMLSourceFile | null = sutR.read(compilerTestCallbacks.loadFile(inputFilename));
-      const validated = sutR.validate(result as Keylayout.KeylayoutXMLSourceFile, inputFilename);
-      assert.isFalse(validated);
-    });
-    it('validate() should return false on null as input', async function () {
-      const sutR = new KeylayoutFileReader(compilerTestCallbacks);
-      const inputFilename = makePathToFixture('../data/Test_noActionWhen.keylayout');
-      const validated = sutR.validate(null, inputFilename);
-      assert.isFalse(validated);
-    });
-    it('validate() should return false on undefined as input', async function () {
-      const sutR = new KeylayoutFileReader(compilerTestCallbacks);
-      const inputFilename = makePathToFixture('../data/Test_noActionWhen.keylayout');
-       const validated = sutR.validate(undefined, inputFilename);
-      assert.isFalse(validated);
-    });
   });
 
   describe("read() ", function () {

developer/src/kmc-convert/test/keylayout-to-kmn-converter.tests.ts

@@ -31,7 +31,7 @@ describe('KeylayoutToKmnConverter', function () {
       [makePathToFixture('../data/Test_OtherOutputName.kmn')],
     ].forEach(function (files) {
       it(infile + " should run ", async function () {
-        await NodeAssert.doesNotReject(async () => sut.run(makePathToFixture(infile), files[0]));
+        await NodeAssert.doesNotReject(async () =>await sut.run(makePathToFixture(infile), files[0]));
         assert.equal(compilerTestCallbacks.messages.length, 0);
       });
     });
@@ -51,7 +51,7 @@ describe('KeylayoutToKmnConverter', function () {
       [makePathToFixture('../data/Test_MissingAllERROR.keylayout')],
     ].forEach(function (files) {
       it(files + " should give an error ", async function () {
-        sut.run(files[0]);
+        await sut.run(files[0]);
         assert.isTrue(compilerTestCallbacks.messages.length > 0);
       });
     });
@@ -96,7 +96,7 @@ describe('KeylayoutToKmnConverter', function () {
       ['../data/Test.keylayout'],
     ].forEach(function (files) {
       it(files + " should give no errors ", async function () {
-        sut.run(makePathToFixture(files[0]));
+        await sut.run(makePathToFixture(files[0]));
         assert.equal(compilerTestCallbacks.messages.length, 0);
       });
     });
@@ -130,7 +130,7 @@ describe('KeylayoutToKmnConverter', function () {
       ['../data/Test_undefinedAction.keylayout'],
     ].forEach(function (files) {
       it(files + " should give Error: undefined action detected", async function () {
-        sut.run(makePathToFixture(files[0]));
+        await sut.run(makePathToFixture(files[0]));
         assert.equal(compilerTestCallbacks.messages.length, 1);
         assert.equal(compilerTestCallbacks.messages[0].code, 5292040);
       });

developer/src/kmc-convert/test/kmn-file-writer.tests.ts

@@ -505,7 +505,7 @@ describe('KmnFileWriter', function () {
       ['typing a &gt', 'typing a &gt'],
 
     ].forEach(function (values) {
-      it(('readXmlOutput should convert "' + values[0] + '"').padEnd(30, " ") + 'to "' + values[1] + '"', async function () {
+      it(('readXmlOutput should convert ' + values[0] + "'").padEnd(30, " ") + " to '" + values[1] + "'", async function () {
         const out: ReplacedOutputString = {
           input: values[0] as string,
           replaced_character: '',
@@ -521,39 +521,42 @@ describe('KmnFileWriter', function () {
 
   describe('unescape_string', function () {
     [
-      ["", ''],
+      ['', ''],
       [undefined, undefined],
       [null, undefined],
-      ["&", '&'],
-      ["&", '&'],
-      ["&", '&'],
-      [">", '>'],
-      ["'", "'"],
-      [">", '>'],
-      ["&lt;", '<'],
-      ["&amp;#x1234;", '&#x1234;'],
-      ["&amp;#x1234;&amp;", '&#x1234;&'],
+      ['&amp;', '&'],
+      ['&amp;amp;', '&'],
+      ['&amp;amp;amp;', '&'],
+      ['&amp;gt;', '>'],
+      ['&apos;', "'"],
+      ['&gt;', '>'],
+      ['&GT;', '&GT;'],
+      ['&Gt;', '&Gt;'],
+      ['&gT;', '&gT;'],
+      ['&lt;', '<'],
+      ['&amp;#x1234;', '&#x1234;'],
+      ['&amp;#x1234;&amp;', '&#x1234;&'],
       ['aሴ&#x1F60F;bẘ&gt;😆z<y&amp;&#97;😎-&#128547;&#x1F60F;', 'aሴ&#x1F60F;bẘ>😆z<y&&#97;😎-&#128547;&#x1F60F;'],
-      ["a&gt;", 'a>'],
-      ["a&gt;b", 'a>b'],
-      ["a &gt; b", 'a > b'],
-      ["typing a '&gt;'", "typing a \'>\'"],
-      ["typing a &gt;", 'typing a >'],
-      ["typing a &amp;gt", 'typing a &gt'],
-      ["a&bcd", "a&bcd"],
+      ['a&gt;', 'a>'],
+      ['a&gt;b', 'a>b'],
+      ['a &gt; b', 'a > b'],
+      ["typing a '&gt;'", "typing a '>'"],
+      ['typing a &gt;', 'typing a >'],
+      ['typing a &amp;gt', 'typing a &gt'],
+      ['a&bcd', 'a&bcd'],
       ['&', '&'],
-      ["&#;", "&#;"],
-      ["&#x;", "&#x;"],
-      ['&##;', "&##;"],
-      ["&#x110000;", '&#x110000;'],
-      ["&#x2000000;", '&#x2000000;'],
+      ['&#;', '&#;'],
+      ['&#x;', '&#x;'],
+      ['&##;', '&##;'],
+      ['&#x110000;', '&#x110000;'],
+      ['&#x2000000;', '&#x2000000;'],
       ['&#1234;56', '&#1234;56'],
       ['&#&#1234;56', '&#&#1234;56'],
       ['a&#&#1234;56', 'a&#&#1234;56'],
-      ["&#x0026;gt;", '&#x0026;gt;'],
-      ["&#x0026;gt", "&#x0026;gt"],
+      ['&#x0026;gt;', '&#x0026;gt;'],
+      ['&#x0026;gt', '&#x0026;gt'],
     ].forEach(function (values) {
-      it(('unescape_string should unescape "' + values[0] + '"').padEnd(30, " ") + 'to "' + values[1] + '"', async function () {
+      it(('unescape_string should unescape ' + values[0] + "'").padEnd(30, ' ') + " to '" + values[1] + "'", async function () {
         const result = UnicodeCharacterConversion.unescape_string(values[0] as string);
         assert.equal(result, values[1]);
       });