Merge pull request #9141 from keymanapp/feat/common-7377-segment-ordr-epic-ldml

feat(common): add segmenter for element strings 🙀

common/web/types/src/ldml-keyboard/pattern-parser.ts

@@ -119,3 +119,60 @@ export class VariableParser {
     return str.trim().split(/\s+/);
   }
 }
+
+/** for ElementParser.segment() */
+export enum ElementType {
+  codepoint = '.',
+  escaped ='\\',
+  uset = '[',
+  string = '*',
+};
+
+/** one portion of a segmented element string */
+export class ElementSegment {
+  public readonly type: ElementType;
+  /**
+   * @param segment the string in the segment
+   * @param type type of segment. Will be calculated if not provided.
+   */
+  constructor(public segment: string, type?: ElementType) {
+    if (type) {
+      this.type = type;
+    } else if (ElementParser.MATCH_USET.test(segment)) {
+      this.type = ElementType.uset;
+    } else if(ElementParser.MATCH_ESCAPED.test(segment)) {
+      this.type = ElementType.escaped;
+    } else {
+      this.type = ElementType.codepoint;
+    }
+  }
+};
+
+/** Class for helping with Element strings (i.e. reorder) */
+export class ElementParser {
+  /**
+   * Matches any complex UnicodeSet that would otherwise be misinterpreted
+   * by `MATCH_ELEMENT_SEGMENTS` due to nested `[]`'s.
+   * For example, `[[a-z]-[aeiou]]` could be
+   * mis-segmented into `[[a-z]`, `-`, `[aeiou]`, `]` */
+  public static readonly MATCH_NESTED_SQUARE_BRACKETS = /\[[^\]]*\[/;
+
+  /** Match (segment) UnicodeSets OR hex escapes OR single Unicode codepoints */
+  public static readonly MATCH_ELEMENT_SEGMENTS =
+    /(?:\[[^\]]*\]|\\u[0-9a-fA-F]{4}|\\u\{[0-9a-fA-F]{1,6}\}|.)/gu;
+
+  /** Does it start with a UnicodeSet? Used to test the segments. */
+  public static readonly MATCH_USET = /^\[/;
+
+  /** Does it start with an escaped char? Used to test the segments. */
+  public static readonly MATCH_ESCAPED = /^\\u/;
+
+  /** Split a string into ElementSegments */
+  public static segment(str: string): ElementSegment[] {
+    if (this.MATCH_NESTED_SQUARE_BRACKETS.test(str)) {
+      throw Error(`Unsupported: nested square brackets in element segment: ${str}`);
+    }
+    return str.match(ElementParser.MATCH_ELEMENT_SEGMENTS)
+      .map(str => new ElementSegment(str));
+  }
+};

common/web/types/src/util/util.ts

@@ -19,12 +19,47 @@ export function boxXmlArray(o: any, x: string): void {
 // TODO-LDML: #7569 the below regex works, but captures more than it should
 // (it would include \u{fffffffffffffffff } which
 // is overlong and has a space at the end.) The second regex does not work yet.
-const MATCH_HEX_ESCAPE = /\\u{([0-9a-fA-F ]{1,})}/g;
+export const MATCH_HEX_ESCAPE = /\\u{([0-9a-fA-F ]{1,})}/g;
 // const MATCH_HEX_ESCAPE = /\\u{((?:(?:[0-9a-fA-F]{1,5})|(?:10[0-9a-fA-F]{4})(?: (?!}))?)+)}/g;
 
+/** regex for single quad escape such as \u0127 */
+export const MATCH_QUAD_ESCAPE = /\\u([0-9a-fA-F]{4})/g;
+
 export class UnescapeError extends Error {
 }
 
+/**
+ * Unescape one codepoint
+ * @param hex one codepoint in hex, such as '0127'
+ * @returns the unescaped codepoint
+ */
+function unescapeOne(hex: string): string {
+  const codepoint = Number.parseInt(hex, 16);
+  return String.fromCodePoint(codepoint);
+}
+
+/**
+ * Unescape one single quad string such as \u0127
+ * @param s input string
+ * @returns output
+ */
+export function unescapeOneQuadString(s: string): string {
+  if (!s) {
+    return s;
+  }
+  /**
+ * process one regex match
+ * @param str ignored
+ * @param matched the entire match such as '0127' or '22 22'
+ * @returns the unescaped match
+ */
+  function processMatch(str: string, matched: string): string {
+    return unescapeOne(matched);
+  }
+  s = s.replace(MATCH_QUAD_ESCAPE, processMatch);
+  return s;
+}
+
 /**
  * Unescapes a string according to UTS#18§1.1, see <https://www.unicode.org/reports/tr18/#Hex_notation>
  * @param s escaped string
@@ -35,15 +70,6 @@ export function unescapeString(s: string): string {
     return s;
   }
   try {
-    /**
-     * Unescape one codepoint
-     * @param hex one codepoint in hex, such as '0127'
-     * @returns the unescaped codepoint
-     */
-    function unescapeOne(hex: string) : string {
-      const codepoint = Number.parseInt(hex, 16);
-      return String.fromCodePoint(codepoint);
-    }
     /**
      * process one regex match
      * @param str ignored

common/web/types/test/ldml-keyboard/test-pattern-parser.ts

@@ -1,6 +1,6 @@
 import 'mocha';
 import { assert } from 'chai';
-import { MarkerParser, VariableParser } from '../../src/ldml-keyboard/pattern-parser.js';
+import { ElementParser, ElementType, MarkerParser, VariableParser } from '../../src/ldml-keyboard/pattern-parser.js';
 
 describe('Test of Pattern Parsers', () => {
   describe('should test MarkerParser', () => {
@@ -77,4 +77,95 @@ describe('Test of Pattern Parsers', () => {
       }
     });
   });
+  describe('ElementParser', () => {
+    const samplePatterns = [
+      `\\u1A60`,
+      `[\\u1A75-\\u1A79]`,
+      `\\u1A60\\u1A45`,
+      `\\u1A60[\\u1A75-\\u1A79]\\u1A45`,
+      `ែ្ម`,
+    ]
+    describe('try out the regexes', () => {
+      it('should detect usets', () => {
+        [
+          `[a-z]`,
+          `[[a-z]-[aeiou]]`,
+        ].forEach(s => assert.ok(ElementParser.MATCH_USET.test(s), `expected true: ${s}`));
+      });
+      it('should detect non usets', () => {
+        [
+          `\\u0127`,
+          `\\u{22}`,
+          `x`,
+        ].forEach(s => assert.notOk(ElementParser.MATCH_USET.test(s), `expected false: ${s}`));
+      });
+      it('should detect escaped', () => {
+        [
+          `\\u0127`,
+          `\\u{22}`,
+        ].forEach(s => assert.ok(ElementParser.MATCH_ESCAPED.test(s), `expected true: ${s}`));
+      });
+      it('should detect non escaped', () => {
+        [
+          `[a-z]`,
+          `[[a-z]-[aeiou]]`,
+          `ê`,
+        ].forEach(s => assert.notOk(ElementParser.MATCH_ESCAPED.test(s), `expected false: ${s}`));
+      });
+      it('should reject nested square brackets', () => {
+        [
+          `[[a-z]-[aeiou]]`,
+        ].forEach(s => assert.ok(ElementParser.MATCH_NESTED_SQUARE_BRACKETS.test(s), `expected true: ${s}`));
+      });
+      it('should allow non-nested square brackets', () => {
+        [
+          `[a-z]`,
+          `ê`,
+          ...samplePatterns,
+        ].forEach(s => assert.notOk(ElementParser.MATCH_NESTED_SQUARE_BRACKETS.test(s), `expected false: ${s}`));
+      });
+    });
+    describe('segment some strings', () => {
+      it('should be able to segment strings from the spec and samples', () => {
+        samplePatterns.forEach(str => assert.ok(ElementParser.segment(str)));
+      });
+      it('should throw on nested brackets', () => {
+        [
+          `[[a-z]-[aeiou]]`,
+        ].forEach(str => assert.throws(() => ElementParser.segment(str)));
+      });
+      [
+        {
+          str: `ê🙀`,
+          expect: [{
+            segment: 'ê',
+            type: ElementType.codepoint
+          },{
+            segment: '🙀',
+            type: ElementType.codepoint
+          }],
+        },
+        {
+          str: `\\u1A60[\\u1A75-\\u1A79]\\u1A45ħ`,
+          expect: [{
+            segment: '\\u1A60',
+            type: ElementType.escaped
+          },{
+            segment: '[\\u1A75-\\u1A79]',
+            type: ElementType.uset
+          },{
+            segment: '\\u1A45',
+            type: ElementType.escaped
+          },{
+            segment: 'ħ',
+            type: ElementType.codepoint
+          }],
+        },
+      ].forEach(({str, expect}) => it(`Segment: ${str}`, () => {
+        const segmented = ElementParser.segment(str);
+        assert.ok(segmented, `segmenting ${str}`);
+        assert.deepEqual(segmented, expect, `segments of ${str}`)
+      }));
+    });
+  });
 });

common/web/types/test/util/test-unescape.ts

@@ -1,6 +1,6 @@
 import 'mocha';
 import {assert} from 'chai';
-import {unescapeString, UnescapeError, isOneChar, toOneChar} from '../../src/util/util.js';
+import {unescapeString, UnescapeError, isOneChar, toOneChar, unescapeOneQuadString} from '../../src/util/util.js';
 
 describe('test UTF32 functions()', function() {
   it('should properly categorize strings', () => {
@@ -56,3 +56,10 @@ describe('test unescapeString()', function() {
     assert.throws(() => unescapeString('\\u{110000}'), UnescapeError);
   });
 });
+
+describe('test unescapeOneQuadString()', () => {
+  it('should be able to convert', () => {
+    // testing that `\u0127` is unescaped correctly (to U+0127: 'ħ')
+    assert.equal(unescapeOneQuadString('\\u0127'), '\u{0127}');
+  });
+});