diff --git a/common/web/types/src/ldml-keyboard/pattern-parser.ts b/common/web/types/src/ldml-keyboard/pattern-parser.ts index 571a605e4e..e8d04c1bb2 100644 --- a/common/web/types/src/ldml-keyboard/pattern-parser.ts +++ b/common/web/types/src/ldml-keyboard/pattern-parser.ts @@ -119,3 +119,60 @@ export class VariableParser { return str.trim().split(/\s+/); } } + +/** for ElementParser.segment() */ +export enum ElementType { + codepoint = '.', + escaped ='\\', + uset = '[', + string = '*', +}; + +/** one portion of a segmented element string */ +export class ElementSegment { + public readonly type: ElementType; + /** + * @param segment the string in the segment + * @param type type of segment. Will be calculated if not provided. + */ + constructor(public segment: string, type?: ElementType) { + if (type) { + this.type = type; + } else if (ElementParser.MATCH_USET.test(segment)) { + this.type = ElementType.uset; + } else if(ElementParser.MATCH_ESCAPED.test(segment)) { + this.type = ElementType.escaped; + } else { + this.type = ElementType.codepoint; + } + } +}; + +/** Class for helping with Element strings (i.e. reorder) */ +export class ElementParser { + /** + * Matches any complex UnicodeSet that would otherwise be misinterpreted + * by `MATCH_ELEMENT_SEGMENTS` due to nested `[]`'s. + * For example, `[[a-z]-[aeiou]]` could be + * mis-segmented into `[[a-z]`, `-`, `[aeiou]`, `]` */ + public static readonly MATCH_NESTED_SQUARE_BRACKETS = /\[[^\]]*\[/; + + /** Match (segment) UnicodeSets OR hex escapes OR single Unicode codepoints */ + public static readonly MATCH_ELEMENT_SEGMENTS = + /(?:\[[^\]]*\]|\\u[0-9a-fA-F]{4}|\\u\{[0-9a-fA-F]{1,6}\}|.)/gu; + + /** Does it start with a UnicodeSet? Used to test the segments. */ + public static readonly MATCH_USET = /^\[/; + + /** Does it start with an escaped char? Used to test the segments. */ + public static readonly MATCH_ESCAPED = /^\\u/; + + /** Split a string into ElementSegments */ + public static segment(str: string): ElementSegment[] { + if (this.MATCH_NESTED_SQUARE_BRACKETS.test(str)) { + throw Error(`Unsupported: nested square brackets in element segment: ${str}`); + } + return str.match(ElementParser.MATCH_ELEMENT_SEGMENTS) + .map(str => new ElementSegment(str)); + } +}; diff --git a/common/web/types/src/util/util.ts b/common/web/types/src/util/util.ts index 6145fb4ceb..39c7b8dbc7 100644 --- a/common/web/types/src/util/util.ts +++ b/common/web/types/src/util/util.ts @@ -19,12 +19,47 @@ export function boxXmlArray(o: any, x: string): void { // TODO-LDML: #7569 the below regex works, but captures more than it should // (it would include \u{fffffffffffffffff } which // is overlong and has a space at the end.) The second regex does not work yet. -const MATCH_HEX_ESCAPE = /\\u{([0-9a-fA-F ]{1,})}/g; +export const MATCH_HEX_ESCAPE = /\\u{([0-9a-fA-F ]{1,})}/g; // const MATCH_HEX_ESCAPE = /\\u{((?:(?:[0-9a-fA-F]{1,5})|(?:10[0-9a-fA-F]{4})(?: (?!}))?)+)}/g; +/** regex for single quad escape such as \u0127 */ +export const MATCH_QUAD_ESCAPE = /\\u([0-9a-fA-F]{4})/g; + export class UnescapeError extends Error { } +/** + * Unescape one codepoint + * @param hex one codepoint in hex, such as '0127' + * @returns the unescaped codepoint + */ +function unescapeOne(hex: string): string { + const codepoint = Number.parseInt(hex, 16); + return String.fromCodePoint(codepoint); +} + +/** + * Unescape one single quad string such as \u0127 + * @param s input string + * @returns output + */ +export function unescapeOneQuadString(s: string): string { + if (!s) { + return s; + } + /** + * process one regex match + * @param str ignored + * @param matched the entire match such as '0127' or '22 22' + * @returns the unescaped match + */ + function processMatch(str: string, matched: string): string { + return unescapeOne(matched); + } + s = s.replace(MATCH_QUAD_ESCAPE, processMatch); + return s; +} + /** * Unescapes a string according to UTS#18§1.1, see * @param s escaped string @@ -35,15 +70,6 @@ export function unescapeString(s: string): string { return s; } try { - /** - * Unescape one codepoint - * @param hex one codepoint in hex, such as '0127' - * @returns the unescaped codepoint - */ - function unescapeOne(hex: string) : string { - const codepoint = Number.parseInt(hex, 16); - return String.fromCodePoint(codepoint); - } /** * process one regex match * @param str ignored diff --git a/common/web/types/test/ldml-keyboard/test-pattern-parser.ts b/common/web/types/test/ldml-keyboard/test-pattern-parser.ts index 9ca1890256..8eee0bf35c 100644 --- a/common/web/types/test/ldml-keyboard/test-pattern-parser.ts +++ b/common/web/types/test/ldml-keyboard/test-pattern-parser.ts @@ -1,6 +1,6 @@ import 'mocha'; import { assert } from 'chai'; -import { MarkerParser, VariableParser } from '../../src/ldml-keyboard/pattern-parser.js'; +import { ElementParser, ElementType, MarkerParser, VariableParser } from '../../src/ldml-keyboard/pattern-parser.js'; describe('Test of Pattern Parsers', () => { describe('should test MarkerParser', () => { @@ -77,4 +77,95 @@ describe('Test of Pattern Parsers', () => { } }); }); + describe('ElementParser', () => { + const samplePatterns = [ + `\\u1A60`, + `[\\u1A75-\\u1A79]`, + `\\u1A60\\u1A45`, + `\\u1A60[\\u1A75-\\u1A79]\\u1A45`, + `ែ្ម`, + ] + describe('try out the regexes', () => { + it('should detect usets', () => { + [ + `[a-z]`, + `[[a-z]-[aeiou]]`, + ].forEach(s => assert.ok(ElementParser.MATCH_USET.test(s), `expected true: ${s}`)); + }); + it('should detect non usets', () => { + [ + `\\u0127`, + `\\u{22}`, + `x`, + ].forEach(s => assert.notOk(ElementParser.MATCH_USET.test(s), `expected false: ${s}`)); + }); + it('should detect escaped', () => { + [ + `\\u0127`, + `\\u{22}`, + ].forEach(s => assert.ok(ElementParser.MATCH_ESCAPED.test(s), `expected true: ${s}`)); + }); + it('should detect non escaped', () => { + [ + `[a-z]`, + `[[a-z]-[aeiou]]`, + `ê`, + ].forEach(s => assert.notOk(ElementParser.MATCH_ESCAPED.test(s), `expected false: ${s}`)); + }); + it('should reject nested square brackets', () => { + [ + `[[a-z]-[aeiou]]`, + ].forEach(s => assert.ok(ElementParser.MATCH_NESTED_SQUARE_BRACKETS.test(s), `expected true: ${s}`)); + }); + it('should allow non-nested square brackets', () => { + [ + `[a-z]`, + `ê`, + ...samplePatterns, + ].forEach(s => assert.notOk(ElementParser.MATCH_NESTED_SQUARE_BRACKETS.test(s), `expected false: ${s}`)); + }); + }); + describe('segment some strings', () => { + it('should be able to segment strings from the spec and samples', () => { + samplePatterns.forEach(str => assert.ok(ElementParser.segment(str))); + }); + it('should throw on nested brackets', () => { + [ + `[[a-z]-[aeiou]]`, + ].forEach(str => assert.throws(() => ElementParser.segment(str))); + }); + [ + { + str: `ê🙀`, + expect: [{ + segment: 'ê', + type: ElementType.codepoint + },{ + segment: '🙀', + type: ElementType.codepoint + }], + }, + { + str: `\\u1A60[\\u1A75-\\u1A79]\\u1A45ħ`, + expect: [{ + segment: '\\u1A60', + type: ElementType.escaped + },{ + segment: '[\\u1A75-\\u1A79]', + type: ElementType.uset + },{ + segment: '\\u1A45', + type: ElementType.escaped + },{ + segment: 'ħ', + type: ElementType.codepoint + }], + }, + ].forEach(({str, expect}) => it(`Segment: ${str}`, () => { + const segmented = ElementParser.segment(str); + assert.ok(segmented, `segmenting ${str}`); + assert.deepEqual(segmented, expect, `segments of ${str}`) + })); + }); + }); }); diff --git a/common/web/types/test/util/test-unescape.ts b/common/web/types/test/util/test-unescape.ts index 704a3e124a..a17b4a85ad 100644 --- a/common/web/types/test/util/test-unescape.ts +++ b/common/web/types/test/util/test-unescape.ts @@ -1,6 +1,6 @@ import 'mocha'; import {assert} from 'chai'; -import {unescapeString, UnescapeError, isOneChar, toOneChar} from '../../src/util/util.js'; +import {unescapeString, UnescapeError, isOneChar, toOneChar, unescapeOneQuadString} from '../../src/util/util.js'; describe('test UTF32 functions()', function() { it('should properly categorize strings', () => { @@ -56,3 +56,10 @@ describe('test unescapeString()', function() { assert.throws(() => unescapeString('\\u{110000}'), UnescapeError); }); }); + +describe('test unescapeOneQuadString()', () => { + it('should be able to convert', () => { + // testing that `\u0127` is unescaped correctly (to U+0127: 'ħ') + assert.equal(unescapeOneQuadString('\\u0127'), '\u{0127}'); + }); +});