diff --git a/developer/js/tests/fixtures/example.qaa.smp/example.qaa.smp.model.ts b/developer/js/tests/fixtures/example.qaa.smp/example.qaa.smp.model.ts new file mode 100644 index 0000000000..2a5715fa94 --- /dev/null +++ b/developer/js/tests/fixtures/example.qaa.smp/example.qaa.smp.model.ts @@ -0,0 +1,6 @@ +const source: LexicalModelSource = { + format: 'trie-1.0', + sources: ['wordlist.tsv'], + wordBreaker: 'default', +}; +export default source; diff --git a/developer/js/tests/fixtures/example.qaa.smp/wordlist.tsv b/developer/js/tests/fixtures/example.qaa.smp/wordlist.tsv new file mode 100644 index 0000000000..12bf3ff333 --- /dev/null +++ b/developer/js/tests/fixtures/example.qaa.smp/wordlist.tsv @@ -0,0 +1,5 @@ +CRAZ🤪 13644 +🙄 9134 +😇 4816 +🇸 +🇺 diff --git a/developer/js/tests/test-compile-trie.ts b/developer/js/tests/test-compile-trie.ts index ff8ae6b9a7..39029da710 100644 --- a/developer/js/tests/test-compile-trie.ts +++ b/developer/js/tests/test-compile-trie.ts @@ -4,7 +4,6 @@ import 'mocha'; import {makePathToFixture, compileModelSourceCode} from './helpers'; - describe('LexicalModelCompiler', function () { describe('#generateLexicalModelCode', function () { it('should compile a trivial word list', function () { @@ -70,4 +69,38 @@ describe('LexicalModelCompiler', function () { // Sanity check: the word breaker is a property of the object. assert.match(code, /\bwordBreaker\b["']?:\s+function\b/); }); + + it('should not generate unpaired surrogate code units', function () { + const MODEL_ID = 'example.qaa.smp'; + const PATH = makePathToFixture(MODEL_ID); + + let compiler = new LexicalModelCompiler; + let code = compiler.generateLexicalModelCode(MODEL_ID, { + format: 'trie-1.0', + sources: ['wordlist.tsv'] + }, PATH) as string; + + let result = compileModelSourceCode(code); + assert.isFalse(result.hasSyntaxError); + assert.isNotNull(result.exportedModel); + assert.equal(result.modelConstructorName, 'TrieModel'); + + // Test every character in the string to make sure we don't have + // unpaired surrogates which destroy everything. + // We can assume that the first and last chars are not SMP + for(var i = 1; i < code.length - 1; i++) { + if(code.charCodeAt(i) >= 0xD800 && code.charCodeAt(i) < 0xDC00) { + assert.isTrue((code.charCodeAt(i+1) >= 0xDC00 && code.charCodeAt(i+1) < 0xE000), + 'Unpaired lead surrogate U+'+code.charCodeAt(i).toString(16)+' at position '+i+' of \''+code+'\''); + } else if(code.charCodeAt(i) >= 0xDC00 && code.charCodeAt(i) < 0xE000) { + assert.isTrue((code.charCodeAt(i-1) >= 0xD800 && code.charCodeAt(i-1) < 0xDC00), + 'Unpaired trail surrogate U+'+code.charCodeAt(i).toString(16)+' at position '+i+' of \''+code+'\''); + } + } + + // Sanity check: the word list has three total unweighted words, with a + // total weight of 44,103! + assert.match(code, /\btotalWeight\b["']?:\s*27596\b/); + }); }); +