erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-08-30 04:07:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

fix(developer): kmlmc generates invalid json with smp

Browse source
This commit is contained in:
Marc Durdin 2019-12-09 13:37:51 +00:00
parent 568b72895f
commit 45c0ea105e
3 changed files with 45 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
developer/js/tests/fixtures/example.qaa.smp/example.qaa.smp.model.ts vendored Normal file
View file

@ -0,0 +1,6 @@
const source: LexicalModelSource = {
format: 'trie-1.0',
sources: ['wordlist.tsv'],
wordBreaker: 'default',
};
export default source;

5
developer/js/tests/fixtures/example.qaa.smp/wordlist.tsv vendored Normal file
View file

@ -0,0 +1,5 @@
CRAZ🤪 13644
🙄 9134
😇 4816
🇸
🇺
Can't render this file because it has a wrong number of fields in line 4.

35
developer/js/tests/test-compile-trie.ts
View file

@ -4,7 +4,6 @@ import 'mocha';
import {makePathToFixture, compileModelSourceCode} from './helpers';
describe('LexicalModelCompiler', function () {
describe('#generateLexicalModelCode', function () {
it('should compile a trivial word list', function () {
@ -70,4 +69,38 @@ describe('LexicalModelCompiler', function () {
// Sanity check: the word breaker is a property of the object.
assert.match(code, /\bwordBreaker\b["']?:\s+function\b/);
});
it('should not generate unpaired surrogate code units', function () {
const MODEL_ID = 'example.qaa.smp';
const PATH = makePathToFixture(MODEL_ID);
let compiler = new LexicalModelCompiler;
let code = compiler.generateLexicalModelCode(MODEL_ID, {
format: 'trie-1.0',
sources: ['wordlist.tsv']
}, PATH) as string;
let result = compileModelSourceCode(code);
assert.isFalse(result.hasSyntaxError);
assert.isNotNull(result.exportedModel);
assert.equal(result.modelConstructorName, 'TrieModel');
// Test every character in the string to make sure we don't have
// unpaired surrogates which destroy everything.
// We can assume that the first and last chars are not SMP
for(var i = 1; i < code.length - 1; i++) {
if(code.charCodeAt(i) >= 0xD800 && code.charCodeAt(i) < 0xDC00) {
assert.isTrue((code.charCodeAt(i+1) >= 0xDC00 && code.charCodeAt(i+1) < 0xE000),
'Unpaired lead surrogate U+'+code.charCodeAt(i).toString(16)+' at position '+i+' of \''+code+'\'');
} else if(code.charCodeAt(i) >= 0xDC00 && code.charCodeAt(i) < 0xE000) {
assert.isTrue((code.charCodeAt(i-1) >= 0xD800 && code.charCodeAt(i-1) < 0xDC00),
'Unpaired trail surrogate U+'+code.charCodeAt(i).toString(16)+' at position '+i+' of \''+code+'\'');
}
}
// Sanity check: the word list has three total unweighted words, with a
// total weight of 44,103!
assert.match(code, /\btotalWeight\b["']?:\s*27596\b/);
});
});