diff --git a/common/models/templates/src/trie-model.ts b/common/models/templates/src/trie-model.ts index 859bb92f96..ca75aa25cf 100644 --- a/common/models/templates/src/trie-model.ts +++ b/common/models/templates/src/trie-model.ts @@ -187,13 +187,16 @@ let charCode = entry.charCodeAt(0); if(charCode >= 0xD800 && charCode <= 0xDBFF) { // First part of a SMP char. - // For now, we'll just assume the second completes such a char. + // For now, we'll just assume the second always completes such a char. + // + // Note: Things get nasty here if this is only sometimes true; in the future, + // we should compile-time enforce that this assumption is always true if possible. if(entryNode.type == 'internal') { let internalNode = entryNode; for(let j = 0; j < entryNode.values.length; j++) { let prefix = this.prefix + entry + internalNode.values[j]; yield { - key: entryNode.values[j], + key: entry + entryNode.values[j], traversal: function() { return new TrieModel.Traversal(internalNode.children[internalNode.values[j]], prefix) } } } @@ -224,9 +227,15 @@ let fullText = root.entries[0].key; let prefix = this.prefix; if(prefix.length < fullText.length) { + let key = fullText[prefix.length]; + let charCode = fullText.charCodeAt(prefix.length); + if(charCode >= 0xD800 && charCode <= 0xDBFF) { + // Merge the other half of an SMP char in! + key = key + fullText[prefix.length+1]; + } yield { - key: fullText[prefix.length], - traversal: function() { return new TrieModel.Traversal(root, prefix + fullText[prefix.length])} + key: key, + traversal: function() { return new TrieModel.Traversal(root, prefix + key)} } } return; diff --git a/common/models/templates/test/fixtures/tries/smp-apple.json b/common/models/templates/test/fixtures/tries/smp-apple.json new file mode 100644 index 0000000000..05b9ec4ed9 --- /dev/null +++ b/common/models/templates/test/fixtures/tries/smp-apple.json @@ -0,0 +1,47 @@ +{ + "totalWeight":2, + "root": { + "type":"internal","weight":1, + "values":["\ud835"], + "children": { + "\ud835": { + "type":"internal", + "weight":1, + "values":["\uddba"], + "children":{ + "\uddba": { + "type":"internal", + "weight":1, + "values":["\ud835"], + "children":{ + "\ud835": { + "type":"internal", + "weight":1, + "values":["\uddc9"], + "children": { + "\uddc9":{ + "type":"internal", + "weight":1, + "values":["p","\ud835"], + "children":{ + "p": { + "type":"leaf", + "weight":1, + "entries":[{"key":"𝖺𝗉pl𝖾","weight":1,"content":"𝖺𝗉pl𝖾"}] + }, + "\ud835": { + "type":"leaf", + "weight":1, + "entries":[{"key":"𝖺𝗉𝗉𝗅𝖾","weight":1,"content":"𝖺𝗉𝗉𝗅𝖾"}] + } + } + } + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/common/models/templates/test/test-trie-traversal.js b/common/models/templates/test/test-trie-traversal.js index 4ab8a3d466..ed757127ce 100644 --- a/common/models/templates/test/test-trie-traversal.js +++ b/common/models/templates/test/test-trie-traversal.js @@ -5,6 +5,14 @@ var assert = require('chai').assert; var TrieModel = require('../').models.TrieModel; +// Useful for tests related to strings with supplementary pairs. +var smpForUnicode = function(code){ + var H = Math.floor((code - 0x10000) / 0x400) + 0xD800; + var L = (code - 0x10000) % 0x400 + 0xDC00; + + return String.fromCharCode(H, L); +} + describe('Trie traversal abstractions', function() { it('root-level iteration over child nodes', function() { var model = new TrieModel(jsonFixture('tries/english-1000')); @@ -24,7 +32,7 @@ describe('Trie traversal abstractions', function() { assert.isEmpty(rootKeys); }); - it('iteration over simple internal node', function() { + it('traversal with simple internal nodes', function() { var model = new TrieModel(jsonFixture('tries/english-1000')); let rootTraversal = model.getRootTraversal(); @@ -77,7 +85,7 @@ describe('Trie traversal abstractions', function() { assert.isEmpty(eKeys); }); - it('iteration over compact leaf node', function() { + it('traversal over compact leaf node', function() { var model = new TrieModel(jsonFixture('tries/english-1000')); let rootTraversal = model.getRootTraversal(); @@ -108,14 +116,14 @@ describe('Trie traversal abstractions', function() { do { assert.isNotEmpty(leafChildSequence); - let oIter = curChild.traversal().children(); - let curr = oIter.next(); + let iter = curChild.traversal().children(); + let curr = iter.next(); curChild = curr.value; // Test generator behavior - there should be one child, then the 'done' state. assert.isDefined(curChild); assert.equal(curChild.key, leafChildSequence[0]); - curr = oIter.next(); + curr = iter.next(); assert.isTrue(curr.done); // Prepare for iteration. @@ -140,4 +148,96 @@ describe('Trie traversal abstractions', function() { assert.isTrue(eSuccess); }); + + + it('traversal with SMP entries', function() { + // Two entries, both of which read "apple" to native English speakers. + // One solely uses SMP characters, the other of which uses a mix of SMP and standard. + var model = new TrieModel(jsonFixture('tries/smp-apple')); + + let rootTraversal = model.getRootTraversal(); + assert.isDefined(rootTraversal); + + let smpA = smpForUnicode(0x1d5ba); + let smpP = smpForUnicode(0x1d5c9); + let smpE = smpForUnicode(0x1d5be); + + // Just to be sure our utility function is working right. + assert.equal(smpA + smpP + 'pl' + smpE, "𝖺𝗉pl𝖾"); + + let pKeys = ['p', smpP]; + let leafChildSequence = ['l', smpE]; + + let aSuccess = false; + let pSuccess = false; + let eSuccess = false; + for(child of rootTraversal.children()) { + if(child.key == smpA) { + aSuccess = true; + let traversalInner1 = child.traversal(); + assert.isDefined(traversalInner1); + assert.isUndefined(child.entries); + + for(aChild of traversalInner1.children()) { + if(aChild.key == smpP) { + pSuccess = true; + let traversalInner2 = aChild.traversal(); + assert.isDefined(traversalInner2); + assert.isUndefined(aChild.entries); + + for(pChild of traversalInner2.children()) { + let keyIndex = pKeys.indexOf(pChild.key); + assert.notEqual(keyIndex, -1, "Did not find char '" + pChild.key + "' in array!"); + pKeys.splice(keyIndex, 1); + + if(pChild.key == 'p') { // We'll test traversal with the 'mixed' entry from here. + let traversalInner3 = pChild.traversal(); + assert.isDefined(traversalInner3); + assert.isUndefined(pChild.entries); + + // Now to handle the rest, knowing it's backed by a leaf node. + let curChild = pChild; + + // At this point, we're already at the trie's actual leaf node for "trouble". + // But for edit-distance trie traversal, we want to decompress this and model + // an uncompacted Trie. + do { + assert.isNotEmpty(leafChildSequence); + + let iter = curChild.traversal().children(); + let curr = iter.next(); + curChild = curr.value; + + // Test generator behavior - there should be one child, then the 'done' state. + assert.isDefined(curChild); + assert.equal(curChild.key, leafChildSequence[0]); + curr = iter.next(); + assert.isTrue(curr.done); + + // Prepare for iteration. + leafChildSequence.shift(); + + // Conditional test - if that was not the final character, entries should be undefined. + if(leafChildSequence.length > 0) { + assert.isUndefined(curChild.traversal().entries); + } else { + let finalTraversal = curChild.traversal(); + assert.isDefined(finalTraversal.entries); + assert.equal(finalTraversal.entries[0], smpA + smpP + 'pl' + smpE); + eSuccess = true; + } + } while (leafChildSequence.length > 0); + } + } + } + } + } + } + + assert.isTrue(aSuccess); + assert.isTrue(pSuccess); + assert.isTrue(eSuccess); + + assert.isEmpty(pKeys); + }); });