feat(common/models): tracked context notes applied Suggestions

common/predictive-text/worker/correction/context-tracker.ts

@@ -8,6 +8,8 @@ namespace correction {
 
   export class TrackedContextToken {
     raw: string;
+    replacementText: string;
+
     transformDistributions: Distribution<Transform>[] = [];
     replacements: TrackedContextSuggestion[];
     activeReplacementId: number = -1;
@@ -15,6 +17,26 @@ namespace correction {
     get isNew(): boolean {
       return this.transformDistributions.length == 0;
     }
+
+    get currentText(): string {
+      if(this.replacementText === undefined || this.replacementText === null) {
+        return this.raw;
+      } else {
+        return this.replacementText;
+      }
+    }
+
+    get replacement(): TrackedContextSuggestion {
+      let replacementId = this.activeReplacementId;
+      return this.replacements.find(function(replacement) {
+        return replacement.suggestion.id == replacementId;
+      });
+    }
+
+    revert() {
+      delete this.activeReplacementId;
+      delete this.replacements;
+    }
   }
 
   export class TrackedContextState {
@@ -46,6 +68,10 @@ namespace correction {
           copy.replacements = token.replacements
           copy.activeReplacementId = token.activeReplacementId;
           copy.transformDistributions = token.transformDistributions;
+
+          if(token.replacementText) {
+            copy.replacementText = token.replacementText;
+          }
   
           return copy;
         });
@@ -149,6 +175,22 @@ namespace correction {
       return this.circle.length;
     }
 
+    get oldest(): Item {
+      if(this.count == 0) {
+        return undefined;
+      }
+
+      return this.item[0];
+    }
+
+    get newest(): Item {
+      if(this.count == 0) {
+        return undefined;
+      }
+
+      return this.item[this.count - 1];
+    }
+
     enqueue(item: Item): Item {
       var prevItem = null;
       let nextHead = (this.currentHead + 1) % this.maxCount;
@@ -204,7 +246,8 @@ namespace correction {
       // Matters greatly when starting from a nil context.
       if(editPath.length > 1) {
         // First entry:  may not be an 'insert' or a 'transpose' op.
-        if(editPath[0] == 'insert' || editPath[0].indexOf('transpose') >= 0) {
+        // 'insert' allowed if the next token is 'substitute', as this may occur with an edit path of length 2.
+        if((editPath[0] == 'insert' && !(editPath[1] == 'substitute' && editPath.length == 2)) || editPath[0].indexOf('transpose') >= 0) {
           return null;
         } else if(editPath[0] == 'delete') {
           poppedHead = true; // a token from the previous state has been wholly removed.
@@ -235,6 +278,15 @@ namespace correction {
       let state: TrackedContextState;
       
       if(pushedTail) {
+        // On suggestion acceptance, we should update the previous final token.
+        // We do it first so that the acceptance is replicated in the new TrackedContextState
+        // as well.
+        if(ignorePenultimateMatch) {
+          // For this case, we were likely called by ModelCompositor.acceptSuggestion(), which
+          // would have marked the accepted suggestion.
+          matchState.tokens[matchState.tokens.length - 1].replacementText = tokenizedContext[tokenizedContext.length-2];
+        }
+
         state = new TrackedContextState(matchState);
       } else {
         // Since we're continuing a previously-cached context, we can reuse the same SearchSpace
@@ -258,13 +310,6 @@ namespace correction {
         }
 
         if(pushedTail) {
-          // On suggestion acceptance, we should update the previous final token.
-          if(ignorePenultimateMatch) {
-            // TODO:  We might should track the accepted suggestion as a final transform here.
-            //        The infrastructure to track this doesn't exist quite yet, though.
-            state.updateTail(null, tokenizedContext[tokenizedContext.length-2]);
-          }
-
           // ASSUMPTION:  any transform that triggers this case is a pure-whitespace Transform, as we
           //              need a word-break before beginning a new word's context.
           //              Worth note:  when invalid, the lm-layer already has problems in other aspects too.

common/predictive-text/worker/model-compositor.ts

@@ -384,13 +384,14 @@ class ModelCompositor {
     reversionTransform = models.buildMergedTransform(reversionTransform, postTransform);
 
     // Step 2:  building the proper 'displayAs' string for the Reversion
+    let postContext = context;
     if(postTransform) {
       // Now that we've built the reversion based upon the Suggestion's original context,
-      // we may safely manipulate it in order to get a proper 'displayAs' string.
-      context = models.applyTransform(postTransform, context);
+      // we manipulate it in order to get a proper 'displayAs' string.
+      postContext = models.applyTransform(postTransform, postContext);
     }
 
-    let postContextTokens = this.lexicalModel.tokenize(context); //.wordbreak(postContext);
+    let postContextTokens = this.lexicalModel.tokenize(postContext); //.wordbreak(postContext);
     let revertedPrefix = postContextTokens[postContextTokens.length - 1] || '';
 
     let firstConversion = models.transformToSuggestion(reversionTransform);
@@ -412,8 +413,15 @@ class ModelCompositor {
     
     // Step 3:  if we track Contexts, update the tracking data as appropriate.
     if(this.contextTracker) {
-      let contextState = this.contextTracker.analyzeState(this.lexicalModel, context);
-      contextState.tokens[contextState.tokens.length - 1].activeReplacementId = suggestion.id;
+      let contextState = this.contextTracker.newest;
+      if(!contextState) {
+        contextState = this.contextTracker.analyzeState(this.lexicalModel, context);
+      }
+      let lastToken = contextState.tokens[contextState.tokens.length - 1];
+      
+      lastToken.activeReplacementId = suggestion.id;
+      let acceptedContext = models.applyTransform(suggestion.transform, context);
+      this.contextTracker.analyzeState(this.lexicalModel, acceptedContext);
     }
 
     return reversion;