From d03cd7f155313b486f802b5b0be93f54cbebb8ea Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 3 Aug 2023 09:32:30 +0700 Subject: [PATCH] feat(web): the bulk of gesture-staging implementation --- common/web/gesture-recognizer/.mocharc.env.js | 5 + common/web/gesture-recognizer/.mocharc.json | 3 +- .../gestures/matchers/gestureMatcher.ts | 67 +++- .../gestures/matchers/matcherSelector.ts | 325 ++++++++++++++++++ .../src/test/auto/tsconfig.json | 4 +- 5 files changed, 383 insertions(+), 21 deletions(-) create mode 100644 common/web/gesture-recognizer/.mocharc.env.js create mode 100644 common/web/gesture-recognizer/src/engine/headless/gestures/matchers/matcherSelector.ts diff --git a/common/web/gesture-recognizer/.mocharc.env.js b/common/web/gesture-recognizer/.mocharc.env.js new file mode 100644 index 0000000000..81e54c7365 --- /dev/null +++ b/common/web/gesture-recognizer/.mocharc.env.js @@ -0,0 +1,5 @@ +import { fileURLToPath } from "url"; +import { dirname } from 'path'; + +// Tells ts-node where to find the tsconfig.json to be used for executing TS unit tests. +process.env.TS_NODE_PROJECT = `${dirname(fileURLToPath(import.meta.url))}/src/test/auto/tsconfig.json`; \ No newline at end of file diff --git a/common/web/gesture-recognizer/.mocharc.json b/common/web/gesture-recognizer/.mocharc.json index c56f24db4c..bf90c42bf4 100644 --- a/common/web/gesture-recognizer/.mocharc.json +++ b/common/web/gesture-recognizer/.mocharc.json @@ -5,5 +5,6 @@ ], "node-option": [ "loader=ts-node/esm" - ] + ], + "require": ".mocharc.env.js" } \ No newline at end of file diff --git a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts index 02e372989c..f40b42e5af 100644 --- a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts +++ b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/gestureMatcher.ts @@ -20,13 +20,11 @@ export class GestureMatcher { public readonly model: GestureModel; public readonly pathMatchers: PathMatcher[]; - private predecessor?: GestureMatcher; + private readonly predecessor?: GestureMatcher; private readonly publishedPromise: ManagedPromise>; // unsure on the actual typing at the moment. private _result: MatchResult; - private baseSources: GestureSource[]; - public get promise() { return this.publishedPromise.corePromise; } @@ -44,8 +42,6 @@ export class GestureMatcher { const predecessor = sourceObj instanceof GestureSource ? null : sourceObj; const source = predecessor ? null : (sourceObj as GestureSource); - this.baseSources = predecessor?.baseSources || [source]; - this.predecessor = predecessor; this.publishedPromise = new ManagedPromise(); @@ -60,35 +56,45 @@ export class GestureMatcher { this.pathMatchers = []; - const sourceTouchpoints: GestureSource[] = source + const unfilteredSourceTouchpoints: GestureSource[] = source ? [ source ] : predecessor.pathMatchers.map((matcher) => matcher.source); - let offset = 0; + const sourceTouchpoints = unfilteredSourceTouchpoints.map((entry) => { + return entry.isPathComplete ? null : entry; + }).reduce((cleansed, entry) => { + return entry ? cleansed.concat(entry) : cleansed; + }, []); + + if(model.sustainTimer && sourceTouchpoints.length > 0) { + // If a sustain timer is set, it's because we expect to have NO gesture-source _initially_. + // If we actually have one, that's cause for rejection. + // + this.finalize(false, 'path'); + return; + } else if(!model.sustainTimer && sourceTouchpoints.length == 0) { + // If no sustain timer is set, we don't start against the specified set; that'll happen + // once there's an actual source to support the modeled gesture. + this.finalize(false, 'path'); + } + for(let touchpointIndex = 0; touchpointIndex < sourceTouchpoints.length; touchpointIndex++) { const srcContact = sourceTouchpoints[touchpointIndex]; - // If a touchpoint's path is already complete, ignore it when modeling a new gesture. - if(srcContact.path.isComplete) { - offset++; - continue; - } - if(srcContact instanceof GestureSourceSubview) { srcContact.disconnect(); // prevent further updates from mangling tracked path info. } - let i = touchpointIndex - offset; - const contactSpec = model.contacts[i]; + const contactSpec = model.contacts[touchpointIndex]; /* c8 ignore next 3 */ if(!contactSpec) { - throw new Error(`No contact model for inherited path: gesture "${model.id}', entry ${i}`); + throw new Error(`No contact model for inherited path: gesture "${model.id}', entry ${touchpointIndex}`); } const inheritancePattern = contactSpec?.model.pathInheritance ?? 'chop'; let preserveBaseItem: boolean = false; - let contact: GestureSource; + let contact: GestureSourceSubview; switch(inheritancePattern) { case 'reject': this.finalize(false, 'path'); @@ -220,6 +226,30 @@ export class GestureMatcher { return bestMatcher.source; } + public get baseItem(): Type { + return this.comparisonStandard.baseItem; + } + + public get currentItem(): Type { + return this.comparisonStandard.currentSample.item; + } + + /* + * Gets the GestureSource identifier corresponding to the gesture being matched + * and all predecessor stages. All are relevant for resolving gesture-selection; + * predecessor IDs become relevant for gesture stages that start without an + * active GestureSource. (One that's not already finished its path) + * + * In theory, just one predecessor previous should be fine, rather than + * 'all'... but that'd take a little extra work. + */ + public get allSourceIds(): string[] { + const currentIds = this.pathMatchers.map((entry) => entry.source.identifier); + const predecessorIds = this.predecessor ? this.predecessor.allSourceIds : []; + + return currentIds.concat(predecessorIds); + } + mayAddContact(): boolean { return this.pathMatchers.length < this.model.contacts.length; } @@ -263,11 +293,10 @@ export class GestureMatcher { } } - this.baseSources.push(simpleSource); this.addContactInternal(simpleSource.constructSubview(false, true)); } - private addContactInternal(simpleSource: GestureSource) { + private addContactInternal(simpleSource: GestureSourceSubview) { const existingContacts = this.pathMatchers.length; // The number of already-active contacts tracked for this gesture diff --git a/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/matcherSelector.ts b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/matcherSelector.ts new file mode 100644 index 0000000000..f4510198ba --- /dev/null +++ b/common/web/gesture-recognizer/src/engine/headless/gestures/matchers/matcherSelector.ts @@ -0,0 +1,325 @@ +import EventEmitter from "eventemitter3"; + +import { ManagedPromise } from "@keymanapp/web-utils"; + +import { GestureSource, GestureSourceSubview } from "../../gestureSource.js"; +import { GestureMatcher, MatchResult } from "./gestureMatcher.js"; +import { GestureModel } from "../specs/gestureModel.js"; +import { QueuedPromisePrioritizer } from "../../queuedPromisePrioritizer.js"; + +interface GestureSourceTracker { + source: GestureSourceSubview; + matchPromise: ManagedPromise>; +} + +export interface MatcherSelection { + matcher: GestureMatcher, + result: MatchResult +} + +interface EventMap { + 'rejectionwithaction': (selection: MatcherSelection, replaceModelWith: (replacementModel: GestureModel) => void) => void; +} + +/** + * This class is used to "select" successfully-matched gesture models from among an + * active set of potential GestureMatchers. There may be multiple GestureSources / + * contact-points active; it is able to resolve when they are correlated and how + * resolution should proceed based upon the "selected" gesture model. + * + * When at least one "match" for a gesture model occurs, this engine ensures that the + * highest-priority one that matched is selected. It will be returned via Promise along + * with the specified match "action". If, instead, no model ends up matching a + * GestureSource, the Promise will resolve when the last potential model is rejected, + * providing values indicating match failure and the action to be taken. + */ +export class MatcherSelector extends EventEmitter> { + private _sourceSelector: GestureSourceTracker[] = []; + private potentialMatchers: GestureMatcher[] = []; + + private readonly promisePrioritizer = new QueuedPromisePrioritizer(); + + /** + * Aims to match the gesture-source's path against the specified set of gesture models. The + * returned Promise will resolve either when a match is found or all models have rejected the path. + * @param source + * @param gestureModelSet + */ + public matchGesture( + source: GestureSource, + gestureModelSet: GestureModel[] + ): Promise>; + + /** + * Facilitates matching a new stage in an ongoing gesture-stage sequence based on a previously- + * matched stage and the specified models for stages that may follow it. + * @param source + * @param gestureModelSet + */ + public matchGesture( + priorStageMatcher: GestureMatcher, + gestureModelSet: GestureModel[] + ): Promise>; + + public matchGesture( + source: GestureSource | GestureMatcher, + gestureModelSet: GestureModel[] + ): Promise> { + /* + * To be clear, this _starts_ the source-tracking process. It's an async process, though. + */ + + const sourceNotYetStaged = source instanceof GestureSource; + const sources = sourceNotYetStaged + ? [source.constructSubview(false, true)] + : source.pathMatchers.map((pathMatch) => pathMatch.source as GestureSourceSubview); + + const matchPromise = new ManagedPromise>(); + + /* + * First... + * 1. Verify no duplicate sources (even if subviews) + * 2. Set up source 'trackers' used for synchronization & result-reporting. + */ + const sourceTrackers = sources.map((src) => { + // TODO: Assertion check - there's no version of the source currently being actively matched. + + // Even if a component path is already completed, TRACK IT. It's by far the easiest way + // to handle gesture stages that start without active sources - such as multitap stages after + // the initial tap. + + // Sets up source selectors - the object that matches a source against its Promise. + // Promises only resolve once, after all - once called, a "selection" has been made. + const sourceSelectors: GestureSourceTracker = { + source: src, + matchPromise: matchPromise + }; + this._sourceSelector.push(sourceSelectors); + + return sourceSelectors; + }); + + const synchronizationSet = sourceTrackers.map((track) => track.matchPromise); + + /** + * If we received a single gesture-source on its own that's just starting out, it may be able + * to fulfill secondary `contacts` entries for in-process gesture-models. + * + * If we're following up a previous gesture stage, meaning the contacts are already part of + * an ongoing gesture-sequence and have known associations already... they're not allowed to + * change their committed links; bypass this section. + */ + if(sourceNotYetStaged) { + const extendableMatcherSet = this.potentialMatchers.filter((matcher) => matcher.mayAddContact()); + extendableMatcherSet.forEach((matcher) => { + // TODO: do we alter the resolution priority in any way, now that there's an extra touchpoint? + // Answer is not yet clear; perhaps work on gesture-staging will help indicate if this would + // be useful... and how it should act, if so. + + matcher.addContact(source); + matcher.promise.then(this.matcherSelectionFilter(matcher, synchronizationSet)); + }); + + // In theory, we _could_ do a quick `await` post-loop to see if anything has instantly resolved, + // shortcutting if there's an instant match... but that does make unit testing a bit less intuitive. + } + + /** + * In either case, time to spin up gesture models limited to new sources, that don't combine with + * already-active ones. This could be the first stage in a sequence or a followup to a prior stage. + */ + const newMatchers = gestureModelSet.map((model) => new GestureMatcher(model, source)); + + for(const matcher of newMatchers) { + matcher.promise.then(this.matcherSelectionFilter(matcher, synchronizationSet)); + } + this.potentialMatchers = this.potentialMatchers.concat(newMatchers); + + /* + * Easiest way to ensure resolution priorities are respected: keep 'em sorted in descending order. + * When we iterate through on update-steps, we go sequentially; the first Promise to be marked + * 'resolved' wins. + */ + this.potentialMatchers.sort((a, b) => b.model.resolutionPriority - a.model.resolutionPriority); + + // Now that all GestureMatchers are built, reset ALL of our sync-update-check hooks. + this.resetSourceHooks(); + + return matchPromise.corePromise; + } + + private readonly attemptSynchronousUpdate = () => { + const sourceCurrentTimestamps = this._sourceSelector.map((tracker) => tracker.source.isPathComplete ? null : tracker.source.currentSample.t); + const t = sourceCurrentTimestamps[0]; + + // Ignore timestamps from already-terminated paths; they should not block synchronicity checks. + if(sourceCurrentTimestamps.find((t2) => (t2 !== null) && (t != t2))) { + return; + } + + this.potentialMatchers.forEach((matcher) => matcher.update()); + }; + + private resetSourceHooks() { + const resetHooks = (gestureSource: GestureSourceSubview) => { + // GestureSourceSubviews stay synchronized with their 'base' via event handlers. + // We want GestureMatchers to receive all updates before we attempt a sync'd update. + const baseSource = gestureSource.baseSource; + + // So, a resetHooks call says to remove the old handler... + baseSource.path.off('step', this.attemptSynchronousUpdate); + baseSource.path.off('complete', this.attemptSynchronousUpdate); + baseSource.path.off('invalidated', this.attemptSynchronousUpdate); + + // And re-add it, but at the end of the handler list. + baseSource.path.on('step', this.attemptSynchronousUpdate); + baseSource.path.on('complete', this.attemptSynchronousUpdate); + baseSource.path.on('invalidated', this.attemptSynchronousUpdate); + } + + // Make sure our source-watching hooks are the last handler for the event; + // matcher-handlers should go first. (Due to how subview synchronization works) + this._sourceSelector.forEach((entry) => resetHooks(entry.source)); + } + + private matchersForSource(source: GestureSource) { + return this.potentialMatchers.filter((matcher) => { + return !!matcher.pathMatchers.find((pathMatch) => pathMatch.source == source) + }); + } + + private matcherSelectionFilter(matcher: GestureMatcher, matchSynchronizers: ManagedPromise[]) { + // Returns a closure-captured Promise-resolution handler used by individual GestureMatchers managed + // by this class instance. + return async (result: MatchResult) => { + // Note: is only called by GestureMatcher Promises that are resolving. + + // Ensure that any essentially-synchronous resolving GestureMatchers resolve in order of + // their specified `resolutionPriority`, with larger values first. + await this.promisePrioritizer.queueWithPriority(matcher.model.resolutionPriority); + + /* + * If we already had a gesture stage match, this will have already been fulfilled; + * bypass all match-handling. Capturing `matchSynchronization` in a closure in this + * manner is important to ensure that the returned handler is "locked" to the + * currently-processing gesture stage. + */ + for(let synchronizer of matchSynchronizers) { + if(synchronizer.isFulfilled) { + return; + } + } + + // Find ALL associated match-promises for sources matched by the matcher. + const matchedContactIds = matcher.allSourceIds; + + const _this = this; + const sourceMetadata = matchedContactIds.map((id) => { + const match = this._sourceSelector.find((metadata) => metadata.source.identifier == id); + /* c8 ignore start */ + if(!match) { + _this._sourceSelector.map(() => {}); + throw Error(`Could not find original tracker-object for source with id ${id}`); + } + /* c8 ignore end */ + return match; + }); + + // We have a result for this matcher; go ahead and remove it from the 'potential' list. + const matcherIndex = this.potentialMatchers.indexOf(matcher); + this.potentialMatchers.splice(matcherIndex, 1); + + /* + * This is the common case for failed gesture matches. It should never be set + * for a successful gesture match. This is a "didn't match" signal, so we don't + * do any gesture-staging stuff here or enter a state where we need to ignore + * other matchers. + */ + if(result.action.type == 'none') { + // Check - are there any remaining matchers compatible with the rejected matcher's sources? + const remainingMatcherStats = sourceMetadata.map((tracker) => { + return { + tracker: tracker, + // We need to inspect each matcher's `contacts` entries for references to the source. + pendingCount: this.potentialMatchers.filter((matcher) => { + return !!matcher.allSourceIds.find((id) => tracker.source.identifier == id); + }).length // and tally up a count at the end. + }; + }); + + // If we just rejected the last possible matcher for a tracked gesture-source... + // then, for each such affected source... + for(const stat of remainingMatcherStats) { + if(stat.pendingCount == 0) { + // ... report the failure and signal to close-out that source / stop tracking it. + stat.tracker.matchPromise.resolve({ + matcher: null, + result: { + matched: false, + action: { + type: 'complete', + item: null + } + } + }); + } + } + + // Again, we allow any other matchers against the represented sources to REMAIN AS THEY ARE. + // This is a "didn't resolve" case - we only matched against a "path reset" case. + return; + } + + if(!result.matched) { + // There is an action to be resolved... + // But we didn't actually MATCH a gesture. + const replacer = (replacementModel: GestureModel) => { + const replacementMatcher = new GestureMatcher(replacementModel, matcher); + replacementMatcher.promise.then(this.matcherSelectionFilter(replacementMatcher, sourceMetadata.map((entry) => entry.matchPromise))); + + this.potentialMatchers.push(replacementMatcher); + this.resetSourceHooks(); + }; + + // So we emit an event to signal the rejection & allow its replacement via the closure above. + this.emit('rejectionwithaction', {matcher, result}, replacer); + return; + } else /* if(result.matched) */ { + for(const tracker of sourceMetadata) { + // If we have a successful gesture match, we should proactively clear out the matchers + // that (a) didn't win and (b) use at least one source in common with the winner. + const losingMatchers = this.matchersForSource(tracker.source); + this.potentialMatchers = this.potentialMatchers.filter((matcher) => { + return !losingMatchers.find((matcher2) => matcher == matcher2); + }); + + // Drop all trackers for the matched sources. + this._sourceSelector = this._sourceSelector.filter((a) => !sourceMetadata.find((b) => a == b)); + + // And now for one line with some "heavy lifting": + + /* + * Does two things: + * 1. Fulfills the contract set by `matchGesture`. + * + * 2. Fulfilling the ManagedPromise acts as a synchronizer, facilitating the guarantee at + * the start of this closure. It's set synchronously, so other gesture-matchers that + * call into this method will know that a match has already fulfilled for the matched + * source(s). Any further matchers will be silently ignored, effectively cancelling them. + * + * If we're within this closure, the closure's synchronizer-promise matches the instance + * currently set on its `tracker` - as are any others affected by the resolving matcher. + * + * It _is_ possible that we may need to resolve a Promise not included in the synchronizer + * set - if a second contact / source was added at a later point in time to something that + * started single-contact. Two separate 'raise' attempts would occur, since the links to + * this method were set for each source independently. The most consistent way to ensure + * synchronization is thus to rely on the instance annotated on the tracker itself for + * each matched source. + */ + tracker.matchPromise.resolve({matcher, result}); + } + } + }; + } +} \ No newline at end of file diff --git a/common/web/gesture-recognizer/src/test/auto/tsconfig.json b/common/web/gesture-recognizer/src/test/auto/tsconfig.json index ae2302aa52..336b2d5f57 100644 --- a/common/web/gesture-recognizer/src/test/auto/tsconfig.json +++ b/common/web/gesture-recognizer/src/test/auto/tsconfig.json @@ -6,6 +6,8 @@ { "extends": "../../../../tsconfig.kmw-main-base.json", "compilerOptions": { - "moduleResolution": "Node16", + // Not needed when testing via Node, and when `true` it seems to desync preset breakpoints + // worse than when `false`. + "importHelpers": false, } }