From fd97412bc44ae468ca1216a4bcbd445a97a38152 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 26 Feb 2024 14:01:52 +0700 Subject: [PATCH] change(web): implements a touch-event sequentialization queue --- .../src/engine/eventSequentializationQueue.ts | 51 +++++ .../src/engine/headless/inputEngineBase.ts | 2 + .../engine/headless/touchpointCoordinator.ts | 16 +- .../src/engine/inputEventEngine.ts | 1 + .../src/engine/touchEventEngine.ts | 201 +++++++++++------- 5 files changed, 194 insertions(+), 77 deletions(-) create mode 100644 common/web/gesture-recognizer/src/engine/eventSequentializationQueue.ts diff --git a/common/web/gesture-recognizer/src/engine/eventSequentializationQueue.ts b/common/web/gesture-recognizer/src/engine/eventSequentializationQueue.ts new file mode 100644 index 0000000000..2cc962c73d --- /dev/null +++ b/common/web/gesture-recognizer/src/engine/eventSequentializationQueue.ts @@ -0,0 +1,51 @@ +import { timedPromise } from "@keymanapp/web-utils"; + +export class EventSequentializationQueue { + private queue: (() => Promise | void)[]; + private defermentPromise: Promise; + + constructor() { + this.queue = []; + } + + private setDeferment(promise: Promise) { + this.defermentPromise = promise; + promise.then(() => { + this.defermentPromise = null; + this.triggerEvent(); + }); + } + + private async triggerEvent() { + while(this.queue.length > 0) { + const functor = this.queue.shift(); + + // Things break _badly_ if we don't keep the queue running if errors are triggered by the functor. + // It's best to ignore the error and let things play out. + try { + // Is either undefined or is a Promise. + const result = functor(); + // We either wait on a manual lock (from within an InputEventEngine) or a macrotask queue wait, + // allowing gesture-matching microtask queue Promises to complete before proceeding. + this.setDeferment(result ? result : timedPromise(0)); + } catch (err) { + const baseMsg = 'Error sequentializing received inputs'; + if(err instanceof Error) { + console.error(`${baseMsg}: ${err.message}\n\n${err.stack}`); + } else { + console.error(baseMsg); + console.error(err); + } + } + } + } + + queueEventFunctor(functor: () => Promise | void) { + this.queue.push(functor); + // We only need to trigger events if the queue has no prior entries and there isn't an + // active deferment that will auto-trigger the event at the appropriate time. + if(this.queue.length == 1 && !this.defermentPromise) { + this.triggerEvent(); + } + } +} \ No newline at end of file diff --git a/common/web/gesture-recognizer/src/engine/headless/inputEngineBase.ts b/common/web/gesture-recognizer/src/engine/headless/inputEngineBase.ts index 3acd723c42..e73c502369 100644 --- a/common/web/gesture-recognizer/src/engine/headless/inputEngineBase.ts +++ b/common/web/gesture-recognizer/src/engine/headless/inputEngineBase.ts @@ -56,6 +56,8 @@ export abstract class InputEngineBase extends return source; } + public unlockTouchpoint?: (touchpoint: GestureSource) => void; + /** * Calls to this method will cancel any touchpoints whose internal IDs are _not_ included in the parameter. * Designed to facilitate recovery from error cases and peculiar states that sometimes arise when debugging. diff --git a/common/web/gesture-recognizer/src/engine/headless/touchpointCoordinator.ts b/common/web/gesture-recognizer/src/engine/headless/touchpointCoordinator.ts index ff81277187..522af5317c 100644 --- a/common/web/gesture-recognizer/src/engine/headless/touchpointCoordinator.ts +++ b/common/web/gesture-recognizer/src/engine/headless/touchpointCoordinator.ts @@ -223,7 +223,21 @@ export class TouchpointCoordinator extends Even const selector = this.currentSelector; touchpoint.setGestureMatchInspector(this.buildGestureMatchInspector(selector)); - this.emit('inputstart', touchpoint); + + // If there's an error in code receiving this event, we must not let that break the flow of + // event input processing here! + try { + this.emit('inputstart', touchpoint); + } catch (err) { + console.error(err); + } + + // In particular, things will break HORRIBLY if this code block does not get to run. + this.inputEngines.forEach((engine) => { + // It is now safe to signal further updates for this touchpoint, as we can be sure + // that each will be received. + engine.unlockTouchpoint?.(touchpoint); + }); const selection = await selectionPromise; diff --git a/common/web/gesture-recognizer/src/engine/inputEventEngine.ts b/common/web/gesture-recognizer/src/engine/inputEventEngine.ts index 8f434b71df..f3cb583e6f 100644 --- a/common/web/gesture-recognizer/src/engine/inputEventEngine.ts +++ b/common/web/gesture-recognizer/src/engine/inputEventEngine.ts @@ -48,6 +48,7 @@ export abstract class InputEventEngine extends Inpu }); this.emit('pointstart', touchpoint); + return touchpoint; } protected onInputMove(identifier: number, sample: InputSample, target: EventTarget) { diff --git a/common/web/gesture-recognizer/src/engine/touchEventEngine.ts b/common/web/gesture-recognizer/src/engine/touchEventEngine.ts index a2fc2b6b2c..1c387834ef 100644 --- a/common/web/gesture-recognizer/src/engine/touchEventEngine.ts +++ b/common/web/gesture-recognizer/src/engine/touchEventEngine.ts @@ -4,6 +4,9 @@ import { InputSample } from "./headless/inputSample.js"; import { Nonoptional } from "./nonoptional.js"; import { ZoneBoundaryChecker } from "./configuration/zoneBoundaryChecker.js"; import { GestureSource } from "./headless/gestureSource.js"; +import { ManagedPromise } from "@keymanapp/web-utils"; +import { EventSequentializationQueue } from "./eventSequentializationQueue.js"; +import { GesturePath } from "./index.js"; function touchListToArray(list: TouchList) { const arr: Touch[] = []; @@ -19,7 +22,11 @@ export class TouchEventEngine extends InputEv private readonly _touchMove: typeof TouchEventEngine.prototype.onTouchMove; private readonly _touchEnd: typeof TouchEventEngine.prototype.onTouchEnd; + protected readonly sequentializer = new EventSequentializationQueue(); + private safeBoundMaskMap: {[id: number]: number} = {}; + private pendingSourceIdentifiers: Map = new Map(); + private inputStartSignalMap: Map, ManagedPromise> = new Map(); public constructor(config: Nonoptional>) { super(config); @@ -35,17 +42,6 @@ export class TouchEventEngine extends InputEv return this.config.touchEventRoot; } - // public static forPredictiveBanner(banner: SuggestionBanner, handlerRoot: SuggestionManager) { - // const config: GestureRecognizerConfiguration = { - // targetRoot: banner.getDiv(), - // // document.body is the event root b/c we need to track the mouse if it leaves - // // the VisualKeyboard's hierarchy. - // eventRoot: banner.getDiv(), - // }; - - // return new TouchEventEngine(config); - // } - registerEventHandlers() { // The 'passive' property ensures we can prevent MouseEvent followups from TouchEvents. // It is only specified during `addEventListener`, not during `removeEventListener`. @@ -86,6 +82,19 @@ export class TouchEventEngine extends InputEv } } + public unlockTouchpoint? = (touchpoint: GestureSource>) => { + const lock = this.inputStartSignalMap.get(touchpoint); + if(lock) { + lock.resolve(); + this.inputStartSignalMap.delete(touchpoint); + } + }; + + public hasActiveTouchpoint(identifier: number): boolean { + const baseResult = super.hasActiveTouchpoint(identifier); + return baseResult || !!this.pendingSourceIdentifiers.has(identifier); + } + private buildSampleFromTouch(touch: Touch, timestamp: number) { // WILL be null for newly-starting `GestureSource`s / contact points. const source = this.getTouchpointWithId(touch.identifier); @@ -106,89 +115,129 @@ export class TouchEventEngine extends InputEv // during a touchstart.) const allTouches = touchListToArray(event.touches); const newTouches = touchListToArray(event.changedTouches); - // Maintain all touches in the `.touches` array that are NOT marked as `.changedTouches` (and therefore, new) - this.maintainTouchpointsWithIds(allTouches - .filter((touch1) => newTouches.findIndex(touch2 => touch1.identifier == touch2.identifier) == -1) - .map((touch) => touch.identifier) - ); - // Ensure the same timestamp is used for all touches being updated. - const timestamp = performance.now(); + this.sequentializer.queueEventFunctor(() => { + // Maintain all touches in the `.touches` array that are NOT marked as `.changedTouches` (and therefore, new) + this.maintainTouchpointsWithIds(allTouches + .filter((touch1) => newTouches.findIndex(touch2 => touch1.identifier == touch2.identifier) == -1) + .map((touch) => touch.identifier) + ); + }); - // During a touch-start, only _new_ touch contact points are listed here; - // we shouldn't signal "input start" for any previously-existing touch points, - // so `.changedTouches` is the best way forward. - for(let i=0; i < event.changedTouches.length; i++) { - const touch = event.changedTouches.item(i); - const sample = this.buildSampleFromTouch(touch, timestamp); + this.sequentializer.queueEventFunctor(() => { + // Ensure the same timestamp is used for all touches being updated. + const timestamp = performance.now(); + let lastValidTouchpoint: GestureSource = null; + let lastValidTouchId: number; + const uniqueObject = {}; - if(!ZoneBoundaryChecker.inputStartOutOfBoundsCheck(sample, this.config)) { - // If we started very close to a safe zone border, remember which one(s). - // This is important for input-sequence cancellation check logic. - this.safeBoundMaskMap[touch.identifier] = ZoneBoundaryChecker.inputStartSafeBoundProximityCheck(sample, this.config); - } else { - // This touchpoint shouldn't be considered; do not signal a touchstart for it. - continue; + // During a touch-start, only _new_ touch contact points are listed here; + // we shouldn't signal "input start" for any previously-existing touch points, + // so `.changedTouches` is the best way forward. + for(let i=0; i < event.changedTouches.length; i++) { + const touch = event.changedTouches.item(i); + const touchId = touch.identifier; + const sample = this.buildSampleFromTouch(touch, timestamp); + + this.pendingSourceIdentifiers.set(touchId, uniqueObject); + + if(!ZoneBoundaryChecker.inputStartOutOfBoundsCheck(sample, this.config)) { + // If we started very close to a safe zone border, remember which one(s). + // This is important for input-sequence cancellation check logic. + this.safeBoundMaskMap[touchId] = ZoneBoundaryChecker.inputStartSafeBoundProximityCheck(sample, this.config); + } else { + // This touchpoint shouldn't be considered; do not signal a touchstart for it. + continue; + } + + lastValidTouchpoint = this.onInputStart(touchId, sample, event.target, true); + lastValidTouchId = touchId; } - this.onInputStart(touch.identifier, sample, event.target, true); - } + if(lastValidTouchpoint) { + // Ensure we only do the cleanup if and when it hasn't already been replaced by new events later. + const cleanup = () => { + if(this.pendingSourceIdentifiers.get(lastValidTouchId) == uniqueObject) { + this.pendingSourceIdentifiers.delete(lastValidTouchId); + } + } + + lastValidTouchpoint.path.on('complete', cleanup); + lastValidTouchpoint.path.on('invalidated', cleanup); + + // This 'lock' should only be released when the last simultaneously-registered touch is published via + // gesture-recognizer event. + let eventSignalPromise = new ManagedPromise(); + this.inputStartSignalMap.set(lastValidTouchpoint, eventSignalPromise); + + return eventSignalPromise.corePromise; + } + }); } onTouchMove(event: TouchEvent) { - let propagationActive = true; - // Ensure the same timestamp is used for all touches being updated. - const timestamp = performance.now(); - - this.maintainTouchpointsWithIds(touchListToArray(event.touches) - .map((touch) => touch.identifier) - ); - - // Do not change to `changedTouches` - we need a sample for all active touches in order - // to facilitate path-update synchronization for multi-touch gestures. - // - // May be worth doing changedTouches _first_ though. - for(let i=0; i < event.touches.length; i++) { + for(let i = 0; i < event.touches.length; i++) { const touch = event.touches.item(i); - - if(!this.hasActiveTouchpoint(touch.identifier)) { - continue; - } - - if(propagationActive) { + if(this.hasActiveTouchpoint(touch.identifier)) { this.preventPropagation(event); - propagationActive = false; - } - - const config = this.getConfigForId(touch.identifier); - - const sample = this.buildSampleFromTouch(touch, timestamp); - - if(!ZoneBoundaryChecker.inputMoveCancellationCheck(sample, config, this.safeBoundMaskMap[touch.identifier])) { - this.onInputMove(touch.identifier, sample, touch.target); - } else { - this.onInputMoveCancel(touch.identifier, sample, touch.target); + break; } } + + this.sequentializer.queueEventFunctor(() => { + this.maintainTouchpointsWithIds(touchListToArray(event.touches) + .map((touch) => touch.identifier) + ); + }); + + this.sequentializer.queueEventFunctor(() => { + // Ensure the same timestamp is used for all touches being updated. + const timestamp = performance.now(); + + // Do not change to `changedTouches` - we need a sample for all active touches in order + // to facilitate path-update synchronization for multi-touch gestures. + // + // May be worth doing changedTouches _first_ though. + for(let i=0; i < event.touches.length; i++) { + const touch = event.touches.item(i); + + if(!this.hasActiveTouchpoint(touch.identifier)) { + continue; + } + + const config = this.getConfigForId(touch.identifier); + const sample = this.buildSampleFromTouch(touch, timestamp); + + if(!ZoneBoundaryChecker.inputMoveCancellationCheck(sample, config, this.safeBoundMaskMap[touch.identifier])) { + this.onInputMove(touch.identifier, sample, touch.target); + } else { + this.onInputMoveCancel(touch.identifier, sample, touch.target); + } + } + }) + } onTouchEnd(event: TouchEvent) { - let propagationActive = true; - - // Only lists touch contact points that have been lifted; touchmove is raised separately if any movement occurred. - for(let i=0; i < event.changedTouches.length; i++) { + for(let i = 0; i < event.changedTouches.length; i++) { const touch = event.changedTouches.item(i); - - if(!this.hasActiveTouchpoint(touch.identifier)) { - continue; - } - - if(propagationActive) { + if(this.hasActiveTouchpoint(touch.identifier)) { this.preventPropagation(event); - propagationActive = false; + break; } - - this.onInputEnd(touch.identifier, event.target); } + + this.sequentializer.queueEventFunctor(() => { + // Only lists touch contact points that have been lifted; touchmove is raised separately if any movement occurred. + for(let i=0; i < event.changedTouches.length; i++) { + const touch = event.changedTouches.item(i); + + if(!this.hasActiveTouchpoint(touch.identifier)) { + continue; + } + + this.onInputEnd(touch.identifier, event.target); + } + }); } } \ No newline at end of file