diff --git a/common/web/gesture-recognizer/src/engine/headless/subsegmentation/README.md b/common/web/gesture-recognizer/src/engine/headless/subsegmentation/README.md new file mode 100644 index 0000000000..12d18554b7 --- /dev/null +++ b/common/web/gesture-recognizer/src/engine/headless/subsegmentation/README.md @@ -0,0 +1,14 @@ +## About this subfolder + +This folder contains code for an advanced touchpath sub-segmentation engine that would facilitate more complex gestures, +such as multi-segment flicks and/or swipe-style input. The decision was made to "ice" it for now in favor of preventing +further delays for the release of the feature. + +To revisit a state when the system was fully connected, the following two commits may provide a useful reference: +- https://github.com/keymanapp/keyman/tree/0c7ac4ff3caf612674602fefb7ff71350b13964a + - The last feature-gestures commit with full subsegmentation integration +- https://github.com/keymanapp/keyman/tree/5c48cc5b9b127ab42a8055b7d960dbc4d39e4df7 + - See #7440 - its description details the basic process for creation of a asynchronous FSM for recognizing & constructing gestures based on subsegments produced by the subsegmentation engine. + - The permalinked commit itself holds the code that said process would connect with. + - This was never fully committed to feature-gestures; #7440 was directly based upon the "last feature-gestures + commit..." mentioned above. diff --git a/common/web/gesture-recognizer/src/engine/headless/subsegmentation/constructingSegment.ts b/common/web/gesture-recognizer/src/engine/headless/subsegmentation/constructingSegment.ts index 343276a826..28aef917a6 100644 --- a/common/web/gesture-recognizer/src/engine/headless/subsegmentation/constructingSegment.ts +++ b/common/web/gesture-recognizer/src/engine/headless/subsegmentation/constructingSegment.ts @@ -1,7 +1,7 @@ import { SegmentClassifier } from "../segmentClassifier.js"; import { CumulativePathStats } from "../cumulativePathStats.js"; import { SegmentationSplit, Subsegmentation } from "./pathSegmenter.js"; -import { SegmentImplementation } from "../segment.js"; +import { SegmentImplementation } from "./segment.js"; import { SubsegmentCompatibilityAnalyzer } from "./subsegmentCompatibilityAnalyzer.js"; /** diff --git a/common/web/gesture-recognizer/src/engine/headless/subsegmentation/pathSegmenter.ts b/common/web/gesture-recognizer/src/engine/headless/subsegmentation/pathSegmenter.ts index 79480d7588..34929ed325 100644 --- a/common/web/gesture-recognizer/src/engine/headless/subsegmentation/pathSegmenter.ts +++ b/common/web/gesture-recognizer/src/engine/headless/subsegmentation/pathSegmenter.ts @@ -1,7 +1,7 @@ import { ConstructingSegment } from "./constructingSegment.js"; import { CumulativePathStats, PathCoordAxis, sigMinus } from "../cumulativePathStats.js"; import { InputSample } from "../inputSample.js"; -import { Segment, SegmentImplementation } from "../segment.js"; +import { Segment, SegmentImplementation } from "./segment.js"; import { SegmentClass, SegmentClassifier, SegmentClassifierConfig } from "../segmentClassifier.js"; // Mostly used here to compare the sum-squared error components of segmented regressions diff --git a/common/web/gesture-recognizer/src/engine/headless/segment.ts b/common/web/gesture-recognizer/src/engine/headless/subsegmentation/segment.ts similarity index 97% rename from common/web/gesture-recognizer/src/engine/headless/segment.ts rename to common/web/gesture-recognizer/src/engine/headless/subsegmentation/segment.ts index 68619cf918..817b05d326 100644 --- a/common/web/gesture-recognizer/src/engine/headless/segment.ts +++ b/common/web/gesture-recognizer/src/engine/headless/subsegmentation/segment.ts @@ -1,6 +1,6 @@ -import { CumulativePathStats } from "./cumulativePathStats.js"; -import { InputSample } from "./inputSample.js"; -import { type SegmentClass } from "./segmentClassifier.js"; +import { CumulativePathStats } from "../cumulativePathStats.js"; +import { InputSample } from "../inputSample.js"; +import { type SegmentClass } from "../segmentClassifier.js"; export interface JSONSegment { type: SegmentClass, diff --git a/common/web/gesture-recognizer/src/engine/headless/trackedPath.ts b/common/web/gesture-recognizer/src/engine/headless/trackedPath.ts index 41bd378dfe..6d731ff5ae 100644 --- a/common/web/gesture-recognizer/src/engine/headless/trackedPath.ts +++ b/common/web/gesture-recognizer/src/engine/headless/trackedPath.ts @@ -1,7 +1,6 @@ import EventEmitter from "eventemitter3"; import { InputSample } from "./inputSample.js"; -import { PathSegmenter } from "./subsegmentation/pathSegmenter.js"; -import { Segment } from "./segment.js"; +import { CumulativePathStats } from "./cumulativePathStats.js"; /** * Documents the expected typing of serialized versions of the `TrackedPoint` class. @@ -9,14 +8,12 @@ import { Segment } from "./segment.js"; export type JSONTrackedPath = { coords: InputSample[]; // ensures type match with public class property. wasCancelled?: boolean; - segments: Segment[]; } interface EventMap { 'step': (sample: InputSample) => void, 'complete': () => void, 'invalidated': () => void - 'segmentation': (segment: Segment) => void } /** @@ -44,25 +41,14 @@ interface EventMap { * - Will precede resolution Promise fulfillment on the `Segment` provided by * the most recently-preceding 'segmentation' event. * - And possibly recognition Promise fulfillment. - * - * `'segmentation'`: a new segmentation boundary has been identified for the - * ongoing touchpath. - * - Provides one parameter - a new `Segment` instance representing the - * still-in-construction part of the touchpath until this event is - * raised again. (That would indicate a new segmentation boundary - * marking the end of the first event's returned `Segment`.) */ export class TrackedPath extends EventEmitter { private samples: InputSample[] = []; - private _segments: Segment[] = []; - - private readonly segmenter: PathSegmenter; private _isComplete: boolean = false; private wasCancelled?: boolean; - // private _segments: Segment[]; - // public get segments(): readonly Segment[] { return this._segments; } + private stats: CumulativePathStats; /** * Initializes an empty path intended for tracking a newly-activated touchpoint. @@ -84,17 +70,7 @@ export class TrackedPath extends EventEmitter { this.wasCancelled = jsonObj.wasCancelled; } - // Keep this as the _final_ statement in the constructor. `PathSegmenter` will - // need a reference to this instance, even if only via closure. - // (Most likely; not yet done.) Kinda awkward, but it's useful for compartmentalization. - // - DO use 'via closure.' That allows us to have the segment passing done via - // `private` method. - const segmentStartClosure = (segment: Segment) => { - this._segments.push(segment); - this.emit('segmentation', segment); - } - - this.segmenter = new PathSegmenter(PathSegmenter.DEFAULT_CONFIG, segmentStartClosure); + this.stats = new CumulativePathStats(); } /** @@ -117,9 +93,8 @@ export class TrackedPath extends EventEmitter { // The tracked path should emit InputSample events before Segment events and // resolution of Segment Promises. this.samples.push(sample); + this.stats = this.stats.extend(sample); this.emit('step', sample); - - this.segmenter.add(sample); } /** @@ -138,8 +113,6 @@ export class TrackedPath extends EventEmitter { this.emit('invalidated'); } - this.segmenter.close(); - // If not cancelling, signal completion after finishing segments. if(!cancel) { this.emit('complete'); @@ -156,38 +129,6 @@ export class TrackedPath extends EventEmitter { return this.samples; } - /** - * Returns the segmented form of the touchpath over its lifetime thus far. It is - * possible for certain parts of the path to go unrepresented if they are detected as - * insignificant. - * - * Also of note: for the common case, the final coordinate of one Segment will usually - * be the initial point of the following Segment. Usually, but not always. This is - * the only overlap that may occur. - * - * Note: segment events and updates will always occur in the following order: - * - * 1. A segment's role is 'recognized' - its classification becomes known. - * 2. The segment is 'resolved' - the segment is marked as completed. - * 3. The next segment is added to this field, with on('segmentation') is raised - * for it. - * - * Note that it is possible for a segment to be 'recognized' and even 'resolved' - * before its event is raised (thus, before it is added here) under some scenarios. - * In particular, 'start' and 'end'-type segments are always 'recognized' and - * 'resolved'. - * - * While the touchpath is active, there is a very high chance that the final - * segment listed will not be resolved. There will also be a distinct chance - * that it is not yet recognized. - * - * The first Segment should always be a 'start', while the final Segment - once - * _all_ input for the ongoing touchpath is complete - will be an 'end'. - */ - public get segments(): readonly Segment[] { - return this._segments; - } - /** * Creates a serialization-friendly version of this instance for use by * `JSON.stringify`. @@ -201,7 +142,6 @@ export class TrackedPath extends EventEmitter { targetY: obj.targetY, t: obj.t }))), - segments: [].concat(this.segments), wasCancelled: this.wasCancelled } diff --git a/common/web/gesture-recognizer/src/engine/index.ts b/common/web/gesture-recognizer/src/engine/index.ts index c7671ece94..913051ddac 100644 --- a/common/web/gesture-recognizer/src/engine/index.ts +++ b/common/web/gesture-recognizer/src/engine/index.ts @@ -10,7 +10,7 @@ export { MouseEventEngine } from "./mouseEventEngine.js"; export { PathSegmenter, Subsegmentation } from "./headless/subsegmentation/pathSegmenter.js"; export { PaddedZoneSource } from './configuration/paddedZoneSource.js'; export { RecognitionZoneSource } from './configuration/recognitionZoneSource.js'; -export { Segment } from "./headless/segment.js"; +export { Segment } from "./headless/subsegmentation/segment.js"; export { SegmentClassifier } from "./headless/segmentClassifier.js"; export { TouchEventEngine } from "./touchEventEngine.js"; export { ViewportZoneSource } from './configuration/viewportZoneSource.js'; diff --git a/common/web/gesture-recognizer/src/test/auto/headless/recordedSegmentations.js b/common/web/gesture-recognizer/src/test/auto/headless/recordedSegmentations.js index 04c68a4b16..7dec8e8492 100644 --- a/common/web/gesture-recognizer/src/test/auto/headless/recordedSegmentations.js +++ b/common/web/gesture-recognizer/src/test/auto/headless/recordedSegmentations.js @@ -19,6 +19,25 @@ const SEGMENT_TEST_JSON_FOLDER = path.resolve(`${scriptFolder}/../../resources/j import { assertSegmentSimilarity } from '../../resources/assertSegmentSimilarity.js'; +/** + * Since we're disconnecting subsegmentation stuff for the first gestures release, our + * previously-recorded subsegmentations aren't retrievable in the same spot as before. + * + * This manually retrieves them from the originally-recorded version in order to preserve + * the unit tests. Makes a few assumptions, but they're valid for the original recordings. + * @param {*} jsonObj + */ +function retrieveSubsegmentations(jsonObj) { + return jsonObj.inputs[0].touchpoints[0].path.segments; +} + +// --------------------------------------- +// NOTE: this suite of tests is for a disconnected subsystem - subsegmentation - designed +// to handle more complicated gestures than supported in our initial release. We had to +// triage it to prevent further delays. +// +// The tests themselves should still be functional. +// --------------------------------------- describe("Segmentation - from recorded sequences", function() { beforeEach(function() { this.fakeClock = sinon.useFakeTimers(); @@ -85,7 +104,7 @@ describe("Segmentation - from recorded sequences", function() { await recognitionTestCapturer.recognizedPromise; // Any post-sequence tests to run. - const originalSegments = testObj.originalSegments; + const originalSegments = retrieveSubsegmentations(jsonObj); const originalSegmentTypeSequence = originalSegments.map((segment) => segment.type); const reproedSegments = spy.getCalls().map((call) => call.args[0]); @@ -151,7 +170,7 @@ describe("Segmentation - from recorded sequences", function() { await recognitionTestCapturer.recognizedPromise; // Any post-sequence tests to run. - const originalSegments = testObj.originalSegments; + const originalSegments = retrieveSubsegmentations(jsonObj); const originalSegmentTypeSequence = originalSegments.map((segment) => segment.type); const reproedSegments = spy.getCalls().map((call) => call.args[0]); @@ -187,7 +206,7 @@ describe("Segmentation - from recorded sequences", function() { await testObj.compositePromise; // Any post-sequence tests to run. - const originalSegments = testObj.originalSegments; + const originalSegments = retrieveSubsegmentations(jsonObj); const originalSegmentTypeSequence = originalSegments.map((segment) => segment.type); const reproedSegments = spy.getCalls().map((call) => call.args[0]); @@ -227,7 +246,7 @@ describe("Segmentation - from recorded sequences", function() { await testObj.compositePromise; // Any post-sequence tests to run. - const originalSegments = testObj.originalSegments; + const originalSegments = retrieveSubsegmentations(jsonObj); const reproedSegments = spy.getCalls().map((call) => call.args[0]); // Because of the sweeping arc motion, we won't assume a perfect match to the segmentation here. @@ -281,7 +300,7 @@ describe("Segmentation - from recorded sequences", function() { const nulls = reproedSegments.filter((segment) => segment.type === null); assert.isEmpty(nulls); - assert.isEmpty(holds.filter((hold) => hold.duration > 200)); // all motions were very quick. + assert.isEmpty(holds.filter((hold) => hold.duration > 300)); // all motions were very quick. // True (intended) motions were 'e' -> 's' -> 'w' -> 'n', but the motions weren't that precise // due to prioritizing speed. diff --git a/common/web/gesture-recognizer/src/test/auto/headless/trackedPath.js b/common/web/gesture-recognizer/src/test/auto/headless/trackedPath.js index 03c7d6000d..6b86ee3ba4 100644 --- a/common/web/gesture-recognizer/src/test/auto/headless/trackedPath.js +++ b/common/web/gesture-recognizer/src/test/auto/headless/trackedPath.js @@ -1,20 +1,10 @@ import { assert } from 'chai' import sinon from 'sinon'; -import fs from 'fs'; -import path from 'path'; -import url from 'url'; import { TrackedPath } from '@keymanapp/gesture-recognizer'; -import { HeadlessRecordingSimulator, timedPromise } from '../../../../build/tools/obj/index.js'; +import { timedPromise } from '../../../../build/tools/obj/index.js'; -// Ensures that the resources are resolved relative to this script, not to the cwd when the test -// runner was launched. -const scriptFolder = path.dirname(url.fileURLToPath(import.meta.url)); -const SEGMENT_TEST_JSON_FOLDER = path.resolve(`${scriptFolder}/../../resources/json/segmentation`); - -import { assertSegmentSimilarity } from '../../resources/assertSegmentSimilarity.js' -; // End of "for the integrated style..." describe("TrackedPath", function() { @@ -22,49 +12,6 @@ describe("TrackedPath", function() { // let testJSONtext = fs.readFileSync('src/test/resources/json/canaryRecording.json'); describe("Single-sample 'sequence'", function() { - // A near-duplicate of the "Segmentation" -> "Single-sample 'sequence'" test. - // Acts as a mild 'integration' test of the segmentation engine + the public-facing - // TrackedPath events it advertises. - it("'segmentation' - expected segment types", function() { - let spy = sinon.fake(); - - const touchpath = new TrackedPath(); - touchpath.on('segmentation', spy); - - const sample = { - targetX: 1, - targetY: 1, - t: 100 - }; - - // Expected sequence: - // 1: on `segmenter.add(sample)`: - // a. 'start' segment - // b. unrecognized (null-type) segment. - // 2: on `segmenter.close()`: - // b. 'end' segment - - touchpath.extend(sample); - try { - assert.isTrue(spy.calledTwice, "'segmentation' event was not raised exactly twice before the .close()."); - } finally { - touchpath.terminate(false); // false = "not cancelled" - } - assert.isTrue(spy.calledThrice, "'segmentation' event was not raised exactly once after the .close()."); - - for(let i=0; i < 3; i++) { - assert.equal(spy.args[i].length, 1, "'segmentation' event was raised with an unexpected number of arguments"); - } - - assert.equal(spy.firstCall.args[0].type, 'start', "First event should provide a 'start'-type segment."); - assert.equal(spy.secondCall.args[0].type, undefined, "Second event's segment should not be classified."); - assert.equal(spy.thirdCall.args[0].type, 'end', "Third event should provide an 'end'-type segment."); - - assert.deepEqual(touchpath.coords, [sample]); - assert.deepEqual(touchpath.segments, spy.getCalls().map((call) => call.args[0]), - "The touchpath's segment array does not match the `Segment`s from raised events."); - }); - it("'step', 'complete' events", function() { const spyEventStep = sinon.fake(); const spyEventComplete = sinon.fake(); @@ -134,13 +81,11 @@ describe("TrackedPath", function() { const spyEventStep = sinon.fake(); const spyEventComplete = sinon.fake(); const spyEventInvalidated = sinon.fake(); - const spyEventSegmentation = sinon.fake(); const touchpath = new TrackedPath(); touchpath.on('step', spyEventStep); touchpath.on('complete', spyEventComplete); touchpath.on('invalidated', spyEventInvalidated); - touchpath.on('segmentation', spyEventSegmentation); const sample = { targetX: 1, @@ -151,28 +96,18 @@ describe("TrackedPath", function() { touchpath.extend(sample); touchpath.terminate(false); // false = "not cancelled" - assert(spyEventStep.firstCall.calledBefore(spyEventSegmentation.firstCall), - "'step' should be raised before 'segmentation'"); - assert(spyEventSegmentation.thirdCall.calledBefore(spyEventComplete.firstCall), - "all 'segmentation' events should be raised before 'complete'"); - - const spy = spyEventSegmentation; assert.deepEqual(touchpath.coords, [sample]); - spy.call - assert.deepEqual(touchpath.segments, spy.getCalls().map((call) => call.args[0])); }); it("event ordering - 'invalidated'", function() { const spyEventStep = sinon.fake(); const spyEventComplete = sinon.fake(); const spyEventInvalidated = sinon.fake(); - const spyEventSegmentation = sinon.fake(); const touchpath = new TrackedPath(); touchpath.on('step', spyEventStep); touchpath.on('complete', spyEventComplete); touchpath.on('invalidated', spyEventInvalidated); - touchpath.on('segmentation', spyEventSegmentation); const sample = { targetX: 1, @@ -183,17 +118,7 @@ describe("TrackedPath", function() { touchpath.extend(sample); touchpath.terminate(true); // false = "not cancelled" - assert(spyEventStep.firstCall.calledBefore(spyEventSegmentation.firstCall), - "'step' should be raised before 'segmentation'"); - assert(spyEventSegmentation.secondCall.calledBefore(spyEventInvalidated.firstCall), - "first 'segmentation' event ('start' segment) not raised before cancellation"); - assert(spyEventSegmentation.thirdCall.calledAfter(spyEventInvalidated.firstCall), - "Cancellation event not raised before cancelled segment completion"); - - // Even though the touchpath was 'cancelled', we should still see the segments that finished processing. - const spy = spyEventSegmentation; assert.deepEqual(touchpath.coords, [sample]); - assert.deepEqual(touchpath.segments, spy.getCalls().map((call) => call.args[0])); }); }); @@ -211,13 +136,11 @@ describe("TrackedPath", function() { const spyEventStep = sinon.fake(); const spyEventComplete = sinon.fake(); const spyEventInvalidated = sinon.fake(); - const spyEventSegmentation = sinon.fake(); const touchpath = new TrackedPath(); touchpath.on('step', spyEventStep); touchpath.on('complete', spyEventComplete); touchpath.on('invalidated', spyEventInvalidated); - touchpath.on('segmentation', spyEventSegmentation); const startSample = { targetX: 1, @@ -247,8 +170,6 @@ describe("TrackedPath", function() { }).then(() => { // The main test assertions. assert.deepEqual(touchpath.coords, [startSample, endSample]); - assert.deepEqual(touchpath.segments, spyEventSegmentation.getCalls().map((call) => call.args[0])); - assert.deepEqual(spyEventSegmentation.getCalls().map((call) => call.args[0].type), ['start', 'hold', 'end']); }); const finalPromise = segmentationEndPromise.catch((reason) => { @@ -278,43 +199,5 @@ describe("TrackedPath", function() { // suddenly restored during investigation can cause some very confusing behavior. this.fakeClock.restore(); }) - - // A near-duplication of the recordedSegments.js version, but integrated with TrackedPath. - it("flick_ne_se.json", async function() { - let testJSONtext = fs.readFileSync(`${SEGMENT_TEST_JSON_FOLDER}/flick_ne_se.json`); - let jsonObj = JSON.parse(testJSONtext); - - // Prepare some of the basic setup. - let spy = sinon.fake(); - const trackedPath = new TrackedPath(); - trackedPath.on('segmentation', spy); - - //(PathSegmenter.DEFAULT_CONFIG, spy); - - const configObj = { - replaySample: (sample) => trackedPath.extend(sample), - endSequence: () => trackedPath.terminate(false) - } - - const testObj = HeadlessRecordingSimulator.prepareTest(jsonObj, configObj); - - await this.fakeClock.runAllAsync(); - await testObj.compositePromise; - - // Any post-sequence tests to run. - const originalSegments = testObj.originalSegments; - const originalSegmentTypeSequence = originalSegments.map((segment) => segment.type); - - const reproedSegments = spy.getCalls().map((call) => call.args[0]); - const reproedSegmentTypeSequence = reproedSegments.map((segment) => segment.type); - - assert.sameOrderedMembers(reproedSegmentTypeSequence, originalSegmentTypeSequence); - - assertSegmentSimilarity(reproedSegments[1], originalSegments[1], 'hold'); // ~820ms - assertSegmentSimilarity(reproedSegments[2], originalSegments[2], 'move'); // 'ne' - assertSegmentSimilarity(reproedSegments[3], originalSegments[3], 'hold'); // ~580ms - assertSegmentSimilarity(reproedSegments[4], originalSegments[4], 'move'); // 'se' - assertSegmentSimilarity(reproedSegments[5], originalSegments[5], 'hold'); // ~300ms - }); }); }); \ No newline at end of file diff --git a/common/web/gesture-recognizer/src/tools/unit-test-resources/src/headlessRecordingSimulator.ts b/common/web/gesture-recognizer/src/tools/unit-test-resources/src/headlessRecordingSimulator.ts index ba80c0680d..2f28330255 100644 --- a/common/web/gesture-recognizer/src/tools/unit-test-resources/src/headlessRecordingSimulator.ts +++ b/common/web/gesture-recognizer/src/tools/unit-test-resources/src/headlessRecordingSimulator.ts @@ -21,6 +21,7 @@ export interface RecordingTestConfig { endSequence: () => void; } +// Note: this class is currently only used by subsegmentation unit tests. export class HeadlessRecordingSimulator { // Designed to test against PathSegmenter and TrackedPath - just implement the config interface appropriately! static prepareTest(recordingObj: RecordedCoordSequenceSet, config: RecordingTestConfig): ProcessedSequenceTest { @@ -30,7 +31,9 @@ export class HeadlessRecordingSimulator { const sourceTrackedPath = recordingObj.inputs[0].touchpoints[0].path; testObj.originalSamples = sourceTrackedPath.coords; - testObj.originalSegments = sourceTrackedPath.segments; + const sampleCount = testObj.originalSamples.length; + // still exists on original recorded sequences that exist before we removed segmentation. + testObj.originalSegments = sourceTrackedPath['segments']; const lastSegment = testObj.originalSegments[testObj.originalSegments.length-1]; // Build promises designed to reproduce the events at the correct times. @@ -40,9 +43,18 @@ export class HeadlessRecordingSimulator { }, sample.t - testObj.originalSamples[0].t); }); + + // Originally we did not record the release-timing of the touch, instead using the timing of + // the last sample for the last subsegmentation's last sample. We've disabled that in order + // to get out a release in a more timely manner, but we'll still use it first if it's available. + // + // If not... we use a more current style. TODO: resolve 'release timing' aspect of input + // sequence recording + playback. + const endTime = lastSegment ? lastSegment.lastCoord.t : sourceTrackedPath.coords[sampleCount-1].t; + testObj.endPromise = timedPromise(() => { config.endSequence(); - }, lastSegment.lastCoord.t - testObj.originalSamples[0].t); + }, endTime - testObj.originalSamples[0].t); // Wrap it all together with a nice little bow. testObj.compositePromise = Promise.all([testObj.endPromise].concat(testObj.samplePromises)).catch((reason) => {