diff --git a/common/web/gesture-recognizer/src/engine/headless/cumulativePathStats.ts b/common/web/gesture-recognizer/src/engine/headless/cumulativePathStats.ts index 8f26dbb796..232382bef5 100644 --- a/common/web/gesture-recognizer/src/engine/headless/cumulativePathStats.ts +++ b/common/web/gesture-recognizer/src/engine/headless/cumulativePathStats.ts @@ -20,168 +20,19 @@ export type PathCoordAxisPair = 'tx' | 'ty' | 'xy'; */ type StatAxis = PathCoordAxis | 'v'; -/** - * Acts as a subtraction operation with a built-in, adaptive "significance" check. - * If the result * 2^30 (~ * 10^9) is still smaller in magnitude than an operand, we - * assume it to be a floating-point error that should have been 0 and act accordingly, - * returning 0. - * - * For reference, (32-bit) floats have 23 bits of significand precision, while (64-bit) - * doubles have 52. Therefore, we'll still be more precise than baseline floats. - */ -export function sigMinus(operand1: number, operand2: number) { - const diff = operand1 - operand2; - const magnitude = Math.max(Math.abs(operand1), Math.abs(operand2)); - - const logDiff = Math.log2(magnitude) - Math.log2(Math.abs(diff)); - // If an operand is 2^30 (or ~10^9) larger than the result of the difference, it's - // nigh-certainly a floating-point error at play. - return logDiff < 30 ? diff : 0; -} - /** * As the name suggests, this class facilitates tracking of cumulative mathematical values, etc - * necessary to perform the statistical operations necessary for path segmentation. + * useful for interpretation of a contact point's path as it relates to gestures. * - * Instances of this class are immutable. + * Instances of this class may be considered immutable externally. + * + * A subclass with properties useful for path segmentation: `RegressiblePathStats`. */ export class CumulativePathStats { - /** - * Provides linear-regression statistics & fitting values based on the underlying `CumulativePathStats` - * object used to generate it. All operations are O(1). - */ - static readonly regression = class RegressionFromSums { - readonly independent: PathCoordAxis; - readonly dependent: PathCoordAxis; - readonly paired: PathCoordAxisPair; - - readonly accumulator: CumulativePathStats; - - /** - * - * @param mainStats The `CumulativePathStats` instance to base all regression data on. - * @param dependentAxis The 'output' axis / dimension; the axis whose behavior should be predicted based on - * existing data of its relationship with the independent axis. - * @param independentAxis The 'input' axis/dimension. - */ - constructor(mainStats: CumulativePathStats, dependentAxis: PathCoordAxis, independentAxis: PathCoordAxis) { - /* c8 ignore next 3 */ - if(dependentAxis == independentAxis) { - throw new Error("Two different axes must be specified for the regression object."); - } - - this.accumulator = mainStats; - - this.dependent = dependentAxis; - this.independent = independentAxis; - - if(dependentAxis < independentAxis) { - this.paired = dependentAxis.concat(independentAxis) as PathCoordAxisPair; - } else { - this.paired = independentAxis.concat(dependentAxis) as PathCoordAxisPair; - } - } - - /** - * The 'slope' of the 'slope-intercept' form of the line that best fits the relationship between - * this regression's selected axes. - */ - get slope(): number { - // The technical definition is the commented-out line, but the denominator component of both - // cancels out - it's 'more efficient' to use the following line as a result. - - // this.accumulator.covariance(this.paired) / this.accumulator.variance(this.independent); - const val = this.accumulator.crossSum(this.paired) / this.accumulator.squaredSum(this.independent); - return val; - } - - /** - * The 'intercept' of the 'slope-intercept' form of the line that best fits the relationship between - * this regression's selected axes. - */ - get intercept(): number { - // Performing a regression based on these pre-summed values means that our obtained intercept is in - // the mapped coordinate system. - const mappedIntercept = this.accumulator.mappedMean(this.dependent) - this.slope * this.accumulator.mappedMean(this.independent); - - const val = mappedIntercept + this.accumulator.mappingConstant(this.dependent) - - this.slope * this.accumulator.mappingConstant(this.independent); - - return val; - } - - /** - * The total summed squared-distances of the best fitting line from actually-observed values; - * in other words, the "sum of the squared errors". - * - * Statistically, this is the portion of the dependent variable's variance (un-normalized) - * that is unexplained by this regression. - */ - get sumOfSquaredError(): number { - return sigMinus(this.accumulator.squaredSum(this.dependent), this.sumOfSquaredModeled); - } - - /** - * The portion of the dependent variable's variance that is successfully explained by this regression. - */ - get sumOfSquaredModeled(): number { - // If we have a perfectly straight vertical line, from the perspective of our independent axis, - // we get infinite slope. That's... not great for the math. - // - // Fortunately, it ALSO means that we can perfectly model the segment. - if(this.accumulator.squaredSum(this.independent) == 0) { - return this.accumulator.squaredSum(this.dependent); - } else { - return this.slope * this.accumulator.crossSum(this.paired); - } - } - - /** - * A statistical term that signals how successful the regression is. Always has values on - * the interval [0, 1], with 1 being a perfect fit. - */ - get coefficientOfDetermination(): number { - if(this.accumulator.squaredSum(this.dependent) == 0 || this.accumulator.squaredSum(this.independent) == 0) { - return 1; - } - - const acc = this.accumulator; - const num = acc.crossSum(this.paired) * acc.crossSum(this.paired); - const denom = acc.squaredSum(this.dependent) * acc.squaredSum(this.independent); - - return num / denom; - } - - /** - * Gets the value of the dependent axis that lies on the regression's fitted line - * for a specified independent axis value. - * - * @param value The input value to use for the independent axis's variable. - * @returns The predicted dependent axis value. - */ - predictFromValue(value: number) { - return this.slope * value + this.intercept; - } - } - - private rawLinearSums = {'x': 0, 'y': 0, 't': 0, 'v': 0}; - private rawSquaredSums = {'x': 0, 'y': 0, 't': 0, 'v': 0}; - // Would 'tv' (time vs velocity) be worth it to track? And possibly even do a regression for? - // If so, maybe throw that in. - private rawCrossSums = {'tx': 0, 'ty': 0, 'xy': 0}; + protected rawLinearSums = {'x': 0, 'y': 0, 't': 0, 'v': 0}; + // Handles raw-distance stuff. private coordArcSum: number = 0; - private arcSampleCount: number = 0; - - // These two are kept separate because of their extreme interconnectedness - after all, - // they actually represent a SINGLE (polar) axis - the angle. - // - // Sadly, there's no straightforward, well-founded way to use these to give a proper - // statistical sense of 'fit' or 'regression' here, _especially_ in regard to segmentation. - // Proper angle-[other] cross-sums are pretty much impossible, at least as efficiently - // as the others are handled [O(1)]. - private cosLinearSum: number = 0; - private sinLinearSum: number = 0; /** * The base sample used to transpose all other received samples. Use of this helps @@ -190,7 +41,7 @@ export class CumulativePathStats { * * Refer to https://en.wikipedia.org/wiki/Catastrophic_cancellation. */ - private baseSample?: InputSample; + protected baseSample?: InputSample; /** * The initial sample included by this instance's computed stats. Needed for @@ -199,12 +50,13 @@ export class CumulativePathStats { private _initialSample?: InputSample; private _lastSample?: InputSample; - private followingSample?: InputSample; + protected followingSample?: InputSample; private _sampleCount = 0; constructor(); constructor(sample: InputSample); constructor(instance: CumulativePathStats); + constructor(obj?: InputSample | CumulativePathStats) constructor(obj?: InputSample | CumulativePathStats) { if(!obj) { return; @@ -215,8 +67,6 @@ export class CumulativePathStats { Object.assign(this, obj); this.rawLinearSums = {...obj.rawLinearSums}; - this.rawCrossSums = {...obj.rawCrossSums}; - this.rawSquaredSums = {...obj.rawSquaredSums}; } else if(isAnInputSample(obj)) { Object.assign(this, this.extend(obj)); /* c8 ignore next 3 */ @@ -233,33 +83,30 @@ export class CumulativePathStats { * newly-sampled point. */ public extend(sample: InputSample): CumulativePathStats { - if(!this._initialSample) { - this._initialSample = sample; - this.baseSample = sample; + return this._extend(new CumulativePathStats(this), sample); + } + + protected _extend(result: CumulativePathStats, sample: InputSample) { + if(!result._initialSample) { + result._initialSample = sample; + result.baseSample = sample; } - const result = new CumulativePathStats(this); + + const baseSample = result.baseSample; // Set _after_ deep-copying this for the result. this.followingSample = sample; // Helps prevent "catastrophic cancellation" issues from floating-point computation // for these statistical properties and properties based upon them. - const x = sample.targetX - this.baseSample.targetX; - const y = sample.targetY - this.baseSample.targetY; - const t = sample.t - this.baseSample.t; + const x = sample.targetX - baseSample.targetX; + const y = sample.targetY - baseSample.targetY; + const t = sample.t - baseSample.t; result.rawLinearSums.x += x; result.rawLinearSums.y += y; result.rawLinearSums.t += t; - result.rawCrossSums.tx += t * x; - result.rawCrossSums.ty += t * y; - result.rawCrossSums.xy += x * y; - - result.rawSquaredSums.x += x * x; - result.rawSquaredSums.y += y * y; - result.rawSquaredSums.t += t * t; - if(this.lastSample) { // arc length stuff! const xDelta = sample.targetX - this.lastSample.targetX; @@ -271,20 +118,8 @@ export class CumulativePathStats { result.coordArcSum += coordArcDelta; - if(xDelta || yDelta) { - // We wish to measure angle clockwise from <0, -1> in the DOM. So, cos values should - // align with that axis, while sin values should align with the positive x-axis. - // - // This provides a mathematical 'transformation' to the axes used by `atan2` in the - // `angleMean` property. - result.cosLinearSum += -yDelta / coordArcDelta; - result.sinLinearSum += xDelta / coordArcDelta; - result.arcSampleCount += 1; - } - if(tDelta) { result.rawLinearSums.v += coordArcDelta / tDelta; - result.rawSquaredSums.v += coordArcDeltaSq / (tDelta * tDelta); } } @@ -302,6 +137,12 @@ export class CumulativePathStats { * @returns */ public deaccumulate(subsetStats?: CumulativePathStats): CumulativePathStats { + const result = new CumulativePathStats(this); + + return this._deaccumulate(result, subsetStats); + } + + public _deaccumulate(result: CumulativePathStats, subsetStats?: CumulativePathStats): CumulativePathStats { // Possible addition: use `this.buildRenormalized` on the returned version // if catastrophic cancellation effects (random, small floating point errors) // are not sufficiently mitigated & handled by the measures currently in place. @@ -316,8 +157,6 @@ export class CumulativePathStats { // the cumulative stats that follow. If any are still active, we can't // recentralize. - const result = new CumulativePathStats(this); - // We actually WILL accept a `null` argument; makes some of the segmentation // logic simpler. if(!subsetStats) { @@ -336,16 +175,6 @@ export class CumulativePathStats { result.rawLinearSums[d] -= subsetStats.rawLinearSums[d]; } - for(let dimPair in result.rawCrossSums) { - const d = dimPair as PathCoordAxisPair; - result.rawCrossSums[d] -= subsetStats.rawCrossSums[d]; - } - - for(let dim in result.rawSquaredSums) { - const d = dim as PathCoordAxis; - result.rawSquaredSums[d] -= subsetStats.rawSquaredSums[d]; - } - // arc length stuff! if(subsetStats.followingSample && subsetStats.lastSample) { const xDelta = subsetStats.followingSample.targetX - subsetStats.lastSample.targetX; @@ -361,13 +190,8 @@ export class CumulativePathStats { result.coordArcSum -= coordArcDelta; result.coordArcSum -= subsetStats.coordArcSum; - result.cosLinearSum -= subsetStats.cosLinearSum; - result.sinLinearSum -= subsetStats.sinLinearSum; - result.arcSampleCount -= (subsetStats.arcSampleCount + 1); - if(tDelta) { result.rawLinearSums.v -= coordArcDelta / tDelta; - result.rawSquaredSums.v -= coordArcDeltaSq / (tDelta * tDelta); } } @@ -412,7 +236,7 @@ export class CumulativePathStats { * @param dim * @returns */ - private mappingConstant(dim: StatAxis) { + protected mappingConstant(dim: StatAxis) { if(!this.baseSample) { return undefined; } @@ -428,16 +252,6 @@ export class CumulativePathStats { } } - /** - * Gets the statistical mean, utilizing the internal 'mapped' coordinate space. - * This is the version compatible with cross-sums and squared-sums. - * @param dim - * @returns - */ - private mappedMean(dim: StatAxis) { - return this.rawLinearSums[dim] / this.sampleCount; - } - /** * Gets the statistical mean value of the samples observed during the represented * interval on the specified axis. @@ -447,112 +261,7 @@ export class CumulativePathStats { public mean(dim: StatAxis) { // This external-facing version needs to provide values in 'external'-friendly // coordinate space. - return this.mappedMean(dim) + this.mappingConstant(dim); - } - - /** - * Gets the sum of the squared distance from the mean seen in samples observed - * during the represented interval on the specified axis. - * @param dim - * @returns - */ - public squaredSum(dim: StatAxis) { - const x2 = this.rawSquaredSums[dim]; - const x1 = this.rawLinearSums[dim]; - - return sigMinus(x2, x1 * x1 / this.sampleCount); - } - - /** - * Gets the sum of the statistical 'cross' term "distance" away from the mean - * observed during the represented interval on the specified axis. - * @param dimPair - * @returns - */ - public crossSum(dimPair: PathCoordAxisPair) { - const dim1 = dimPair.charAt(0) as PathCoordAxis; - const dim2 = dimPair.charAt(1) as PathCoordAxis; - - let orderedDims = dimPair as PathCoordAxisPair; - - const ab = this.rawCrossSums[orderedDims]; - const a = this.rawLinearSums[dim1]; - const b = this.rawLinearSums[dim2]; - - return sigMinus(ab, a * b / this.sampleCount); - } - - /** - * Gets the unbiased covariance between the specified pair of axes for samples - * observed during the represented interval. - * @param dimPair - * @returns - */ - public covariance(dimPair: PathCoordAxisPair) { - return this.crossSum(dimPair) / (this.sampleCount - 1); - } - - /** - * Gets the unbiased variance on the specified axis for samples observed - * during the represented interval. - */ - public variance(dim: StatAxis) { - return this.squaredSum(dim) / (this.sampleCount - 1); - } - - /** - * Utilizing (and possibly abusing) statistical identities, this function produces - * an equivalent, but-recentered copy of this instance's statistical accumulations - * that will be less prone to catastrophic cancellation. - * - * In non-stats speak, the new instance will suffer smaller floating-point - * errors than the old instance whenever they do occur. - * @returns - */ - public buildRenormalized(): CumulativePathStats { - // Other (internal) notes: the internal mapping of the new instance will not - // match that of the old instance. This should not affect the practical - // results of any mapping to and from the external coordinate space, however. - let result = new CumulativePathStats(this); - - let newBase: InputSample = { - targetX: this.mappedMean('x') + this.baseSample.targetX, - targetY: this.mappedMean('y') + this.baseSample.targetY, - t: this.mappedMean('t') + this.baseSample.t - }; - - result.baseSample = newBase; - - for(const dimPair in result.rawCrossSums) { - const d = dimPair as PathCoordAxisPair; - result.rawCrossSums[d] = this.crossSum(d); - } - - for(const dim in result.rawSquaredSums) { - // 'v' does not need renormalization. - if(dim == 'v') { - break; - } - - // The identity we're using to renormalize rawCrossSums and rawSquaredSums - // automatically guarantees a mean of 0 after the renormalization. - const d = dim as PathCoordAxis; - result.rawLinearSums[d] = 0; - result.rawSquaredSums[d] = this.squaredSum(d); - } - - return result; - } - - /** - * Provides a linear-regression perspective on two specified axes over the represented - * interval. - * @param dependent - * @param independent - * @returns - */ - public fitRegression(dependent: PathCoordAxis, independent: PathCoordAxis) { - return new CumulativePathStats.regression(this, dependent, independent); + return this.rawLinearSums[dim] / this.sampleCount + this.mappingConstant(dim); } /** @@ -645,77 +354,6 @@ export class CumulativePathStats { return this.duration ? this.netDistance / this.duration : 0; } - /** - * Returns the represented interval's 'mean angle' clockwise from the DOM's - * <0, -1> (the unit vector toward the top of the screen) in radians. - * - * Based upon the 'circular mean'. Refer to https://en.wikipedia.org/wiki/Circular_mean. - * - * Note that very slow-moving segments may be heavily affected by pixel aliasing - * effects; mouse and touch events usually do not provide sub-pixel resolution. - */ - public get angleMean() { - if(this.arcSampleCount == 0) { - return Number.NaN; - } - - // Neato reference: https://rosettacode.org/wiki/Averages/Mean_angle - // But we don't actually need to divide by sample count; `atan2` handles that! - const sinMean = this.sinLinearSum; - const cosMean = this.cosLinearSum; - - let angle = Math.atan2(sinMean, cosMean); // result: on the interval (-pi, pi] - // Convert to [0, 2*pi). - if(angle < 0) { - angle = angle + 2 * Math.PI; - } - - return angle; - } - - /** - * Provides the rSquared value needed internally for circular-statistic properties. - * - * Range: floating-point values on the interval [0, 1]. - */ - private get angleRSquared() { - if(this.arcSampleCount == 0) { - return 1; - } - - // Refer to https://en.wikipedia.org/wiki/Directional_statistics#Distribution_of_the_mean. - // We're computing the squared value of that page's R-bar stat. - // - // Now, why it's called that? ... good question. My best guess is that it's meant to - // correspond to linear regression's 'r' stat, which when squared serves as the - // coefficient of determination for the regression. Intuitively, that does seem to - // match what this represents - though for normal regressions, the c.o.d isn't normally - // used to compute deviation or variance! - const rSquaredBase = this.cosLinearSum * this.cosLinearSum + this.sinLinearSum * this.sinLinearSum; - return rSquaredBase / (this.arcSampleCount * this.arcSampleCount); - } - - /** - * The **circular standard deviation** of the represented interval's angle observations. - * - * Refer to https://en.wikipedia.org/wiki/Directional_statistics#Standard_deviation. - * - * Note that very slow-moving segments may be heavily affected by pixel aliasing - * effects; mouse and touch events usually do not provide sub-pixel resolution. - * This can result in very high deviation values. - * - * In less-technical terms - the "stair-stepping" effect seen on high zoom levels means we don't - * get perfectly straight lines, and that can cause this value to be unexpectedly high. - */ - public get angleDeviation() { - if(this.arcSampleCount == 0) { - return Number.NaN; - } - - const val = Math.sqrt(-Math.log(this.angleRSquared)); - return isNaN(val) ? 0 : val; - } - /** * Provides the actual, pixel-based distance actually traveled by the represented segment. * May not be an integer (because diagonals are a thing). @@ -737,10 +375,7 @@ export class CumulativePathStats { netDistance: this.netDistance, duration: this.duration, sampleCount: this.sampleCount, - angleMeanDegrees: this.angleMean * 180 / Math.PI, - angleDeviation: this.angleDeviation, - rawDistance: this.rawDistance, - speedVariance: this.variance('v') + rawDistance: this.rawDistance } } /* c8 ignore end */ diff --git a/common/web/gesture-recognizer/src/engine/headless/subsegmentation/README.md b/common/web/gesture-recognizer/src/engine/headless/subsegmentation/README.md deleted file mode 100644 index 12d18554b7..0000000000 --- a/common/web/gesture-recognizer/src/engine/headless/subsegmentation/README.md +++ /dev/null @@ -1,14 +0,0 @@ -## About this subfolder - -This folder contains code for an advanced touchpath sub-segmentation engine that would facilitate more complex gestures, -such as multi-segment flicks and/or swipe-style input. The decision was made to "ice" it for now in favor of preventing -further delays for the release of the feature. - -To revisit a state when the system was fully connected, the following two commits may provide a useful reference: -- https://github.com/keymanapp/keyman/tree/0c7ac4ff3caf612674602fefb7ff71350b13964a - - The last feature-gestures commit with full subsegmentation integration -- https://github.com/keymanapp/keyman/tree/5c48cc5b9b127ab42a8055b7d960dbc4d39e4df7 - - See #7440 - its description details the basic process for creation of a asynchronous FSM for recognizing & constructing gestures based on subsegments produced by the subsegmentation engine. - - The permalinked commit itself holds the code that said process would connect with. - - This was never fully committed to feature-gestures; #7440 was directly based upon the "last feature-gestures - commit..." mentioned above. diff --git a/common/web/gesture-recognizer/src/engine/headless/subsegmentation/constructingSegment.ts b/common/web/gesture-recognizer/src/engine/headless/subsegmentation/constructingSegment.ts deleted file mode 100644 index b31e403a2d..0000000000 --- a/common/web/gesture-recognizer/src/engine/headless/subsegmentation/constructingSegment.ts +++ /dev/null @@ -1,309 +0,0 @@ -import { SegmentClassifier } from "../segmentClassifier.js"; -import { CumulativePathStats } from "../cumulativePathStats.js"; -import { SegmentationSplit, Subsegmentation } from "./pathSegmenter.js"; -import { SegmentImplementation } from "./segment.js"; -import { SubsegmentCompatibilityAnalyzer } from "./subsegmentCompatibilityAnalyzer.js"; - -/** - * This class is responsible for managing the construction of public-facing Segments while keeping - * all the internal parts... internal. As such, it includes state management for parts of - * PathSegmenter's operations. Note that it makes many assumptions based upon its usage within - * PathSegmenter. - * - * See also `SubsegmentCompatibilityAnalyzer`, which defines the criteria used within this class - * for determining when to recombine subsegments and when to uphold segmentation decisions. - */ -export class ConstructingSegment { - readonly classifier: SegmentClassifier; - - /** - * Marks previously-accumulated stats on the touchpath for the portion that precedes - * the in-construction Segment. - */ - readonly baseAccumulation: CumulativePathStats; - - /** - * Notes all subsegments identified as part of the overall segment being constructed. - */ - private subsegmentations: Subsegmentation[] = []; - - /** - * Marks a potential follow-up subsegment. This portion is not automatically - * committed during finalization. - */ - private pendingSubsegmentation: Subsegmentation = null; - - /** - * A flag that is only set once the in-construction segment is first recognized as a 'wait'. - * If set, this indicates that the current - and _only_ the current - `pendingSubsegmentation` - * may only be replaced by an update that would still be compatible if the field's current - * value were already committed. (Goal: prevent the 'locked' pending section from becoming - * not-committed in to the 'wait' role that triggered the 'lock'.) - */ - private _pendingLocked: boolean = false; - - private _pathSegment: SegmentImplementation; - - constructor(initialPendingSubsegment: Subsegmentation, classifier: SegmentClassifier) { - // Note: may be null! Occurs for the first processed subsegment. - this.baseAccumulation = initialPendingSubsegment.baseAccumulation; - this.classifier = classifier; - - this.subsegmentations = []; - this.updatePendingSubsegment(initialPendingSubsegment); - } - - /** - * Constructs a stats object for the interval ending with the specified accumulation - * state and beginning at the start of the in-construction Segment. - * @param accumulation - * @returns - */ - private buildIntervalFromBase(accumulation: CumulativePathStats) { - return accumulation.deaccumulate(this.baseAccumulation); - } - - /** - * Returns the stats accumulation covering all currently-committed subsegments. - * May be `null` if there are no committed subsegments. - */ - public get committedInterval(): CumulativePathStats { - return this.committedIntervalAsSubsegmentation.stats; - } - - /** - * Returns the number of already-committed subsegments that comprise the - * in-construction Segment thus far. - */ - public get subsegmentCount(): number { - return this.subsegmentations.length; - } - - /** - * Returns a merged form of all currently-committed subsegments in - * Subsegmentation form. - */ - public get committedIntervalAsSubsegmentation(): Subsegmentation { - if(this.subsegmentations.length == 0) { - return { - stats: null, - endingAccumulation: null, - baseAccumulation: this.baseAccumulation - }; - } else { - const tailSub = this.subsegmentations[this.subsegmentations.length - 1]; - const tail = tailSub.endingAccumulation; - - return { - stats: this.buildIntervalFromBase(tail), - endingAccumulation: tail, - baseAccumulation: this.baseAccumulation - }; - } - } - - /** - * Checks if the committed portion of the constructing segment are "compatible" with - * a potential following subsegment. Appending said subsegment may not change the - * 'type' of segment being constructed or certain properties of it. - * - * @returns `true` if compatible, `false` if not. - */ - public isCompatible(subsegmentation: Subsegmentation): boolean { - const pendingAnalyzer = this.analyzeCompatibility(subsegmentation); - return pendingAnalyzer.isCompatible; - } - - /** - * Returns the analyzer instance used for the `isCompatible` check. The returned - * object provides additional helper properties useful for inspecting compatibility - * logic. - */ - public analyzeCompatibility(subsegmentation: Subsegmentation): SubsegmentCompatibilityAnalyzer { - const committed = this.committedIntervalAsSubsegmentation; - const segmentationSplit = new SegmentationSplit(committed, subsegmentation); - - return new SubsegmentCompatibilityAnalyzer(segmentationSplit, this.classifier); - } - - /** - * Returns the stats accumulation covering all currently-committed subsegments + - * the submitted `Subsegmentation` object (as if it were committed). - * - * Assumes that its parameter immediately follows any previously-committed - * subsegments. - */ - private wholeSegmentation(subsegmentation: Subsegmentation): Subsegmentation { - const whole: Subsegmentation = { - stats: subsegmentation.endingAccumulation.deaccumulate(this.baseAccumulation), - baseAccumulation: this.baseAccumulation, - endingAccumulation: subsegmentation.endingAccumulation - }; - - return whole; - } - - /** - * A subsegment is pre-committed when it is required to successfully classify the in-construction - * segment after reaching the configured time threshold for segment recognition - that is, - * when previously-committed subsegments are insufficient to support the classification of a - * 'recognized' Segment. - * - * In such a case, the precommitted portion must be maintained. In the event that extending the - * subsegment (on future updates) would render it incompatible, the subsegment must be forceably - * segmented in order to commit the precommitted portion. - */ - public get hasPrecommittedSubsegment() { - return this._pendingLocked; - } - - private set hasPrecommittedSubsegment(flag: boolean) { - this._pendingLocked = flag; - } - - public get hasPendingSubsegment() { - return !!this.pendingSubsegmentation; - } - - /** - * Call this to denote the current state of an in-construction subsegment that **will** be included - * in the final Segment once completed if the pending subsegment does not diverge in a later update. - * - * If the pending subsegment appears to belong to the current Segment, it contains valuable info - * about the state of the as-of-yet unresolved Segment, including the most recent location - * of the corresponding touchpoint. - * - * NOTE: Assumes that .isCompatible has been checked first! This method (currently) does not - * perform the related check! - * - It is currently called when and where relevant in PathSegmenter, the only thing currently - * calling this method. - * - * @param subsegmentation - * @returns `true` unless the subsegment - */ - public updatePendingSubsegment(subsegmentation: Subsegmentation): boolean { - const isFirstUpdate = !this.pendingSubsegmentation && this.subsegmentations.length == 0; - const fullStatsWithIncoming = this.wholeSegmentation(subsegmentation).stats; - - // Do we have a locked pending section? If so, check to be sure that an update won't break things. - if(this.hasPrecommittedSubsegment) { - const classification = this.classifier.classifySegment(fullStatsWithIncoming); - - // If the classification would change after updating the pending subsegment, block the update & - // report update failure. - if(this.pathSegment.type != classification) { - return false; - } - } - - this.pendingSubsegmentation = subsegmentation; - - if(isFirstUpdate) { - this.pathSegment = new SegmentImplementation(); - } - - this.pathSegment.updateStats(fullStatsWithIncoming); - - // Check the length of time that's elapsed. If we've surpassed the recognition threshold, - // it's time to commit to classifying the in-construction Segment. - const alreadyElapsed = fullStatsWithIncoming.duration; - const recognitionWaitTime = this.classifier.config.holdMinimumDuration - alreadyElapsed; - const recognitionFromMove = (this.pathSegment.distance > this.classifier.config.holdMoveTolerance); - - // `undefined` if and only if still unrecognized. - if((recognitionFromMove || recognitionWaitTime <= 0) && !this._pathSegment.isRecognized) { - const classification = this.classifier.classifySegment(fullStatsWithIncoming); - - // Based on the specification for segment classification, there WILL be a classification - // assigned here. It's an implementation error if not. - if(!classification) { - throw new Error("Implementation error - segment was not properly recognized"); - } - - // auto-resolve the recognition promise & 'lock' the classification (and also the portion - // of the touchpath that triggered it). - this.pathSegment.classifyType(classification); - this.hasPrecommittedSubsegment = true; - } - - return true; - } - - /** - * Call to clear a previously-tracked in-construction subsegment that has diverged and is no - * longer compatible. - */ - public clearPendingSubsegment() { - if(this.hasPrecommittedSubsegment) { - throw new Error("Invalid state: must fully commit a subsection due to a precommitted portion!"); - } - - // Probably does not need to trigger an event; if called, there's a new, follow-up segment - // coming that will maintain the current coordinate with its events instead. Assuming - // the need to rely on Segment events for that; even that's better off handled with path.coords - // events instead. - this.pendingSubsegmentation = null; - - // If there was a pending subsegment, we need to remove its components from the published segment stats. - this.pathSegment.updateStats(this.committedInterval); - } - - /** - * Commits a currently-pending subsegment as part of the in-construction Segment. - */ - public commitPendingSubsegment() { - if(this.pendingSubsegmentation) { - this.subsegmentations.push(this.pendingSubsegmentation); - - // Check the peak speed & update if needed. - const commitStats = this.pendingSubsegmentation.stats; - let peakSpeed = commitStats.mean('v') + Math.sqrt(commitStats.variance('v')); - if(peakSpeed > this.pathSegment.peakSpeed) { - this.pathSegment.setPeakSpeed(peakSpeed); - } - - // Clear 'pending'-related fields. - this.pendingSubsegmentation = null; - this.hasPrecommittedSubsegment = false; - - // Recognition check! - if(!this.pathSegment.isRecognized) { - const classification = this.classifier.classifySegment(this.committedInterval); - if(classification) { - this.pathSegment.classifyType(classification); - } - } - } else { - throw new Error("Illegal state - `commitPendingPortion` should never be called with nothing pending."); - } - } - - /** - * Finalizes the Segment, committing any pending portions, classifying ('recognizing') it if - * necessary and resolving it. - */ - public finalize() { - if(this.pendingSubsegmentation) { - this.commitPendingSubsegment(); - } - - // Fully 'recognize' the Segment if it somehow hasn't yet been recognized. - if(!this.pathSegment.isRecognized) { - this.pathSegment.classifyType(this.classifier.classifySegment(this.committedInterval)); - } - - this.pathSegment.resolve(); - } - - /** - * The in-construction Segment, as published to `GesturePath.segments` & `GesturePath`'s - * 'segmentation' event. - */ - public get pathSegment() { - return this._pathSegment; - } - - private set pathSegment(segment: SegmentImplementation) { - this._pathSegment = segment; - } -} \ No newline at end of file diff --git a/common/web/gesture-recognizer/src/engine/headless/subsegmentation/pathSegmenter.ts b/common/web/gesture-recognizer/src/engine/headless/subsegmentation/pathSegmenter.ts deleted file mode 100644 index accd989b8d..0000000000 --- a/common/web/gesture-recognizer/src/engine/headless/subsegmentation/pathSegmenter.ts +++ /dev/null @@ -1,983 +0,0 @@ -import { ConstructingSegment } from "./constructingSegment.js"; -import { CumulativePathStats, PathCoordAxis, sigMinus } from "../cumulativePathStats.js"; -import { InputSample } from "../inputSample.js"; -import { Segment, SegmentImplementation } from "./segment.js"; -import { SegmentClass, SegmentClassifier, SegmentClassifierConfig } from "../segmentClassifier.js"; - -// Mostly used here to compare the sum-squared error components of segmented regressions -// to the sum-squared "modeled" components of their overall variance. Those are the two -// "independent random variables" we're examining. Variances (which are sums of squared -// values themselves) tend to be chi-squared distributed, fulfilling the conditions. -// That's stats-speak for "what this module does with it has a solid theoretical basis." -// -// We do NOT want to be computing this thing at run-time. (Just take one look at the -// equations found in that Wikipedia article!) Fortunately, it's very common in stats -// circles to just... use a lookup table for finding p-values for things following -// the f-distribution. -// -// For future non-stats-inclined maintainers: -// One could say that the p-value = the chance that what we have just "randomly" happened -// to occur without our expectations actually being met. It's a super-common -// stats term; they'd then say that if the p-value is sufficiently low, then we\ -// "reject the null hypothesis" - the theory that it actually DID happen randomly - in -// favor of the high likelihood that we really are onto something. - -/** - * Acts as a lookup table for null-hypothesis tests utilizing the F-distribution from the - * domain of statistics. Refer to https://en.wikipedia.org/wiki/F-distribution for details - * on its properties. - * - * This is only a partial implementation; only a couple of numerator degrees-of-freedom are - * worth our consideration, and there's only marginal gain to be found supporting more - * denominator degrees-of-freedom than are already supported here. Keeping it small also - * helps with maintenance and readability. - */ -class FDistribution { - - /** - * Indexing: [threshold-index][num-2][denom-1]. - * - * If numerator has only 1 degree of freedom, that means we don't really have - * segments so much as different averages between the two segments - there's - * no slope on either side. In other words, "just a bump". No segmentation - * there. - * - * 2 degrees of freedom: no movement on the axis on only one side. - * - * threshold-index: - * - p-value of .100: 0 - * - p-value of .050: 1 - * - * The f-statistic value must match or exceed its corresponding entry in - * the table below, based on the detected DoF (degrees of freedom) for the - * 'numerator' and 'denominator' components. Exceeding the threshold specified - * for the p = .100 section indicates less than a 10% probability of the statistic - * being tested having arisen by purely random chance. Failure to exceed even - * that implies an unacceptably high chance of being "convenient but meaningless" - - * the "null hypothesis", in stats speak. - */ - private static readonly table = [ - // p = .100 - [ - // numerator: 2 - [ - // denom: 1-10 - 49.50, 9.00, 5.46, 4.32, 3.78, 3.46, 3.26, 3.11, 3.01, 2.92, - // denom: 11-20 - 2.86, 2.81, 2.76, 2.73, 2.70, 2.67, 2.64, 2.62, 2.61, 2.59 - // we COULD continue with threshold-dropping after that, - // but I doubt we'll need it in practicality. - // At 30: 2.49 - // limit -> infinity: 2.30 - ], - // numerator: 3 - [ - // denom: 1-10 - 53.59, 9.16, 5.39, 4.19, 3.62, 3.29, 3.07, 2.92, 2.81, 2.73, - // denom: 11-20 - 2.66, 2.61, 2.56, 2.52, 2.49, 2.46, 2.44, 2.42, 2.40, 2.38 - // Likewise, from above. - // At 30: 2.28 - // limit -> infinity: 2.08 - ] - ], - // p = .050 - [ - // numerator: 2 - [ - // denom: 1-10 - 199.5, 19.0, 9.55, 6.94, 5.79, 5.14, 4.74, 4.46, 4.26, 4.10, - // denom: 10-20 - 3.98, 3.89, 3.81, 3.74, 3.68, 3.63, 3.59, 3.55, 3.52, 3.49 - // At 30: 3.32 - // limit -> infinity: 3.00 - ], - // numerator: 3 - [ - // denom: 1-10 - 215.7,19.16, 9.28, 6.59, 5.41, 4.76, 4.35, 4.07, 3.86, 3.71, - // denom: 10-20 - 3.59, 3.49, 3.41, 3.34, 3.29, 3.24, 3.20, 3.16, 3.13, 3.10 - // At 30: 2.92 - // limit -> infinity: 2.60 - ] - ] - ] - - /** - * Determines a threshold tier for segmentation based on the segmented - * regression f-statistic. - * @param statistic - * @param numDoF - * @param denomDoF - * - * Returns 0.050 if the statistic indicates a p-value of 0.050 or lower. - * Failing that, returns 0.100 if a p-value of 0.100 or lower is indicated. - * Otherwise, returns 1.0 - we have no basis to "reject the null hypothesis". - */ - static thresholdTier(statistic: number, numDoF: number, denomDoF: number) { - // Former case: we'd never segment anyway - // Latter case: it's currently wrong to statistically test. - // The F-distribution is not defined for this case. - if(numDoF < 2 || denomDoF < 1) { - return 1; - } - - const numIndex = (numDoF > 3 ? 3 : numDoF) - 2; - const denomIndex = (denomDoF > 20 ? 20 : denomDoF) - 1; - - const tier2Threshold = FDistribution.table[1][numIndex][denomIndex]; - if(statistic >= tier2Threshold) { - return 0.05; - } - - const tier1Threshold = FDistribution.table[0][numIndex][denomIndex]; - // If arising purely randomly isn't at least less than 10% likely, - // we'll categorically say it happened randomly with full certainty (1). - // Obviously not actually true, but it works for our thresholding. - return statistic >= tier1Threshold ? 0.10 : 1; - } -} - -/** - * Represents a "subsegmentation" - one part of a phase 1 segmentation split. - */ -export interface Subsegmentation { - /** - * The stats for observations comprising the subsegment. - */ - stats: CumulativePathStats; - - /** - * The cumulative stats up to the subsegment's endpoint, including - * accumulated components that precede the subsegment entirely. - */ - endingAccumulation: CumulativePathStats; - - /** - * The cumulative stats up to the point before the subsegment's start - * point, not including any accumulation for components within the subsegment. - */ - baseAccumulation: CumulativePathStats; -} - -/** - * Represents the result of a segmentation of the search path and the related - * statistical accumulations needed to properly test the segmentation's - * validity. - */ -export class SegmentationSplit { - public static readonly SPLIT_CRITERION_THRESHOLD = 1.5; - - /** - * The properties of the "left-hand" / earlier half of the time interval - * being segmented. - */ - readonly pre: Subsegmentation; - - /** - * The properties of the "right-hand" / later half of the time interval - * being segmented. - */ - readonly post: Subsegmentation; - - /** - * The properties of the full interval being segmented, before the split. - */ - readonly union: CumulativePathStats; - - /** - * Provides segmented-regression statistics & fitting values on the two specified axes / - * dimensions based on the subsegments and their corresponding linear-regression - * statistics. All operations are O(1). - */ - static readonly segmentationComparison = class SegmentedRegression { - host: SegmentationSplit; - readonly independent: PathCoordAxis; - readonly dependent: PathCoordAxis; - readonly paired: 'tx' | 'ty' | 'xy'; - - /** - * The linear regression for the underlying `pre` (earlier) subsegment. - */ - pre: typeof CumulativePathStats.regression.prototype; - - /** - * The linear regression for the underlying `post` (later) subsegment. - */ - post: typeof CumulativePathStats.regression.prototype; - - /** - * The linear, unsegmented regression for the underlying `union` (combined) subsegment. - */ - union: typeof CumulativePathStats.regression.prototype; - - constructor(host: SegmentationSplit, dependentAxis: PathCoordAxis, independentAxis: PathCoordAxis) { - if(dependentAxis == independentAxis) { - throw new Error("Two different axes must be specified for the regression object."); - } - - this.host = host; - - this.dependent = dependentAxis; - this.independent = independentAxis; - - if(dependentAxis < independentAxis) { - this.paired = dependentAxis.concat(independentAxis) as 'tx' | 'ty' | 'xy'; - } else { - this.paired = independentAxis.concat(dependentAxis) as 'tx' | 'ty' | 'xy'; - } - - this.pre = this.host.pre .stats.fitRegression(dependentAxis, independentAxis); - this.post = this.host.post .stats.fitRegression(dependentAxis, independentAxis); - this.union = this.host.union.fitRegression(dependentAxis, independentAxis); - } - - /** - * The total summed squared-distances of the best fitting line from actually-observed values - * for their corresponding subsegment; in other words, the "sum of the squared errors" when - * utilizing segmented regression. - * - * Statistically, this is the portion of the dependent variable's variance (un-normalized) - * that is unexplained even after dividing the interval into separate subsegments. - */ - get remainingSumSquaredError(): number { - return this.pre.sumOfSquaredError + this.post.sumOfSquaredError; - } - - /** - * Provides the coordinate that belongs to BOTH subsegments, as it ends one and begins - * the other. - */ - get splitPoint() { - let splitPoint = this.host.pre.stats.lastSample; - - let indep = splitPoint.t; - if(this.independent == 'x') { - indep = splitPoint.targetX; - } else if(this.independent == 'y') { - indep = splitPoint.targetY; - } - - let dep = splitPoint.t; - if(this.dependent == 'x') { - dep = splitPoint.targetX; - } else if(this.dependent == 'y') { - dep = splitPoint.targetY; - } - - return { - dep: dep, - indep: indep - }; - } - - /** - * The total summed squared-distances of the best fitting line from actually-observed values - * when **not** utilizing segmented regression. Double-counts the subsegment's common point, as - * it is counted within both subsegments' sum-of-squared error term. - */ - get unsegmentedSumSquaredError(): number { - // What was our remaining error for the unsegmented regression? - let baseSSE = this.union.sumOfSquaredError; - - // We double-count the split point when segmenting, so we should add an - // extra copy of its residual. - const splitPoint = this.splitPoint; - const splitPointError = this.splitPoint.dep - this.union.predictFromValue(splitPoint.indep); - - // This is the total sum-squared error to be handled by segmenting. - return baseSSE + splitPointError * splitPointError; - } - - /** - * Gets the amount of (unnormalized) variance not explained by a standard regression on - * the full interval but succesfully explained by splitting the interval in twain via - * segmented regression. - */ - get sumSquaredGainFromSegmentation(): number { - return sigMinus(this.unsegmentedSumSquaredError, this.remainingSumSquaredError); - } - - /** - * A statistical term that signals how successful the segmented regression is. Always - * has values on the interval [0, 1], with 1 being a perfect fit. - */ - get coefficientOfDetermination(): number { - if(this.host.union.squaredSum(this.dependent) == 0 || this.host.union.squaredSum(this.independent) == 0) { - return 1; - } - - return 1 - this.remainingSumSquaredError / this.host.union.squaredSum(this.dependent); - } - - /** - * Gets the raw statistic value needed to test for validity of segmentation based on the axes - * under examination. The higher the proportion of newly-explained variance to still-unexplained - * variance is, the more confident we can _formally_ be in our segmentation. - */ - get fStat(): number { - const val = this.sumSquaredGainFromSegmentation / this.remainingSumSquaredError; - - // We're fine with Infinity. Just... not so much NaN. - return isNaN(val) ? 0 : val; - } - - /** - * Gets the degrees-of-freedom to be used for the numerator DoF parameter of - * the F-distribution needed for validity testing. - */ - get fDoF1(): number { - let numDoF = 3; - // Cases where there clearly may as well be 'no slope' at all. - // This saves us a degree of freedom, which is VERY useful for segmenting - // the boundary between a 'hold' and a 'move'. - if(this.host.pre.stats.squaredSum(this.dependent) < 1e-8) { - numDoF--; - } - if(this.host.post.stats.squaredSum(this.dependent) < 1e-8) { - numDoF--; - } - - return numDoF; - } - - /** - * Gets the degrees-of-freedom to be used for the denominator DoF parameter of - * the F-distribution needed for validity testing. - */ - get fDoF2(): number { - return this.host.union.sampleCount - 2 - this.fDoF1; - } - - /** - * States a statistically-founded level of confidence we may place in upholding - * the represented segmentation, as based on the regressions and axes under examination. - * - * If we cannot hold any confidence whatsoever (due to being unable to "reject the - * null hypothesis" from this specific regression), will return 0. - */ - get certaintyThreshold() { - return 1 - FDistribution.thresholdTier(this.fStat, this.fDoF1, this.fDoF2); - } - - get prettyPrint(): string { - return `F_(${this.fDoF1}, ${this.fDoF2}) = ${this.fStat} @ ${this.certaintyThreshold}` - } - } - - constructor(pre: Subsegmentation, - post: Subsegmentation) { - this.pre = pre; - this.post = post; - this.union = post.endingAccumulation.deaccumulate(pre.baseAccumulation); - } - - static fromTrackedStats(steppedStats: CumulativePathStats[], - baseAccumulation: CumulativePathStats, - splitIndex: number) { - let intervalEndStats = steppedStats[splitIndex]; - const pre: Subsegmentation = { - stats: intervalEndStats.deaccumulate(baseAccumulation), - endingAccumulation: intervalEndStats, - baseAccumulation: baseAccumulation - }; - - // Keep stats value components based on the final point of the 'pre' segment. - intervalEndStats = steppedStats[steppedStats.length-1]; - const postBaseAccumulation = steppedStats[splitIndex-1]; - const post: Subsegmentation = { - stats: intervalEndStats.deaccumulate(postBaseAccumulation), - endingAccumulation: intervalEndStats, - baseAccumulation: postBaseAccumulation - } - - return new SegmentationSplit(pre, post); - } - - /** - * Provides a segmented-regression perspective for the represented interval and proposed - * split point based upon the two specified axes. - * @param dependent - * @param independent - * @returns - */ - public segReg(dependentAxis: PathCoordAxis, independentAxis: PathCoordAxis) { - return new SegmentationSplit.segmentationComparison(this, dependentAxis, independentAxis); - } - - /** - * Defines our test for upholding a minimum of sub-segmentation based upon changes in the - * interval's coordinates over time. This may occur even with no change in spacial direction - * if there is significant enough change in _speed_, as that "curves the line" when one axis - * is time. - */ - get segmentationMerited(): boolean { - let totalThreshold = 0; - const xTest = new SegmentationSplit.segmentationComparison(this, 'x', 't'); - const yTest = new SegmentationSplit.segmentationComparison(this, 'y', 't'); - - // Our testing thresholds are for p=0.05 and p=0.10, which correspond to certainties of 95% and - // 90% that our segmentation did not arrive from random chance based on the axis being tested. - const p05 = 0.95; // 1 - 0.05 - const p10 = 0.9; // 1 - 0.10 - - totalThreshold += xTest.certaintyThreshold >= p05 ? 2 : (xTest.certaintyThreshold >= p10 ? 1 : 0) ; - totalThreshold += yTest.certaintyThreshold >= p05 ? 2 : (yTest.certaintyThreshold >= p10 ? 1 : 0) ; - - return totalThreshold >= 2; - } - - /** - * Defines our test for considering two (or more) sub-segments as part of the same overall - * segment. This seeks to 'link' geometrically-related subsegments - especially those that - * maintain the same direction but differ only in observed speed. - */ - get mergeMerited(): boolean { - // Because of caret-like motions (as in, in the '^' shape), we need to text for - // regression on both axes. One may have notably higher variance than the other. - // - // These tests ignore time, and therefore speed. Only the raw geometry of the motion - // matters for this test. - const xTest = new SegmentationSplit.segmentationComparison(this, 'x', 'y'); - const yTest = new SegmentationSplit.segmentationComparison(this, 'y', 'x'); - - // If we don't get a p-value less than .100, then as far as x & y are concerned - and thus the user - // is concerned - it's the same segment. Speed may be different, but not the direction. - if(xTest.certaintyThreshold > 0) { - return false; - } - - return yTest.certaintyThreshold == 0; - } - - public _debugLogSplitReport(demarcate?: boolean) { - if(demarcate) { - console.log("------------------------------------------------------------------"); - } - - console.log("Split object: "); - console.log(this); - - console.log(); - - const xF = this.segReg('x', 't'); - const yF = this.segReg('y', 't'); - console.log(`x F-test: ${xF.prettyPrint}`); - console.log(`y F-test: ${yF.prettyPrint}`); - - console.log(); - - if(demarcate) { - console.log("------------------------------------------------------------------"); - } - } - - public _debugLogAlignmentReport() { - console.log("Desegmentation under consideration: "); - console.log(this); - - console.log(); - - const xF = this.segReg('x', 'y'); - const yF = this.segReg('y', 'x'); - console.log(`merger F-test (xy): ${xF.prettyPrint}`); - console.log(`merger F-test (yx): ${yF.prettyPrint}`); - console.log(`will remerge: ${this.mergeMerited}`); - } -} - -export interface SegmentationConfiguration extends SegmentClassifierConfig { - // See `PathSegmenter.segmentationConfig`'s comment; we may wish to - // define & provide extra configuration parameters here... or wherever - // this type's final location ends up. -} - -/** - * The core logic, algorithm, and manager for touchpath segmentation and - * subsegmentation. - * - * This class itself directly handles 'subsegmentation', as the first pass of - * our algorithm errs on the side of segmenting too much in order to never - * 'undersegment'. It then delegates to other classes to recombine the - * results as appropriate before producing completed `Segment`s. - */ -export class PathSegmenter { - /** - * The minimum amount of time (in ms) to wait between sample repetitions - * once inputs stop arriving, so long as the path is still active. - */ - private readonly REPEAT_INTERVAL = 33; - - /** - * The time-interval length (in ms) at the end of the path to consider - * when determining whether or not to trigger path segmentation. - */ - private readonly SLIDING_WINDOW_INTERVAL = 50; - - /** - * Tracks the mathematical values used to provide path segment stats - * at each point on the path. Individual steps may be removed (and - * batched into `choppedStats`) once their respective points on the - * path have been fully processed. - */ - private steppedCumulativeStats: CumulativePathStats[]; - - /** - * Tracks the segment currently under construction. - * - * May also model an "empty" tail on the touchpath in order to track - * which part of the path's accumulation is no longer under consideration. - */ - private constructingSegment: ConstructingSegment; - - /** - * Used to 'repeat' the most-recently observed incoming sample if no - * other replaces it before it triggers. - * - * A repeating sample indicates lack of motion, which is valuable - * information for stats-based segmentation. - */ - private repeatTimer: number | NodeJS.Timeout; - - /** - * The timestamp of observation of the most recently observed sample's - * most recent repetition - even if it's only the first evaluation. - */ - private repeatTimestamp: number; - - /** - * Represents the cumulative statistics of all points on the path - * that lie on already-fully-segmented parts of it. - */ - private choppedStats: CumulativePathStats = null; - - /** - * A closure used to 'forward' generated Segments, generally to their public-facing - * location on GesturePath.segments. - */ - private readonly segmentForwarder: (segment: Segment) => void; - - /** - * Denotes whether or not a first touchpath sample has been provided. - */ - private hasStarted: boolean = false; - - // For consideration: should REPEAT_INTERVAL, SLIDING_WINDOW_INTERVAL, and other - // related values be defined here? - // - // Note: - // - hardcoded 2 * SLIDING_WINDOW_INTERVAL: minimum interval required for a - // subsegmentation attempt (to ensure sufficient observations on each side) - // - hardcoded SLIDING_WINDOW_INTERVAL / 2: minimum interval that must - // remain on each side after the sliding "optimum split point" search. - // - // NOTE: this will likely (eventually) be defined elsewhere, at a "higher level" - // within this module / package. - private segmentationConfig: SegmentationConfiguration; - - public static readonly DEFAULT_CONFIG: SegmentationConfiguration = { - holdMinimumDuration: 100, - holdMoveTolerance: 5 - } - - constructor(segmentationConfig: SegmentationConfiguration, segmentForwarder: (segment: Segment) => void) { - this.steppedCumulativeStats = []; - this.segmentForwarder = segmentForwarder; - - this.segmentationConfig = segmentationConfig; - } - - /** - * Appends a new coordinate to the touch path and also kick-starts a timer for replicating it - * should neither further inputs be received nor termination of the touchpoint be indicated. - * @param sample - */ - public add(sample: InputSample) { - // If this is the first received input sample, generate & publish a "start" segment. - // As ConstructingSegment is designed to work with -sequences- of samples, it's less - // useful here... and unnecessary, as we already have all the info we need. - if(!this.hasStarted) { - this.hasStarted = true; - - const startSegment = new SegmentImplementation(); - startSegment.updateStats(new CumulativePathStats(sample)); - startSegment.classifyType(SegmentClass.START); - startSegment.resolve(); - - this.segmentForwarder(startSegment); - } - - // Set up the input-repeater (in case we don't get further feedback but remain active) - const repeater = (timeDelta: number) => { - this.observe(sample, timeDelta); - } - - // If we previously set up an input-repeater, cancel it. We've got a more up-to-date - // coordinate on the touchpath now. - if(this.repeatTimer) { - // @ts-ignore - clearInterval(this.repeatTimer); - this.repeatTimer = null; - } - - this.repeatTimer = setInterval(() => { - const timeDelta = Date.now() - this.repeatTimestamp; - repeater(timeDelta); - }, this.REPEAT_INTERVAL); - this.repeatTimestamp = Date.now(); - repeater(0); - } - - /** - * Used to finalize all segmentation for the touchpath whenever termination of the corresponding - * touchpoint has been terminated (mouse-up, touch-end). - */ - public close() { - // The Node clearTimeout & DOM clearTimeout appear to TS as overloads of each other, - // and their type definitions will conflict. A simple @ts-ignore will bypass this issue. - // @ts-ignore - clearInterval(this.repeatTimer); // Cancels the input-repeater. - this.repeatTimer = null; - - if(this.steppedCumulativeStats.length == 0) { - return; - } - - // Make sure that the final part of the touchpath is given a subsegment & then handled. - const finalAccumulation = this.steppedCumulativeStats[this.steppedCumulativeStats.length - 1] - let finalSubsegment: Subsegmentation = { - stats: finalAccumulation.deaccumulate(this.choppedStats), - endingAccumulation: finalAccumulation, - baseAccumulation: this.choppedStats - } - - // No need to check if this matches any predecessor subsegments; that already happened during - // the last `performSubsegmentation` call. There's no new data since then. - // (Actually... this should already be in place now, after recent changes!) - this.constructingSegment?.updatePendingSubsegment(finalSubsegment); - this.finalizeSegment(); // also commits pending subsegment - - // Using the last-received sample, generate & publish an "end" segment. - // As ConstructingSegment is designed to work with -sequences- of samples, it's less - // useful here... and unnecessary, as we already have all the info we need. - const endSegment = new SegmentImplementation(); - endSegment.updateStats(new CumulativePathStats(finalAccumulation.lastSample)); - endSegment.classifyType(SegmentClass.END); - endSegment.resolve(); - - this.segmentForwarder(endSegment); - } - - /** - * Adds a statistic 'observation' of the state of the touchpath. - * - * This is either to be called directly upon reception of a new input-coordinate or - * upon replication of a prior input should the touchpath still be active without any - * indication of motion. - * @param sample - * @param timeDelta - */ - private observe(sample: InputSample, timeDelta: number) { - let cumulativeStats: CumulativePathStats; - if(this.steppedCumulativeStats.length) { - cumulativeStats = this.steppedCumulativeStats[this.steppedCumulativeStats.length-1]; - } else { - cumulativeStats = new CumulativePathStats(); - } - - sample = {... sample, t: sample.t + timeDelta}; - const extendedStats = cumulativeStats.extend(sample); - this.steppedCumulativeStats.push(extendedStats); - - this.performSubsegmentation(); - } - - // The "reported via event" aspect mentioned below is necessary because this may be - // called via setTimeout callback on a held touchpoint. We need that `setTimeout` - // (which replicates the most recently-seen input coordinate) in order to have good - // sample-data for detecting the boundary between motion and lack thereof during - // segmentation. - /** - * This is the "main method" for touchpath segmentation. Should a new segment result - * from its analysis, it will be reported via event. - * @returns - */ - private performSubsegmentation() { - const cumulativeStats = this.steppedCumulativeStats[this.steppedCumulativeStats.length - 1]; - const unsegmentedDuration = cumulativeStats.lastTimestamp - this.steppedCumulativeStats[0].lastTimestamp; - const unsegmentedForm: Subsegmentation = { - stats: cumulativeStats.deaccumulate(this.choppedStats), - endingAccumulation: cumulativeStats, - baseAccumulation: this.choppedStats - } - - // STEP 1: Determine the range of the initial sliding time window for the most recent samples. - - if(unsegmentedDuration < this.SLIDING_WINDOW_INTERVAL * 2) { - // If it returns false, that only makes the interval _even shorter_. - if(!this.updateSegmentConstruction(unsegmentedForm)) { - // Updates our internal state tracking; no infinite loop will result. - this.performSubsegmentation(); - // return will automatically happen via fall-through. - } - return; - } - - let splitPoint = 0; - // Do not consider the just-added `extendedStats` entry when building the sliding - // time window. - for(let i = this.steppedCumulativeStats.length-2; i >=0; i--) { - if(this.steppedCumulativeStats[i].lastTimestamp < cumulativeStats.lastTimestamp - this.SLIDING_WINDOW_INTERVAL) { - splitPoint = i+1; - break; - } - } - - - // We split the cumulative stats on a specific point, which then resides on the edge - // of both of the resulting intervals. This completes "step 1". - let candidateSplit = SegmentationSplit.fromTrackedStats(this.steppedCumulativeStats, this.choppedStats, splitPoint); - - // STEP 2: given the proposed time window, do we have a basis for segmentation? And is there a better - // candidate split point nearby? - - // We either have the conditions to trigger segmentation or just became long enough to consider it. - // If we're only just long enough to consider it, there may be a better segmentation point to start with. - // Now... is there a better segmentation point? But first... how do we facilitate searching for one? - - // Start: split-point search helper local-class. - class SplitSearchState { - readonly xTest: typeof SegmentationSplit.segmentationComparison.prototype; - readonly yTest: typeof SegmentationSplit.segmentationComparison.prototype; - readonly candidate: SegmentationSplit; - - constructor(candidate?: SegmentationSplit) { - this.candidate = candidate; - - if(candidate == null) { - return; - } - - this.xTest = candidate.segReg('x', 't'); - this.yTest = candidate.segReg('y', 't'); - } - - // The value for an 'objective function' to optimize in our search for a better candidate. - // The higher this is, the more we like its potential as the segmentation point. - get segRating() { - if(this.candidate) { - return Math.max(this.xTest.coefficientOfDetermination, this.yTest.coefficientOfDetermination); - } else { - return 0; - } - } - - // But, if it turns out this potential point wouldn't actually result in segmentation, well... - // "abandon ship". - get segmentationMerited() { - return this.candidate?.segmentationMerited ?? false; - } - } - - // Start: split-point search - - // NOTE: During initial development, I actually had a hard rule about not even searching if segmentation - // failed on the initially-constructed sliding window. Some analysis may be wise here, as it'd be nice - // to optimize away the 'need to search' if we can find hard & fast rules about when we'll never need - // to go looking. But... being too aggressive can cause nasty problems. - - /* We'll use our initial sliding window as a starting point. It's close to the active location - * of the touch, so we're unlikely to miss a proper segmentation point if it lies close. - * We'll find the local maximum (best split point for the local part of the path) - * via gradient descent. - * - * Note: binary search is a 'bad idea', as the touchpath is likely not following a - * monotonous path, mathematically speaking. (It'd be possible to overshoot a "peak" or - * "valley" quite easily.) - */ - let currentSplit = new SplitSearchState(candidateSplit); - - let leftSplit = new SplitSearchState(SegmentationSplit.fromTrackedStats(this.steppedCumulativeStats, this.choppedStats, splitPoint-1)); - let rightCandidate: SegmentationSplit = null; - if(splitPoint+1 < this.steppedCumulativeStats.length) { - rightCandidate = SegmentationSplit.fromTrackedStats(this.steppedCumulativeStats, this.choppedStats, splitPoint+1); - } - let rightSplit = new SplitSearchState(rightCandidate); - - // Step 2a: detect which direction gives the best improvement in segmentation potential. - const criteria = [leftSplit.segRating, currentSplit.segRating, rightSplit.segRating]; - let sortedCriteria = [].concat(criteria).sort(); - - const delta = criteria.indexOf(sortedCriteria[2])-1; // -1 if 'left' is best, 1 if 'right' is best. - - // Step 2b: we've found the direction: go searching! - if(delta != 0) { - // We can get better segmentation by shifting. Proceed in the optimal direction. - do { - const nextSplitIndex = splitPoint + delta; - if(nextSplitIndex >= this.steppedCumulativeStats.length || nextSplitIndex < 0) { - break; - } - - let nextCandidate = SegmentationSplit.fromTrackedStats(this.steppedCumulativeStats, this.choppedStats, splitPoint + delta); - let nextSplit = new SplitSearchState(nextCandidate); - - // Prevent overly-short intervals / over-segmentation. - if(nextCandidate.pre.stats.duration < this.SLIDING_WINDOW_INTERVAL / 2) { - break; - } else if(nextCandidate.post.stats.duration < this.SLIDING_WINDOW_INTERVAL / 2) { - break; - } - - // Not an improvement? Guess we found the best spot. - if(nextSplit.segRating <= currentSplit.segRating) { - break; - } else if(!nextSplit.segmentationMerited && currentSplit.segmentationMerited) { - // Note: this can happen if segmentation is triggered due to divergence on both axes - // if one of them drops a threshold tier and the other fails to improve sufficiently. - break; - } else { - splitPoint += delta; - currentSplit = nextSplit; - candidateSplit = nextCandidate; - } - // If we found a new best segmentation point, we then ask if we can get even better by shifting further. - } while(true); - } - // Step 2 complete. - - // Step 3: evaluate validity of the left-hand subsegment reasonably continuing the in-construction segment. - // (In sort, subsegment "compatibility" with what came before.) - // A classic example case: same direction, different speeds. - // - - if(!this.updateSegmentConstruction(candidateSplit.pre)) { - // It's quite possible that segmentation is possible in the leftover section post-reversion. - // There is a pretty good chance that it'll instantly abort, but no guarantee. - this.performSubsegmentation(); - return; - } - - // First phase of segmentation: complete! - // Step 4: basic bookkeeping. - - /* - * NOTE: this marks a very good location to call _debugLogSplitReport() if a deep-dive - * inspection of the subsegmentation algorithm and its decisions is needed. - */ - - if(!candidateSplit.segmentationMerited) { - if(!this.updateSegmentConstruction(unsegmentedForm)) { - this.performSubsegmentation(); - //return will happen via fall-through here. - } - return; - } - - // A successful update will ensure a valid .constructingSegment instance exists. - this.commitSubsegmentation(); - - // Step 5: now that subsegmentation & its bookkeeping is done... process the implications. - // Does the right-hand (still-constructing) subsegment appear reasonable to - // 'link' with its predecessors? If not, finalize the predecessors. - - // As this occurs after a `commitPending`, there is currently no pending subsegment. - // As such, this update will either: - // - initialize a new pending subsegment for the in-construction Segment if compatible - // - or finalize the Segment if incompatible, then use this subsegment to start a new Segment. - // As a result, this update call will always succeed; there's no prior pending state that may be reverted to. - this.updateSegmentConstruction(candidateSplit.post); - } - - private finalizeSegment() { - if(this.constructingSegment) { - if(this.constructingSegment.subsegmentCount == 0 && !this.constructingSegment.hasPendingSubsegment) { - throw new Error("Implementation error!"); - } - this.constructingSegment.finalize(); - - /* - * NOTE: if a deep-dive investigation is needed, it may prove helpful to emit each finalized - * `constructingSegment` instance to the console here. Like, _**precisely**_ here, immediately - * after this multiline comment. - * - * From there, note that in many modern browsers, you can right-click a logged object and say - * to "Store object as global variable", giving you console access to any such logged instances. - * - * `SubsegmentCompatibilityAnalyzer` is designed for ease-of-use with the members of - * `ConstructingSegment.subsegmentations` via `SegmentationSplit`, facilitating interactive - * inspection of which subsegments are and are not recombined into `Segment`s and why. - */ - - this.constructingSegment = null; - } - } - - /** - * Updates the in-construction Segment with the specified subsegment's accumulation - * data. If no Segment is currently under construction, it also creates a new one. - * - * In the case that an update must be blocked due to a "pending commit" (from having - * recognized the Segment while depending on the currently-constructing subsegment), - * it will return `false` to signal that it has overriden the subsegmentation with - * the most recently-valid prior version, committed that, and that the algorithm's - * current analysis is no longer valid. - * @param subsegment - * @returns `false` if the caller should restart due to forced change of segmentation - * state. - */ - private updateSegmentConstruction(subsegment: Subsegmentation): boolean { - let updateFlag: boolean; - if(this.constructingSegment) { - if(!this.constructingSegment.isCompatible(subsegment)) { - // unknown: is pending locked or not? - updateFlag = false; - } else { - // Note that this call may semi-silently precommit the subsegment if this update is the - // first to surpass the configured segment recognition timer threshold. - // - // Alternatively, if updating the pending subsegment results in incompatibility while - // a pre-existing precomitted version does not, the update will fail. - - updateFlag = this.constructingSegment.updatePendingSubsegment(subsegment); - if(updateFlag) { - return true; - } - } - } - - // If the segment under construction was registered as a 'hold' segment in the - // middle of the subsegment, we now require the 'pending subsegment' to maintain - // the same type. But, reaching here means it's no longer compatible - so we - // use the previously-registered subsegmentation here instead. - if(updateFlag === false && this.constructingSegment.hasPrecommittedSubsegment) { - this.commitSubsegmentation(); - - // Signal our caller to refresh itself and restart segmentation for the round. - // Kinda dirty, but it's 100% internal to this class, at least. - return false; - } - - this.constructingSegment?.clearPendingSubsegment(); - this.finalizeSegment(); - - this.constructingSegment = new ConstructingSegment(subsegment, new SegmentClassifier(this.segmentationConfig)); - this.segmentForwarder(this.constructingSegment.pathSegment); - - return true; - } - - private commitSubsegmentation() { - this.constructingSegment.commitPendingSubsegment(); - - // Based on what we just committed, we can find the split point that led to the subsegmentation commit: - const splitPointAccumulation = this.constructingSegment.committedIntervalAsSubsegmentation.endingAccumulation; - const splitPoint = this.steppedCumulativeStats.indexOf(splitPointAccumulation); - - // And given that split point, we can maintain our internal state accordingly. - // The exact point of the split is duplicated; we remove accumulation from everything before it. - this.choppedStats = this.steppedCumulativeStats[splitPoint-1]; - this.steppedCumulativeStats = this.steppedCumulativeStats.slice(splitPoint); - } -} \ No newline at end of file diff --git a/common/web/gesture-recognizer/src/engine/headless/subsegmentation/segment.ts b/common/web/gesture-recognizer/src/engine/headless/subsegmentation/segment.ts deleted file mode 100644 index 676eec0bc7..0000000000 --- a/common/web/gesture-recognizer/src/engine/headless/subsegmentation/segment.ts +++ /dev/null @@ -1,287 +0,0 @@ -import { CumulativePathStats } from "../cumulativePathStats.js"; -import { InputSample } from "../inputSample.js"; -import { type SegmentClass } from "../segmentClassifier.js"; - -export interface JSONSegment { - type: SegmentClass, - duration: number, - distance: number, - speed: number, - peakSpeed: number - angle: number, - cardinalDirection: string, - initialCoord: InputSample, - lastCoord: InputSample -} - -/** - * Denotes one "segment" of the ongoing touchpath. These segments are then utilized as the basis - * for detecting and synthesizing gestures. - * - * By first breaking the touchpath into gestures, we gain the ability to define something of a - * finite-state-machine (FSM) model for each type of gesture we wish to support. While technically - * possible to do even without this step... we find that this makes such a goal far less complex. - */ -export class Segment { - /** - * Denotes the highest (mean + 1-sigma) speed seen among all subsegments comprising - * this segment. - * - * This is likely not the maximum speed observed, as that observation is likely an outlier - * and is not internally recorded. We rebuild this from stats observations. - * - * Think of it as acting like a "smoothed peak speed". - */ - private _peakSpeed: number; - - private _type?: SegmentClass; - - private _stats: CumulativePathStats | JSONSegment; - - private _recognitionPromise: Promise; - private _recognitionPromiseResolver: (type: SegmentClass | PromiseLike) => void; - - private _resolutionPromise: Promise; - private _resolutionPromiseResolver: () => void; - - private _isResolved: boolean = false; - - /** - * Intended for internal-use only; this is utilized during the segmentation process. - */ - public constructor(); - /** - * Reconstructs an instance based upon a previously-serialized `.toJSON()` call on this - * object. - * @param serializedObj - */ - public constructor(serializedObj: JSONSegment); - public constructor(serializedObj?: JSONSegment) { - if(serializedObj) { - // We'll use the loaded object as our core reference. - this._stats = serializedObj; - this._peakSpeed = serializedObj.peakSpeed; // Is retrieved from `this`, not `this._stats`. - this._type = serializedObj.type; // This one too. - - // Ensure that the two promises are properly initialized for this construction type. - this._recognitionPromise = Promise.resolve(serializedObj.type); - this._resolutionPromise = Promise.resolve(); - } else { - this._peakSpeed = 0; - - this._recognitionPromise = new Promise((resolve) => { - this._recognitionPromiseResolver = resolve; - }); - - this._resolutionPromise = new Promise((resolve) => { - this._resolutionPromiseResolver = () => { - this._isResolved = true; - resolve(); - } - }) - } - } - - /** - * Indicates the likely "highest sustained" speed observed within the `Segment`'s path. - * - * To be more precise, the mean + 1-sigma (average + one standard deviation) of the speed - * for the fastest component of the `Segment`. - */ - get peakSpeed(): number { - return this._peakSpeed; - } - - protected setPeakSpeed(speed: number) { - if(this._isResolved) { - throw new Error("May not modify a resolved segment!"); - } - this._peakSpeed = speed; - } - - protected updateStats(totalStats: CumulativePathStats) { - if(this._isResolved) { - throw new Error("May not modify a resolved segment!"); - } - this._stats = totalStats; - } - - protected classifyType(type: SegmentClass) { - if(!this._stats) { - throw new Error("Cannot recognize the segment - lacking critical metadata"); - } - - if(this._isResolved) { - throw new Error("May not modify a resolved segment!"); - } - - if(this._type === undefined) { - this._type = type; - - this._recognitionPromiseResolver(type); - } else if(this._type != type) { - throw new Error("May not change segment type once set!"); - } - } - - /** - * The starting point of this touchpath segment. - */ - get initialCoord(): InputSample { - if(this._stats instanceof CumulativePathStats) { - return this._stats.initialSample; - } else { - return this._stats.initialCoord; - } - }; - - /** - * The ending point of this touchpath segment. - */ - get lastCoord(): InputSample { - if(this._stats instanceof CumulativePathStats) { - return this._stats.lastSample; - } else { - return this._stats.lastCoord; - } - }; - - /** - * The duration of this touchpath segment, measured in ms. - */ - get duration(): number { - return this._stats.duration; - } - - /** - * The net distance traveled by this touchpath segment. In other words, - * the distance between `initialCoord` and `lastCoord` (ignoring time). - */ - get distance(): number { - if(this._stats instanceof CumulativePathStats) { - return this._stats.netDistance; - } else { - return this._stats.distance; - } - } - - /** - * The average speed of this touchpath segment - distance / duration. - */ - get speed(): number { - return this._stats.speed; - }; - - /** - * The angle traveled by this touchpath segment. Measured in radians. - */ - get angle(): number { - return this._stats.angle; - }; - - /** - * The 'directional bucket' for the detected angle. This will - * correspond to a cardinal or an intercardinal ('n', 'ne', etc.) - */ - get direction(): string { - return this._stats.cardinalDirection; - }; - - /** - * The classification of this Segment: 'start', 'hold', 'move', or 'end'. - * - * May be null if the Segment has not yet been `recognized`. - * - * @see `SegmentClass` - */ - get type(): SegmentClass { - return this._type || null; - } - - protected _isRecognized(): boolean { - return this._type !== undefined; - } - - /** - * A `Promise` that resolves when the segment's classification is determined - * - when we "recognize" its role in the touchpath. - */ - get whenRecognized(): Promise { - return this._recognitionPromise; - } - - /** - * A `Promise` that resolves when the segment and its role within the touchpath - * are fully determined, as segmentation has noted the need for a new boundary - - * and thus, distinct and different behavior after the endpoint of this segment - * of the touchpath. - */ - get whenResolved(): Promise { - return this._resolutionPromise; - } - - protected resolve() { - if(!this._stats) { - throw new Error("Cannot resolve the segment - illegal state!"); - } - this._resolutionPromiseResolver(); - } - - /** - * Provides a human-readable yet serialization-friendly version of this instance. - * @returns - */ - public toJSON(): JSONSegment { - const cleanSample: (sample: InputSample) => InputSample = (sample) => { - const clone = {... sample}; - delete clone.clientX; - delete clone.clientY; - - return clone; - } - - return { - type: this.type, - duration: this.duration, - cardinalDirection: this.direction, - speed: this.speed, - distance: this.distance, - angle: this.angle, - peakSpeed: this.peakSpeed, - initialCoord: cleanSample(this.initialCoord), - lastCoord: cleanSample(this.lastCoord) - }; - } -} - -/** - * Provides an editable typing for Segment utilized during the segmentation process; - * casting to the superclass provides an uneditable view into the Segment instead. - * - * This is intended for internal use only. - */ -export class SegmentImplementation extends Segment { - constructor() { - super(); - } - - public setPeakSpeed(speed: number) { - super.setPeakSpeed(speed); - } - - public updateStats(stats: CumulativePathStats) { - super.updateStats(stats); - } - - public classifyType(type: SegmentClass) { - super.classifyType(type); - } - - public resolve() { - super.resolve(); - } - - public get isRecognized() { - return super._isRecognized(); - } -} \ No newline at end of file diff --git a/common/web/gesture-recognizer/src/engine/headless/subsegmentation/subsegmentCompatibilityAnalyzer.ts b/common/web/gesture-recognizer/src/engine/headless/subsegmentation/subsegmentCompatibilityAnalyzer.ts deleted file mode 100644 index cfa2f7c5a9..0000000000 --- a/common/web/gesture-recognizer/src/engine/headless/subsegmentation/subsegmentCompatibilityAnalyzer.ts +++ /dev/null @@ -1,159 +0,0 @@ -import { SegmentationSplit } from "./pathSegmenter.js"; -import { SegmentClass, SegmentClassifier } from "../segmentClassifier.js"; - -/** - * This class fulfills two roles: - * - * 1. The `.isCompatible()` function provides the core definition used by - * `ConstructingSegment` to re-merge compatible subsegments, 'correcting' any - * 'oversegmentation' that may have resulted from `PathSegmenter`'s algorithm. - * - * 2. This class's other properties and methods facilitate inspection of - * the algorithm and its decision-making process during interactive debugging - * sessions. - * - * `SegmentationSplit` objects are constructed from Subsegmentation objects, - * which are made in abundance throughout the segmentation algorithms. So, it - * shouldn't be difficult to construct the necessary parameters to dynamically - * construct instances of this class during an interactive debugging session. - */ -export class SubsegmentCompatibilityAnalyzer { - readonly classifier: SegmentClassifier; - private split: SegmentationSplit; - - constructor(subsegmentationSplit: SegmentationSplit, classifier: SegmentClassifier) { - this.split = subsegmentationSplit; - - this.classifier = classifier; - } - - private get preStats() { - return this.split.pre.stats; - } - - private get postStats() { - return this.split.post.stats; - } - - private get unionStats() { - return this.split.union; - } - - /** - * This property summarizes subsegment compatibility on the basis of direction. - * - * 1. If both subsegments are compatible as determined by geometry-based regression, we consider - * them compatible. (The first phase considers both geometry _and time_; we leave the "time" - * part out here.) - * - * 2. If regression upholds the segmentation, but both subsegments are classified as - * 'move'-compatible and their directions both fall within the same "direction bucket", we - * consider them compatible. - * - * 3. If both subsegments are classified as 'hold's, angle doesn't matter - the user's intent - * is "no motion". - */ - get directionCompatible(): boolean { - if(!this.preStats || this.regressionDirectionCompatible) { - return true; - } - - let preMatchesMove = this.classifier.classifySubsegment(this.preStats) != SegmentClass.HOLD; - let postMatchesMove = this.classifier.classifySubsegment(this.postStats) != SegmentClass.HOLD; - - if(preMatchesMove != postMatchesMove) { - // If one subsegment appears to be a 'hold' while the other does not, well... holds - // don't (practically) have a direction, which mismatches with the 'move' that does - // have a direction. - return false; - } else if(!preMatchesMove) { - // If both are 'hold' subsegments, both should be treated as directionless. - return true; - } else { - // If both are 'move' / 'move'-like subsegments, only merge if their directional - // classification falls into the same 'direction bucket'. - return this.cardinalDirectionCompatible; - } - } - - /** - * Indicates whether or not the two halves of the represented segmentation fall within - * the same cardinal-direction "bucket". - */ - get cardinalDirectionCompatible(): boolean { - return !this.preStats || this.preStats.cardinalDirection == this.postStats.cardinalDirection; - } - - /** - * Indicates whether or not geometry-based regression of the two subsegments (without - * respect to speed and/or time) provides sufficient evidence to uphold the segmentation. - */ - get regressionDirectionCompatible(): boolean { - return !this.preStats || this.split.mergeMerited; - } - - /** - * Indicates whether or not the classifications of the two subsegments is compatible; - * if so, it returns the corresponding `SegmentClass` (or `null`, if no classification - * may be committed yet). - * - * If the classifications are incompatible, ths function will return `undefined` - * instead. - */ - get classificationIfCompatible(): SegmentClass { - // Get the baseline subsegment classification for each subsegment. - let leftClass = this.classifier.classifySubsegment(this.preStats); - let rightClass = this.classifier.classifySubsegment(this.postStats); - let unionClass = this.classifier.classifySubsegment(this.unionStats); - - // Choose the first non-null one as a fallback, then apply it. - let fallbackClass = leftClass || rightClass || unionClass; - - leftClass = leftClass || fallbackClass; - rightClass = rightClass || fallbackClass; - unionClass = unionClass || fallbackClass; - - // If all classes (post-fallback) match, that's the class if compatible. - if(leftClass == rightClass && leftClass == unionClass) { - return fallbackClass; // can technically still be null. - } else { - return undefined; - } - } - - /** - * The full, complete test for subsegment 'compatibility'. If `true`, our - * criteria indicate no merit in upholding the segmentation under review. - * `false` indicates that the segmentation should be upheld instead. - */ - get isCompatible(): boolean { - const commonClass = this.classificationIfCompatible; - - // If two adjacent hold subsegments also make a hold when combined... - // just merge the two holds & call 'em compatible. - if(!this.preStats || commonClass == 'hold') { - return true; - } else if(commonClass === undefined) { // `null`: pending; `undefined`: incompatible - return false; - } - - // if(null || "move"): as `null` "looks like" a not-quite-there-yet "move", - // we treat it as such here. Such subsegments are only compatible if - // their directions are compatible. - return this.directionCompatible; - } - - /** - * Intended only for use during interactive debugging sessions; prints useful logging - * information to the developer console. - */ - public debugLogCompabilityReport() { - if(!this.preStats) { - console.log("No prior subsegments - thus, no compatibility conflicts are possible."); - return; - } - - console.log("Regression-based compatibility check:") - this.split._debugLogAlignmentReport(); - } -} \ No newline at end of file diff --git a/common/web/gesture-recognizer/src/engine/index.ts b/common/web/gesture-recognizer/src/engine/index.ts index 066aaedebb..50b8f12285 100644 --- a/common/web/gesture-recognizer/src/engine/index.ts +++ b/common/web/gesture-recognizer/src/engine/index.ts @@ -1,6 +1,5 @@ import { validateModelDefs } from './headless/gestures/specs/modelDefValidator.js'; -export { ConstructingSegment } from './headless/subsegmentation/constructingSegment.js'; export { CumulativePathStats } from './headless/cumulativePathStats.js'; export { GestureModelDefs } from './headless/gestures/specs/gestureModelDefs.js'; export { GestureRecognizer } from "./gestureRecognizer.js"; @@ -11,10 +10,8 @@ export { SerializedGesturePath, GesturePath } from "./headless/gesturePath.js"; export { ConfigChangeClosure, GestureStageReport, GestureSequence } from "./headless/gestures/matchers/gestureSequence.js"; export { SerializedGestureSource, GestureSource, buildGestureMatchInspector } from "./headless/gestureSource.js"; export { MouseEventEngine } from "./mouseEventEngine.js"; -export { PathSegmenter, Subsegmentation } from "./headless/subsegmentation/pathSegmenter.js"; export { PaddedZoneSource } from './configuration/paddedZoneSource.js'; export { RecognitionZoneSource } from './configuration/recognitionZoneSource.js'; -export { Segment } from "./headless/subsegmentation/segment.js"; export { SegmentClassifier } from "./headless/segmentClassifier.js"; export { TouchEventEngine } from "./touchEventEngine.js"; export { TouchpointCoordinator } from "./headless/touchpointCoordinator.js"; diff --git a/common/web/gesture-recognizer/src/test/auto/headless/basicSegmentations.js b/common/web/gesture-recognizer/src/test/auto/headless/basicSegmentations.js deleted file mode 100644 index 06c580ed03..0000000000 --- a/common/web/gesture-recognizer/src/test/auto/headless/basicSegmentations.js +++ /dev/null @@ -1,390 +0,0 @@ -import { assert } from 'chai' -import sinon from 'sinon'; - -import * as PromiseStatusModule from 'promise-status-async'; -const promiseStatus = PromiseStatusModule.promiseStatus; -const PromiseStatuses = PromiseStatusModule.PromiseStatuses; - -import { PathSegmenter } from '@keymanapp/gesture-recognizer'; -import { timedPromise } from '@keymanapp/web-utils'; - -describe("Basic segmentation cases", function() { - describe("Single-sample 'sequence'", function() { - it("expected segment types", function() { - let spy = sinon.fake(); - - const segmenter = new PathSegmenter(PathSegmenter.DEFAULT_CONFIG, spy); - - const sample = { - targetX: 1, - targetY: 1, - t: 100 - }; - - // Expected sequence: - // 1: on `segmenter.add(sample)`: - // a. 'start' segment - // b. unrecognized (null-type) segment. - // 2: on `segmenter.close()`: - // b. 'end' segment - - segmenter.add(sample); - try { - assert.isTrue(spy.calledTwice, "Segmenter callback was not called exactly twice before the .close()."); - } finally { - segmenter.close(); - } - assert.isTrue(spy.calledThrice, "Segmenter callback was not called exactly once after the .close()."); - - for(let i=0; i < 3; i++) { - assert.equal(spy.args[i].length, 1, "Segmenter callback received an unexpected number of arguments"); - } - - // Is the first argument of each call's argument set. - assert.equal(spy.firstCall .args[0].type, 'start', "First call should receive a 'start'-type segment."); - assert.equal(spy.secondCall.args[0].type, undefined, "Second call's segment should not be classified."); - assert.equal(spy.thirdCall .args[0].type, 'end', "Third call should receive an 'end'-type segment."); - }); - - it("segment recognition + resolution", async function() { - let spy = sinon.fake(); - - const segmenter = new PathSegmenter(PathSegmenter.DEFAULT_CONFIG, spy); - - const sample = { - targetX: 1, - targetY: 1, - t: 100 - }; - - segmenter.add(sample); - try { - // It's still best to verify this before proceeding. - assert.isTrue(spy.calledTwice, "Segmenter callback was not called exactly twice before the .close()."); - - // 'start': should be fully resolved, right out of the gate. - assert.equal(await promiseStatus(spy.firstCall.args[0].whenRecognized), PromiseStatuses.PROMISE_RESOLVED); - assert.equal(await promiseStatus(spy.firstCall.args[0].whenResolved), PromiseStatuses.PROMISE_RESOLVED); - - // null-typed, just-starting segment: neither recognized nor resolved. - assert.equal(await promiseStatus(spy.secondCall.args[0].whenRecognized), PromiseStatuses.PROMISE_PENDING); - assert.equal(await promiseStatus(spy.secondCall.args[0].whenResolved), PromiseStatuses.PROMISE_PENDING); - } finally { - segmenter.close(); - } - - assert.isTrue(spy.calledThrice, "Segmenter callback was not called exactly once after the .close()."); - - // null-typed segment should now be recognized and resolved. - assert.equal(await promiseStatus(spy.secondCall.args[0].whenRecognized), PromiseStatuses.PROMISE_RESOLVED); - assert.equal(await promiseStatus(spy.secondCall.args[0].whenResolved), PromiseStatuses.PROMISE_RESOLVED); - - // 'end': should be fully resolved, right out of the gate. - assert.equal(await promiseStatus(spy.thirdCall.args[0].whenRecognized), PromiseStatuses.PROMISE_RESOLVED); - assert.equal(await promiseStatus(spy.thirdCall.args[0].whenResolved), PromiseStatuses.PROMISE_RESOLVED); - }); - }); - - describe("Single held point", function() { - beforeEach(function() { - this.fakeClock = sinon.useFakeTimers(); - }) - - afterEach(function() { - this.fakeClock.restore(); - }) - - it("'hold' segment recognition + resolution", async function() { - let spy = sinon.fake(); - - const segmenter = new PathSegmenter(PathSegmenter.DEFAULT_CONFIG, spy); - - const startSample = { - targetX: 1, - targetY: 1, - t: 100 - }; - - const endSample = { - targetX: 1, - targetY: 1, - t: 1100 // total duration: 1 sec. - }; - - const segment2Recognition = sinon.fake(); - const segment2Resolution = sinon.fake(); - - // Timestamp 1: segmentation begins, with an initial Sample recorded. - const firstPromise = timedPromise(0).then(() => { - segmenter.add(startSample); - assert.isTrue(spy.calledTwice, "Segmenter callback was not called exactly twice upon adding the first point."); - - // The focus of this unit test is the two `Promise`s provided by this specific `Segment`. - const pendingSegment = spy.secondCall.args[0]; - assert.exists(pendingSegment); - - pendingSegment.whenRecognized.then(segment2Recognition); - pendingSegment.whenResolved.then(segment2Resolution); - }).then(() => { - // At time = 0, neither recognition nor resolution should have triggered. - assert.isFalse(segment2Recognition.called); - - const pendingSegment = spy.secondCall.args[0]; - assert.isNull(pendingSegment.type); // not yet recognized. - - assert.isFalse(segment2Resolution.called); - }); - - // Timestamp 2: segmentation continues... a second later. A second Sample is recorded. - const secondSampleTestPromise = firstPromise.then(() => { - return timedPromise(1000).then(() => { - // This should have occurred already, despite no new sample having been provided - // to the segmenter. - assert.isTrue(segment2Recognition.called); - - const pendingSegment = spy.secondCall.args[0]; - assert.equal(pendingSegment.type, 'hold'); // now has a type, as it's been recognized. - - // And now to update with a new sample. - segmenter.add(endSample); - }).then(() => { - assert.isFalse(segment2Resolution.called); - - const pendingSegment = spy.secondCall.args[0]; - assert.isAtLeast(pendingSegment.duration, 1000); // Latest sample's timestamp gives exactly 1000. - }); - }); - - // Timestamp 3: segmentation is then ended via a followup event. - const segmentationEndPromise = secondSampleTestPromise.then(() => { - assert.isFalse(segment2Resolution.called); - segmenter.close(); - // Fun fact: assert.isFalse(segment2Resolution.called) still holds here - // (after `segmenter.close()`) because Promises are async. - }).then(() => { - assert.isTrue(segment2Resolution.called); - assert.isTrue(spy.calledThrice); - }) - - const finalPromise = segmentationEndPromise.catch((reason) => { - segmenter.close(); - throw reason; - }); - - this.fakeClock.runAllAsync(); - - // This is the one that reports all of our async assertion failures. - return finalPromise; - }); - - it("motionless 'hold' segment properties", function() { - const spy = sinon.fake(); - - const segmenter = new PathSegmenter(PathSegmenter.DEFAULT_CONFIG, spy); - - const startSample = { - targetX: 1, - targetY: 1, - t: 100 - }; - - const endSample = { - targetX: 1, - targetY: 1, - t: 1100 // total duration: 1 sec. - }; - - // Sample 'playback' Promise setup - const samples = [startSample, endSample]; - - const samplePromises = samples.map((sample) => { - return timedPromise(sample.t - startSample.t).then(() => { - segmenter.add(sample); - }); - }); - - // Timestamp 3: segmentation is then ended via a followup event. - const segmentationEndPromise = Promise.all(samplePromises).then(() => { - segmenter.close(); - }); - - const finalPromise = segmentationEndPromise.catch((reason) => { - segmenter.close(); - throw reason; - }); - - this.fakeClock.runAllAsync(); - - // This is the one that reports all of our async assertion failures. - return finalPromise.then(() => { - const holdSegment = spy.secondCall.args[0]; - - assert.equal(holdSegment.type, 'hold'); - assert.isAtLeast(holdSegment.duration, endSample.t - startSample.t); - assert.equal(holdSegment.speed, 0); - assert.isUndefined(holdSegment.angle); - assert.isUndefined(holdSegment.direction); - assert.equal(holdSegment.peakSpeed, 0); - assert.equal(holdSegment.distance, 0); - - // Samples may be cloned when passed through the segmenter! - assert.deepEqual(holdSegment.initialCoord, startSample); - assert.deepEqual(holdSegment.lastCoord, endSample); - }); - }); - }); - - describe("Idealized 'move'", function() { - beforeEach(function() { - this.fakeClock = sinon.useFakeTimers(); - }) - - afterEach(function() { - this.fakeClock.restore(); - }) - - /** - * Builds a diagonal sequence traveling 40 pixels 's', 40 pixels 'e' in a perfect diagonal lock-step. - */ - const buildSampleSequence = () => { - const startSample = { - targetX: 1, - targetY: 1, - t: 100 - }; - - let samples = [startSample]; - - for(let i=1; i <=20; i++) { - samples.push({ - targetX: startSample.targetX + 2 * i, - targetY: startSample.targetY + 2 * i, - t: startSample.t + 10 * i - }); - } - - return samples; - } - - it("'move' segment recognition + resolution", async function() { - let spy = sinon.fake(); - - const segmenter = new PathSegmenter(PathSegmenter.DEFAULT_CONFIG, spy); - - const samples = buildSampleSequence(); - - const samplePromises = samples.map((sample) => { - return timedPromise(sample.t - samples[0].t).then(() => { - segmenter.add(sample); - }); - }); - - // Find and replace select promises with then'd versions of themselves to 'hook in' as needed. - // We know the 'move' segment will appear upon completion of the first sample's Promise. - - const segment2Recognition = sinon.fake(); - const segment2Resolution = sinon.fake(); - - samplePromises[0] = samplePromises[0].then(async () => { - assert.isAtLeast(spy.callCount, 2); - const moveSegment = spy.secondCall.args[0]; - - assert.equal(await promiseStatus(moveSegment.whenRecognized), PromiseStatuses.PROMISE_PENDING); - assert.equal(await promiseStatus(moveSegment.whenResolved), PromiseStatuses.PROMISE_PENDING); - - // Now that we're sure they haven't already been called, let's set the `fake` calls in place. - moveSegment.whenRecognized.then(segment2Recognition); - moveSegment.whenResolved.then(segment2Resolution); - - assert.isNotNull(moveSegment); - }); - - // Given the current engine setup, we know that after 5 samples, we pass the movement-based - // recognition threshold and thus should be classified as a 'move'. - samplePromises[5] = samplePromises[5].then(() => { - const moveSegment = spy.secondCall.args[0]; - - assert.isTrue(segment2Recognition.called); - assert.isFalse(segment2Resolution.called); - - assert.equal(moveSegment.type, 'move'); - }); - - samplePromises[20] = samplePromises[20].then(() => { - const moveSegment = spy.secondCall.args[0]; - - assert.isTrue(segment2Recognition.called); - assert.isFalse(segment2Resolution.called); - - assert.isAtLeast(moveSegment.duration, samples[20].t - samples[0].t); - }); - - // Timestamp 3: segmentation is then ended via a followup event. - const segmentationEndPromise = Promise.all(samplePromises).then(() => { - assert.isFalse(segment2Resolution.called); - segmenter.close(); - }).then(() => { - assert.isTrue(segment2Resolution.called); - assert.equal(spy.callCount, 3); - }); - - const finalPromise = segmentationEndPromise.catch((reason) => { - segmenter.close(); - throw reason; - }); - - this.fakeClock.runAllAsync(); - - // This is the one that reports all of our async assertion failures. - return finalPromise; - }); - - it("idealized 'move' segment properties", function() { - let spy = sinon.fake(); - - const segmenter = new PathSegmenter(PathSegmenter.DEFAULT_CONFIG, spy); - - const samples = buildSampleSequence(); - - const samplePromises = samples.map((sample) => { - return timedPromise(sample.t - samples[0].t).then(() => { - segmenter.add(sample); - }); - }); - - // Timestamp 3: segmentation is then ended via a followup event. - const segmentationEndPromise = Promise.all(samplePromises).then(() => { - segmenter.close(); - }); - - const finalPromise = segmentationEndPromise.catch((reason) => { - segmenter.close(); - throw reason; - }); - - this.fakeClock.runAllAsync(); - - // This is the one that reports all of our async assertion failures. - return finalPromise.then(() => { - // The real checks. - const moveSegment = spy.secondCall.args[0]; - - assert.equal(moveSegment.type, 'move'); - assert.isAtLeast(moveSegment.duration, samples[20].t - samples[0].t); - assert.equal(moveSegment.direction, 'se'); - assert.equal(moveSegment.distance, 40 * Math.SQRT2); - assert.equal(moveSegment.angle, (135 / 180) * Math.PI); - - // Speed: our idealized sequence moves 2*sqrt(2) px distance every 10ms. - const idealSpeed = 2 * Math.sqrt(2) / 10; - - assert.isAtLeast(moveSegment.speed, 0.99 * idealSpeed); - assert.isAtLeast(moveSegment.peakSpeed, 0.99 * idealSpeed); - - // Samples may be cloned when passed through the segmenter! - assert.deepEqual(moveSegment.initialCoord, samples[0]); - assert.deepEqual(moveSegment.lastCoord, samples[20]); - }); - }); - }); -}); \ No newline at end of file diff --git a/common/web/gesture-recognizer/src/test/auto/headless/pathStats.ts b/common/web/gesture-recognizer/src/test/auto/headless/pathStats.ts index ffea495a97..fb985622d0 100644 --- a/common/web/gesture-recognizer/src/test/auto/headless/pathStats.ts +++ b/common/web/gesture-recognizer/src/test/auto/headless/pathStats.ts @@ -9,7 +9,6 @@ describe("CumulativePathStats", function() { assert.equal(stats.duration, 0); assert.equal(stats.angle, undefined); - assert.isNaN(stats.angleDeviation); assert.equal(stats.cardinalDirection, undefined); assert.equal(stats.rawDistance, 0); assert.equal(stats.netDistance, 0); @@ -27,10 +26,7 @@ describe("CumulativePathStats", function() { assert.equal(stats.duration, 0); assert.equal(stats.angle, undefined); - assert.isNaN(stats.angleDeviation); assert.equal(stats.cardinalDirection, undefined); - assert.isNaN(stats.variance('x')); - assert.isNaN(stats.variance('y')); assert.equal(stats.mean('x'), 4); assert.equal(stats.mean('y'), 8); }); @@ -58,14 +54,6 @@ describe("CumulativePathStats", function() { assert.equal(stats.mean('y'), samples[2].targetY); assert.equal(stats.mean('t'), samples[2].t); - - // The two are perfectly correlated and have the same variance. - // Since we renormalize samples, the base 'intercept' of each variable's linear form should have no effect.. - assert.equal(stats.squaredSum('x'), stats.crossSum('xy')); - assert.equal(stats.squaredSum('y'), stats.crossSum('xy')); - // As x and y are perfectly correlated with the same variance, their cross-sum with t should match. - assert.equal(stats.crossSum('tx'), stats.crossSum('ty')); - assert.equal(stats.angle, 135 * Math.PI / 180); assert.equal(stats.rawDistance, 16 * Math.SQRT2); // 4 intervals of length 4 * sqrt(2) assert.equal(stats.cardinalDirection, 'se'); @@ -111,44 +99,6 @@ describe("CumulativePathStats", function() { assert.sameDeepOrderedMembers(postStats, preStats); }); - it("Basic regressions (perfect correlation)", function() { - const samples: InputSample[] = []; - - // Exactly 5 points, evenly spaced and linear. - // So, the arithmetic mean should be very obvious - it's the middle sample. - for(let i = 0; i < 5; i++) { - samples.push({ - targetX: 4 * i + 10, - targetY: 4 * i + 20, - t: 100 * i // since it makes head-math simpler for the regressions we'll be doing. - }); - } - - let stats = new CumulativePathStats(); - - for(const sample of samples) { - stats = stats.extend(sample); - } - - const xtRegression = stats.fitRegression('x', 't'); - assert.equal(xtRegression.sumOfSquaredError, 0); - assert.equal(xtRegression.slope, 4 / 100); - assert.equal(xtRegression.intercept, 10); - assert.equal(xtRegression.predictFromValue(1000), 40 + 10); - - const ytRegression = stats.fitRegression('y', 't'); - assert.equal(ytRegression.sumOfSquaredError, 0); - assert.equal(ytRegression.slope, 4 / 100); - assert.equal(ytRegression.intercept, 20); - assert.equal(ytRegression.predictFromValue(1000), 40 + 20); - - const xyRegression = stats.fitRegression('y', 'x'); - assert.equal(xyRegression.sumOfSquaredError, 0); - assert.equal(xyRegression.slope, 1); - assert.equal(xyRegression.intercept, 10); - assert.equal(xyRegression.predictFromValue(50), 60); - }); - it("Deaccumulation", function() { const sampleSet1: InputSample[] = []; @@ -209,14 +159,6 @@ describe("CumulativePathStats", function() { assert.equal(deaccumulatedSecondHalfStats.mean('y'), secondHalfStats.mean('y')); assert.equal(deaccumulatedSecondHalfStats.mean('t'), secondHalfStats.mean('t')); - assert.equal(deaccumulatedSecondHalfStats.variance('x'), secondHalfStats.variance('x')); - assert.equal(deaccumulatedSecondHalfStats.variance('y'), secondHalfStats.variance('y')); - assert.equal(deaccumulatedSecondHalfStats.variance('t'), secondHalfStats.variance('t')); - - assert.equal(deaccumulatedSecondHalfStats.covariance('tx'), secondHalfStats.covariance('tx')); - assert.equal(deaccumulatedSecondHalfStats.covariance('ty'), secondHalfStats.covariance('ty')); - assert.equal(deaccumulatedSecondHalfStats.covariance('xy'), secondHalfStats.covariance('xy')); - // Floating-point "equality". assert.closeTo(deaccumulatedSecondHalfStats.netDistance, secondHalfStats.netDistance, 1e-8); assert.closeTo(deaccumulatedSecondHalfStats.rawDistance, secondHalfStats.rawDistance, 1e-8); @@ -226,69 +168,5 @@ describe("CumulativePathStats", function() { assert.equal(deaccumulatedSecondHalfStats.initialSample, secondHalfStats.initialSample); assert.equal(deaccumulatedSecondHalfStats.lastSample, secondHalfStats.lastSample); - - // Angle deviation stuff doesn't currently have the same level of catastrophic cancellation protection - // as the main statistical properties. But... even still, we can add a reasonable test. - assert.isBelow(Math.abs(deaccumulatedSecondHalfStats.angleDeviation - secondHalfStats.angleDeviation), 1e-7); - }); - - it("Renormalization", () => { - let stats = new CumulativePathStats(); - - const turtle = new TouchpathTurtle({ - targetX: 50, - targetY: -25, - t: 500, - item: 'a' - }); - - turtle.on('sample', (sample) => { - stats = stats.extend(sample); - }); - - function assertMatchingStats(actual: CumulativePathStats, expected: CumulativePathStats) { - assert.equal(actual.duration, expected.duration); - assert.equal(actual.sampleCount, expected.sampleCount); - - assert.closeTo(actual.angle, expected.angle, 1e-8); - assert.closeTo(actual.mean('x'), expected.mean('x'), 1e-8); - assert.closeTo(actual.mean('y'), expected.mean('y'), 1e-8); - assert.closeTo(actual.mean('t'), expected.mean('t'), 1e-8); - - assert.closeTo(actual.variance('x'), expected.variance('x'), 1e-8); - assert.closeTo(actual.variance('y'), expected.variance('y'), 1e-8); - assert.closeTo(actual.variance('t'), expected.variance('t'), 1e-8); - - assert.closeTo(actual.rawDistance, expected.rawDistance, 1e-8); - assert.closeTo(actual.netDistance, expected.netDistance, 1e-8); - - const actualRegression = actual.fitRegression('x', 'y'); - const expectedRegression = expected.fitRegression('x', 'y'); - assert.closeTo( - actualRegression.coefficientOfDetermination, - expectedRegression.coefficientOfDetermination, - 1e-8 - ); - assert.closeTo(actualRegression.slope, expectedRegression.slope, 1e-8); - } - - turtle.move(90, 4, 20, 2); - turtle.move(60, 5, 20, 2); // net move: <4, -3>. - turtle.commitPending(); - - let renormalizedStats = stats.buildRenormalized(); - assertMatchingStats(renormalizedStats, stats); - - // Continue to accumulate stats for both. - turtle.on('sample', (sample) => { - renormalizedStats = renormalizedStats.extend(sample); - }) - - turtle.move(0, 10, 40, 4); - turtle.move(45, 3 * Math.sqrt(2), 40, 4); - turtle.commitPending(); - - // After the renormalization, there should be no externally-visible distinction between the two stats objects. - assertMatchingStats(renormalizedStats, stats); }); }); \ No newline at end of file diff --git a/common/web/gesture-recognizer/src/test/auto/headless/recordedSegmentations.js b/common/web/gesture-recognizer/src/test/auto/headless/recordedSegmentations.js deleted file mode 100644 index e9f046a381..0000000000 --- a/common/web/gesture-recognizer/src/test/auto/headless/recordedSegmentations.js +++ /dev/null @@ -1,342 +0,0 @@ -import { assert } from 'chai'; -import sinon from 'sinon'; -import fs from 'fs'; -import path from 'path'; -import url from 'url'; - -import * as PromiseStatusModule from 'promise-status-async'; -const promiseStatus = PromiseStatusModule.promiseStatus; -const PromiseStatuses = PromiseStatusModule.PromiseStatuses; - -import { PathSegmenter } from '@keymanapp/gesture-recognizer'; - -import { HeadlessInputEngine } from '#tools'; - -// Ensures that the resources are resolved relative to this script, not to the cwd when the test -// runner was launched. -const scriptFolder = path.dirname(url.fileURLToPath(import.meta.url)); -const SEGMENT_TEST_JSON_FOLDER = path.resolve(`${scriptFolder}/../../resources/json/segmentation`); - -import { assertSegmentSimilarity } from '../../resources/assertSegmentSimilarity.js'; - -/** - * Since we're disconnecting subsegmentation stuff for the first gestures release, our - * previously-recorded subsegmentations aren't retrievable in the same spot as before. - * - * This manually retrieves them from the originally-recorded version in order to preserve - * the unit tests. Makes a few assumptions, but they're valid for the original recordings. - * @param {*} jsonObj - */ -function retrieveSubsegmentations(jsonObj) { - return jsonObj.inputs[0].path.segments; -} - -// --------------------------------------- -// NOTE: this suite of tests is for a disconnected subsystem - subsegmentation - designed -// to handle more complicated gestures than supported in our initial release. We had to -// triage it to prevent further delays. -// -// The tests themselves should still be functional. -// --------------------------------------- -describe("Segmentation - from recorded sequences", function() { - beforeEach(function() { - this.fakeClock = sinon.useFakeTimers(); - }) - - afterEach(function() { - // NOTE: for debugging investigations, it may be necessary to use .only on - // the test under investigation and to disable the `this.fakeClock.restore()` line. - // - // Tests tend to timeout when interactively debugging, and having unmocked timers - // suddenly restored during investigation can cause some very confusing behavior. - this.fakeClock.restore(); - }) - - it("simple_ne_move.json", async function() { - // NOTE: this recording's final 'hold' segment is somewhat tightly attuned to the DEFAULT_CONFIG - // hold time setting. Changing default values there may necessitate a hand-edit tweak to the - // test recording's data in order for this test to continue passing as-is. - - let testJSONtext = fs.readFileSync(`${SEGMENT_TEST_JSON_FOLDER}/simple_ne_move.json`); - let jsonObj = JSON.parse(testJSONtext); - - // Some special setup - we're going to capture the 'move' segment in-process and run a check that way. - const recognitionTestCapturer = { - recognizedPromise: null - } - - // Spy: the method we pass in is _actually called_ while _also_ capturing metadata for every call. - let spy = sinon.spy((segment) => { - if(segment.type == 'move' && !recognitionTestCapturer.recognizedPromise) { - segment.whenRecognized.then(() => { - recognitionTestCapturer.recognizedPromise = new Promise((resolve, reject) => { - // Verify that we resolved because of distance, not time. - assert.isAtLeast(segment.distance, PathSegmenter.DEFAULT_CONFIG.holdMoveTolerance); - assert.isBelow(segment.duration, PathSegmenter.DEFAULT_CONFIG.holdMinimumDuration); - - // Makes sure the segment isn't resolved at the same time it is recognized. - promiseStatus(segment.whenResolved).then((status) => { - assert.equal(status, PromiseStatuses.PROMISE_PENDING); - resolve(); - }).catch((err) => { - reject(err); - }); - }); - - return recognitionTestCapturer.recognizedPromise; - }); - } - }); - - // Now the normal basic setup. - const segmenter = new PathSegmenter(PathSegmenter.DEFAULT_CONFIG, spy); - - let engine = new HeadlessInputEngine(); - engine.once('pointstart', (point) => { - // The first point is always available initially, rather than via event. - segmenter.add(point.path.coords[0]); - point.path.on('step', (val) => { - segmenter.add(val); - }); - }); - - const testPromise = engine.playbackRecording(jsonObj).then(() => segmenter.close()); - await this.fakeClock.runAllAsync(); - await testPromise; - await recognitionTestCapturer.recognizedPromise; - - // Any post-sequence tests to run. - const originalSegments = retrieveSubsegmentations(jsonObj); - const originalSegmentTypeSequence = originalSegments.map((segment) => segment.type); - - const reproedSegments = spy.getCalls().map((call) => call.args[0]); - const reproedSegmentTypeSequence = reproedSegments.map((segment) => segment.type); - - assert.sameOrderedMembers(reproedSegmentTypeSequence, originalSegmentTypeSequence); - - // Ensure all relevant Promises resolved. - for(let segment of reproedSegments) { - assert.isTrue(await promiseStatus(segment.whenRecognized) == PromiseStatuses.PROMISE_RESOLVED); - assert.isTrue(await promiseStatus(segment.whenResolved) == PromiseStatuses.PROMISE_RESOLVED); - } - - assertSegmentSimilarity(reproedSegments[1], originalSegments[1], 'hold'); // ~200ms - assertSegmentSimilarity(reproedSegments[2], originalSegments[2], 'move'); // 'ne' - }); - - it("nonstationary_hold.json", async function() { - let testJSONtext = fs.readFileSync(`${SEGMENT_TEST_JSON_FOLDER}/nonstationary_hold.json`); - let jsonObj = JSON.parse(testJSONtext); - - // Some special setup - we're going to capture the 'hold' segment in-process and run a check that way. - const recognitionTestCapturer = { - recognizedPromise: null - } - - // Spy: the method we pass in is _actually called_ while _also_ capturing metadata for every call. - let spy = sinon.spy((segment) => { - if(segment.type == 'hold' && !recognitionTestCapturer.recognizedPromise) { - segment.whenRecognized.then(() => { - recognitionTestCapturer.recognizedPromise = new Promise((resolve, reject) => { - // Verify that we resolved because of time, not distance. - assert.isBelow(segment.distance, PathSegmenter.DEFAULT_CONFIG.holdMoveTolerance); - assert.isAtLeast(segment.duration, PathSegmenter.DEFAULT_CONFIG.holdMinimumDuration); - - // Makes sure the segment isn't resolved at the same time it is recognized. - promiseStatus(segment.whenResolved).then((status) => { - assert.equal(status, PromiseStatuses.PROMISE_PENDING); - resolve(); - }).catch((err) => { - reject(err); - }); - }); - - return recognitionTestCapturer.recognizedPromise; - }); - } - }); - - // Now the normal basic setup. - const segmenter = new PathSegmenter(PathSegmenter.DEFAULT_CONFIG, spy); - - let engine = new HeadlessInputEngine(); - engine.once('pointstart', (point) => { - // The first point is always available initially, rather than via event. - segmenter.add(point.path.coords[0]); - point.path.on('step', (val) => { - segmenter.add(val); - }); - }); - - const testPromise = engine.playbackRecording(jsonObj).then(() => segmenter.close()); - await this.fakeClock.runAllAsync(); - await testPromise; - await recognitionTestCapturer.recognizedPromise; - - // Any post-sequence tests to run. - const originalSegments = retrieveSubsegmentations(jsonObj); - const originalSegmentTypeSequence = originalSegments.map((segment) => segment.type); - - const reproedSegments = spy.getCalls().map((call) => call.args[0]); - const reproedSegmentTypeSequence = reproedSegments.map((segment) => segment.type); - - assert.sameOrderedMembers(reproedSegmentTypeSequence, originalSegmentTypeSequence); - - // Ensure all relevant Promises resolved. - for(let segment of reproedSegments) { - assert.isTrue(await promiseStatus(segment.whenRecognized) == PromiseStatuses.PROMISE_RESOLVED); - assert.isTrue(await promiseStatus(segment.whenResolved) == PromiseStatuses.PROMISE_RESOLVED); - } - - assertSegmentSimilarity(reproedSegments[1], originalSegments[1], 'hold'); // ~1200ms - }); - - it("flick_ne_se.json", async function() { - let testJSONtext = fs.readFileSync(`${SEGMENT_TEST_JSON_FOLDER}/flick_ne_se.json`); - let jsonObj = JSON.parse(testJSONtext); - - // Prepares some basic setup - we won't set up segment-specific recognition tests here. - let spy = sinon.fake(); - const segmenter = new PathSegmenter(PathSegmenter.DEFAULT_CONFIG, spy); - - let engine = new HeadlessInputEngine(); - engine.once('pointstart', (point) => { - // The first point is always available initially, rather than via event. - segmenter.add(point.path.coords[0]); - point.path.on('step', (val) => { - segmenter.add(val); - }); - }); - - const testPromise = engine.playbackRecording(jsonObj).then(() => segmenter.close()); - await this.fakeClock.runAllAsync(); - await testPromise; - - // Any post-sequence tests to run. - const originalSegments = retrieveSubsegmentations(jsonObj); - const originalSegmentTypeSequence = originalSegments.map((segment) => segment.type); - - const reproedSegments = spy.getCalls().map((call) => call.args[0]); - const reproedSegmentTypeSequence = reproedSegments.map((segment) => segment.type); - - assert.sameOrderedMembers(reproedSegmentTypeSequence, originalSegmentTypeSequence); - - // Ensure all relevant Promises resolved. - for(let segment of reproedSegments) { - assert.isTrue(await promiseStatus(segment.whenRecognized) == PromiseStatuses.PROMISE_RESOLVED); - assert.isTrue(await promiseStatus(segment.whenResolved) == PromiseStatuses.PROMISE_RESOLVED); - } - - assertSegmentSimilarity(reproedSegments[1], originalSegments[1], 'hold'); // ~820ms - assertSegmentSimilarity(reproedSegments[2], originalSegments[2], 'move'); // 'ne' - assertSegmentSimilarity(reproedSegments[3], originalSegments[3], 'hold'); // ~580ms - assertSegmentSimilarity(reproedSegments[4], originalSegments[4], 'move'); // 'se' - assertSegmentSimilarity(reproedSegments[5], originalSegments[5], 'hold'); // ~300ms - }); - - it("longpress_to_ne.json", async function() { - let testJSONtext = fs.readFileSync(`${SEGMENT_TEST_JSON_FOLDER}/longpress_to_ne.json`); - let jsonObj = JSON.parse(testJSONtext); - - // Prepares some basic setup - we won't set up segment-specific recognition tests here. - let spy = sinon.fake(); - const segmenter = new PathSegmenter(PathSegmenter.DEFAULT_CONFIG, spy); - - let engine = new HeadlessInputEngine(); - engine.once('pointstart', (point) => { - // The first point is always available initially, rather than via event. - segmenter.add(point.path.coords[0]); - point.path.on('step', (val) => { - segmenter.add(val); - }); - }); - - const testPromise = engine.playbackRecording(jsonObj).then(() => segmenter.close()); - await this.fakeClock.runAllAsync(); - await testPromise; - - // Any post-sequence tests to run. - const originalSegments = retrieveSubsegmentations(jsonObj); - const reproedSegments = spy.getCalls().map((call) => call.args[0]); - - // Because of the sweeping arc motion, we won't assume a perfect match to the segmentation here. - - // Ensure all relevant Promises resolved. - for(let segment of reproedSegments) { - assert.isTrue(await promiseStatus(segment.whenRecognized) == PromiseStatuses.PROMISE_RESOLVED); - assert.isTrue(await promiseStatus(segment.whenResolved) == PromiseStatuses.PROMISE_RESOLVED); - } - - assertSegmentSimilarity(reproedSegments[1], originalSegments[1], 'hold'); // ~1460ms - // there are actually a LOT of move segments here; the motion was in an arc. - // will make a good test case for the eventual longpress gesture. - assert.equal(reproedSegments[2].type, 'move'); - assert.equal(reproedSegments[2].direction, 'n'); // the initial direction once motion started. - }); - - it("quick_small_square.json", async function() { - // NOTE: this recording has a number of borderline-duration 'hold' segments between its moves. - // Naturally, there has to be SOME kind of transition when sharply changing direction. - // NOTE: We wish to ensure that each 'edge' of the 'square' remains reasonably distinct. - - let testJSONtext = fs.readFileSync(`${SEGMENT_TEST_JSON_FOLDER}/quick_small_square.json`); - let jsonObj = JSON.parse(testJSONtext); - - // Prepares some basic setup - we won't set up segment-specific recognition tests here. - let spy = sinon.fake(); - const segmenter = new PathSegmenter(PathSegmenter.DEFAULT_CONFIG, spy); - - let engine = new HeadlessInputEngine(); - engine.once('pointstart', (point) => { - // The first point is always available initially, rather than via event. - segmenter.add(point.path.coords[0]); - point.path.on('step', (val) => { - segmenter.add(val); - }); - }); - - const testPromise = engine.playbackRecording(jsonObj).then(() => segmenter.close()); - await this.fakeClock.runAllAsync(); - await testPromise; - - // Any post-sequence tests to run. - const reproedSegments = spy.getCalls().map((call) => call.args[0]); - - // Ensure all relevant Promises resolved. - for(let segment of reproedSegments) { - assert.isTrue(await promiseStatus(segment.whenRecognized) == PromiseStatuses.PROMISE_RESOLVED); - assert.isTrue(await promiseStatus(segment.whenResolved) == PromiseStatuses.PROMISE_RESOLVED); - } - - const holds = reproedSegments.filter((segment) => segment.type == 'hold'); - const moves = reproedSegments.filter((segment) => segment.type == 'move'); - const nulls = reproedSegments.filter((segment) => segment.type === null); - - assert.isEmpty(nulls); - assert.isEmpty(holds.filter((hold) => hold.duration > 300)); // all motions were very quick. - - // True (intended) motions were 'e' -> 's' -> 'w' -> 'n', but the motions weren't that precise - // due to prioritizing speed. - const eMoveIndex = moves.findIndex((move) => move.direction.includes('e')); - const sMoveIndex = moves.findIndex((move) => move.direction.includes('s')); - const wMoveIndex = moves.findIndex((move) => move.direction.includes('w')); - const nMoveIndex = moves.findIndex((move) => move.direction.includes('n')); - - assert.notEqual(eMoveIndex, -1); - assert.notEqual(sMoveIndex, -1); - assert.notEqual(wMoveIndex, -1); - assert.notEqual(nMoveIndex, -1); - - // Again, the true (intended) motions were 'e' -> 's' -> 'w' -> 'n'. This verifies the relative ordering. - assert.isBelow(eMoveIndex, sMoveIndex); - assert.isBelow(sMoveIndex, wMoveIndex); - assert.isBelow(wMoveIndex, nMoveIndex); - - // Original's length: 12, but there are some borderline holds there & we don't want this test - // to be too rigid. - assert.isAtLeast(reproedSegments.length, 9); // 2 = 'start' + 'end' - // 2 = 1 hold after start, 1 hold before end - // 4 cardinal directions - // At least one notable hold during direction changes - }); -}); \ No newline at end of file