erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-08-10 19:05:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 19

chore(web): Merge branch 'feature-gestures' into change/web/floating-tablet-preview

Browse source
This commit is contained in:
Joshua A. Horton 2023-11-30 13:45:20 +07:00
commit aa72c8d712
10 changed files with 30 additions and 3004 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

425
common/web/gesture-recognizer/src/engine/headless/cumulativePathStats.ts
View file

@ -20,168 +20,19 @@ export type PathCoordAxisPair = 'tx' | 'ty' | 'xy';
*/
type StatAxis = PathCoordAxis | 'v';
/**
* Acts as a subtraction operation with a built-in, adaptive "significance" check.
* If the result * 2^30 (~ * 10^9) is still smaller in magnitude than an operand, we
* assume it to be a floating-point error that should have been 0 and act accordingly,
* returning 0.
*
* For reference, (32-bit) floats have 23 bits of significand precision, while (64-bit)
* doubles have 52. Therefore, we'll still be more precise than baseline floats.
*/
export function sigMinus(operand1: number, operand2: number) {
const diff = operand1 - operand2;
const magnitude = Math.max(Math.abs(operand1), Math.abs(operand2));
const logDiff = Math.log2(magnitude) - Math.log2(Math.abs(diff));
// If an operand is 2^30 (or ~10^9) larger than the result of the difference, it's
// nigh-certainly a floating-point error at play.
return logDiff < 30 ? diff : 0;
}
/**
* As the name suggests, this class facilitates tracking of cumulative mathematical values, etc
* necessary to perform the statistical operations necessary for path segmentation.
* useful for interpretation of a contact point's path as it relates to gestures.
*
* Instances of this class are immutable.
* Instances of this class may be considered immutable externally.
*
* A subclass with properties useful for path segmentation: `RegressiblePathStats`.
*/
export class CumulativePathStats<Type = any> {
/**
* Provides linear-regression statistics & fitting values based on the underlying `CumulativePathStats`
* object used to generate it. All operations are O(1).
*/
static readonly regression = class RegressionFromSums {
readonly independent: PathCoordAxis;
readonly dependent: PathCoordAxis;
readonly paired: PathCoordAxisPair;
readonly accumulator: CumulativePathStats;
/**
*
* @param mainStats The `CumulativePathStats` instance to base all regression data on.
* @param dependentAxis The 'output' axis / dimension; the axis whose behavior should be predicted based on
* existing data of its relationship with the independent axis.
* @param independentAxis The 'input' axis/dimension.
*/
constructor(mainStats: CumulativePathStats, dependentAxis: PathCoordAxis, independentAxis: PathCoordAxis) {
/* c8 ignore next 3 */
if(dependentAxis == independentAxis) {
throw new Error("Two different axes must be specified for the regression object.");
}
this.accumulator = mainStats;
this.dependent = dependentAxis;
this.independent = independentAxis;
if(dependentAxis < independentAxis) {
this.paired = dependentAxis.concat(independentAxis) as PathCoordAxisPair;
} else {
this.paired = independentAxis.concat(dependentAxis) as PathCoordAxisPair;
}
}
/**
* The 'slope' of the 'slope-intercept' form of the line that best fits the relationship between
* this regression's selected axes.
*/
get slope(): number {
// The technical definition is the commented-out line, but the denominator component of both
// cancels out - it's 'more efficient' to use the following line as a result.
// this.accumulator.covariance(this.paired) / this.accumulator.variance(this.independent);
const val = this.accumulator.crossSum(this.paired) / this.accumulator.squaredSum(this.independent);
return val;
}
/**
* The 'intercept' of the 'slope-intercept' form of the line that best fits the relationship between
* this regression's selected axes.
*/
get intercept(): number {
// Performing a regression based on these pre-summed values means that our obtained intercept is in
// the mapped coordinate system.
const mappedIntercept = this.accumulator.mappedMean(this.dependent) - this.slope * this.accumulator.mappedMean(this.independent);
const val = mappedIntercept + this.accumulator.mappingConstant(this.dependent) -
this.slope * this.accumulator.mappingConstant(this.independent);
return val;
}
/**
* The total summed squared-distances of the best fitting line from actually-observed values;
* in other words, the "sum of the squared errors".
*
* Statistically, this is the portion of the dependent variable's variance (un-normalized)
* that is unexplained by this regression.
*/
get sumOfSquaredError(): number {
return sigMinus(this.accumulator.squaredSum(this.dependent), this.sumOfSquaredModeled);
}
/**
* The portion of the dependent variable's variance that is successfully explained by this regression.
*/
get sumOfSquaredModeled(): number {
// If we have a perfectly straight vertical line, from the perspective of our independent axis,
// we get infinite slope. That's... not great for the math.
//
// Fortunately, it ALSO means that we can perfectly model the segment.
if(this.accumulator.squaredSum(this.independent) == 0) {
return this.accumulator.squaredSum(this.dependent);
} else {
return this.slope * this.accumulator.crossSum(this.paired);
}
}
/**
* A statistical term that signals how successful the regression is. Always has values on
* the interval [0, 1], with 1 being a perfect fit.
*/
get coefficientOfDetermination(): number {
if(this.accumulator.squaredSum(this.dependent) == 0 || this.accumulator.squaredSum(this.independent) == 0) {
return 1;
}
const acc = this.accumulator;
const num = acc.crossSum(this.paired) * acc.crossSum(this.paired);
const denom = acc.squaredSum(this.dependent) * acc.squaredSum(this.independent);
return num / denom;
}
/**
* Gets the value of the dependent axis that lies on the regression's fitted line
* for a specified independent axis value.
*
* @param value The input value to use for the independent axis's variable.
* @returns The predicted dependent axis value.
*/
predictFromValue(value: number) {
return this.slope * value + this.intercept;
}
}
private rawLinearSums = {'x': 0, 'y': 0, 't': 0, 'v': 0};
private rawSquaredSums = {'x': 0, 'y': 0, 't': 0, 'v': 0};
// Would 'tv' (time vs velocity) be worth it to track? And possibly even do a regression for?
// If so, maybe throw that in.
private rawCrossSums = {'tx': 0, 'ty': 0, 'xy': 0};
protected rawLinearSums = {'x': 0, 'y': 0, 't': 0, 'v': 0};
// Handles raw-distance stuff.
private coordArcSum: number = 0;
private arcSampleCount: number = 0;
// These two are kept separate because of their extreme interconnectedness - after all,
// they actually represent a SINGLE (polar) axis - the angle.
//
// Sadly, there's no straightforward, well-founded way to use these to give a proper
// statistical sense of 'fit' or 'regression' here, _especially_ in regard to segmentation.
// Proper angle-[other] cross-sums are pretty much impossible, at least as efficiently
// as the others are handled [O(1)].
private cosLinearSum: number = 0;
private sinLinearSum: number = 0;
/**
* The base sample used to transpose all other received samples. Use of this helps
@ -190,7 +41,7 @@ export class CumulativePathStats<Type = any> {
*
* Refer to https://en.wikipedia.org/wiki/Catastrophic_cancellation.
*/
private baseSample?: InputSample<Type>;
protected baseSample?: InputSample<Type>;
/**
* The initial sample included by this instance's computed stats. Needed for
@ -199,12 +50,13 @@ export class CumulativePathStats<Type = any> {
private _initialSample?: InputSample<Type>;
private _lastSample?: InputSample<Type>;
private followingSample?: InputSample<Type>;
protected followingSample?: InputSample<Type>;
private _sampleCount = 0;
constructor();
constructor(sample: InputSample<Type>);
constructor(instance: CumulativePathStats<Type>);
constructor(obj?: InputSample<Type> | CumulativePathStats<Type>)
constructor(obj?: InputSample<Type> | CumulativePathStats<Type>) {
if(!obj) {
return;
@ -215,8 +67,6 @@ export class CumulativePathStats<Type = any> {
Object.assign(this, obj);
this.rawLinearSums = {...obj.rawLinearSums};
this.rawCrossSums = {...obj.rawCrossSums};
this.rawSquaredSums = {...obj.rawSquaredSums};
} else if(isAnInputSample(obj)) {
Object.assign(this, this.extend(obj));
/* c8 ignore next 3 */
@ -233,33 +83,30 @@ export class CumulativePathStats<Type = any> {
* newly-sampled point.
*/
public extend(sample: InputSample<any>): CumulativePathStats<Type> {
if(!this._initialSample) {
this._initialSample = sample;
this.baseSample = sample;
return this._extend(new CumulativePathStats(this), sample);
}
protected _extend(result: CumulativePathStats<Type>, sample: InputSample<any>) {
if(!result._initialSample) {
result._initialSample = sample;
result.baseSample = sample;
}
const result = new CumulativePathStats(this);
const baseSample = result.baseSample;
// Set _after_ deep-copying this for the result.
this.followingSample = sample;
// Helps prevent "catastrophic cancellation" issues from floating-point computation
// for these statistical properties and properties based upon them.
const x = sample.targetX - this.baseSample.targetX;
const y = sample.targetY - this.baseSample.targetY;
const t = sample.t - this.baseSample.t;
const x = sample.targetX - baseSample.targetX;
const y = sample.targetY - baseSample.targetY;
const t = sample.t - baseSample.t;
result.rawLinearSums.x += x;
result.rawLinearSums.y += y;
result.rawLinearSums.t += t;
result.rawCrossSums.tx += t * x;
result.rawCrossSums.ty += t * y;
result.rawCrossSums.xy += x * y;
result.rawSquaredSums.x += x * x;
result.rawSquaredSums.y += y * y;
result.rawSquaredSums.t += t * t;
if(this.lastSample) {
// arc length stuff!
const xDelta = sample.targetX - this.lastSample.targetX;
@ -271,20 +118,8 @@ export class CumulativePathStats<Type = any> {
result.coordArcSum += coordArcDelta;
if(xDelta || yDelta) {
// We wish to measure angle clockwise from <0, -1> in the DOM. So, cos values should
// align with that axis, while sin values should align with the positive x-axis.
//
// This provides a mathematical 'transformation' to the axes used by `atan2` in the
// `angleMean` property.
result.cosLinearSum += -yDelta / coordArcDelta;
result.sinLinearSum += xDelta / coordArcDelta;
result.arcSampleCount += 1;
}
if(tDelta) {
result.rawLinearSums.v += coordArcDelta / tDelta;
result.rawSquaredSums.v += coordArcDeltaSq / (tDelta * tDelta);
}
}
@ -302,6 +137,12 @@ export class CumulativePathStats<Type = any> {
* @returns
*/
public deaccumulate(subsetStats?: CumulativePathStats<Type>): CumulativePathStats<Type> {
const result = new CumulativePathStats(this);
return this._deaccumulate(result, subsetStats);
}
public _deaccumulate(result: CumulativePathStats<Type>, subsetStats?: CumulativePathStats<Type>): CumulativePathStats<Type> {
// Possible addition: use `this.buildRenormalized` on the returned version
// if catastrophic cancellation effects (random, small floating point errors)
// are not sufficiently mitigated & handled by the measures currently in place.
@ -316,8 +157,6 @@ export class CumulativePathStats<Type = any> {
// the cumulative stats that follow. If any are still active, we can't
// recentralize.
const result = new CumulativePathStats(this);
// We actually WILL accept a `null` argument; makes some of the segmentation
// logic simpler.
if(!subsetStats) {
@ -336,16 +175,6 @@ export class CumulativePathStats<Type = any> {
result.rawLinearSums[d] -= subsetStats.rawLinearSums[d];
}
for(let dimPair in result.rawCrossSums) {
const d = dimPair as PathCoordAxisPair;
result.rawCrossSums[d] -= subsetStats.rawCrossSums[d];
}
for(let dim in result.rawSquaredSums) {
const d = dim as PathCoordAxis;
result.rawSquaredSums[d] -= subsetStats.rawSquaredSums[d];
}
// arc length stuff!
if(subsetStats.followingSample && subsetStats.lastSample) {
const xDelta = subsetStats.followingSample.targetX - subsetStats.lastSample.targetX;
@ -361,13 +190,8 @@ export class CumulativePathStats<Type = any> {
result.coordArcSum -= coordArcDelta;
result.coordArcSum -= subsetStats.coordArcSum;
result.cosLinearSum -= subsetStats.cosLinearSum;
result.sinLinearSum -= subsetStats.sinLinearSum;
result.arcSampleCount -= (subsetStats.arcSampleCount + 1);
if(tDelta) {
result.rawLinearSums.v -= coordArcDelta / tDelta;
result.rawSquaredSums.v -= coordArcDeltaSq / (tDelta * tDelta);
}
}
@ -412,7 +236,7 @@ export class CumulativePathStats<Type = any> {
* @param dim
* @returns
*/
private mappingConstant(dim: StatAxis) {
protected mappingConstant(dim: StatAxis) {
if(!this.baseSample) {
return undefined;
}
@ -428,16 +252,6 @@ export class CumulativePathStats<Type = any> {
}
}
/**
* Gets the statistical mean, utilizing the internal 'mapped' coordinate space.
* This is the version compatible with cross-sums and squared-sums.
* @param dim
* @returns
*/
private mappedMean(dim: StatAxis) {
return this.rawLinearSums[dim] / this.sampleCount;
}
/**
* Gets the statistical mean value of the samples observed during the represented
* interval on the specified axis.
@ -447,112 +261,7 @@ export class CumulativePathStats<Type = any> {
public mean(dim: StatAxis) {
// This external-facing version needs to provide values in 'external'-friendly
// coordinate space.
return this.mappedMean(dim) + this.mappingConstant(dim);
}
/**
* Gets the sum of the squared distance from the mean seen in samples observed
* during the represented interval on the specified axis.
* @param dim
* @returns
*/
public squaredSum(dim: StatAxis) {
const x2 = this.rawSquaredSums[dim];
const x1 = this.rawLinearSums[dim];
return sigMinus(x2, x1 * x1 / this.sampleCount);
}
/**
* Gets the sum of the statistical 'cross' term "distance" away from the mean
* observed during the represented interval on the specified axis.
* @param dimPair
* @returns
*/
public crossSum(dimPair: PathCoordAxisPair) {
const dim1 = dimPair.charAt(0) as PathCoordAxis;
const dim2 = dimPair.charAt(1) as PathCoordAxis;
let orderedDims = dimPair as PathCoordAxisPair;
const ab = this.rawCrossSums[orderedDims];
const a = this.rawLinearSums[dim1];
const b = this.rawLinearSums[dim2];
return sigMinus(ab, a * b / this.sampleCount);
}
/**
* Gets the unbiased covariance between the specified pair of axes for samples
* observed during the represented interval.
* @param dimPair
* @returns
*/
public covariance(dimPair: PathCoordAxisPair) {
return this.crossSum(dimPair) / (this.sampleCount - 1);
}
/**
* Gets the unbiased variance on the specified axis for samples observed
* during the represented interval.
*/
public variance(dim: StatAxis) {
return this.squaredSum(dim) / (this.sampleCount - 1);
}
/**
* Utilizing (and possibly abusing) statistical identities, this function produces
* an equivalent, but-recentered copy of this instance's statistical accumulations
* that will be less prone to catastrophic cancellation.
*
* In non-stats speak, the new instance will suffer smaller floating-point
* errors than the old instance whenever they do occur.
* @returns
*/
public buildRenormalized(): CumulativePathStats<Type> {
// Other (internal) notes: the internal mapping of the new instance will not
// match that of the old instance. This should not affect the practical
// results of any mapping to and from the external coordinate space, however.
let result = new CumulativePathStats(this);
let newBase: InputSample<any> = {
targetX: this.mappedMean('x') + this.baseSample.targetX,
targetY: this.mappedMean('y') + this.baseSample.targetY,
t: this.mappedMean('t') + this.baseSample.t
};
result.baseSample = newBase;
for(const dimPair in result.rawCrossSums) {
const d = dimPair as PathCoordAxisPair;
result.rawCrossSums[d] = this.crossSum(d);
}
for(const dim in result.rawSquaredSums) {
// 'v' does not need renormalization.
if(dim == 'v') {
break;
}
// The identity we're using to renormalize rawCrossSums and rawSquaredSums
// automatically guarantees a mean of 0 after the renormalization.
const d = dim as PathCoordAxis;
result.rawLinearSums[d] = 0;
result.rawSquaredSums[d] = this.squaredSum(d);
}
return result;
}
/**
* Provides a linear-regression perspective on two specified axes over the represented
* interval.
* @param dependent
* @param independent
* @returns
*/
public fitRegression(dependent: PathCoordAxis, independent: PathCoordAxis) {
return new CumulativePathStats.regression(this, dependent, independent);
return this.rawLinearSums[dim] / this.sampleCount + this.mappingConstant(dim);
}
/**
@ -645,77 +354,6 @@ export class CumulativePathStats<Type = any> {
return this.duration ? this.netDistance / this.duration : 0;
}
/**
* Returns the represented interval's 'mean angle' clockwise from the DOM's
* <0, -1> (the unit vector toward the top of the screen) in radians.
*
* Based upon the 'circular mean'. Refer to https://en.wikipedia.org/wiki/Circular_mean.
*
* Note that very slow-moving segments may be heavily affected by pixel aliasing
* effects; mouse and touch events usually do not provide sub-pixel resolution.
*/
public get angleMean() {
if(this.arcSampleCount == 0) {
return Number.NaN;
}
// Neato reference: https://rosettacode.org/wiki/Averages/Mean_angle
// But we don't actually need to divide by sample count; `atan2` handles that!
const sinMean = this.sinLinearSum;
const cosMean = this.cosLinearSum;
let angle = Math.atan2(sinMean, cosMean); // result: on the interval (-pi, pi]
// Convert to [0, 2*pi).
if(angle < 0) {
angle = angle + 2 * Math.PI;
}
return angle;
}
/**
* Provides the rSquared value needed internally for circular-statistic properties.
*
* Range: floating-point values on the interval [0, 1].
*/
private get angleRSquared() {
if(this.arcSampleCount == 0) {
return 1;
}
// Refer to https://en.wikipedia.org/wiki/Directional_statistics#Distribution_of_the_mean.
// We're computing the squared value of that page's R-bar stat.
//
// Now, why it's called that? ... good question. My best guess is that it's meant to
// correspond to linear regression's 'r' stat, which when squared serves as the
// coefficient of determination for the regression. Intuitively, that does seem to
// match what this represents - though for normal regressions, the c.o.d isn't normally
// used to compute deviation or variance!
const rSquaredBase = this.cosLinearSum * this.cosLinearSum + this.sinLinearSum * this.sinLinearSum;
return rSquaredBase / (this.arcSampleCount * this.arcSampleCount);
}
/**
* The **circular standard deviation** of the represented interval's angle observations.
*
* Refer to https://en.wikipedia.org/wiki/Directional_statistics#Standard_deviation.
*
* Note that very slow-moving segments may be heavily affected by pixel aliasing
* effects; mouse and touch events usually do not provide sub-pixel resolution.
* This can result in very high deviation values.
*
* In less-technical terms - the "stair-stepping" effect seen on high zoom levels means we don't
* get perfectly straight lines, and that can cause this value to be unexpectedly high.
*/
public get angleDeviation() {
if(this.arcSampleCount == 0) {
return Number.NaN;
}
const val = Math.sqrt(-Math.log(this.angleRSquared));
return isNaN(val) ? 0 : val;
}
/**
* Provides the actual, pixel-based distance actually traveled by the represented segment.
* May not be an integer (because diagonals are a thing).
@ -737,10 +375,7 @@ export class CumulativePathStats<Type = any> {
netDistance: this.netDistance,
duration: this.duration,
sampleCount: this.sampleCount,
angleMeanDegrees: this.angleMean * 180 / Math.PI,
angleDeviation: this.angleDeviation,
rawDistance: this.rawDistance,
speedVariance: this.variance('v')
rawDistance: this.rawDistance
}
}
/* c8 ignore end */

14
common/web/gesture-recognizer/src/engine/headless/subsegmentation/README.md
View file

@ -1,14 +0,0 @@
## About this subfolder
This folder contains code for an advanced touchpath sub-segmentation engine that would facilitate more complex gestures,
such as multi-segment flicks and/or swipe-style input. The decision was made to "ice" it for now in favor of preventing
further delays for the release of the feature.
To revisit a state when the system was fully connected, the following two commits may provide a useful reference:
- https://github.com/keymanapp/keyman/tree/0c7ac4ff3caf612674602fefb7ff71350b13964a
- The last feature-gestures commit with full subsegmentation integration
- https://github.com/keymanapp/keyman/tree/5c48cc5b9b127ab42a8055b7d960dbc4d39e4df7
- See #7440 - its description details the basic process for creation of a asynchronous FSM for recognizing & constructing gestures based on subsegments produced by the subsegmentation engine.
- The permalinked commit itself holds the code that said process would connect with.
- This was never fully committed to feature-gestures; #7440 was directly based upon the "last feature-gestures
commit..." mentioned above.

309
common/web/gesture-recognizer/src/engine/headless/subsegmentation/constructingSegment.ts
View file

@ -1,309 +0,0 @@
import { SegmentClassifier } from "../segmentClassifier.js";
import { CumulativePathStats } from "../cumulativePathStats.js";
import { SegmentationSplit, Subsegmentation } from "./pathSegmenter.js";
import { SegmentImplementation } from "./segment.js";
import { SubsegmentCompatibilityAnalyzer } from "./subsegmentCompatibilityAnalyzer.js";
/**
* This class is responsible for managing the construction of public-facing Segments while keeping
* all the internal parts... internal. As such, it includes state management for parts of
* PathSegmenter's operations. Note that it makes many assumptions based upon its usage within
* PathSegmenter.
*
* See also `SubsegmentCompatibilityAnalyzer`, which defines the criteria used within this class
* for determining when to recombine subsegments and when to uphold segmentation decisions.
*/
export class ConstructingSegment {
readonly classifier: SegmentClassifier;
/**
* Marks previously-accumulated stats on the touchpath for the portion that precedes
* the in-construction Segment.
*/
readonly baseAccumulation: CumulativePathStats;
/**
* Notes all subsegments identified as part of the overall segment being constructed.
*/
private subsegmentations: Subsegmentation[] = [];
/**
* Marks a potential follow-up subsegment. This portion is not automatically
* committed during finalization.
*/
private pendingSubsegmentation: Subsegmentation = null;
/**
* A flag that is only set once the in-construction segment is first recognized as a 'wait'.
* If set, this indicates that the current - and _only_ the current - `pendingSubsegmentation`
* may only be replaced by an update that would still be compatible if the field's current
* value were already committed. (Goal: prevent the 'locked' pending section from becoming
* not-committed in to the 'wait' role that triggered the 'lock'.)
*/
private _pendingLocked: boolean = false;
private _pathSegment: SegmentImplementation;
constructor(initialPendingSubsegment: Subsegmentation, classifier: SegmentClassifier) {
// Note: may be null! Occurs for the first processed subsegment.
this.baseAccumulation = initialPendingSubsegment.baseAccumulation;
this.classifier = classifier;
this.subsegmentations = [];
this.updatePendingSubsegment(initialPendingSubsegment);
}
/**
* Constructs a stats object for the interval ending with the specified accumulation
* state and beginning at the start of the in-construction Segment.
* @param accumulation
* @returns
*/
private buildIntervalFromBase(accumulation: CumulativePathStats) {
return accumulation.deaccumulate(this.baseAccumulation);
}
/**
* Returns the stats accumulation covering all currently-committed subsegments.
* May be `null` if there are no committed subsegments.
*/
public get committedInterval(): CumulativePathStats {
return this.committedIntervalAsSubsegmentation.stats;
}
/**
* Returns the number of already-committed subsegments that comprise the
* in-construction Segment thus far.
*/
public get subsegmentCount(): number {
return this.subsegmentations.length;
}
/**
* Returns a merged form of all currently-committed subsegments in
* Subsegmentation form.
*/
public get committedIntervalAsSubsegmentation(): Subsegmentation {
if(this.subsegmentations.length == 0) {
return {
stats: null,
endingAccumulation: null,
baseAccumulation: this.baseAccumulation
};
} else {
const tailSub = this.subsegmentations[this.subsegmentations.length - 1];
const tail = tailSub.endingAccumulation;
return {
stats: this.buildIntervalFromBase(tail),
endingAccumulation: tail,
baseAccumulation: this.baseAccumulation
};
}
}
/**
* Checks if the committed portion of the constructing segment are "compatible" with
* a potential following subsegment. Appending said subsegment may not change the
* 'type' of segment being constructed or certain properties of it.
*
* @returns `true` if compatible, `false` if not.
*/
public isCompatible(subsegmentation: Subsegmentation): boolean {
const pendingAnalyzer = this.analyzeCompatibility(subsegmentation);
return pendingAnalyzer.isCompatible;
}
/**
* Returns the analyzer instance used for the `isCompatible` check. The returned
* object provides additional helper properties useful for inspecting compatibility
* logic.
*/
public analyzeCompatibility(subsegmentation: Subsegmentation): SubsegmentCompatibilityAnalyzer {
const committed = this.committedIntervalAsSubsegmentation;
const segmentationSplit = new SegmentationSplit(committed, subsegmentation);
return new SubsegmentCompatibilityAnalyzer(segmentationSplit, this.classifier);
}
/**
* Returns the stats accumulation covering all currently-committed subsegments +
* the submitted `Subsegmentation` object (as if it were committed).
*
* Assumes that its parameter immediately follows any previously-committed
* subsegments.
*/
private wholeSegmentation(subsegmentation: Subsegmentation): Subsegmentation {
const whole: Subsegmentation = {
stats: subsegmentation.endingAccumulation.deaccumulate(this.baseAccumulation),
baseAccumulation: this.baseAccumulation,
endingAccumulation: subsegmentation.endingAccumulation
};
return whole;
}
/**
* A subsegment is pre-committed when it is required to successfully classify the in-construction
* segment after reaching the configured time threshold for segment recognition - that is,
* when previously-committed subsegments are insufficient to support the classification of a
* 'recognized' Segment.
*
* In such a case, the precommitted portion must be maintained. In the event that extending the
* subsegment (on future updates) would render it incompatible, the subsegment must be forceably
* segmented in order to commit the precommitted portion.
*/
public get hasPrecommittedSubsegment() {
return this._pendingLocked;
}
private set hasPrecommittedSubsegment(flag: boolean) {
this._pendingLocked = flag;
}
public get hasPendingSubsegment() {
return !!this.pendingSubsegmentation;
}
/**
* Call this to denote the current state of an in-construction subsegment that **will** be included
* in the final Segment once completed if the pending subsegment does not diverge in a later update.
*
* If the pending subsegment appears to belong to the current Segment, it contains valuable info
* about the state of the as-of-yet unresolved Segment, including the most recent location
* of the corresponding touchpoint.
*
* NOTE: Assumes that .isCompatible has been checked first! This method (currently) does not
* perform the related check!
* - It is currently called when and where relevant in PathSegmenter, the only thing currently
* calling this method.
*
* @param subsegmentation
* @returns `true` unless the subsegment
*/
public updatePendingSubsegment(subsegmentation: Subsegmentation): boolean {
const isFirstUpdate = !this.pendingSubsegmentation && this.subsegmentations.length == 0;
const fullStatsWithIncoming = this.wholeSegmentation(subsegmentation).stats;
// Do we have a locked pending section? If so, check to be sure that an update won't break things.
if(this.hasPrecommittedSubsegment) {
const classification = this.classifier.classifySegment(fullStatsWithIncoming);
// If the classification would change after updating the pending subsegment, block the update &
// report update failure.
if(this.pathSegment.type != classification) {
return false;
}
}
this.pendingSubsegmentation = subsegmentation;
if(isFirstUpdate) {
this.pathSegment = new SegmentImplementation();
}
this.pathSegment.updateStats(fullStatsWithIncoming);
// Check the length of time that's elapsed. If we've surpassed the recognition threshold,
// it's time to commit to classifying the in-construction Segment.
const alreadyElapsed = fullStatsWithIncoming.duration;
const recognitionWaitTime = this.classifier.config.holdMinimumDuration - alreadyElapsed;
const recognitionFromMove = (this.pathSegment.distance > this.classifier.config.holdMoveTolerance);
// `undefined` if and only if still unrecognized.
if((recognitionFromMove || recognitionWaitTime <= 0) && !this._pathSegment.isRecognized) {
const classification = this.classifier.classifySegment(fullStatsWithIncoming);
// Based on the specification for segment classification, there WILL be a classification
// assigned here. It's an implementation error if not.
if(!classification) {
throw new Error("Implementation error - segment was not properly recognized");
}
// auto-resolve the recognition promise & 'lock' the classification (and also the portion
// of the touchpath that triggered it).
this.pathSegment.classifyType(classification);
this.hasPrecommittedSubsegment = true;
}
return true;
}
/**
* Call to clear a previously-tracked in-construction subsegment that has diverged and is no
* longer compatible.
*/
public clearPendingSubsegment() {
if(this.hasPrecommittedSubsegment) {
throw new Error("Invalid state: must fully commit a subsection due to a precommitted portion!");
}
// Probably does not need to trigger an event; if called, there's a new, follow-up segment
// coming that will maintain the current coordinate with its events instead. Assuming
// the need to rely on Segment events for that; even that's better off handled with path.coords
// events instead.
this.pendingSubsegmentation = null;
// If there was a pending subsegment, we need to remove its components from the published segment stats.
this.pathSegment.updateStats(this.committedInterval);
}
/**
* Commits a currently-pending subsegment as part of the in-construction Segment.
*/
public commitPendingSubsegment() {
if(this.pendingSubsegmentation) {
this.subsegmentations.push(this.pendingSubsegmentation);
// Check the peak speed & update if needed.
const commitStats = this.pendingSubsegmentation.stats;
let peakSpeed = commitStats.mean('v') + Math.sqrt(commitStats.variance('v'));
if(peakSpeed > this.pathSegment.peakSpeed) {
this.pathSegment.setPeakSpeed(peakSpeed);
}
// Clear 'pending'-related fields.
this.pendingSubsegmentation = null;
this.hasPrecommittedSubsegment = false;
// Recognition check!
if(!this.pathSegment.isRecognized) {
const classification = this.classifier.classifySegment(this.committedInterval);
if(classification) {
this.pathSegment.classifyType(classification);
}
}
} else {
throw new Error("Illegal state - `commitPendingPortion` should never be called with nothing pending.");
}
}
/**
* Finalizes the Segment, committing any pending portions, classifying ('recognizing') it if
* necessary and resolving it.
*/
public finalize() {
if(this.pendingSubsegmentation) {
this.commitPendingSubsegment();
}
// Fully 'recognize' the Segment if it somehow hasn't yet been recognized.
if(!this.pathSegment.isRecognized) {
this.pathSegment.classifyType(this.classifier.classifySegment(this.committedInterval));
}
this.pathSegment.resolve();
}
/**
* The in-construction Segment, as published to `GesturePath.segments` & `GesturePath`'s
* 'segmentation' event.
*/
public get pathSegment() {
return this._pathSegment;
}
private set pathSegment(segment: SegmentImplementation) {
this._pathSegment = segment;
}
}

983
common/web/gesture-recognizer/src/engine/headless/subsegmentation/pathSegmenter.ts
View file

@ -1,983 +0,0 @@
import { ConstructingSegment } from "./constructingSegment.js";
import { CumulativePathStats, PathCoordAxis, sigMinus } from "../cumulativePathStats.js";
import { InputSample } from "../inputSample.js";
import { Segment, SegmentImplementation } from "./segment.js";
import { SegmentClass, SegmentClassifier, SegmentClassifierConfig } from "../segmentClassifier.js";
// Mostly used here to compare the sum-squared error components of segmented regressions
// to the sum-squared "modeled" components of their overall variance. Those are the two
// "independent random variables" we're examining. Variances (which are sums of squared
// values themselves) tend to be chi-squared distributed, fulfilling the conditions.
// That's stats-speak for "what this module does with it has a solid theoretical basis."
//
// We do NOT want to be computing this thing at run-time. (Just take one look at the
// equations found in that Wikipedia article!) Fortunately, it's very common in stats
// circles to just... use a lookup table for finding p-values for things following
// the f-distribution.
//
// For future non-stats-inclined maintainers:
// One could say that the p-value = the chance that what we have just "randomly" happened
// to occur without our expectations actually being met. It's a super-common
// stats term; they'd then say that if the p-value is sufficiently low, then we\
// "reject the null hypothesis" - the theory that it actually DID happen randomly - in
// favor of the high likelihood that we really are onto something.
/**
* Acts as a lookup table for null-hypothesis tests utilizing the F-distribution from the
* domain of statistics. Refer to https://en.wikipedia.org/wiki/F-distribution for details
* on its properties.
*
* This is only a partial implementation; only a couple of numerator degrees-of-freedom are
* worth our consideration, and there's only marginal gain to be found supporting more
* denominator degrees-of-freedom than are already supported here. Keeping it small also
* helps with maintenance and readability.
*/
class FDistribution {
/**
* Indexing: [threshold-index][num-2][denom-1].
*
* If numerator has only 1 degree of freedom, that means we don't really have
* segments so much as different averages between the two segments - there's
* no slope on either side. In other words, "just a bump". No segmentation
* there.
*
* 2 degrees of freedom: no movement on the axis on only one side.
*
* threshold-index:
* - p-value of .100: 0
* - p-value of .050: 1
*
* The f-statistic value must match or exceed its corresponding entry in
* the table below, based on the detected DoF (degrees of freedom) for the
* 'numerator' and 'denominator' components. Exceeding the threshold specified
* for the p = .100 section indicates less than a 10% probability of the statistic
* being tested having arisen by purely random chance. Failure to exceed even
* that implies an unacceptably high chance of being "convenient but meaningless" -
* the "null hypothesis", in stats speak.
*/
private static readonly table = [
// p = .100
[
// numerator: 2
[
// denom: 1-10
49.50, 9.00, 5.46, 4.32, 3.78, 3.46, 3.26, 3.11, 3.01, 2.92,
// denom: 11-20
2.86, 2.81, 2.76, 2.73, 2.70, 2.67, 2.64, 2.62, 2.61, 2.59
// we COULD continue with threshold-dropping after that,
// but I doubt we'll need it in practicality.
// At 30: 2.49
// limit -> infinity: 2.30
],
// numerator: 3
[
// denom: 1-10
53.59, 9.16, 5.39, 4.19, 3.62, 3.29, 3.07, 2.92, 2.81, 2.73,
// denom: 11-20
2.66, 2.61, 2.56, 2.52, 2.49, 2.46, 2.44, 2.42, 2.40, 2.38
// Likewise, from above.
// At 30: 2.28
// limit -> infinity: 2.08
]
],
// p = .050
[
// numerator: 2
[
// denom: 1-10
199.5, 19.0, 9.55, 6.94, 5.79, 5.14, 4.74, 4.46, 4.26, 4.10,
// denom: 10-20
3.98, 3.89, 3.81, 3.74, 3.68, 3.63, 3.59, 3.55, 3.52, 3.49
// At 30: 3.32
// limit -> infinity: 3.00
],
// numerator: 3
[
// denom: 1-10
215.7,19.16, 9.28, 6.59, 5.41, 4.76, 4.35, 4.07, 3.86, 3.71,
// denom: 10-20
3.59, 3.49, 3.41, 3.34, 3.29, 3.24, 3.20, 3.16, 3.13, 3.10
// At 30: 2.92
// limit -> infinity: 2.60
]
]
]
/**
* Determines a threshold tier for segmentation based on the segmented
* regression f-statistic.
* @param statistic
* @param numDoF
* @param denomDoF
*
* Returns 0.050 if the statistic indicates a p-value of 0.050 or lower.
* Failing that, returns 0.100 if a p-value of 0.100 or lower is indicated.
* Otherwise, returns 1.0 - we have no basis to "reject the null hypothesis".
*/
static thresholdTier(statistic: number, numDoF: number, denomDoF: number) {
// Former case: we'd never segment anyway
// Latter case: it's currently wrong to statistically test.
// The F-distribution is not defined for this case.
if(numDoF < 2 || denomDoF < 1) {
return 1;
}
const numIndex = (numDoF > 3 ? 3 : numDoF) - 2;
const denomIndex = (denomDoF > 20 ? 20 : denomDoF) - 1;
const tier2Threshold = FDistribution.table[1][numIndex][denomIndex];
if(statistic >= tier2Threshold) {
return 0.05;
}
const tier1Threshold = FDistribution.table[0][numIndex][denomIndex];
// If arising purely randomly isn't at least less than 10% likely,
// we'll categorically say it happened randomly with full certainty (1).
// Obviously not actually true, but it works for our thresholding.
return statistic >= tier1Threshold ? 0.10 : 1;
}
}
/**
* Represents a "subsegmentation" - one part of a phase 1 segmentation split.
*/
export interface Subsegmentation {
/**
* The stats for observations comprising the subsegment.
*/
stats: CumulativePathStats;
/**
* The cumulative stats up to the subsegment's endpoint, including
* accumulated components that precede the subsegment entirely.
*/
endingAccumulation: CumulativePathStats;
/**
* The cumulative stats up to the point before the subsegment's start
* point, not including any accumulation for components within the subsegment.
*/
baseAccumulation: CumulativePathStats;
}
/**
* Represents the result of a segmentation of the search path and the related
* statistical accumulations needed to properly test the segmentation's
* validity.
*/
export class SegmentationSplit {
public static readonly SPLIT_CRITERION_THRESHOLD = 1.5;
/**
* The properties of the "left-hand" / earlier half of the time interval
* being segmented.
*/
readonly pre: Subsegmentation;
/**
* The properties of the "right-hand" / later half of the time interval
* being segmented.
*/
readonly post: Subsegmentation;
/**
* The properties of the full interval being segmented, before the split.
*/
readonly union: CumulativePathStats;
/**
* Provides segmented-regression statistics & fitting values on the two specified axes /
* dimensions based on the subsegments and their corresponding linear-regression
* statistics. All operations are O(1).
*/
static readonly segmentationComparison = class SegmentedRegression {
host: SegmentationSplit;
readonly independent: PathCoordAxis;
readonly dependent: PathCoordAxis;
readonly paired: 'tx' | 'ty' | 'xy';
/**
* The linear regression for the underlying `pre` (earlier) subsegment.
*/
pre: typeof CumulativePathStats.regression.prototype;
/**
* The linear regression for the underlying `post` (later) subsegment.
*/
post: typeof CumulativePathStats.regression.prototype;
/**
* The linear, unsegmented regression for the underlying `union` (combined) subsegment.
*/
union: typeof CumulativePathStats.regression.prototype;
constructor(host: SegmentationSplit, dependentAxis: PathCoordAxis, independentAxis: PathCoordAxis) {
if(dependentAxis == independentAxis) {
throw new Error("Two different axes must be specified for the regression object.");
}
this.host = host;
this.dependent = dependentAxis;
this.independent = independentAxis;
if(dependentAxis < independentAxis) {
this.paired = dependentAxis.concat(independentAxis) as 'tx' | 'ty' | 'xy';
} else {
this.paired = independentAxis.concat(dependentAxis) as 'tx' | 'ty' | 'xy';
}
this.pre = this.host.pre .stats.fitRegression(dependentAxis, independentAxis);
this.post = this.host.post .stats.fitRegression(dependentAxis, independentAxis);
this.union = this.host.union.fitRegression(dependentAxis, independentAxis);
}
/**
* The total summed squared-distances of the best fitting line from actually-observed values
* for their corresponding subsegment; in other words, the "sum of the squared errors" when
* utilizing segmented regression.
*
* Statistically, this is the portion of the dependent variable's variance (un-normalized)
* that is unexplained even after dividing the interval into separate subsegments.
*/
get remainingSumSquaredError(): number {
return this.pre.sumOfSquaredError + this.post.sumOfSquaredError;
}
/**
* Provides the coordinate that belongs to BOTH subsegments, as it ends one and begins
* the other.
*/
get splitPoint() {
let splitPoint = this.host.pre.stats.lastSample;
let indep = splitPoint.t;
if(this.independent == 'x') {
indep = splitPoint.targetX;
} else if(this.independent == 'y') {
indep = splitPoint.targetY;
}
let dep = splitPoint.t;
if(this.dependent == 'x') {
dep = splitPoint.targetX;
} else if(this.dependent == 'y') {
dep = splitPoint.targetY;
}
return {
dep: dep,
indep: indep
};
}
/**
* The total summed squared-distances of the best fitting line from actually-observed values
* when **not** utilizing segmented regression. Double-counts the subsegment's common point, as
* it is counted within both subsegments' sum-of-squared error term.
*/
get unsegmentedSumSquaredError(): number {
// What was our remaining error for the unsegmented regression?
let baseSSE = this.union.sumOfSquaredError;
// We double-count the split point when segmenting, so we should add an
// extra copy of its residual.
const splitPoint = this.splitPoint;
const splitPointError = this.splitPoint.dep - this.union.predictFromValue(splitPoint.indep);
// This is the total sum-squared error to be handled by segmenting.
return baseSSE + splitPointError * splitPointError;
}
/**
* Gets the amount of (unnormalized) variance not explained by a standard regression on
* the full interval but succesfully explained by splitting the interval in twain via
* segmented regression.
*/
get sumSquaredGainFromSegmentation(): number {
return sigMinus(this.unsegmentedSumSquaredError, this.remainingSumSquaredError);
}
/**
* A statistical term that signals how successful the segmented regression is. Always
* has values on the interval [0, 1], with 1 being a perfect fit.
*/
get coefficientOfDetermination(): number {
if(this.host.union.squaredSum(this.dependent) == 0 || this.host.union.squaredSum(this.independent) == 0) {
return 1;
}
return 1 - this.remainingSumSquaredError / this.host.union.squaredSum(this.dependent);
}
/**
* Gets the raw statistic value needed to test for validity of segmentation based on the axes
* under examination. The higher the proportion of newly-explained variance to still-unexplained
* variance is, the more confident we can _formally_ be in our segmentation.
*/
get fStat(): number {
const val = this.sumSquaredGainFromSegmentation / this.remainingSumSquaredError;
// We're fine with Infinity. Just... not so much NaN.
return isNaN(val) ? 0 : val;
}
/**
* Gets the degrees-of-freedom to be used for the numerator DoF parameter of
* the F-distribution needed for validity testing.
*/
get fDoF1(): number {
let numDoF = 3;
// Cases where there clearly may as well be 'no slope' at all.
// This saves us a degree of freedom, which is VERY useful for segmenting
// the boundary between a 'hold' and a 'move'.
if(this.host.pre.stats.squaredSum(this.dependent) < 1e-8) {
numDoF--;
}
if(this.host.post.stats.squaredSum(this.dependent) < 1e-8) {
numDoF--;
}
return numDoF;
}
/**
* Gets the degrees-of-freedom to be used for the denominator DoF parameter of
* the F-distribution needed for validity testing.
*/
get fDoF2(): number {
return this.host.union.sampleCount - 2 - this.fDoF1;
}
/**
* States a statistically-founded level of confidence we may place in upholding
* the represented segmentation, as based on the regressions and axes under examination.
*
* If we cannot hold any confidence whatsoever (due to being unable to "reject the
* null hypothesis" from this specific regression), will return 0.
*/
get certaintyThreshold() {
return 1 - FDistribution.thresholdTier(this.fStat, this.fDoF1, this.fDoF2);
}
get prettyPrint(): string {
return `F_(${this.fDoF1}, ${this.fDoF2}) = ${this.fStat} @ ${this.certaintyThreshold}`
}
}
constructor(pre: Subsegmentation,
post: Subsegmentation) {
this.pre = pre;
this.post = post;
this.union = post.endingAccumulation.deaccumulate(pre.baseAccumulation);
}
static fromTrackedStats(steppedStats: CumulativePathStats[],
baseAccumulation: CumulativePathStats,
splitIndex: number) {
let intervalEndStats = steppedStats[splitIndex];
const pre: Subsegmentation = {
stats: intervalEndStats.deaccumulate(baseAccumulation),
endingAccumulation: intervalEndStats,
baseAccumulation: baseAccumulation
};
// Keep stats value components based on the final point of the 'pre' segment.
intervalEndStats = steppedStats[steppedStats.length-1];
const postBaseAccumulation = steppedStats[splitIndex-1];
const post: Subsegmentation = {
stats: intervalEndStats.deaccumulate(postBaseAccumulation),
endingAccumulation: intervalEndStats,
baseAccumulation: postBaseAccumulation
}
return new SegmentationSplit(pre, post);
}
/**
* Provides a segmented-regression perspective for the represented interval and proposed
* split point based upon the two specified axes.
* @param dependent
* @param independent
* @returns
*/
public segReg(dependentAxis: PathCoordAxis, independentAxis: PathCoordAxis) {
return new SegmentationSplit.segmentationComparison(this, dependentAxis, independentAxis);
}
/**
* Defines our test for upholding a minimum of sub-segmentation based upon changes in the
* interval's coordinates over time. This may occur even with no change in spacial direction
* if there is significant enough change in _speed_, as that "curves the line" when one axis
* is time.
*/
get segmentationMerited(): boolean {
let totalThreshold = 0;
const xTest = new SegmentationSplit.segmentationComparison(this, 'x', 't');
const yTest = new SegmentationSplit.segmentationComparison(this, 'y', 't');
// Our testing thresholds are for p=0.05 and p=0.10, which correspond to certainties of 95% and
// 90% that our segmentation did not arrive from random chance based on the axis being tested.
const p05 = 0.95; // 1 - 0.05
const p10 = 0.9; // 1 - 0.10
totalThreshold += xTest.certaintyThreshold >= p05 ? 2 : (xTest.certaintyThreshold >= p10 ? 1 : 0) ;
totalThreshold += yTest.certaintyThreshold >= p05 ? 2 : (yTest.certaintyThreshold >= p10 ? 1 : 0) ;
return totalThreshold >= 2;
}
/**
* Defines our test for considering two (or more) sub-segments as part of the same overall
* segment. This seeks to 'link' geometrically-related subsegments - especially those that
* maintain the same direction but differ only in observed speed.
*/
get mergeMerited(): boolean {
// Because of caret-like motions (as in, in the '^' shape), we need to text for
// regression on both axes. One may have notably higher variance than the other.
//
// These tests ignore time, and therefore speed. Only the raw geometry of the motion
// matters for this test.
const xTest = new SegmentationSplit.segmentationComparison(this, 'x', 'y');
const yTest = new SegmentationSplit.segmentationComparison(this, 'y', 'x');
// If we don't get a p-value less than .100, then as far as x & y are concerned - and thus the user
// is concerned - it's the same segment. Speed may be different, but not the direction.
if(xTest.certaintyThreshold > 0) {
return false;
}
return yTest.certaintyThreshold == 0;
}
public _debugLogSplitReport(demarcate?: boolean) {
if(demarcate) {
console.log("------------------------------------------------------------------");
}
console.log("Split object: ");
console.log(this);
console.log();
const xF = this.segReg('x', 't');
const yF = this.segReg('y', 't');
console.log(`x F-test: ${xF.prettyPrint}`);
console.log(`y F-test: ${yF.prettyPrint}`);
console.log();
if(demarcate) {
console.log("------------------------------------------------------------------");
}
}
public _debugLogAlignmentReport() {
console.log("Desegmentation under consideration: ");
console.log(this);
console.log();
const xF = this.segReg('x', 'y');
const yF = this.segReg('y', 'x');
console.log(`merger F-test (xy): ${xF.prettyPrint}`);
console.log(`merger F-test (yx): ${yF.prettyPrint}`);
console.log(`will remerge: ${this.mergeMerited}`);
}
}
export interface SegmentationConfiguration extends SegmentClassifierConfig {
// See `PathSegmenter.segmentationConfig`'s comment; we may wish to
// define & provide extra configuration parameters here... or wherever
// this type's final location ends up.
}
/**
* The core logic, algorithm, and manager for touchpath segmentation and
* subsegmentation.
*
* This class itself directly handles 'subsegmentation', as the first pass of
* our algorithm errs on the side of segmenting too much in order to never
* 'undersegment'. It then delegates to other classes to recombine the
* results as appropriate before producing completed `Segment`s.
*/
export class PathSegmenter {
/**
* The minimum amount of time (in ms) to wait between sample repetitions
* once inputs stop arriving, so long as the path is still active.
*/
private readonly REPEAT_INTERVAL = 33;
/**
* The time-interval length (in ms) at the end of the path to consider
* when determining whether or not to trigger path segmentation.
*/
private readonly SLIDING_WINDOW_INTERVAL = 50;
/**
* Tracks the mathematical values used to provide path segment stats
* at each point on the path. Individual steps may be removed (and
* batched into `choppedStats`) once their respective points on the
* path have been fully processed.
*/
private steppedCumulativeStats: CumulativePathStats[];
/**
* Tracks the segment currently under construction.
*
* May also model an "empty" tail on the touchpath in order to track
* which part of the path's accumulation is no longer under consideration.
*/
private constructingSegment: ConstructingSegment;
/**
* Used to 'repeat' the most-recently observed incoming sample if no
* other replaces it before it triggers.
*
* A repeating sample indicates lack of motion, which is valuable
* information for stats-based segmentation.
*/
private repeatTimer: number | NodeJS.Timeout;
/**
* The timestamp of observation of the most recently observed sample's
* most recent repetition - even if it's only the first evaluation.
*/
private repeatTimestamp: number;
/**
* Represents the cumulative statistics of all points on the path
* that lie on already-fully-segmented parts of it.
*/
private choppedStats: CumulativePathStats = null;
/**
* A closure used to 'forward' generated Segments, generally to their public-facing
* location on GesturePath.segments.
*/
private readonly segmentForwarder: (segment: Segment) => void;
/**
* Denotes whether or not a first touchpath sample has been provided.
*/
private hasStarted: boolean = false;
// For consideration: should REPEAT_INTERVAL, SLIDING_WINDOW_INTERVAL, and other
// related values be defined here?
//
// Note:
// - hardcoded 2 * SLIDING_WINDOW_INTERVAL: minimum interval required for a
// subsegmentation attempt (to ensure sufficient observations on each side)
// - hardcoded SLIDING_WINDOW_INTERVAL / 2: minimum interval that must
// remain on each side after the sliding "optimum split point" search.
//
// NOTE: this will likely (eventually) be defined elsewhere, at a "higher level"
// within this module / package.
private segmentationConfig: SegmentationConfiguration;
public static readonly DEFAULT_CONFIG: SegmentationConfiguration = {
holdMinimumDuration: 100,
holdMoveTolerance: 5
}
constructor(segmentationConfig: SegmentationConfiguration, segmentForwarder: (segment: Segment) => void) {
this.steppedCumulativeStats = [];
this.segmentForwarder = segmentForwarder;
this.segmentationConfig = segmentationConfig;
}
/**
* Appends a new coordinate to the touch path and also kick-starts a timer for replicating it
* should neither further inputs be received nor termination of the touchpoint be indicated.
* @param sample
*/
public add(sample: InputSample<any>) {
// If this is the first received input sample, generate & publish a "start" segment.
// As ConstructingSegment is designed to work with -sequences- of samples, it's less
// useful here... and unnecessary, as we already have all the info we need.
if(!this.hasStarted) {
this.hasStarted = true;
const startSegment = new SegmentImplementation();
startSegment.updateStats(new CumulativePathStats(sample));
startSegment.classifyType(SegmentClass.START);
startSegment.resolve();
this.segmentForwarder(startSegment);
}
// Set up the input-repeater (in case we don't get further feedback but remain active)
const repeater = (timeDelta: number) => {
this.observe(sample, timeDelta);
}
// If we previously set up an input-repeater, cancel it. We've got a more up-to-date
// coordinate on the touchpath now.
if(this.repeatTimer) {
// @ts-ignore
clearInterval(this.repeatTimer);
this.repeatTimer = null;
}
this.repeatTimer = setInterval(() => {
const timeDelta = Date.now() - this.repeatTimestamp;
repeater(timeDelta);
}, this.REPEAT_INTERVAL);
this.repeatTimestamp = Date.now();
repeater(0);
}
/**
* Used to finalize all segmentation for the touchpath whenever termination of the corresponding
* touchpoint has been terminated (mouse-up, touch-end).
*/
public close() {
// The Node clearTimeout & DOM clearTimeout appear to TS as overloads of each other,
// and their type definitions will conflict. A simple @ts-ignore will bypass this issue.
// @ts-ignore
clearInterval(this.repeatTimer); // Cancels the input-repeater.
this.repeatTimer = null;
if(this.steppedCumulativeStats.length == 0) {
return;
}
// Make sure that the final part of the touchpath is given a subsegment & then handled.
const finalAccumulation = this.steppedCumulativeStats[this.steppedCumulativeStats.length - 1]
let finalSubsegment: Subsegmentation = {
stats: finalAccumulation.deaccumulate(this.choppedStats),
endingAccumulation: finalAccumulation,
baseAccumulation: this.choppedStats
}
// No need to check if this matches any predecessor subsegments; that already happened during
// the last `performSubsegmentation` call. There's no new data since then.
// (Actually... this should already be in place now, after recent changes!)
this.constructingSegment?.updatePendingSubsegment(finalSubsegment);
this.finalizeSegment(); // also commits pending subsegment
// Using the last-received sample, generate & publish an "end" segment.
// As ConstructingSegment is designed to work with -sequences- of samples, it's less
// useful here... and unnecessary, as we already have all the info we need.
const endSegment = new SegmentImplementation();
endSegment.updateStats(new CumulativePathStats(finalAccumulation.lastSample));
endSegment.classifyType(SegmentClass.END);
endSegment.resolve();
this.segmentForwarder(endSegment);
}
/**
* Adds a statistic 'observation' of the state of the touchpath.
*
* This is either to be called directly upon reception of a new input-coordinate or
* upon replication of a prior input should the touchpath still be active without any
* indication of motion.
* @param sample
* @param timeDelta
*/
private observe(sample: InputSample<any>, timeDelta: number) {
let cumulativeStats: CumulativePathStats;
if(this.steppedCumulativeStats.length) {
cumulativeStats = this.steppedCumulativeStats[this.steppedCumulativeStats.length-1];
} else {
cumulativeStats = new CumulativePathStats();
}
sample = {... sample, t: sample.t + timeDelta};
const extendedStats = cumulativeStats.extend(sample);
this.steppedCumulativeStats.push(extendedStats);
this.performSubsegmentation();
}
// The "reported via event" aspect mentioned below is necessary because this may be
// called via setTimeout callback on a held touchpoint. We need that `setTimeout`
// (which replicates the most recently-seen input coordinate) in order to have good
// sample-data for detecting the boundary between motion and lack thereof during
// segmentation.
/**
* This is the "main method" for touchpath segmentation. Should a new segment result
* from its analysis, it will be reported via event.
* @returns
*/
private performSubsegmentation() {
const cumulativeStats = this.steppedCumulativeStats[this.steppedCumulativeStats.length - 1];
const unsegmentedDuration = cumulativeStats.lastTimestamp - this.steppedCumulativeStats[0].lastTimestamp;
const unsegmentedForm: Subsegmentation = {
stats: cumulativeStats.deaccumulate(this.choppedStats),
endingAccumulation: cumulativeStats,
baseAccumulation: this.choppedStats
}
// STEP 1: Determine the range of the initial sliding time window for the most recent samples.
if(unsegmentedDuration < this.SLIDING_WINDOW_INTERVAL * 2) {
// If it returns false, that only makes the interval _even shorter_.
if(!this.updateSegmentConstruction(unsegmentedForm)) {
// Updates our internal state tracking; no infinite loop will result.
this.performSubsegmentation();
// return will automatically happen via fall-through.
}
return;
}
let splitPoint = 0;
// Do not consider the just-added `extendedStats` entry when building the sliding
// time window.
for(let i = this.steppedCumulativeStats.length-2; i >=0; i--) {
if(this.steppedCumulativeStats[i].lastTimestamp < cumulativeStats.lastTimestamp - this.SLIDING_WINDOW_INTERVAL) {
splitPoint = i+1;
break;
}
}
// We split the cumulative stats on a specific point, which then resides on the edge
// of both of the resulting intervals. This completes "step 1".
let candidateSplit = SegmentationSplit.fromTrackedStats(this.steppedCumulativeStats, this.choppedStats, splitPoint);
// STEP 2: given the proposed time window, do we have a basis for segmentation? And is there a better
// candidate split point nearby?
// We either have the conditions to trigger segmentation or just became long enough to consider it.
// If we're only just long enough to consider it, there may be a better segmentation point to start with.
// Now... is there a better segmentation point? But first... how do we facilitate searching for one?
// Start: split-point search helper local-class.
class SplitSearchState {
readonly xTest: typeof SegmentationSplit.segmentationComparison.prototype;
readonly yTest: typeof SegmentationSplit.segmentationComparison.prototype;
readonly candidate: SegmentationSplit;
constructor(candidate?: SegmentationSplit) {
this.candidate = candidate;
if(candidate == null) {
return;
}
this.xTest = candidate.segReg('x', 't');
this.yTest = candidate.segReg('y', 't');
}
// The value for an 'objective function' to optimize in our search for a better candidate.
// The higher this is, the more we like its potential as the segmentation point.
get segRating() {
if(this.candidate) {
return Math.max(this.xTest.coefficientOfDetermination, this.yTest.coefficientOfDetermination);
} else {
return 0;
}
}
// But, if it turns out this potential point wouldn't actually result in segmentation, well...
// "abandon ship".
get segmentationMerited() {
return this.candidate?.segmentationMerited ?? false;
}
}
// Start: split-point search
// NOTE: During initial development, I actually had a hard rule about not even searching if segmentation
// failed on the initially-constructed sliding window. Some analysis may be wise here, as it'd be nice
// to optimize away the 'need to search' if we can find hard & fast rules about when we'll never need
// to go looking. But... being too aggressive can cause nasty problems.
/* We'll use our initial sliding window as a starting point. It's close to the active location
* of the touch, so we're unlikely to miss a proper segmentation point if it lies close.
* We'll find the local maximum (best split point for the local part of the path)
* via gradient descent.
*
* Note: binary search is a 'bad idea', as the touchpath is likely not following a
* monotonous path, mathematically speaking. (It'd be possible to overshoot a "peak" or
* "valley" quite easily.)
*/
let currentSplit = new SplitSearchState(candidateSplit);
let leftSplit = new SplitSearchState(SegmentationSplit.fromTrackedStats(this.steppedCumulativeStats, this.choppedStats, splitPoint-1));
let rightCandidate: SegmentationSplit = null;
if(splitPoint+1 < this.steppedCumulativeStats.length) {
rightCandidate = SegmentationSplit.fromTrackedStats(this.steppedCumulativeStats, this.choppedStats, splitPoint+1);
}
let rightSplit = new SplitSearchState(rightCandidate);
// Step 2a: detect which direction gives the best improvement in segmentation potential.
const criteria = [leftSplit.segRating, currentSplit.segRating, rightSplit.segRating];
let sortedCriteria = [].concat(criteria).sort();
const delta = criteria.indexOf(sortedCriteria[2])-1; // -1 if 'left' is best, 1 if 'right' is best.
// Step 2b: we've found the direction: go searching!
if(delta != 0) {
// We can get better segmentation by shifting. Proceed in the optimal direction.
do {
const nextSplitIndex = splitPoint + delta;
if(nextSplitIndex >= this.steppedCumulativeStats.length || nextSplitIndex < 0) {
break;
}
let nextCandidate = SegmentationSplit.fromTrackedStats(this.steppedCumulativeStats, this.choppedStats, splitPoint + delta);
let nextSplit = new SplitSearchState(nextCandidate);
// Prevent overly-short intervals / over-segmentation.
if(nextCandidate.pre.stats.duration < this.SLIDING_WINDOW_INTERVAL / 2) {
break;
} else if(nextCandidate.post.stats.duration < this.SLIDING_WINDOW_INTERVAL / 2) {
break;
}
// Not an improvement? Guess we found the best spot.
if(nextSplit.segRating <= currentSplit.segRating) {
break;
} else if(!nextSplit.segmentationMerited && currentSplit.segmentationMerited) {
// Note: this can happen if segmentation is triggered due to divergence on both axes
// if one of them drops a threshold tier and the other fails to improve sufficiently.
break;
} else {
splitPoint += delta;
currentSplit = nextSplit;
candidateSplit = nextCandidate;
}
// If we found a new best segmentation point, we then ask if we can get even better by shifting further.
} while(true);
}
// Step 2 complete.
// Step 3: evaluate validity of the left-hand subsegment reasonably continuing the in-construction segment.
// (In sort, subsegment "compatibility" with what came before.)
// A classic example case: same direction, different speeds.
//
if(!this.updateSegmentConstruction(candidateSplit.pre)) {
// It's quite possible that segmentation is possible in the leftover section post-reversion.
// There is a pretty good chance that it'll instantly abort, but no guarantee.
this.performSubsegmentation();
return;
}
// First phase of segmentation: complete!
// Step 4: basic bookkeeping.
/*
* NOTE: this marks a very good location to call _debugLogSplitReport() if a deep-dive
* inspection of the subsegmentation algorithm and its decisions is needed.
*/
if(!candidateSplit.segmentationMerited) {
if(!this.updateSegmentConstruction(unsegmentedForm)) {
this.performSubsegmentation();
//return will happen via fall-through here.
}
return;
}
// A successful update will ensure a valid .constructingSegment instance exists.
this.commitSubsegmentation();
// Step 5: now that subsegmentation & its bookkeeping is done... process the implications.
// Does the right-hand (still-constructing) subsegment appear reasonable to
// 'link' with its predecessors? If not, finalize the predecessors.
// As this occurs after a `commitPending`, there is currently no pending subsegment.
// As such, this update will either:
// - initialize a new pending subsegment for the in-construction Segment if compatible
// - or finalize the Segment if incompatible, then use this subsegment to start a new Segment.
// As a result, this update call will always succeed; there's no prior pending state that may be reverted to.
this.updateSegmentConstruction(candidateSplit.post);
}
private finalizeSegment() {
if(this.constructingSegment) {
if(this.constructingSegment.subsegmentCount == 0 && !this.constructingSegment.hasPendingSubsegment) {
throw new Error("Implementation error!");
}
this.constructingSegment.finalize();
/*
* NOTE: if a deep-dive investigation is needed, it may prove helpful to emit each finalized
* `constructingSegment` instance to the console here. Like, _**precisely**_ here, immediately
* after this multiline comment.
*
* From there, note that in many modern browsers, you can right-click a logged object and say
* to "Store object as global variable", giving you console access to any such logged instances.
*
* `SubsegmentCompatibilityAnalyzer` is designed for ease-of-use with the members of
* `ConstructingSegment.subsegmentations` via `SegmentationSplit`, facilitating interactive
* inspection of which subsegments are and are not recombined into `Segment`s and why.
*/
this.constructingSegment = null;
}
}
/**
* Updates the in-construction Segment with the specified subsegment's accumulation
* data. If no Segment is currently under construction, it also creates a new one.
*
* In the case that an update must be blocked due to a "pending commit" (from having
* recognized the Segment while depending on the currently-constructing subsegment),
* it will return `false` to signal that it has overriden the subsegmentation with
* the most recently-valid prior version, committed that, and that the algorithm's
* current analysis is no longer valid.
* @param subsegment
* @returns `false` if the caller should restart due to forced change of segmentation
* state.
*/
private updateSegmentConstruction(subsegment: Subsegmentation): boolean {
let updateFlag: boolean;
if(this.constructingSegment) {
if(!this.constructingSegment.isCompatible(subsegment)) {
// unknown: is pending locked or not?
updateFlag = false;
} else {
// Note that this call may semi-silently precommit the subsegment if this update is the
// first to surpass the configured segment recognition timer threshold.
//
// Alternatively, if updating the pending subsegment results in incompatibility while
// a pre-existing precomitted version does not, the update will fail.
updateFlag = this.constructingSegment.updatePendingSubsegment(subsegment);
if(updateFlag) {
return true;
}
}
}
// If the segment under construction was registered as a 'hold' segment in the
// middle of the subsegment, we now require the 'pending subsegment' to maintain
// the same type. But, reaching here means it's no longer compatible - so we
// use the previously-registered subsegmentation here instead.
if(updateFlag === false && this.constructingSegment.hasPrecommittedSubsegment) {
this.commitSubsegmentation();
// Signal our caller to refresh itself and restart segmentation for the round.
// Kinda dirty, but it's 100% internal to this class, at least.
return false;
}
this.constructingSegment?.clearPendingSubsegment();
this.finalizeSegment();
this.constructingSegment = new ConstructingSegment(subsegment, new SegmentClassifier(this.segmentationConfig));
this.segmentForwarder(this.constructingSegment.pathSegment);
return true;
}
private commitSubsegmentation() {
this.constructingSegment.commitPendingSubsegment();
// Based on what we just committed, we can find the split point that led to the subsegmentation commit:
const splitPointAccumulation = this.constructingSegment.committedIntervalAsSubsegmentation.endingAccumulation;
const splitPoint = this.steppedCumulativeStats.indexOf(splitPointAccumulation);
// And given that split point, we can maintain our internal state accordingly.
// The exact point of the split is duplicated; we remove accumulation from everything before it.
this.choppedStats = this.steppedCumulativeStats[splitPoint-1];
this.steppedCumulativeStats = this.steppedCumulativeStats.slice(splitPoint);
}
}

287
common/web/gesture-recognizer/src/engine/headless/subsegmentation/segment.ts
View file

@ -1,287 +0,0 @@
import { CumulativePathStats } from "../cumulativePathStats.js";
import { InputSample } from "../inputSample.js";
import { type SegmentClass } from "../segmentClassifier.js";
export interface JSONSegment {
type: SegmentClass,
duration: number,
distance: number,
speed: number,
peakSpeed: number
angle: number,
cardinalDirection: string,
initialCoord: InputSample<any>,
lastCoord: InputSample<any>
}
/**
* Denotes one "segment" of the ongoing touchpath. These segments are then utilized as the basis
* for detecting and synthesizing gestures.
*
* By first breaking the touchpath into gestures, we gain the ability to define something of a
* finite-state-machine (FSM) model for each type of gesture we wish to support. While technically
* possible to do even without this step... we find that this makes such a goal far less complex.
*/
export class Segment {
/**
* Denotes the highest (mean + 1-sigma) speed seen among all subsegments comprising
* this segment.
*
* This is likely not the maximum speed observed, as that observation is likely an outlier
* and is not internally recorded. We rebuild this from stats observations.
*
* Think of it as acting like a "smoothed peak speed".
*/
private _peakSpeed: number;
private _type?: SegmentClass;
private _stats: CumulativePathStats | JSONSegment;
private _recognitionPromise: Promise<SegmentClass>;
private _recognitionPromiseResolver: (type: SegmentClass | PromiseLike<SegmentClass>) => void;
private _resolutionPromise: Promise<void>;
private _resolutionPromiseResolver: () => void;
private _isResolved: boolean = false;
/**
* Intended for internal-use only; this is utilized during the segmentation process.
*/
public constructor();
/**
* Reconstructs an instance based upon a previously-serialized `.toJSON()` call on this
* object.
* @param serializedObj
*/
public constructor(serializedObj: JSONSegment);
public constructor(serializedObj?: JSONSegment) {
if(serializedObj) {
// We'll use the loaded object as our core reference.
this._stats = serializedObj;
this._peakSpeed = serializedObj.peakSpeed; // Is retrieved from `this`, not `this._stats`.
this._type = serializedObj.type; // This one too.
// Ensure that the two promises are properly initialized for this construction type.
this._recognitionPromise = Promise.resolve(serializedObj.type);
this._resolutionPromise = Promise.resolve();
} else {
this._peakSpeed = 0;
this._recognitionPromise = new Promise<SegmentClass>((resolve) => {
this._recognitionPromiseResolver = resolve;
});
this._resolutionPromise = new Promise<void>((resolve) => {
this._resolutionPromiseResolver = () => {
this._isResolved = true;
resolve();
}
})
}
}
/**
* Indicates the likely "highest sustained" speed observed within the `Segment`'s path.
*
* To be more precise, the mean + 1-sigma (average + one standard deviation) of the speed
* for the fastest component of the `Segment`.
*/
get peakSpeed(): number {
return this._peakSpeed;
}
protected setPeakSpeed(speed: number) {
if(this._isResolved) {
throw new Error("May not modify a resolved segment!");
}
this._peakSpeed = speed;
}
protected updateStats(totalStats: CumulativePathStats) {
if(this._isResolved) {
throw new Error("May not modify a resolved segment!");
}
this._stats = totalStats;
}
protected classifyType(type: SegmentClass) {
if(!this._stats) {
throw new Error("Cannot recognize the segment - lacking critical metadata");
}
if(this._isResolved) {
throw new Error("May not modify a resolved segment!");
}
if(this._type === undefined) {
this._type = type;
this._recognitionPromiseResolver(type);
} else if(this._type != type) {
throw new Error("May not change segment type once set!");
}
}
/**
* The starting point of this touchpath segment.
*/
get initialCoord(): InputSample<any> {
if(this._stats instanceof CumulativePathStats) {
return this._stats.initialSample;
} else {
return this._stats.initialCoord;
}
};
/**
* The ending point of this touchpath segment.
*/
get lastCoord(): InputSample<any> {
if(this._stats instanceof CumulativePathStats) {
return this._stats.lastSample;
} else {
return this._stats.lastCoord;
}
};
/**
* The duration of this touchpath segment, measured in ms.
*/
get duration(): number {
return this._stats.duration;
}
/**
* The net distance traveled by this touchpath segment. In other words,
* the distance between `initialCoord` and `lastCoord` (ignoring time).
*/
get distance(): number {
if(this._stats instanceof CumulativePathStats) {
return this._stats.netDistance;
} else {
return this._stats.distance;
}
}
/**
* The average speed of this touchpath segment - distance / duration.
*/
get speed(): number {
return this._stats.speed;
};
/**
* The angle traveled by this touchpath segment. Measured in radians.
*/
get angle(): number {
return this._stats.angle;
};
/**
* The 'directional bucket' for the detected angle. This will
* correspond to a cardinal or an intercardinal ('n', 'ne', etc.)
*/
get direction(): string {
return this._stats.cardinalDirection;
};
/**
* The classification of this Segment: 'start', 'hold', 'move', or 'end'.
*
* May be null if the Segment has not yet been `recognized`.
*
* @see `SegmentClass`
*/
get type(): SegmentClass {
return this._type || null;
}
protected _isRecognized(): boolean {
return this._type !== undefined;
}
/**
* A `Promise` that resolves when the segment's classification is determined
* - when we "recognize" its role in the touchpath.
*/
get whenRecognized(): Promise<SegmentClass> {
return this._recognitionPromise;
}
/**
* A `Promise` that resolves when the segment and its role within the touchpath
* are fully determined, as segmentation has noted the need for a new boundary -
* and thus, distinct and different behavior after the endpoint of this segment
* of the touchpath.
*/
get whenResolved(): Promise<void> {
return this._resolutionPromise;
}
protected resolve() {
if(!this._stats) {
throw new Error("Cannot resolve the segment - illegal state!");
}
this._resolutionPromiseResolver();
}
/**
* Provides a human-readable yet serialization-friendly version of this instance.
* @returns
*/
public toJSON(): JSONSegment {
const cleanSample: (sample: InputSample<any>) => InputSample<any> = (sample) => {
const clone = {... sample};
delete clone.clientX;
delete clone.clientY;
return clone;
}
return {
type: this.type,
duration: this.duration,
cardinalDirection: this.direction,
speed: this.speed,
distance: this.distance,
angle: this.angle,
peakSpeed: this.peakSpeed,
initialCoord: cleanSample(this.initialCoord),
lastCoord: cleanSample(this.lastCoord)
};
}
}
/**
* Provides an editable typing for Segment utilized during the segmentation process;
* casting to the superclass provides an uneditable view into the Segment instead.
*
* This is intended for internal use only.
*/
export class SegmentImplementation extends Segment {
constructor() {
super();
}
public setPeakSpeed(speed: number) {
super.setPeakSpeed(speed);
}
public updateStats(stats: CumulativePathStats) {
super.updateStats(stats);
}
public classifyType(type: SegmentClass) {
super.classifyType(type);
}
public resolve() {
super.resolve();
}
public get isRecognized() {
return super._isRecognized();
}
}

159
common/web/gesture-recognizer/src/engine/headless/subsegmentation/subsegmentCompatibilityAnalyzer.ts
View file

@ -1,159 +0,0 @@
import { SegmentationSplit } from "./pathSegmenter.js";
import { SegmentClass, SegmentClassifier } from "../segmentClassifier.js";
/**
* This class fulfills two roles:
*
* 1. The `.isCompatible()` function provides the core definition used by
* `ConstructingSegment` to re-merge compatible subsegments, 'correcting' any
* 'oversegmentation' that may have resulted from `PathSegmenter`'s algorithm.
*
* 2. This class's other properties and methods facilitate inspection of
* the algorithm and its decision-making process during interactive debugging
* sessions.
*
* `SegmentationSplit` objects are constructed from Subsegmentation objects,
* which are made in abundance throughout the segmentation algorithms. So, it
* shouldn't be difficult to construct the necessary parameters to dynamically
* construct instances of this class during an interactive debugging session.
*/
export class SubsegmentCompatibilityAnalyzer {
readonly classifier: SegmentClassifier;
private split: SegmentationSplit;
constructor(subsegmentationSplit: SegmentationSplit, classifier: SegmentClassifier) {
this.split = subsegmentationSplit;
this.classifier = classifier;
}
private get preStats() {
return this.split.pre.stats;
}
private get postStats() {
return this.split.post.stats;
}
private get unionStats() {
return this.split.union;
}
/**
* This property summarizes subsegment compatibility on the basis of direction.
*
* 1. If both subsegments are compatible as determined by geometry-based regression, we consider
* them compatible. (The first phase considers both geometry _and time_; we leave the "time"
* part out here.)
*
* 2. If regression upholds the segmentation, but both subsegments are classified as
* 'move'-compatible and their directions both fall within the same "direction bucket", we
* consider them compatible.
*
* 3. If both subsegments are classified as 'hold's, angle doesn't matter - the user's intent
* is "no motion".
*/
get directionCompatible(): boolean {
if(!this.preStats || this.regressionDirectionCompatible) {
return true;
}
let preMatchesMove = this.classifier.classifySubsegment(this.preStats) != SegmentClass.HOLD;
let postMatchesMove = this.classifier.classifySubsegment(this.postStats) != SegmentClass.HOLD;
if(preMatchesMove != postMatchesMove) {
// If one subsegment appears to be a 'hold' while the other does not, well... holds
// don't (practically) have a direction, which mismatches with the 'move' that does
// have a direction.
return false;
} else if(!preMatchesMove) {
// If both are 'hold' subsegments, both should be treated as directionless.
return true;
} else {
// If both are 'move' / 'move'-like subsegments, only merge if their directional
// classification falls into the same 'direction bucket'.
return this.cardinalDirectionCompatible;
}
}
/**
* Indicates whether or not the two halves of the represented segmentation fall within
* the same cardinal-direction "bucket".
*/
get cardinalDirectionCompatible(): boolean {
return !this.preStats || this.preStats.cardinalDirection == this.postStats.cardinalDirection;
}
/**
* Indicates whether or not geometry-based regression of the two subsegments (without
* respect to speed and/or time) provides sufficient evidence to uphold the segmentation.
*/
get regressionDirectionCompatible(): boolean {
return !this.preStats || this.split.mergeMerited;
}
/**
* Indicates whether or not the classifications of the two subsegments is compatible;
* if so, it returns the corresponding `SegmentClass` (or `null`, if no classification
* may be committed yet).
*
* If the classifications are incompatible, ths function will return `undefined`
* instead.
*/
get classificationIfCompatible(): SegmentClass {
// Get the baseline subsegment classification for each subsegment.
let leftClass = this.classifier.classifySubsegment(this.preStats);
let rightClass = this.classifier.classifySubsegment(this.postStats);
let unionClass = this.classifier.classifySubsegment(this.unionStats);
// Choose the first non-null one as a fallback, then apply it.
let fallbackClass = leftClass || rightClass || unionClass;
leftClass = leftClass || fallbackClass;
rightClass = rightClass || fallbackClass;
unionClass = unionClass || fallbackClass;
// If all classes (post-fallback) match, that's the class if compatible.
if(leftClass == rightClass && leftClass == unionClass) {
return fallbackClass; // can technically still be null.
} else {
return undefined;
}
}
/**
* The full, complete test for subsegment 'compatibility'. If `true`, our
* criteria indicate no merit in upholding the segmentation under review.
* `false` indicates that the segmentation should be upheld instead.
*/
get isCompatible(): boolean {
const commonClass = this.classificationIfCompatible;
// If two adjacent hold subsegments also make a hold when combined...
// just merge the two holds & call 'em compatible.
if(!this.preStats || commonClass == 'hold') {
return true;
} else if(commonClass === undefined) { // `null`: pending; `undefined`: incompatible
return false;
}
// if(null || "move"): as `null` "looks like" a not-quite-there-yet "move",
// we treat it as such here. Such subsegments are only compatible if
// their directions are compatible.
return this.directionCompatible;
}
/**
* Intended only for use during interactive debugging sessions; prints useful logging
* information to the developer console.
*/
public debugLogCompabilityReport() {
if(!this.preStats) {
console.log("No prior subsegments - thus, no compatibility conflicts are possible.");
return;
}
console.log("Regression-based compatibility check:")
this.split._debugLogAlignmentReport();
}
}

3
common/web/gesture-recognizer/src/engine/index.ts
View file

@ -1,6 +1,5 @@
import { validateModelDefs } from './headless/gestures/specs/modelDefValidator.js';
export { ConstructingSegment } from './headless/subsegmentation/constructingSegment.js';
export { CumulativePathStats } from './headless/cumulativePathStats.js';
export { GestureModelDefs } from './headless/gestures/specs/gestureModelDefs.js';
export { GestureRecognizer } from "./gestureRecognizer.js";
@ -11,10 +10,8 @@ export { SerializedGesturePath, GesturePath } from "./headless/gesturePath.js";
export { ConfigChangeClosure, GestureStageReport, GestureSequence } from "./headless/gestures/matchers/gestureSequence.js";
export { SerializedGestureSource, GestureSource, buildGestureMatchInspector } from "./headless/gestureSource.js";
export { MouseEventEngine } from "./mouseEventEngine.js";
export { PathSegmenter, Subsegmentation } from "./headless/subsegmentation/pathSegmenter.js";
export { PaddedZoneSource } from './configuration/paddedZoneSource.js';
export { RecognitionZoneSource } from './configuration/recognitionZoneSource.js';
export { Segment } from "./headless/subsegmentation/segment.js";
export { SegmentClassifier } from "./headless/segmentClassifier.js";
export { TouchEventEngine } from "./touchEventEngine.js";
export { TouchpointCoordinator } from "./headless/touchpointCoordinator.js";

390
common/web/gesture-recognizer/src/test/auto/headless/basicSegmentations.js
View file

@ -1,390 +0,0 @@
import { assert } from 'chai'
import sinon from 'sinon';
import * as PromiseStatusModule from 'promise-status-async';
const promiseStatus = PromiseStatusModule.promiseStatus;
const PromiseStatuses = PromiseStatusModule.PromiseStatuses;
import { PathSegmenter } from '@keymanapp/gesture-recognizer';
import { timedPromise } from '@keymanapp/web-utils';
describe("Basic segmentation cases", function() {
describe("Single-sample 'sequence'", function() {
it("expected segment types", function() {
let spy = sinon.fake();
const segmenter = new PathSegmenter(PathSegmenter.DEFAULT_CONFIG, spy);
const sample = {
targetX: 1,
targetY: 1,
t: 100
};
// Expected sequence:
// 1: on `segmenter.add(sample)`:
// a. 'start' segment
// b. unrecognized (null-type) segment.
// 2: on `segmenter.close()`:
// b. 'end' segment
segmenter.add(sample);
try {
assert.isTrue(spy.calledTwice, "Segmenter callback was not called exactly twice before the .close().");
} finally {
segmenter.close();
}
assert.isTrue(spy.calledThrice, "Segmenter callback was not called exactly once after the .close().");
for(let i=0; i < 3; i++) {
assert.equal(spy.args[i].length, 1, "Segmenter callback received an unexpected number of arguments");
}
// Is the first argument of each call's argument set.
assert.equal(spy.firstCall .args[0].type, 'start', "First call should receive a 'start'-type segment.");
assert.equal(spy.secondCall.args[0].type, undefined, "Second call's segment should not be classified.");
assert.equal(spy.thirdCall .args[0].type, 'end', "Third call should receive an 'end'-type segment.");
});
it("segment recognition + resolution", async function() {
let spy = sinon.fake();
const segmenter = new PathSegmenter(PathSegmenter.DEFAULT_CONFIG, spy);
const sample = {
targetX: 1,
targetY: 1,
t: 100
};
segmenter.add(sample);
try {
// It's still best to verify this before proceeding.
assert.isTrue(spy.calledTwice, "Segmenter callback was not called exactly twice before the .close().");
// 'start': should be fully resolved, right out of the gate.
assert.equal(await promiseStatus(spy.firstCall.args[0].whenRecognized), PromiseStatuses.PROMISE_RESOLVED);
assert.equal(await promiseStatus(spy.firstCall.args[0].whenResolved), PromiseStatuses.PROMISE_RESOLVED);
// null-typed, just-starting segment: neither recognized nor resolved.
assert.equal(await promiseStatus(spy.secondCall.args[0].whenRecognized), PromiseStatuses.PROMISE_PENDING);
assert.equal(await promiseStatus(spy.secondCall.args[0].whenResolved), PromiseStatuses.PROMISE_PENDING);
} finally {
segmenter.close();
}
assert.isTrue(spy.calledThrice, "Segmenter callback was not called exactly once after the .close().");
// null-typed segment should now be recognized and resolved.
assert.equal(await promiseStatus(spy.secondCall.args[0].whenRecognized), PromiseStatuses.PROMISE_RESOLVED);
assert.equal(await promiseStatus(spy.secondCall.args[0].whenResolved), PromiseStatuses.PROMISE_RESOLVED);
// 'end': should be fully resolved, right out of the gate.
assert.equal(await promiseStatus(spy.thirdCall.args[0].whenRecognized), PromiseStatuses.PROMISE_RESOLVED);
assert.equal(await promiseStatus(spy.thirdCall.args[0].whenResolved), PromiseStatuses.PROMISE_RESOLVED);
});
});
describe("Single held point", function() {
beforeEach(function() {
this.fakeClock = sinon.useFakeTimers();
})
afterEach(function() {
this.fakeClock.restore();
})
it("'hold' segment recognition + resolution", async function() {
let spy = sinon.fake();
const segmenter = new PathSegmenter(PathSegmenter.DEFAULT_CONFIG, spy);
const startSample = {
targetX: 1,
targetY: 1,
t: 100
};
const endSample = {
targetX: 1,
targetY: 1,
t: 1100 // total duration: 1 sec.
};
const segment2Recognition = sinon.fake();
const segment2Resolution = sinon.fake();
// Timestamp 1: segmentation begins, with an initial Sample recorded.
const firstPromise = timedPromise(0).then(() => {
segmenter.add(startSample);
assert.isTrue(spy.calledTwice, "Segmenter callback was not called exactly twice upon adding the first point.");
// The focus of this unit test is the two `Promise`s provided by this specific `Segment`.
const pendingSegment = spy.secondCall.args[0];
assert.exists(pendingSegment);
pendingSegment.whenRecognized.then(segment2Recognition);
pendingSegment.whenResolved.then(segment2Resolution);
}).then(() => {
// At time = 0, neither recognition nor resolution should have triggered.
assert.isFalse(segment2Recognition.called);
const pendingSegment = spy.secondCall.args[0];
assert.isNull(pendingSegment.type); // not yet recognized.
assert.isFalse(segment2Resolution.called);
});
// Timestamp 2: segmentation continues... a second later. A second Sample is recorded.
const secondSampleTestPromise = firstPromise.then(() => {
return timedPromise(1000).then(() => {
// This should have occurred already, despite no new sample having been provided
// to the segmenter.
assert.isTrue(segment2Recognition.called);
const pendingSegment = spy.secondCall.args[0];
assert.equal(pendingSegment.type, 'hold'); // now has a type, as it's been recognized.
// And now to update with a new sample.
segmenter.add(endSample);
}).then(() => {
assert.isFalse(segment2Resolution.called);
const pendingSegment = spy.secondCall.args[0];
assert.isAtLeast(pendingSegment.duration, 1000); // Latest sample's timestamp gives exactly 1000.
});
});
// Timestamp 3: segmentation is then ended via a followup event.
const segmentationEndPromise = secondSampleTestPromise.then(() => {
assert.isFalse(segment2Resolution.called);
segmenter.close();
// Fun fact: assert.isFalse(segment2Resolution.called) still holds here
// (after `segmenter.close()`) because Promises are async.
}).then(() => {
assert.isTrue(segment2Resolution.called);
assert.isTrue(spy.calledThrice);
})
const finalPromise = segmentationEndPromise.catch((reason) => {
segmenter.close();
throw reason;
});
this.fakeClock.runAllAsync();
// This is the one that reports all of our async assertion failures.
return finalPromise;
});
it("motionless 'hold' segment properties", function() {
const spy = sinon.fake();
const segmenter = new PathSegmenter(PathSegmenter.DEFAULT_CONFIG, spy);
const startSample = {
targetX: 1,
targetY: 1,
t: 100
};
const endSample = {
targetX: 1,
targetY: 1,
t: 1100 // total duration: 1 sec.
};
// Sample 'playback' Promise setup
const samples = [startSample, endSample];
const samplePromises = samples.map((sample) => {
return timedPromise(sample.t - startSample.t).then(() => {
segmenter.add(sample);
});
});
// Timestamp 3: segmentation is then ended via a followup event.
const segmentationEndPromise = Promise.all(samplePromises).then(() => {
segmenter.close();
});
const finalPromise = segmentationEndPromise.catch((reason) => {
segmenter.close();
throw reason;
});
this.fakeClock.runAllAsync();
// This is the one that reports all of our async assertion failures.
return finalPromise.then(() => {
const holdSegment = spy.secondCall.args[0];
assert.equal(holdSegment.type, 'hold');
assert.isAtLeast(holdSegment.duration, endSample.t - startSample.t);
assert.equal(holdSegment.speed, 0);
assert.isUndefined(holdSegment.angle);
assert.isUndefined(holdSegment.direction);
assert.equal(holdSegment.peakSpeed, 0);
assert.equal(holdSegment.distance, 0);
// Samples may be cloned when passed through the segmenter!
assert.deepEqual(holdSegment.initialCoord, startSample);
assert.deepEqual(holdSegment.lastCoord, endSample);
});
});
});
describe("Idealized 'move'", function() {
beforeEach(function() {
this.fakeClock = sinon.useFakeTimers();
})
afterEach(function() {
this.fakeClock.restore();
})
/**
* Builds a diagonal sequence traveling 40 pixels 's', 40 pixels 'e' in a perfect diagonal lock-step.
*/
const buildSampleSequence = () => {
const startSample = {
targetX: 1,
targetY: 1,
t: 100
};
let samples = [startSample];
for(let i=1; i <=20; i++) {
samples.push({
targetX: startSample.targetX + 2 * i,
targetY: startSample.targetY + 2 * i,
t: startSample.t + 10 * i
});
}
return samples;
}
it("'move' segment recognition + resolution", async function() {
let spy = sinon.fake();
const segmenter = new PathSegmenter(PathSegmenter.DEFAULT_CONFIG, spy);
const samples = buildSampleSequence();
const samplePromises = samples.map((sample) => {
return timedPromise(sample.t - samples[0].t).then(() => {
segmenter.add(sample);
});
});
// Find and replace select promises with then'd versions of themselves to 'hook in' as needed.
// We know the 'move' segment will appear upon completion of the first sample's Promise.
const segment2Recognition = sinon.fake();
const segment2Resolution = sinon.fake();
samplePromises[0] = samplePromises[0].then(async () => {
assert.isAtLeast(spy.callCount, 2);
const moveSegment = spy.secondCall.args[0];
assert.equal(await promiseStatus(moveSegment.whenRecognized), PromiseStatuses.PROMISE_PENDING);
assert.equal(await promiseStatus(moveSegment.whenResolved), PromiseStatuses.PROMISE_PENDING);
// Now that we're sure they haven't already been called, let's set the `fake` calls in place.
moveSegment.whenRecognized.then(segment2Recognition);
moveSegment.whenResolved.then(segment2Resolution);
assert.isNotNull(moveSegment);
});
// Given the current engine setup, we know that after 5 samples, we pass the movement-based
// recognition threshold and thus should be classified as a 'move'.
samplePromises[5] = samplePromises[5].then(() => {
const moveSegment = spy.secondCall.args[0];
assert.isTrue(segment2Recognition.called);
assert.isFalse(segment2Resolution.called);
assert.equal(moveSegment.type, 'move');
});
samplePromises[20] = samplePromises[20].then(() => {
const moveSegment = spy.secondCall.args[0];
assert.isTrue(segment2Recognition.called);
assert.isFalse(segment2Resolution.called);
assert.isAtLeast(moveSegment.duration, samples[20].t - samples[0].t);
});
// Timestamp 3: segmentation is then ended via a followup event.
const segmentationEndPromise = Promise.all(samplePromises).then(() => {
assert.isFalse(segment2Resolution.called);
segmenter.close();
}).then(() => {
assert.isTrue(segment2Resolution.called);
assert.equal(spy.callCount, 3);
});
const finalPromise = segmentationEndPromise.catch((reason) => {
segmenter.close();
throw reason;
});
this.fakeClock.runAllAsync();
// This is the one that reports all of our async assertion failures.
return finalPromise;
});
it("idealized 'move' segment properties", function() {
let spy = sinon.fake();
const segmenter = new PathSegmenter(PathSegmenter.DEFAULT_CONFIG, spy);
const samples = buildSampleSequence();
const samplePromises = samples.map((sample) => {
return timedPromise(sample.t - samples[0].t).then(() => {
segmenter.add(sample);
});
});
// Timestamp 3: segmentation is then ended via a followup event.
const segmentationEndPromise = Promise.all(samplePromises).then(() => {
segmenter.close();
});
const finalPromise = segmentationEndPromise.catch((reason) => {
segmenter.close();
throw reason;
});
this.fakeClock.runAllAsync();
// This is the one that reports all of our async assertion failures.
return finalPromise.then(() => {
// The real checks.
const moveSegment = spy.secondCall.args[0];
assert.equal(moveSegment.type, 'move');
assert.isAtLeast(moveSegment.duration, samples[20].t - samples[0].t);
assert.equal(moveSegment.direction, 'se');
assert.equal(moveSegment.distance, 40 * Math.SQRT2);
assert.equal(moveSegment.angle, (135 / 180) * Math.PI);
// Speed: our idealized sequence moves 2*sqrt(2) px distance every 10ms.
const idealSpeed = 2 * Math.sqrt(2) / 10;
assert.isAtLeast(moveSegment.speed, 0.99 * idealSpeed);
assert.isAtLeast(moveSegment.peakSpeed, 0.99 * idealSpeed);
// Samples may be cloned when passed through the segmenter!
assert.deepEqual(moveSegment.initialCoord, samples[0]);
assert.deepEqual(moveSegment.lastCoord, samples[20]);
});
});
});
});

122
common/web/gesture-recognizer/src/test/auto/headless/pathStats.ts
View file

@ -9,7 +9,6 @@ describe("CumulativePathStats", function() {
assert.equal(stats.duration, 0);
assert.equal(stats.angle, undefined);
assert.isNaN(stats.angleDeviation);
assert.equal(stats.cardinalDirection, undefined);
assert.equal(stats.rawDistance, 0);
assert.equal(stats.netDistance, 0);
@ -27,10 +26,7 @@ describe("CumulativePathStats", function() {
assert.equal(stats.duration, 0);
assert.equal(stats.angle, undefined);
assert.isNaN(stats.angleDeviation);
assert.equal(stats.cardinalDirection, undefined);
assert.isNaN(stats.variance('x'));
assert.isNaN(stats.variance('y'));
assert.equal(stats.mean('x'), 4);
assert.equal(stats.mean('y'), 8);
});
@ -58,14 +54,6 @@ describe("CumulativePathStats", function() {
assert.equal(stats.mean('y'), samples[2].targetY);
assert.equal(stats.mean('t'), samples[2].t);
// The two are perfectly correlated and have the same variance.
// Since we renormalize samples, the base 'intercept' of each variable's linear form should have no effect..
assert.equal(stats.squaredSum('x'), stats.crossSum('xy'));
assert.equal(stats.squaredSum('y'), stats.crossSum('xy'));
// As x and y are perfectly correlated with the same variance, their cross-sum with t should match.
assert.equal(stats.crossSum('tx'), stats.crossSum('ty'));
assert.equal(stats.angle, 135 * Math.PI / 180);
assert.equal(stats.rawDistance, 16 * Math.SQRT2); // 4 intervals of length 4 * sqrt(2)
assert.equal(stats.cardinalDirection, 'se');
@ -111,44 +99,6 @@ describe("CumulativePathStats", function() {
assert.sameDeepOrderedMembers(postStats, preStats);
});
it("Basic regressions (perfect correlation)", function() {
const samples: InputSample<any>[] = [];
// Exactly 5 points, evenly spaced and linear.
// So, the arithmetic mean should be very obvious - it's the middle sample.
for(let i = 0; i < 5; i++) {
samples.push({
targetX: 4 * i + 10,
targetY: 4 * i + 20,
t: 100 * i // since it makes head-math simpler for the regressions we'll be doing.
});
}
let stats = new CumulativePathStats();
for(const sample of samples) {
stats = stats.extend(sample);
}
const xtRegression = stats.fitRegression('x', 't');
assert.equal(xtRegression.sumOfSquaredError, 0);
assert.equal(xtRegression.slope, 4 / 100);
assert.equal(xtRegression.intercept, 10);
assert.equal(xtRegression.predictFromValue(1000), 40 + 10);
const ytRegression = stats.fitRegression('y', 't');
assert.equal(ytRegression.sumOfSquaredError, 0);
assert.equal(ytRegression.slope, 4 / 100);
assert.equal(ytRegression.intercept, 20);
assert.equal(ytRegression.predictFromValue(1000), 40 + 20);
const xyRegression = stats.fitRegression('y', 'x');
assert.equal(xyRegression.sumOfSquaredError, 0);
assert.equal(xyRegression.slope, 1);
assert.equal(xyRegression.intercept, 10);
assert.equal(xyRegression.predictFromValue(50), 60);
});
it("Deaccumulation", function() {
const sampleSet1: InputSample<any>[] = [];
@ -209,14 +159,6 @@ describe("CumulativePathStats", function() {
assert.equal(deaccumulatedSecondHalfStats.mean('y'), secondHalfStats.mean('y'));
assert.equal(deaccumulatedSecondHalfStats.mean('t'), secondHalfStats.mean('t'));
assert.equal(deaccumulatedSecondHalfStats.variance('x'), secondHalfStats.variance('x'));
assert.equal(deaccumulatedSecondHalfStats.variance('y'), secondHalfStats.variance('y'));
assert.equal(deaccumulatedSecondHalfStats.variance('t'), secondHalfStats.variance('t'));
assert.equal(deaccumulatedSecondHalfStats.covariance('tx'), secondHalfStats.covariance('tx'));
assert.equal(deaccumulatedSecondHalfStats.covariance('ty'), secondHalfStats.covariance('ty'));
assert.equal(deaccumulatedSecondHalfStats.covariance('xy'), secondHalfStats.covariance('xy'));
// Floating-point "equality".
assert.closeTo(deaccumulatedSecondHalfStats.netDistance, secondHalfStats.netDistance, 1e-8);
assert.closeTo(deaccumulatedSecondHalfStats.rawDistance, secondHalfStats.rawDistance, 1e-8);
@ -226,69 +168,5 @@ describe("CumulativePathStats", function() {
assert.equal(deaccumulatedSecondHalfStats.initialSample, secondHalfStats.initialSample);
assert.equal(deaccumulatedSecondHalfStats.lastSample, secondHalfStats.lastSample);
// Angle deviation stuff doesn't currently have the same level of catastrophic cancellation protection
// as the main statistical properties. But... even still, we can add a reasonable test.
assert.isBelow(Math.abs(deaccumulatedSecondHalfStats.angleDeviation - secondHalfStats.angleDeviation), 1e-7);
});
it("Renormalization", () => {
let stats = new CumulativePathStats();
const turtle = new TouchpathTurtle({
targetX: 50,
targetY: -25,
t: 500,
item: 'a'
});
turtle.on('sample', (sample) => {
stats = stats.extend(sample);
});
function assertMatchingStats(actual: CumulativePathStats, expected: CumulativePathStats) {
assert.equal(actual.duration, expected.duration);
assert.equal(actual.sampleCount, expected.sampleCount);
assert.closeTo(actual.angle, expected.angle, 1e-8);
assert.closeTo(actual.mean('x'), expected.mean('x'), 1e-8);
assert.closeTo(actual.mean('y'), expected.mean('y'), 1e-8);
assert.closeTo(actual.mean('t'), expected.mean('t'), 1e-8);
assert.closeTo(actual.variance('x'), expected.variance('x'), 1e-8);
assert.closeTo(actual.variance('y'), expected.variance('y'), 1e-8);
assert.closeTo(actual.variance('t'), expected.variance('t'), 1e-8);
assert.closeTo(actual.rawDistance, expected.rawDistance, 1e-8);
assert.closeTo(actual.netDistance, expected.netDistance, 1e-8);
const actualRegression = actual.fitRegression('x', 'y');
const expectedRegression = expected.fitRegression('x', 'y');
assert.closeTo(
actualRegression.coefficientOfDetermination,
expectedRegression.coefficientOfDetermination,
1e-8
);
assert.closeTo(actualRegression.slope, expectedRegression.slope, 1e-8);
}
turtle.move(90, 4, 20, 2);
turtle.move(60, 5, 20, 2); // net move: <4, -3>.
turtle.commitPending();
let renormalizedStats = stats.buildRenormalized();
assertMatchingStats(renormalizedStats, stats);
// Continue to accumulate stats for both.
turtle.on('sample', (sample) => {
renormalizedStats = renormalizedStats.extend(sample);
})
turtle.move(0, 10, 40, 4);
turtle.move(45, 3 * Math.sqrt(2), 40, 4);
turtle.commitPending();
// After the renormalization, there should be no externally-visible distinction between the two stats objects.
assertMatchingStats(renormalizedStats, stats);
});
});

342
common/web/gesture-recognizer/src/test/auto/headless/recordedSegmentations.js
View file

@ -1,342 +0,0 @@
import { assert } from 'chai';
import sinon from 'sinon';
import fs from 'fs';
import path from 'path';
import url from 'url';
import * as PromiseStatusModule from 'promise-status-async';
const promiseStatus = PromiseStatusModule.promiseStatus;
const PromiseStatuses = PromiseStatusModule.PromiseStatuses;
import { PathSegmenter } from '@keymanapp/gesture-recognizer';
import { HeadlessInputEngine } from '#tools';
// Ensures that the resources are resolved relative to this script, not to the cwd when the test
// runner was launched.
const scriptFolder = path.dirname(url.fileURLToPath(import.meta.url));
const SEGMENT_TEST_JSON_FOLDER = path.resolve(`${scriptFolder}/../../resources/json/segmentation`);
import { assertSegmentSimilarity } from '../../resources/assertSegmentSimilarity.js';
/**
* Since we're disconnecting subsegmentation stuff for the first gestures release, our
* previously-recorded subsegmentations aren't retrievable in the same spot as before.
*
* This manually retrieves them from the originally-recorded version in order to preserve
* the unit tests. Makes a few assumptions, but they're valid for the original recordings.
* @param {*} jsonObj
*/
function retrieveSubsegmentations(jsonObj) {
return jsonObj.inputs[0].path.segments;
}
// ---------------------------------------
// NOTE: this suite of tests is for a disconnected subsystem - subsegmentation - designed
// to handle more complicated gestures than supported in our initial release. We had to
// triage it to prevent further delays.
//
// The tests themselves should still be functional.
// ---------------------------------------
describe("Segmentation - from recorded sequences", function() {
beforeEach(function() {
this.fakeClock = sinon.useFakeTimers();
})
afterEach(function() {
// NOTE: for debugging investigations, it may be necessary to use .only on
// the test under investigation and to disable the `this.fakeClock.restore()` line.
//
// Tests tend to timeout when interactively debugging, and having unmocked timers
// suddenly restored during investigation can cause some very confusing behavior.
this.fakeClock.restore();
})
it("simple_ne_move.json", async function() {
// NOTE: this recording's final 'hold' segment is somewhat tightly attuned to the DEFAULT_CONFIG
// hold time setting. Changing default values there may necessitate a hand-edit tweak to the
// test recording's data in order for this test to continue passing as-is.
let testJSONtext = fs.readFileSync(`${SEGMENT_TEST_JSON_FOLDER}/simple_ne_move.json`);
let jsonObj = JSON.parse(testJSONtext);
// Some special setup - we're going to capture the 'move' segment in-process and run a check that way.
const recognitionTestCapturer = {
recognizedPromise: null
}
// Spy: the method we pass in is _actually called_ while _also_ capturing metadata for every call.
let spy = sinon.spy((segment) => {
if(segment.type == 'move' && !recognitionTestCapturer.recognizedPromise) {
segment.whenRecognized.then(() => {
recognitionTestCapturer.recognizedPromise = new Promise((resolve, reject) => {
// Verify that we resolved because of distance, not time.
assert.isAtLeast(segment.distance, PathSegmenter.DEFAULT_CONFIG.holdMoveTolerance);
assert.isBelow(segment.duration, PathSegmenter.DEFAULT_CONFIG.holdMinimumDuration);
// Makes sure the segment isn't resolved at the same time it is recognized.
promiseStatus(segment.whenResolved).then((status) => {
assert.equal(status, PromiseStatuses.PROMISE_PENDING);
resolve();
}).catch((err) => {
reject(err);
});
});
return recognitionTestCapturer.recognizedPromise;
});
}
});
// Now the normal basic setup.
const segmenter = new PathSegmenter(PathSegmenter.DEFAULT_CONFIG, spy);
let engine = new HeadlessInputEngine();
engine.once('pointstart', (point) => {
// The first point is always available initially, rather than via event.
segmenter.add(point.path.coords[0]);
point.path.on('step', (val) => {
segmenter.add(val);
});
});
const testPromise = engine.playbackRecording(jsonObj).then(() => segmenter.close());
await this.fakeClock.runAllAsync();
await testPromise;
await recognitionTestCapturer.recognizedPromise;
// Any post-sequence tests to run.
const originalSegments = retrieveSubsegmentations(jsonObj);
const originalSegmentTypeSequence = originalSegments.map((segment) => segment.type);
const reproedSegments = spy.getCalls().map((call) => call.args[0]);
const reproedSegmentTypeSequence = reproedSegments.map((segment) => segment.type);
assert.sameOrderedMembers(reproedSegmentTypeSequence, originalSegmentTypeSequence);
// Ensure all relevant Promises resolved.
for(let segment of reproedSegments) {
assert.isTrue(await promiseStatus(segment.whenRecognized) == PromiseStatuses.PROMISE_RESOLVED);
assert.isTrue(await promiseStatus(segment.whenResolved) == PromiseStatuses.PROMISE_RESOLVED);
}
assertSegmentSimilarity(reproedSegments[1], originalSegments[1], 'hold'); // ~200ms
assertSegmentSimilarity(reproedSegments[2], originalSegments[2], 'move'); // 'ne'
});
it("nonstationary_hold.json", async function() {
let testJSONtext = fs.readFileSync(`${SEGMENT_TEST_JSON_FOLDER}/nonstationary_hold.json`);
let jsonObj = JSON.parse(testJSONtext);
// Some special setup - we're going to capture the 'hold' segment in-process and run a check that way.
const recognitionTestCapturer = {
recognizedPromise: null
}
// Spy: the method we pass in is _actually called_ while _also_ capturing metadata for every call.
let spy = sinon.spy((segment) => {
if(segment.type == 'hold' && !recognitionTestCapturer.recognizedPromise) {
segment.whenRecognized.then(() => {
recognitionTestCapturer.recognizedPromise = new Promise((resolve, reject) => {
// Verify that we resolved because of time, not distance.
assert.isBelow(segment.distance, PathSegmenter.DEFAULT_CONFIG.holdMoveTolerance);
assert.isAtLeast(segment.duration, PathSegmenter.DEFAULT_CONFIG.holdMinimumDuration);
// Makes sure the segment isn't resolved at the same time it is recognized.
promiseStatus(segment.whenResolved).then((status) => {
assert.equal(status, PromiseStatuses.PROMISE_PENDING);
resolve();
}).catch((err) => {
reject(err);
});
});
return recognitionTestCapturer.recognizedPromise;
});
}
});
// Now the normal basic setup.
const segmenter = new PathSegmenter(PathSegmenter.DEFAULT_CONFIG, spy);
let engine = new HeadlessInputEngine();
engine.once('pointstart', (point) => {
// The first point is always available initially, rather than via event.
segmenter.add(point.path.coords[0]);
point.path.on('step', (val) => {
segmenter.add(val);
});
});
const testPromise = engine.playbackRecording(jsonObj).then(() => segmenter.close());
await this.fakeClock.runAllAsync();
await testPromise;
await recognitionTestCapturer.recognizedPromise;
// Any post-sequence tests to run.
const originalSegments = retrieveSubsegmentations(jsonObj);
const originalSegmentTypeSequence = originalSegments.map((segment) => segment.type);
const reproedSegments = spy.getCalls().map((call) => call.args[0]);
const reproedSegmentTypeSequence = reproedSegments.map((segment) => segment.type);
assert.sameOrderedMembers(reproedSegmentTypeSequence, originalSegmentTypeSequence);
// Ensure all relevant Promises resolved.
for(let segment of reproedSegments) {
assert.isTrue(await promiseStatus(segment.whenRecognized) == PromiseStatuses.PROMISE_RESOLVED);
assert.isTrue(await promiseStatus(segment.whenResolved) == PromiseStatuses.PROMISE_RESOLVED);
}
assertSegmentSimilarity(reproedSegments[1], originalSegments[1], 'hold'); // ~1200ms
});
it("flick_ne_se.json", async function() {
let testJSONtext = fs.readFileSync(`${SEGMENT_TEST_JSON_FOLDER}/flick_ne_se.json`);
let jsonObj = JSON.parse(testJSONtext);
// Prepares some basic setup - we won't set up segment-specific recognition tests here.
let spy = sinon.fake();
const segmenter = new PathSegmenter(PathSegmenter.DEFAULT_CONFIG, spy);
let engine = new HeadlessInputEngine();
engine.once('pointstart', (point) => {
// The first point is always available initially, rather than via event.
segmenter.add(point.path.coords[0]);
point.path.on('step', (val) => {
segmenter.add(val);
});
});
const testPromise = engine.playbackRecording(jsonObj).then(() => segmenter.close());
await this.fakeClock.runAllAsync();
await testPromise;
// Any post-sequence tests to run.
const originalSegments = retrieveSubsegmentations(jsonObj);
const originalSegmentTypeSequence = originalSegments.map((segment) => segment.type);
const reproedSegments = spy.getCalls().map((call) => call.args[0]);
const reproedSegmentTypeSequence = reproedSegments.map((segment) => segment.type);
assert.sameOrderedMembers(reproedSegmentTypeSequence, originalSegmentTypeSequence);
// Ensure all relevant Promises resolved.
for(let segment of reproedSegments) {
assert.isTrue(await promiseStatus(segment.whenRecognized) == PromiseStatuses.PROMISE_RESOLVED);
assert.isTrue(await promiseStatus(segment.whenResolved) == PromiseStatuses.PROMISE_RESOLVED);
}
assertSegmentSimilarity(reproedSegments[1], originalSegments[1], 'hold'); // ~820ms
assertSegmentSimilarity(reproedSegments[2], originalSegments[2], 'move'); // 'ne'
assertSegmentSimilarity(reproedSegments[3], originalSegments[3], 'hold'); // ~580ms
assertSegmentSimilarity(reproedSegments[4], originalSegments[4], 'move'); // 'se'
assertSegmentSimilarity(reproedSegments[5], originalSegments[5], 'hold'); // ~300ms
});
it("longpress_to_ne.json", async function() {
let testJSONtext = fs.readFileSync(`${SEGMENT_TEST_JSON_FOLDER}/longpress_to_ne.json`);
let jsonObj = JSON.parse(testJSONtext);
// Prepares some basic setup - we won't set up segment-specific recognition tests here.
let spy = sinon.fake();
const segmenter = new PathSegmenter(PathSegmenter.DEFAULT_CONFIG, spy);
let engine = new HeadlessInputEngine();
engine.once('pointstart', (point) => {
// The first point is always available initially, rather than via event.
segmenter.add(point.path.coords[0]);
point.path.on('step', (val) => {
segmenter.add(val);
});
});
const testPromise = engine.playbackRecording(jsonObj).then(() => segmenter.close());
await this.fakeClock.runAllAsync();
await testPromise;
// Any post-sequence tests to run.
const originalSegments = retrieveSubsegmentations(jsonObj);
const reproedSegments = spy.getCalls().map((call) => call.args[0]);
// Because of the sweeping arc motion, we won't assume a perfect match to the segmentation here.
// Ensure all relevant Promises resolved.
for(let segment of reproedSegments) {
assert.isTrue(await promiseStatus(segment.whenRecognized) == PromiseStatuses.PROMISE_RESOLVED);
assert.isTrue(await promiseStatus(segment.whenResolved) == PromiseStatuses.PROMISE_RESOLVED);
}
assertSegmentSimilarity(reproedSegments[1], originalSegments[1], 'hold'); // ~1460ms
// there are actually a LOT of move segments here; the motion was in an arc.
// will make a good test case for the eventual longpress gesture.
assert.equal(reproedSegments[2].type, 'move');
assert.equal(reproedSegments[2].direction, 'n'); // the initial direction once motion started.
});
it("quick_small_square.json", async function() {
// NOTE: this recording has a number of borderline-duration 'hold' segments between its moves.
// Naturally, there has to be SOME kind of transition when sharply changing direction.
// NOTE: We wish to ensure that each 'edge' of the 'square' remains reasonably distinct.
let testJSONtext = fs.readFileSync(`${SEGMENT_TEST_JSON_FOLDER}/quick_small_square.json`);
let jsonObj = JSON.parse(testJSONtext);
// Prepares some basic setup - we won't set up segment-specific recognition tests here.
let spy = sinon.fake();
const segmenter = new PathSegmenter(PathSegmenter.DEFAULT_CONFIG, spy);
let engine = new HeadlessInputEngine();
engine.once('pointstart', (point) => {
// The first point is always available initially, rather than via event.
segmenter.add(point.path.coords[0]);
point.path.on('step', (val) => {
segmenter.add(val);
});
});
const testPromise = engine.playbackRecording(jsonObj).then(() => segmenter.close());
await this.fakeClock.runAllAsync();
await testPromise;
// Any post-sequence tests to run.
const reproedSegments = spy.getCalls().map((call) => call.args[0]);
// Ensure all relevant Promises resolved.
for(let segment of reproedSegments) {
assert.isTrue(await promiseStatus(segment.whenRecognized) == PromiseStatuses.PROMISE_RESOLVED);
assert.isTrue(await promiseStatus(segment.whenResolved) == PromiseStatuses.PROMISE_RESOLVED);
}
const holds = reproedSegments.filter((segment) => segment.type == 'hold');
const moves = reproedSegments.filter((segment) => segment.type == 'move');
const nulls = reproedSegments.filter((segment) => segment.type === null);
assert.isEmpty(nulls);
assert.isEmpty(holds.filter((hold) => hold.duration > 300)); // all motions were very quick.
// True (intended) motions were 'e' -> 's' -> 'w' -> 'n', but the motions weren't that precise
// due to prioritizing speed.
const eMoveIndex = moves.findIndex((move) => move.direction.includes('e'));
const sMoveIndex = moves.findIndex((move) => move.direction.includes('s'));
const wMoveIndex = moves.findIndex((move) => move.direction.includes('w'));
const nMoveIndex = moves.findIndex((move) => move.direction.includes('n'));
assert.notEqual(eMoveIndex, -1);
assert.notEqual(sMoveIndex, -1);
assert.notEqual(wMoveIndex, -1);
assert.notEqual(nMoveIndex, -1);
// Again, the true (intended) motions were 'e' -> 's' -> 'w' -> 'n'. This verifies the relative ordering.
assert.isBelow(eMoveIndex, sMoveIndex);
assert.isBelow(sMoveIndex, wMoveIndex);
assert.isBelow(wMoveIndex, nMoveIndex);
// Original's length: 12, but there are some borderline holds there & we don't want this test
// to be too rigid.
assert.isAtLeast(reproedSegments.length, 9); // 2 = 'start' + 'end'
// 2 = 1 hold after start, 1 hold before end
// 4 cardinal directions
// At least one notable hold during direction changes
});
});