diff --git a/common/web/gesture-recognizer/src/cumulativePathStats.ts b/common/web/gesture-recognizer/src/cumulativePathStats.ts
new file mode 100644
index 0000000000..e148012d22
--- /dev/null
+++ b/common/web/gesture-recognizer/src/cumulativePathStats.ts
@@ -0,0 +1,746 @@
+namespace com.keyman.osk {
+ /**
+ * Denotes one dimension utilized by touchpath input coordinates - 'x' and y' for space,
+ * 't' for time.
+ */
+ export type PathCoordAxis = 'x' | 'y' | 't';
+
+ /**
+ * Denotes a pair of dimensions utilized by touchpath input coordinates. The two axes
+ * (see `PathCoordAxis`) must be specified in alphabetical order.
+ */
+ export type PathCoordAxisPair = 'tx' | 'ty' | 'xy';
+
+ /**
+ * Denotes one dimension or feature (velocity) that this class tracks statistics for.
+ *
+ * Sine and Cosine stats are currently excluded due to their necessary lack of statistical
+ * independence.
+ */
+ type StatAxis = PathCoordAxis | 'v';
+
+ /**
+ * Acts as a subtraction operation with a built-in, adaptive "significance" check.
+ * If the result * 2^30 (~ * 10^9) is still smaller in magnitude than an operand, we
+ * assume it to be a floating-point error that should have been 0 and act accordingly,
+ * returning 0.
+ *
+ * For reference, (32-bit) floats have 23 bits of significand precision, while (64-bit)
+ * doubles have 52. Therefore, we'll still be more precise than baseline floats.
+ */
+ function sigMinus(operand1: number, operand2: number) {
+ const diff = operand1 - operand2;
+
+ const logDiff = Math.log2(Math.abs(operand1)) - Math.log2(Math.abs(operand2));
+ // If an operand is 2^30 (or ~10^9) larger than the result of the difference, it's
+ // nigh-certainly a floating-point error at play.
+ return logDiff < 30 ? diff : 0;
+ }
+
+ /**
+ * As the name suggests, this class facilitates tracking of cumulative mathematical values, etc
+ * necessary to perform the statistical operations necessary for path segmentation.
+ *
+ * Instances of this class are immutable.
+ */
+ export class CumulativePathStats {
+ /**
+ * Provides linear-regression statistics & fitting values based on the underlying `CumulativePathStats`
+ * object used to generate it. All operations are O(1).
+ */
+ static readonly regression = class RegressionFromSums {
+ readonly independent: PathCoordAxis;
+ readonly dependent: PathCoordAxis;
+ readonly paired: PathCoordAxisPair;
+
+ readonly accumulator: CumulativePathStats;
+
+ /**
+ *
+ * @param mainStats The `CumulativePathStats` instance to base all regression data on.
+ * @param dependentAxis The 'output' axis / dimension; the axis whose behavior should be predicted based on
+ * existing data of its relationship with the independent axis.
+ * @param independentAxis The 'input' axis/dimension.
+ */
+ constructor(mainStats: CumulativePathStats, dependentAxis: PathCoordAxis, independentAxis: PathCoordAxis) {
+ if(dependentAxis == independentAxis) {
+ throw "Two different axes must be specified for the regression object.";
+ }
+
+ this.accumulator = mainStats;
+
+ this.dependent = dependentAxis;
+ this.independent = independentAxis;
+
+ if(dependentAxis < independentAxis) {
+ this.paired = dependentAxis.concat(independentAxis) as PathCoordAxisPair;
+ } else {
+ this.paired = independentAxis.concat(dependentAxis) as PathCoordAxisPair;
+ }
+ }
+
+ /**
+ * The 'slope' of the 'slope-intercept' form of the line that best fits the relationship between
+ * this regression's selected axes.
+ */
+ get slope(): number {
+ // The technical definition is the commented-out line, but the denominator component of both
+ // cancels out - it's 'more efficient' to use the following line as a result.
+
+ // this.accumulator.covariance(this.paired) / this.accumulator.variance(this.independent);
+ const val = this.accumulator.crossSum(this.paired) / this.accumulator.squaredSum(this.independent);
+ return val;
+ }
+
+ /**
+ * The 'intercept' of the 'slope-intercept' form of the line that best fits the relationship between
+ * this regression's selected axes.
+ */
+ get intercept(): number {
+ // Performing a regression based on these pre-summed values means that our obtained intercept is in
+ // the mapped coordinate system.
+ const mappedIntercept = this.accumulator.mappedMean(this.dependent) - this.slope * this.accumulator.mappedMean(this.independent);
+
+ const val = mappedIntercept + this.accumulator.mappingConstant(this.dependent) -
+ this.slope * this.accumulator.mappingConstant(this.independent);
+
+ return val;
+ }
+
+ /**
+ * The total summed squared-distances of the best fitting line from actually-observed values;
+ * in other words, the "sum of the squared errors".
+ *
+ * Statistically, this is the portion of the dependent variable's variance (un-normalized)
+ * that is unexplained by this regression.
+ */
+ get sumOfSquaredError(): number {
+ return this.accumulator.squaredSum(this.dependent) - this.sumOfSquaredModeled;
+ }
+
+ /**
+ * The portion of the dependent variable's variance that is successfully explained by this regression.
+ */
+ get sumOfSquaredModeled(): number {
+ // If we have a perfectly straight vertical line, from the perspective of our independent axis,
+ // we get infinite slope. That's... not great for the math.
+ //
+ // Fortunately, it ALSO means that we can perfectly model the segment.
+ if(this.accumulator.squaredSum(this.independent) == 0) {
+ return this.accumulator.squaredSum(this.dependent);
+ } else {
+ return this.slope * this.accumulator.crossSum(this.paired);
+ }
+ }
+
+ /**
+ * A statistical term that signals how successful the regression is. Always has values on
+ * the interval [0, 1], with 1 being a perfect fit.
+ */
+ get coefficientOfDetermination(): number {
+ if(this.accumulator.squaredSum(this.dependent) == 0 || this.accumulator.squaredSum(this.independent) == 0) {
+ return 1;
+ }
+
+ const acc = this.accumulator;
+ const num = acc.crossSum(this.paired) * acc.crossSum(this.paired);
+ const denom = acc.squaredSum(this.dependent) * acc.squaredSum(this.independent);
+
+ return num / denom;
+ }
+
+ /**
+ * Gets the value of the dependent axis that lies on the regression's fitted line
+ * for a specified independent axis value.
+ *
+ * @param value The input value to use for the independent axis's variable.
+ * @returns The predicted dependent axis value.
+ */
+ predictFromValue(value: number) {
+ return this.slope * value + this.intercept;
+ }
+ }
+
+ /**
+ * Floating-point errors may result from cross-sum calculations, and they may be slightly larger than
+ * Number.EPSILON as the sums grow. (Taking the difference of cross-sums)
+ */
+ private static readonly CANCELLATION_EPSILON = Math.sqrt(Number.EPSILON);
+
+ private rawLinearSums = {'x': 0, 'y': 0, 't': 0, 'v': 0};
+ private rawSquaredSums = {'x': 0, 'y': 0, 't': 0, 'v': 0};
+ // Would 'tv' (time vs velocity) be worth it to track? And possibly even do a regression for?
+ // If so, maybe throw that in.
+ private rawCrossSums = {'tx': 0, 'ty': 0, 'xy': 0};
+
+ private coordArcSum: number = 0;
+ private arcSampleCount: number = 0;
+
+ // These two are kept separate because of their extreme interconnectedness - after all,
+ // they actually represent a SINGLE (polar) axis - the angle.
+ //
+ // Sadly, there's no straightforward, well-founded way to use these to give a proper
+ // statistical sense of 'fit' or 'regression' here, _especially_ in regard to segmentation.
+ // Proper angle-[other] cross-sums are pretty much impossible, at least as efficiently
+ // as the others are handled [O(1)].
+ private cosLinearSum: number = 0;
+ private sinLinearSum: number = 0;
+
+ /**
+ * The base sample used to transpose all other received samples. Use of this helps
+ * avoid potential "catastrophic cancellation" effects that can occur when diffing
+ * two numbers far from the sample-space's mathematical origin.
+ *
+ * Refer to https://en.wikipedia.org/wiki/Catastrophic_cancellation.
+ */
+ private baseSample?: InputSample;
+
+ /**
+ * The initial sample included by this instance's computed stats. Needed for
+ * the 'directness' properties.
+ */
+ private initialSample?: InputSample;
+
+ private _lastSample?: InputSample;
+ private followingSample?: InputSample;
+ private _sampleCount = 0;
+
+ constructor();
+ constructor(sample: InputSample);
+ constructor(instance: CumulativePathStats);
+ constructor(obj?: InputSample | CumulativePathStats) {
+ if(!obj) {
+ return;
+ }
+
+ // Will worry about JSON form later.
+ if(obj instanceof CumulativePathStats) {
+ Object.assign(this, obj);
+
+ this.rawLinearSums = {...obj.rawLinearSums};
+ this.rawCrossSums = {...obj.rawCrossSums};
+ this.rawSquaredSums = {...obj.rawSquaredSums};
+ } else if(isAnInputSample(obj)) {
+ Object.assign(this, this.extend(obj));
+ } else {
+ throw "A constructor for this input pattern has not yet been implemented";
+ }
+ }
+
+ /**
+ * Statistically "observes" a new sample point on the touchpath, accumulating values
+ * useful for provision of relevant statistical properties.
+ * @param sample A newly-sampled point on the touchpath.
+ * @returns A new, separate instance for the cumulative properties up to the
+ * newly-sampled point.
+ */
+ public extend(sample: InputSample): CumulativePathStats {
+ if(!this.initialSample) {
+ this.initialSample = sample;
+ this.baseSample = sample;
+ }
+ const result = new CumulativePathStats(this);
+
+ // Set _after_ deep-copying this for the result.
+ this.followingSample = sample;
+
+ // Helps prevent "catastrophic cancellation" issues from floating-point computation
+ // for these statistical properties and properties based upon them.
+ const x = sample.targetX - this.baseSample.targetX;
+ const y = sample.targetY - this.baseSample.targetY;
+ const t = sample.t - this.baseSample.t;
+
+ result.rawLinearSums.x += x;
+ result.rawLinearSums.y += y;
+ result.rawLinearSums.t += t;
+
+ result.rawCrossSums.tx += t * x;
+ result.rawCrossSums.ty += t * y;
+ result.rawCrossSums.xy += x * y;
+
+ result.rawSquaredSums.x += x * x;
+ result.rawSquaredSums.y += y * y;
+ result.rawSquaredSums.t += t * t;
+
+ if(this.lastSample) {
+ // arc length stuff!
+ const xDelta = sample.targetX - this.lastSample.targetX;
+ const yDelta = sample.targetY - this.lastSample.targetY;
+ const tDelta = sample.t - this.lastSample.t;
+
+ const coordArcDeltaSq = xDelta * xDelta + yDelta * yDelta;
+ const coordArcDelta = Math.sqrt(coordArcDeltaSq);
+
+ result.coordArcSum += coordArcDelta;
+
+ if(xDelta || yDelta) {
+ // We wish to measure angle clockwise from <0, -1> in the DOM. So, cos values should
+ // align with that axis, while sin values should align with the positive x-axis.
+ //
+ // This provides a mathematical 'transformation' to the axes used by `atan2` in the
+ // `angleMean` property.
+ result.cosLinearSum += -yDelta / coordArcDelta;
+ result.sinLinearSum += xDelta / coordArcDelta;
+ result.arcSampleCount += 1;
+ }
+
+ if(tDelta) {
+ result.rawLinearSums.v += coordArcDelta / tDelta;
+ result.rawSquaredSums.v += coordArcDeltaSq / (tDelta * tDelta);
+ }
+ }
+
+ result._lastSample = sample;
+ result.sampleCount = this.sampleCount + 1;
+
+ return result;
+ }
+
+ /**
+ * "De-accumulates" currently-accumulated values corresponding to the specified
+ * subset, which should represent an earlier, previously-observed part of the path.
+ * @param subsetStats The accumulated stats for the part of the path being removed
+ * from this instance's current accumulation.
+ * @returns
+ */
+ public deaccumulate(subsetStats?: CumulativePathStats): CumulativePathStats {
+ // Possible addition: use `this.buildRenormalized` on the returned version
+ // if catastrophic cancellation effects (random, small floating point errors)
+ // are not sufficiently mitigated & handled by the measures currently in place.
+ //
+ // Even then, we'd need to apply such generated objects carefully - we can't
+ // re-merge the accumulated values or remap them to their old coordinate system
+ // afterward.`buildRenormalize`'s remapping maneuver is a one-way stats-abuse trick.
+ //
+ // Hint: we'd need to pay attention to the "lingering segments" aspects in which
+ // detected sub-segments might be "re-merged".
+ // - Whenever they're merged & cleared, we should be clear to recentralize
+ // the cumulative stats that follow. If any are still active, we can't
+ // recentralize.
+
+ const result = new CumulativePathStats(this);
+
+ // We actually WILL accept a `null` argument; makes some of the segmentation
+ // logic simpler.
+ if(!subsetStats) {
+ return result;
+ }
+
+ if(!subsetStats.followingSample || !subsetStats.lastSample) {
+ throw 'Invalid argument: stats missing necessary tracking variable.';
+ }
+
+ for(let dim in result.rawLinearSums) {
+ result.rawLinearSums[dim] -= subsetStats.rawLinearSums[dim];
+ }
+
+ for(let dimPair in result.rawCrossSums) {
+ result.rawCrossSums[dimPair] -= subsetStats.rawCrossSums[dimPair];
+ }
+
+ for(let dim in result.rawSquaredSums) {
+ result.rawSquaredSums[dim] -= subsetStats.rawSquaredSums[dim];
+ }
+
+ // arc length stuff!
+ if(subsetStats.followingSample && subsetStats.lastSample) {
+ const xDelta = subsetStats.followingSample.targetX - subsetStats.lastSample.targetX;
+ const yDelta = subsetStats.followingSample.targetY - subsetStats.lastSample.targetY;
+ const tDelta = subsetStats.followingSample.t - subsetStats.lastSample.t;
+
+ const coordArcDeltaSq = xDelta * xDelta + yDelta * yDelta;
+ const coordArcDelta = Math.sqrt(coordArcDeltaSq);
+
+ // Due to how arc length stuff gets segmented.
+ // There's the arc length within the prefix subset (operand 2 below) AND the part connecting it to the
+ // 'remaining' subset (operand 1 below) before the portion wholly within what remains (the result)
+ result.coordArcSum -= coordArcDelta;
+ result.coordArcSum -= subsetStats.coordArcSum;
+
+ result.cosLinearSum -= subsetStats.cosLinearSum;
+ result.sinLinearSum -= subsetStats.sinLinearSum;
+ result.arcSampleCount -= subsetStats.arcSampleCount;
+
+ if(tDelta) {
+ result.rawLinearSums.v -= coordArcDelta / tDelta;
+ result.rawSquaredSums.v -= coordArcDeltaSq / (tDelta * tDelta);
+ }
+ }
+
+ result.sampleCount -= subsetStats.sampleCount;
+
+ // NOTE: baseSample MUST REMAIN THE SAME. All math is based on the corresponding diff.
+ // Though... very long touchpoint interactions could start being affected by that "catastrophic
+ // cancellation" effect without further adjustment. (If it matters, we'll get to that later.)
+ // But _probably_ not; we don't go far beyond a couple of orders of magnitude from the origin in
+ // ANY case except the timestamp (.t) - and even then, not far from the baseSample's timestamp value.
+
+ // initialSample, though, we need to update b/c of the 'directness' properties.
+ result.initialSample = subsetStats.followingSample;
+
+ return result;
+ }
+
+ public get lastSample() {
+ return this._lastSample;
+ }
+
+ public get lastTimestamp(): number {
+ return this.lastSample?.t;
+ }
+
+ public get sampleCount() {
+ return this._sampleCount;
+ }
+
+ private set sampleCount(value: number) {
+ this._sampleCount = value;
+ }
+
+ /**
+ * In order to mitigate the accumulation of small floating-point errors during the
+ * various accumulations performed by this class, the domain of incoming values
+ * is remapped near to the origin via axis-specific mapping constants.
+ * @param dim
+ * @returns
+ */
+ private mappingConstant(dim: StatAxis) {
+ if(!this.baseSample) {
+ return undefined;
+ }
+
+ if(dim == 't') {
+ return this.baseSample.t;
+ } else if(dim == 'x') {
+ return this.baseSample.targetX;
+ } else if(dim == 'y') {
+ return this.baseSample.targetY;
+ } else {
+ return 0;
+ }
+ }
+
+ /**
+ * Gets the statistical mean, utilizing the internal 'mapped' coordinate space.
+ * This is the version compatible with cross-sums and squared-sums.
+ * @param dim
+ * @returns
+ */
+ private mappedMean(dim: StatAxis) {
+ return this.rawLinearSums[dim] / this.sampleCount;
+ }
+
+ /**
+ * Gets the statistical mean value of the samples observed during the represented
+ * interval on the specified axis.
+ * @param dim
+ * @returns
+ */
+ public mean(dim: StatAxis) {
+ // This external-facing version needs to provide values in 'external'-friendly
+ // coordinate space.
+ return this.mappedMean(dim) + this.mappingConstant(dim);
+ }
+
+ /**
+ * Gets the sum of the squared distance from the mean seen in samples observed
+ * during the represented interval on the specified axis.
+ * @param dim
+ * @returns
+ */
+ public squaredSum(dim: StatAxis) {
+ const x2 = this.rawSquaredSums[dim];
+ const x1 = this.rawLinearSums[dim];
+
+ return sigMinus(x2, x1 * x1 / this.sampleCount);
+ }
+
+ /**
+ * Gets the sum of the statistical 'cross' term "distance" away from the mean
+ * observed during the represented interval on the specified axis.
+ * @param dimPair
+ * @returns
+ */
+ public crossSum(dimPair: PathCoordAxisPair) {
+ const dim1 = dimPair.charAt(0);
+ const dim2 = dimPair.charAt(1);
+
+ let orderedDims: string = dimPair;
+
+ const ab = this.rawCrossSums[orderedDims];
+ const a = this.rawLinearSums[dim1];
+ const b = this.rawLinearSums[dim2];
+
+ return sigMinus(ab, a * b / this.sampleCount);
+ }
+
+ /**
+ * Gets the unbiased covariance between the specified pair of axes for samples
+ * observed during the represented interval.
+ * @param dimPair
+ * @returns
+ */
+ public covariance(dimPair: PathCoordAxisPair) {
+ return this.crossSum(dimPair) / (this.sampleCount - 1);
+ }
+
+ /**
+ * Gets the unbiased variance on the specified axis for samples observed
+ * during the represented interval.
+ */
+ public variance(dim: StatAxis) {
+ return this.squaredSum(dim) / (this.sampleCount - 1);
+ }
+
+ /**
+ * Utilizing (and possibly abusing) statistical identities, this function produces
+ * an equivalent, but-recentered copy of this instance's statistical accumulations
+ * that will be less prone to catastrophic cancellation.
+ *
+ * In non-stats speak, the new instance will suffer smaller floating-point
+ * errors than the old instance whenever they do occur.
+ * @returns
+ */
+ public buildRenormalized(): CumulativePathStats {
+ // Other (internal) notes: the internal mapping of the new instance will not
+ // match that of the old instance. This should not affect the practical
+ // results of any mapping to and from the external coordinate space, however.
+ let result = new CumulativePathStats(this);
+
+ let newBase: InputSample = {
+ targetX: this.mappedMean('x') + this.baseSample.targetX,
+ targetY: this.mappedMean('y') + this.baseSample.targetY,
+ t: this.mappedMean('t') + this.baseSample.t
+ };
+
+ result.baseSample = newBase;
+
+ for(const dimPair in result.rawCrossSums) {
+ result.rawCrossSums[dimPair] = this.crossSum(dimPair as PathCoordAxisPair);
+ }
+
+ for(const dim in result.rawSquaredSums) {
+ // 'v' does not need renormalization.
+ if(dim == 'v') {
+ break;
+ }
+
+ // The identity we're using to renormalize rawCrossSums and rawSquaredSums
+ // automatically guarantees a mean of 0 after the renormalization.
+ result.rawLinearSums[dim] = 0;
+ result.rawSquaredSums[dim] = this.squaredSum(dim as PathCoordAxis);
+ }
+
+ return result;
+ }
+
+ /**
+ * Provides a linear-regression perspective on two specified axes over the represented
+ * interval.
+ * @param dependent
+ * @param independent
+ * @returns
+ */
+ public fitRegression(dependent: PathCoordAxis, independent: PathCoordAxis) {
+ return new CumulativePathStats.regression(this, dependent, independent);
+ }
+
+ /**
+ * Provides the direct Euclidean distance between the start and end points of the segment
+ * (or curve) of the interval represented by this instance.
+ *
+ * This will likely not match the actual pixel distance traveled.
+ */
+ public get netDistance() {
+ // No issue with a net distance of 0 due to a single point.
+ if(!this.lastSample || !this.initialSample) {
+ return 0;
+ }
+
+ const xDelta = this.lastSample.targetX - this.initialSample.targetX;
+ const yDelta = this.lastSample.targetY - this.initialSample.targetY;
+
+ return Math.sqrt(xDelta * xDelta + yDelta * yDelta);
+ }
+
+ /**
+ * Gets the duration of the represented interval in milliseconds.
+ */
+ public get duration() {
+ // no issue with a duration of zero from just one sample.
+ if(!this.lastSample || !this.initialSample) {
+ return 0;
+ }
+ return (this.lastSample.t - this.initialSample.t);
+ }
+
+ /**
+ * Returns the angle (in radians) traveled by the corresponding segment clockwise
+ * from the unit vector <0, -1> in the DOM (the unit "upward" direction).
+ */
+ public get angle() {
+ if(this.sampleCount == 1 || !this.lastSample || !this.initialSample) {
+ return Number.NaN;
+ } else if(this.netDistance < 1) {
+ // < 1 px, thus sub-pixel, means we have nothing relevant enough to base an angle on.
+ return Number.NaN;
+ }
+
+ const xDelta = this.lastSample.targetX - this.initialSample.targetX;
+ const yDelta = this.lastSample.targetY - this.initialSample.targetY;
+ const yAngleDiff = Math.acos(-yDelta / this.netDistance);
+
+ return xDelta < 0 ? (2 * Math.PI - yAngleDiff) : yAngleDiff;
+ }
+
+ /**
+ * Returns the angle (in degrees) traveled by the corresponding segment clockwise
+ * from the unit vector <0, -1> in the DOM (the unit "upward" direction).
+ */
+ public get angleInDegrees() {
+ return this.angle * 180 / Math.PI;
+ }
+
+ /**
+ * Returns the cardinal or intercardinal direction on the screen that most
+ * closely matches the direction of movement represented by the represented
+ * segment.
+ *
+ * @return A string one or two letters in length (e.g: 'n', 'sw'), or
+ `undefined` if not enough data to determine a direction.
+ */
+ public get cardinalDirection() {
+ if(this.sampleCount == 1 || !this.lastSample || !this.initialSample) {
+ return undefined;
+ }
+
+ const buckets = ['n', 'ne', 'e', 'se', 's', 'sw', 'w', 'nw', 'n'];
+
+ // We could be 'more efficient' and use radians here instead, but this
+ // version helps a bit more with easy maintainability.
+ const bucketIndex = Math.ceil((this.angleInDegrees - 22.5)/45);
+ return buckets[bucketIndex];
+ }
+
+ /**
+ * Measured in pixels per second.
+ * @return a speed in pixels per millisecond, or `Number.NaN` if no data
+ */
+ public get speed() {
+ return this.duration ? this.netDistance / this.duration : Number.NaN;
+ }
+
+ /**
+ * Returns the represented interval's 'mean angle' clockwise from the DOM's
+ * <0, -1> (the unit vector toward the top of the screen) in radians.
+ *
+ * Based upon the 'circular mean'. Refer to https://en.wikipedia.org/wiki/Circular_mean.
+ *
+ * Note that very slow-moving segments may be heavily affected by pixel aliasing
+ * effects; mouse and touch events usually do not provide sub-pixel resolution.
+ */
+ public get angleMean() {
+ if(this.arcSampleCount == 0) {
+ return Number.NaN;
+ }
+
+ // Neato reference: https://rosettacode.org/wiki/Averages/Mean_angle
+ // But we don't actually need to divide by sample count; `atan2` handles that!
+ const sinMean = this.sinLinearSum;
+ const cosMean = this.cosLinearSum;
+
+ let angle = Math.atan2(sinMean, cosMean); // result: on the interval (-pi, pi]
+ // Convert to [0, 2*pi).
+ if(angle < 0) {
+ angle = angle + 2 * Math.PI;
+ }
+
+ return angle;
+ }
+
+ /**
+ * Provides the rSquared value needed internally for circular-statistic properties.
+ *
+ * Range: floating-point values on the interval [0, 1].
+ */
+ private get angleRSquared() {
+ // Refer to https://en.wikipedia.org/wiki/Directional_statistics#Distribution_of_the_mean.
+ // We're computing the squared value of that page's R-bar stat.
+ //
+ // Now, why it's called that? ... good question. My best guess is that it's meant to
+ // correspond to linear regression's 'r' stat, which when squared serves as the
+ // coefficient of determination for the regression. Intuitively, that does seem to
+ // match what this represents - though for normal regressions, the c.o.d isn't normally
+ // used to compute deviation or variance!
+ const rSquaredBase = this.cosLinearSum * this.cosLinearSum + this.sinLinearSum * this.sinLinearSum;
+ return rSquaredBase / (this.arcSampleCount * this.arcSampleCount);
+ }
+
+ /**
+ * The **circular standard deviation** of the represented interval's angle observations.
+ *
+ * Refer to https://en.wikipedia.org/wiki/Directional_statistics#Standard_deviation.
+ *
+ * Note that very slow-moving segments may be heavily affected by pixel aliasing
+ * effects; mouse and touch events usually do not provide sub-pixel resolution.
+ * This can result in very high deviation values.
+ *
+ * In less-technical terms - the "stair-stepping" effect seen on high zoom levels means we don't
+ * get perfectly straight lines, and that can cause this value to be unexpectedly high.
+ */
+ public get angleDeviation() {
+ if(this.arcSampleCount == 0) {
+ return Number.NaN;
+ }
+
+ return Math.sqrt(-Math.log(this.angleRSquared));
+ }
+
+ /**
+ * Provides the actual, pixel-based distance actually traveled by the represented segment.
+ * May not be an integer (because diagonals are a thing).
+ */
+ public get rawDistance() {
+ return this.coordArcSum;
+ }
+
+ // Convert to a `toJSON` method for use during investigative debugging.
+ private toDebuggingJSON() {
+ // This `likelyState` value is extremely prototyped & just here for reviewer/tester convenience.
+ // It'll need to be developed a bit more fully, but follows my intuitions from development &
+ // testing.
+ let likelyType = 'unknown';
+
+ if(this.mean('v') < 0.08 && this.rawDistance < 12 && this.duration > 100) {
+ likelyType = 'hold';
+ } else if(this.mean('v') < 0.08 && this.rawDistance < 6) {
+ likelyType = 'hold';
+ }
+
+ if(this.mean('v') > 0.4 || (this.mean('v') > 0.2 && this.duration > 80) || this.netDistance > 20) {
+ likelyType = 'move';
+ }
+
+ return {
+ angle: this.angle,
+ cardinal: this.cardinalDirection,
+ likelyType: likelyType,
+ speedMean: this.mean('v'),
+ rawDistance: this.rawDistance,
+ duration: this.duration,
+ sampleCount: this.sampleCount,
+ angleMeanDegrees: this.angleMean * 180 / Math.PI,
+ angleDeviation: this.angleDeviation,
+ speedVariance: this.variance('v')
+ }
+ }
+
+ public toJSON() {
+ // We're not actually saving the JSON out to anything yet or loading/parsing it
+ // for any use beyond direct human interpretation, so... it's "okay" to
+ // leave like this for now.
+ return this.toDebuggingJSON();
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/common/web/gesture-recognizer/src/inputSample.ts b/common/web/gesture-recognizer/src/inputSample.ts
index cc2a0fa0fe..f0b5b39981 100644
--- a/common/web/gesture-recognizer/src/inputSample.ts
+++ b/common/web/gesture-recognizer/src/inputSample.ts
@@ -36,4 +36,8 @@ namespace com.keyman.osk {
}
export type InputSampleSequence = InputSample[];
+
+ export function isAnInputSample(obj: any): obj is InputSample {
+ return 'targetX' in obj && 'targetY' in obj && 't' in obj;
+ }
}
\ No newline at end of file
diff --git a/common/web/gesture-recognizer/src/pathSegmenter.ts b/common/web/gesture-recognizer/src/pathSegmenter.ts
new file mode 100644
index 0000000000..40c8b58a27
--- /dev/null
+++ b/common/web/gesture-recognizer/src/pathSegmenter.ts
@@ -0,0 +1,955 @@
+///
+
+/* FIXME: Too much `console`.
+ *
+ * There's an AWFUL LOT of `console.log`-ing in this file at present.
+ * It was definitely useful for prototyping & development of its (and
+ * of `cumulativePathStats`'s) code, but it's GOTTA be cleaned up
+ * in the next PR... if not sooner.
+ */
+
+namespace com.keyman.osk {
+ // Note: the only `export`-ed class from this file is `PathSegmenter`, hence the helper
+ // classes being placed within the same file. Not to say we _couldn't_ put them elsewhere.
+
+ // ------------
+
+ // Mostly used here to compare the sum-squared error components of segmented regressions
+ // to the sum-squared "modeled" components of their overall variance. Those are the two
+ // "independent random variables" we're examining. Variances (which are sums of squared
+ // values themselves) tend to be chi-squared distributed, fulfilling the conditions.
+ // That's stats-speak for "what this module does with it has a solid theoretical basis."
+ //
+ // We do NOT want to be computing this thing at run-time. (Just take one look at the
+ // equations found in that Wikipedia article!) Fortunately, it's very common in stats
+ // circles to just... use a lookup table for finding p-values for things following
+ // the f-distribution.
+ //
+ // For future non-stats-inclined maintainers:
+ // One could say that the p-value = the chance that what we have just "randomly" happened
+ // to occur without our expectations actually being met. It's a super-common
+ // stats term; they'd then say that if the p-value is sufficiently low, then we\
+ // "reject the null hypothesis" - the theory that it actually DID happen randomly - in
+ // favor of the high likelihood that we really are onto something.
+
+ /**
+ * Acts as a lookup table for null-hypothesis tests utilizing the F-distribution from the
+ * domain of statistics. Refer to https://en.wikipedia.org/wiki/F-distribution for details
+ * on its properties.
+ *
+ * This is only a partial implementation; only a couple of numerator degrees-of-freedom are
+ * worth our consideration, and there's only marginal gain to be found supporting more
+ * denominator degrees-of-freedom than are already supported here. Keeping it small also
+ * helps with maintenance and readability.
+ */
+ class FDistribution {
+
+ /**
+ * Indexing: [threshold-index][num-2][denom-1].
+ *
+ * If numerator has only 1 degree of freedom, that means we don't really have
+ * segments so much as different averages between the two segments - there's
+ * no slope on either side. In other words, "just a bump". No segmentation
+ * there.
+ *
+ * 2 degrees of freedom: no movement on the axis on only one side.
+ *
+ * threshold-index:
+ * - p-value of .100: 0
+ * - p-value of .050: 1
+ *
+ * The f-statistic value must match or exceed its corresponding entry in
+ * the table below, based on the detected DoF (degrees of freedom) for the
+ * 'numerator' and 'denominator' components. Exceeding the threshold specified
+ * for the p = .100 section indicates less than a 10% probability of the statistic
+ * being tested having arisen by purely random chance. Failure to exceed even
+ * that implies an unacceptably high chance of being "convenient but meaningless" -
+ * the "null hypothesis", in stats speak.
+ */
+ private static readonly table = [
+ // p = .100
+ [
+ // numerator: 2
+ [
+ // denom: 1-10
+ 49.50, 9.00, 5.46, 4.32, 3.78, 3.46, 3.26, 3.11, 3.01, 2.92,
+ // denom: 11-20
+ 2.86, 2.81, 2.76, 2.73, 2.70, 2.67, 2.64, 2.62, 2.61, 2.59
+ // we COULD continue with threshold-dropping after that,
+ // but I doubt we'll need it in practicality.
+ // At 30: 2.49
+ // limit -> infinity: 2.30
+ ],
+ // numerator: 3
+ [
+ // denom: 1-10
+ 53.59, 9.16, 5.39, 4.19, 3.62, 3.29, 3.07, 2.92, 2.81, 2.73,
+ // denom: 11-20
+ 2.66, 2.61, 2.56, 2.52, 2.49, 2.46, 2.44, 2.42, 2.40, 2.38
+ // Likewise, from above.
+ // At 30: 2.28
+ // limit -> infinity: 2.08
+ ]
+ ],
+ // p = .050
+ [
+ // numerator: 2
+ [
+ // denom: 1-10
+ 199.5, 19.0, 9.55, 6.94, 5.79, 5.14, 4.74, 4.46, 4.26, 4.10,
+ // denom: 10-20
+ 3.98, 3.89, 3.81, 3.74, 3.68, 3.63, 3.59, 3.55, 3.52, 3.49
+ // At 30: 3.32
+ // limit -> infinity: 3.00
+ ],
+ // numerator: 3
+ [
+ // denom: 1-10
+ 215.7,19.16, 9.28, 6.59, 5.41, 4.76, 4.35, 4.07, 3.86, 3.71,
+ // denom: 10-20
+ 3.59, 3.49, 3.41, 3.34, 3.29, 3.24, 3.20, 3.16, 3.13, 3.10
+ // At 30: 2.92
+ // limit -> infinity: 2.60
+ ]
+ ]
+ ]
+
+ /**
+ * Determines a threshold tier for segmentation based on the segmented
+ * regression f-statistic.
+ * @param statistic
+ * @param numDoF
+ * @param denomDoF
+ *
+ * Returns 0.050 if the statistic indicates a p-value of 0.050 or lower.
+ * Failing that, returns 0.100 if a p-value of 0.100 or lower is indicated.
+ * Otherwise, returns 1.0 - we have no basis to "reject the null hypothesis".
+ */
+ static thresholdTier(statistic: number, numDoF: number, denomDoF: number) {
+ // Former case: we'd never segment anyway
+ // Latter case: it's currently wrong to statistically test.
+ // The F-distribution is not defined for this case.
+ if(numDoF < 2 || denomDoF < 1) {
+ return 1;
+ }
+
+ const numIndex = (numDoF > 3 ? 3 : numDoF) - 2;
+ const denomIndex = (denomDoF > 20 ? 20 : denomDoF) - 1;
+
+ const tier2Threshold = FDistribution.table[1][numIndex][denomIndex];
+ if(statistic >= tier2Threshold) {
+ return 0.05;
+ }
+
+ const tier1Threshold = FDistribution.table[0][numIndex][denomIndex];
+ // If arising purely randomly isn't at least less than 10% likely,
+ // we'll categorically say it happened randomly with full certainty (1).
+ // Obviously not actually true, but it works for our thresholding.
+ return statistic >= tier1Threshold ? 0.10 : 1;
+ }
+ }
+
+ /**
+ * Represents the result of a segmentation of the search path and the related
+ * statistical accumulations needed to properly test the segmentation's
+ * validity.
+ */
+ class Segmentation {
+ public static readonly SPLIT_CRITERION_THRESHOLD = 1.5;
+
+ /**
+ * The properties of the "left-hand" / earlier half of the time interval
+ * being segmented.
+ */
+ readonly pre: CumulativePathStats;
+
+ /**
+ * The properties of the "right-hand" / later half of the time interval
+ * being segmented.
+ */
+ readonly post: CumulativePathStats;
+
+ /**
+ * The properties of the full interval being segmented, before the split.
+ */
+ readonly union: CumulativePathStats;
+
+ /**
+ * The full running accumulation for the ongoing touch path. May include
+ * accumulations from before the interval under examination.
+ */
+ readonly endpoint: CumulativePathStats;
+
+ /**
+ * Provides segmented-regression statistics & fitting values on the two specified axes /
+ * dimensions based on the subsegments and their corresponding linear-regression
+ * statistics. All operations are O(1).
+ */
+ static readonly segmentationComparison = class SegmentedRegression {
+ host: Segmentation;
+ readonly independent: PathCoordAxis;
+ readonly dependent: PathCoordAxis;
+ readonly paired: 'tx' | 'ty' | 'xy';
+
+ /**
+ * The linear regression for the underlying `pre` (earlier) subsegment.
+ */
+ pre: typeof CumulativePathStats.regression.prototype;
+
+ /**
+ * The linear regression for the underlying `post` (later) subsegment.
+ */
+ post: typeof CumulativePathStats.regression.prototype;
+
+ /**
+ * The linear, unsegmented regression for the underlying `union` (combined) subsegment.
+ */
+ union: typeof CumulativePathStats.regression.prototype;
+
+ constructor(host: Segmentation, dependentAxis: PathCoordAxis, independentAxis: PathCoordAxis) {
+ if(dependentAxis == independentAxis) {
+ throw "Two different axes must be specified for the regression object.";
+ }
+
+ this.host = host;
+
+ this.dependent = dependentAxis;
+ this.independent = independentAxis;
+
+ if(dependentAxis < independentAxis) {
+ this.paired = dependentAxis.concat(independentAxis) as 'tx' | 'ty' | 'xy';
+ } else {
+ this.paired = independentAxis.concat(dependentAxis) as 'tx' | 'ty' | 'xy';
+ }
+
+ this.pre = this.host.pre .fitRegression(dependentAxis, independentAxis);
+ this.post = this.host.post .fitRegression(dependentAxis, independentAxis);
+ this.union = this.host.union.fitRegression(dependentAxis, independentAxis);
+ }
+
+ /**
+ * The total summed squared-distances of the best fitting line from actually-observed values
+ * for their corresponding subsegment; in other words, the "sum of the squared errors" when
+ * utilizing segmented regression.
+ *
+ * Statistically, this is the portion of the dependent variable's variance (un-normalized)
+ * that is unexplained even after dividing the interval into separate subsegments.
+ */
+ get remainingSumSquaredError(): number {
+ return this.pre.sumOfSquaredError + this.post.sumOfSquaredError;
+ }
+
+ /**
+ * Provides the coordinate that belongs to BOTH subsegments, as it ends one and begins
+ * the other.
+ */
+ private get splitPoint() {
+ let splitPoint = this.host.pre.lastSample;
+
+ let indep = splitPoint.t;
+ if(this.independent == 'x') {
+ indep = splitPoint.targetX;
+ } else if(this.independent == 'y') {
+ indep = splitPoint.targetY;
+ }
+
+ let dep = splitPoint.t;
+ if(this.dependent == 'x') {
+ dep = splitPoint.targetX;
+ } else if(this.dependent == 'y') {
+ dep = splitPoint.targetY;
+ }
+
+ return {
+ dep: dep,
+ indep: indep
+ };
+ }
+
+ /**
+ * The total summed squared-distances of the best fitting line from actually-observed values
+ * when **not** utilizing segmented regression. Double-counts the subsegment's common point, as
+ * it is counted within both subsegments' sum-of-squared error term.
+ */
+ get unsegmentedSumSquaredError(): number {
+ // What was our remaining error for the unsegmented regression?
+ let baseSSE = this.union.sumOfSquaredError;
+
+ // We double-count the split point when segmenting, so we should add an
+ // extra copy of its residual.
+ const splitPoint = this.splitPoint;
+ const splitPointError = this.splitPoint.dep - this.union.predictFromValue(splitPoint.indep);
+
+ // This is the total sum-squared error to be handled by segmenting.
+ return baseSSE + splitPointError * splitPointError;
+ }
+
+ /**
+ * Gets the amount of (unnormalized) variance not explained by a standard regression on
+ * the full interval but succesfully explained by splitting the interval in twain via
+ * segmented regression.
+ */
+ get sumSquaredGainFromSegmentation(): number {
+ return this.unsegmentedSumSquaredError - this.remainingSumSquaredError;
+ }
+
+ /**
+ * A statistical term that signals how successful the segmented regression is. Always
+ * has values on the interval [0, 1], with 1 being a perfect fit.
+ */
+ get coefficientOfDetermination(): number {
+ if(this.host.union.squaredSum(this.dependent) == 0 || this.host.union.squaredSum(this.independent) == 0) {
+ return 1;
+ }
+
+ return 1 - this.remainingSumSquaredError / this.host.union.squaredSum(this.dependent);
+ }
+
+ /**
+ * Gets the raw statistic value needed to test for validity of segmentation based on the axes
+ * under examination. The higher the proportion of newly-explained variance to still-unexplained
+ * variance is, the more confident we can _formally_ be in our segmentation.
+ */
+ get fStat(): number {
+ const val = this.sumSquaredGainFromSegmentation / this.remainingSumSquaredError;
+
+ // We're fine with Infinity. Just... not so much NaN.
+ return isNaN(val) ? 0 : val;
+ }
+
+ /**
+ * Gets the degrees-of-freedom to be used for the numerator DoF parameter of
+ * the F-distribution needed for validity testing.
+ */
+ get fDoF1(): number {
+ let numDoF = 3;
+ // Cases where there clearly may as well be 'no slope' at all.
+ // This saves us a degree of freedom, which is VERY useful for segmenting
+ // the boundary between a 'hold' and a 'move'.
+ if(this.host.pre.squaredSum(this.dependent) < 1e-8) {
+ numDoF--;
+ }
+ if(this.host.post.squaredSum(this.dependent) < 1e-8) {
+ numDoF--;
+ }
+
+ return numDoF;
+ }
+
+ /**
+ * Gets the degrees-of-freedom to be used for the denominator DoF parameter of
+ * the F-distribution needed for validity testing.
+ */
+ get fDoF2(): number {
+ return this.host.union.sampleCount - 2 - this.fDoF1;
+ }
+
+ /**
+ * States a statistically-founded level of confidence we may place in upholding
+ * the represented segmentation, as based on the regressions and axes under examination.
+ *
+ * If we cannot hold any confidence whatsoever (due to being unable to "reject the
+ * null hypothesis" from this specific regression), will return 0.
+ */
+ get certaintyThreshold() {
+ return 1 - FDistribution.thresholdTier(this.fStat, this.fDoF1, this.fDoF2);
+ }
+ }
+
+ constructor(pre: CumulativePathStats,
+ post: CumulativePathStats,
+ union: CumulativePathStats,
+ cumulativeEndpoint: CumulativePathStats) {
+ this.pre = pre;
+ this.post = post;
+ this.union = union;
+ this.endpoint = cumulativeEndpoint;
+ }
+
+ /**
+ * Provides a segmented-regression perspective for the represented interval and proposed
+ * split point based upon the two specified axes.
+ * @param dependent
+ * @param independent
+ * @returns
+ */
+ public segReg(dependentAxis: PathCoordAxis, independentAxis: PathCoordAxis) {
+ return new Segmentation.segmentationComparison(this, dependentAxis, independentAxis);
+ }
+
+ /**
+ * Defines our test for upholding a minimum of sub-segmentation based upon changes in the
+ * interval's coordinates over time. This may occur even with no change in spacial direction
+ * if there is significant enough change in _speed_, as that "curves the line" when one axis
+ * is time.
+ */
+ get segmentationMerited(): boolean {
+ let totalThreshold = 0;
+ const xTest = new Segmentation.segmentationComparison(this, 'x', 't');
+ const yTest = new Segmentation.segmentationComparison(this, 'y', 't');
+
+ // Our testing thresholds are for p=0.05 and p=0.10, which correspond to certainties of 95% and
+ // 90% that our segmentation did not arrive from random chance based on the axis being tested.
+ const p05 = 0.95; // 1 - 0.05
+ const p10 = 0.9; // 1 - 0.10
+
+ totalThreshold += xTest.certaintyThreshold >= p05 ? 2 : (xTest.certaintyThreshold >= p10 ? 1 : 0) ;
+ totalThreshold += yTest.certaintyThreshold >= p05 ? 2 : (yTest.certaintyThreshold >= p10 ? 1 : 0) ;
+
+ return totalThreshold >= 2;
+ }
+
+ /**
+ * Defines our test for considering two (or more) sub-segments as part of the same overall
+ * segment. This seeks to 'link' geometrically-related subsegments - especially those that
+ * maintain the same direction but differ only in observed speed.
+ */
+ get mergeMerited(): boolean {
+ // Because of caret-like motions (as in, in the '^' shape), we need to text for
+ // regression on both axes. One may have notably higher variance than the other.
+ //
+ // These tests ignore time, and therefore speed. Only the raw geometry of the motion
+ // matters for this test.
+ const xTest = new Segmentation.segmentationComparison(this, 'x', 'y');
+ const yTest = new Segmentation.segmentationComparison(this, 'y', 'x');
+
+ // If we don't get a p-value less than .100, then as far as x & y are concerned - and thus the user
+ // is concerned - it's the same segment. Speed may be different, but not the direction.
+ if(xTest.certaintyThreshold > 0) {
+ return false;
+ }
+
+ return yTest.certaintyThreshold == 0;
+ }
+ }
+
+ /**
+ * A specialized form of the `Segmentation` class that assists in its
+ * construction from the interval directly under examination by the
+ * segmentation algorithm.
+ */
+ class PotentialSegmentation extends Segmentation {
+ /**
+ * The full running accumulation for the ongoing touch path until the point
+ * before `post` / the later subsegment. May include
+ * accumulations from before the interval under examination.
+ */
+ readonly chopPoint: CumulativePathStats;
+
+ /**
+ * The full running accumulation for the ongoing touch path until the point
+ * at the end of `pre` / the earlier subsegment. May include
+ * accumulations from before the interval under examination, and will
+ * include accumulations from the first point represented by `post`.
+ */
+ readonly endOfPre: CumulativePathStats;
+
+ /**
+ * The full running accumulation for the ongoing touch path until the point
+ * before `pre` / the earlier subsegment. Will only include
+ * accumulations from before the interval under examination.
+ */
+ readonly baseChop: CumulativePathStats;
+
+ constructor(steppedStats: CumulativePathStats[],
+ choppedStats: CumulativePathStats,
+ splitIndex: number) {
+ const pre = steppedStats[splitIndex].deaccumulate(choppedStats);
+
+ // Keep stats value components based on the final point of the 'pre' segment.
+ const finalStats = steppedStats[steppedStats.length-1];
+ const post = finalStats.deaccumulate(steppedStats[splitIndex-1]);
+ const union = finalStats.deaccumulate(choppedStats);
+
+ super(pre, post, union, finalStats);
+ this.baseChop = choppedStats;
+ this.chopPoint = steppedStats[splitIndex-1];
+ this.endOfPre = steppedStats[splitIndex];
+ }
+ }
+
+ /**
+ * The core logic, algorithm, and manager for touchpath segmentation.
+ */
+ export class PathSegmenter {
+ /**
+ * The minimum amount of time (in ms) to wait between sample repetitions
+ * once inputs stop arriving, so long as the path is still active.
+ */
+ private readonly REPEAT_INTERVAL = 33;
+
+ /**
+ * The time-interval length (in ms) at the end of the path to consider
+ * when determining whether or not to trigger path segmentation.
+ */
+ private readonly SLIDING_WINDOW_INTERVAL = 50;
+
+ /**
+ * Tracks the mathematical values used to provide path segment stats
+ * at each point on the path. Individual steps may be removed (and
+ * batched into `choppedStats`) once their respective points on the
+ * path have been fully processed.
+ */
+ private steppedCumulativeStats: CumulativePathStats[];
+
+ /**
+ * Tracks all subsegments awaiting completion of their overall segment.
+ */
+ private lingeringSubsegmentations: PotentialSegmentation[];
+
+ /**
+ * TODO: These directly represent path segments produced by the prototype
+ * algorithm. Just... the stats analysis of the path segment, without
+ * obvious / public members to relevant coordinates.
+ *
+ * In follow-up work, this should be used to build & to update
+ * cleaner, public-facing segment objects for the touch path.
+ */
+ private _protoSegments: CumulativePathStats[] = [];
+
+ /**
+ * TODO: These directly represent the subsegments comprising their
+ * corresponding path segments (in `_protoSegments`) produced by the
+ * prototype algorithm. Just... the stats analysis of each subsegment.
+ *
+ * In follow-up work, these should be used internally to build & update
+ * the internals of TBD cleaner, public-facing segment objects for the
+ * touch path.
+ */
+ private _protoSegmentSets: CumulativePathStats[][] = [];
+
+ /**
+ * Used to 'repeat' the most-recently observed incoming sample if no
+ * other replaces it before it triggers.
+ *
+ * A repeating sample indicates lack of motion, which is valuable
+ * information for stats-based segmentation.
+ */
+ private repeatTimer: number | NodeJS.Timeout;
+
+ /**
+ * The timestamp of observation of the most recently observed sample's
+ * most recent repetition - even if it's only the first evaluation.
+ */
+ private repeatTimestamp: number;
+
+ /**
+ * Represents the cumulative statistics of all points on the path
+ * that lie on already-fully-segmented parts of it.
+ */
+ private choppedStats: CumulativePathStats = null;
+
+ constructor() {
+ this.steppedCumulativeStats = [];
+ this.lingeringSubsegmentations = [];
+ }
+
+ /**
+ * Appends a new coordinate to the touch path and also kick-starts a timer for replicating it
+ * should neither further inputs be received nor termination of the touchpoint be indicated.
+ * @param sample
+ */
+ public add(sample: InputSample) {
+ // Set up the input-repeater (in case we don't get further feedback but remain active)
+ const repeater = (timeDelta: number) => {
+ this.observe(sample, timeDelta);
+ }
+
+ // If we previously set up an input-repeater, cancel it. We've got a more up-to-date
+ // coordinate on the touchpath now.
+ if(this.repeatTimer) {
+ // @ts-ignore
+ clearInterval(this.repeatTimer);
+ this.repeatTimer = null;
+ }
+
+ this.repeatTimer = setInterval(() => {
+ const timeDelta = Date.now() - this.repeatTimestamp;
+ repeater(timeDelta);
+ }, this.REPEAT_INTERVAL);
+ this.repeatTimestamp = Date.now();
+ repeater(0);
+ }
+
+ /**
+ * Used to finalize all segmentation for the touchpath whenever termination of the corresponding
+ * touchpoint has been terminated (mouse-up, touch-end).
+ */
+ public close() {
+ // The Node clearTimeout & DOM clearTimeout appear to TS as overloads of each other,
+ // and their type definitions will conflict. A simple @ts-ignore will bypass this issue.
+ // @ts-ignore
+ clearInterval(this.repeatTimer); // Cancels the input-repeater.
+ this.repeatTimer = null;
+
+ // The way things are structured, finalization.pre = final segment. It's a bit hacky, but
+ // it _is_ happy, intentional 'fallout' from the implementation's design. `post` is effectively
+ // a single-point 'subsegment' and will go effectively unutilized by the phase 2 subsegment linker.
+ let finalization = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, this.steppedCumulativeStats.length-1);
+
+ console.log("! Finalization !");
+ this.processSubsegmentation(finalization, true);
+
+ // TODO: these are temporary statements to facilitate exploration, experimentation, & debugging.
+ // We should be providing output to the touchpath object (`.path.segments`).
+ // But... that'll be left for a follow-up PR.
+ console.log(this._protoSegments);
+ console.log(this._protoSegmentSets);
+ console.log(this._protoSegments.map((val) => (val.toJSON())));
+ }
+
+ /**
+ * Adds a statistic 'observation' of the state of the touchpath.
+ *
+ * This is either to be called directly upon reception of a new input-coordinate or
+ * upon replication of a prior input should the touchpath still be active without any
+ * indication of motion.
+ * @param sample
+ * @param timeDelta
+ */
+ private observe(sample: InputSample, timeDelta: number) {
+ let cumulativeStats: CumulativePathStats;
+ if(this.steppedCumulativeStats.length) {
+ cumulativeStats = this.steppedCumulativeStats[this.steppedCumulativeStats.length-1];
+ } else {
+ cumulativeStats = new CumulativePathStats();
+ }
+
+ sample = {... sample};
+ sample.t += timeDelta;
+ const extendedStats = cumulativeStats.extend(sample);
+ this.steppedCumulativeStats.push(extendedStats);
+
+ this.performSubsegmentation();
+ }
+
+ private _debugLogSegmentationReport(candidateSplit: PotentialSegmentation) {
+ console.log("------------------------------------------------------------------");
+
+ console.log("Combined: ");
+ console.log(candidateSplit.union.toJSON());
+ console.log("Pre: ")
+ console.log(candidateSplit.pre.toJSON());
+ console.log("Post: ");
+ console.log(candidateSplit.post.toJSON());
+
+ console.log();
+
+ const xF = candidateSplit.segReg('x', 't');
+ const yF = candidateSplit.segReg('y', 't');
+ console.log(`x F-test: F_(${xF.fDoF1}, ${xF.fDoF2}) = ${xF.fStat} @ ${xF.certaintyThreshold}`);
+ console.log(`y F-test: F_(${yF.fDoF1}, ${yF.fDoF2}) = ${yF.fStat} @ ${yF.certaintyThreshold}`);
+
+ console.log();
+
+ console.log("Split object: ");
+ console.log(candidateSplit);
+
+ console.log("------------------------------------------------------------------");
+ // END: DO NOT RELEASE.
+ }
+
+ // The "reported via event" aspect mentioned below is necessary because this may be
+ // called via setTimeout callback on a held touchpoint. We need that `setTimeout`
+ // (which replicates the most recently-seen input coordinate) in order to have good
+ // sample-data for detecting the boundary between motion and lack thereof during
+ // segmentation.
+ /**
+ * This is the "main method" for touchpath segmentation. Should a new segment result
+ * from its analysis, it will be reported via event.
+ * @returns
+ */
+ private performSubsegmentation() {
+ const cumulativeStats = this.steppedCumulativeStats[this.steppedCumulativeStats.length - 1];
+ const unsegmentedDuration = cumulativeStats.lastTimestamp - this.steppedCumulativeStats[0].lastTimestamp;
+
+ // STEP 1: Determine the range of the sliding time window for the most recent samples.
+
+ if(unsegmentedDuration < this.SLIDING_WINDOW_INTERVAL * 2) {
+ console.log("Interval too short for segmentation.");
+ return;
+ }
+
+ let splitPoint = 0;
+ // Do not consider the just-added `extendedStats` entry when building the sliding
+ // time window.
+ for(let i = this.steppedCumulativeStats.length-2; i >=0; i--) {
+ if(this.steppedCumulativeStats[i].lastTimestamp < cumulativeStats.lastTimestamp - this.SLIDING_WINDOW_INTERVAL) {
+ splitPoint = i+1;
+ break;
+ }
+ }
+
+
+ // We split the cumulative stats on a specific point, which then resides on the edge
+ // of both of the resulting intervals. This completes "step 1".
+ let candidateSplit = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint);
+
+ // STEP 2: given the proposed time window, do we have a basis for segmentation? And is there a better
+ // candidate split point nearby?
+
+ // NOTE: During initial development, I actually had a hard rule about not even searching if segmentation
+ // failed on the initially-constructed sliding window. Some analysis may be wise here, as it'd be nice
+ // to optimize away the 'need to search' if we can find hard & fast rules about when we'll never need
+ // to go looking. But... being too aggressive can cause nasty problems.
+
+ if(!candidateSplit.segmentationMerited) {
+ // // Debug logging statements:
+ const xTest = candidateSplit.segReg('x', 't');
+ const yTest = candidateSplit.segReg('y', 't');
+ console.log(`x F-test: F_(${xTest.fDoF1}, ${xTest.fDoF2}) = ${xTest.fStat} @ ${xTest.certaintyThreshold}`);
+ console.log(`y F-test: F_(${yTest.fDoF1}, ${yTest.fDoF2}) = ${yTest.fStat} @ ${yTest.certaintyThreshold}`);
+
+ console.log("candidate split: " );
+ console.log(candidateSplit);
+ }
+
+ // We either have the conditions to trigger segmentation or just became long enough to consider it.
+ // If we're only just long enough to consider it, there may be a better segmentation point to start with.
+ // Now... is there a better segmentation point?
+ class SplitSearchState {
+ readonly xTest: typeof Segmentation.segmentationComparison.prototype;
+ readonly yTest: typeof Segmentation.segmentationComparison.prototype;
+ readonly candidate: PotentialSegmentation;
+
+ constructor(candidate?: PotentialSegmentation) {
+ this.candidate = candidate;
+
+ if(candidate == null) {
+ return;
+ }
+
+ this.xTest = candidate.segReg('x', 't');
+ this.yTest = candidate.segReg('y', 't');
+ }
+
+ // The value for an 'objective function' to optimize in our search for a better candidate.
+ // The higher this is, the more we like its potential as the segmentation point.
+ get segRating() {
+ if(this.candidate) {
+ return Math.max(this.xTest.coefficientOfDetermination, this.yTest.coefficientOfDetermination);
+ } else {
+ return 0;
+ }
+ }
+
+ // But, if it turns out this potential point wouldn't actually result in segmentation, well...
+ // "abandon ship".
+ get segmentationMerited() {
+ return this.candidate?.segmentationMerited ?? false;
+ }
+ }
+
+ // Start: split-point search
+
+ /* We'll use our initial sliding window as a starting point. It's close to the active location
+ * of the touch, so we're unlikely to miss a proper segmentation point if it lies close.
+ * We'll find the local maximum (best split point for the local part of the path)
+ * via gradient descent.
+ *
+ * Note: binary search is a 'bad idea', as the touchpath is likely not following a
+ * monotonous path, mathematically speaking. (It'd be possible to overshoot a "peak" or
+ * "valley" quite easily.)
+ */
+ let currentSplit = new SplitSearchState(candidateSplit);
+
+ let leftSplit = new SplitSearchState(new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint-1));
+ let rightCandidate: PotentialSegmentation = null;
+ if(splitPoint+1 < this.steppedCumulativeStats.length) {
+ rightCandidate = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint+1);
+ }
+ let rightSplit = new SplitSearchState(rightCandidate);
+
+ // Step 2a: detect which direction gives the best improvement in segmentation potential.
+ const criteria = [leftSplit.segRating, currentSplit.segRating, rightSplit.segRating];
+ let sortedCriteria = [...criteria].sort();
+
+ const delta = criteria.indexOf(sortedCriteria[2])-1; // -1 if 'left' is best, 1 if 'right' is best.
+
+ // Step 2b: we've found the direction: go searching!
+ if(delta != 0) {
+ // We can get better segmentation by shifting. Proceed in the optimal direction.
+ do {
+ const nextSplitIndex = splitPoint + delta;
+ if(nextSplitIndex >= this.steppedCumulativeStats.length || nextSplitIndex < 0) {
+ break;
+ }
+
+ let nextCandidate = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint + delta);
+ let nextSplit = new SplitSearchState(nextCandidate);
+
+ // Prevent overly-short intervals / over-segmentation.
+ if(nextCandidate.pre.duration * 1000 < this.SLIDING_WINDOW_INTERVAL / 2) {
+ break;
+ } else if(nextCandidate.post.duration * 1000 < this.SLIDING_WINDOW_INTERVAL / 2) {
+ break;
+ }
+
+ // Not an improvement? Guess we found the best spot.
+ if(nextSplit.segRating <= currentSplit.segRating) {
+ break;
+ } else if(!nextSplit.segmentationMerited && currentSplit.segmentationMerited) {
+ // Note: this can happen if segmentation is triggered due to divergence on both axes
+ // if one of them drops a threshold tier and the other fails to improve sufficiently.
+ console.warn("aborting split-point relocation due to no longer segmenting");
+ break;
+ } else {
+ splitPoint += delta;
+ currentSplit = nextSplit;
+ candidateSplit = nextCandidate;
+ }
+ // If we found a new best segmentation point, we then ask if we can get even better by shifting further.
+ } while(true);
+ }
+ // Step 2 complete.
+
+ // Step 3: evaluate our candidate split-point - do we sub-segment? If so, do so.
+ console.log("best split: ");
+ console.log(candidateSplit);
+
+ if(!candidateSplit.segmentationMerited) {
+ return;
+ }
+
+ // First phase of segmentation: complete!
+
+ // FIXME: DO NOT RELEASE.
+ // This is exploratory / diagnostic code assisting development of the path segmentation
+ // algorithm.
+ this._debugLogSegmentationReport(candidateSplit);
+ // END: DO NOT RELEASE.
+
+ this.steppedCumulativeStats = this.steppedCumulativeStats.slice(splitPoint);
+ this.choppedStats = candidateSplit.chopPoint;
+
+ // Step 4: process the implications.
+
+ // There are some cases where we want to prevent segmentation from fully happening.
+ // The next method will both handle that and the signal of any completed segments that
+ // may result.
+ this.processSubsegmentation(candidateSplit);
+ }
+
+ /**
+ * Given the results of the subsegmentation process, this method analyzes the results
+ * in order to produce and update the detected segments.
+ *
+ * TODO: Should produce actual `Segment` objects and trigger their emission
+ * them via EventEmitter mechanics.
+ * @param subsegmentation
+ * @param finalize Set to `true` for a terminating touchpath, indicating that
+ * no further subsegmentation will occur for this touchpath.
+ */
+ private processSubsegmentation(subsegmentation: PotentialSegmentation, finalize?: boolean) {
+ finalize = !!finalize;
+
+ // This function makes a LOT of assumptions particular to this method and class.
+ // It is not safe to make this class-level, and it's especially unsafe to make it `public`.
+ const mergeSubsegmentationAccumulations = function(array: PotentialSegmentation[]) {
+ if(array.length == 1) {
+ return array[0].pre; // It's pre-calculated, so just use it.
+ } else {
+ const finalSubsegmentation = array[array.length-1];
+ return finalSubsegmentation.endOfPre.deaccumulate(array[0].baseChop);
+ }
+ }
+
+ // Step 1: did we previously note that any previous subsegments are likely to be 'linked' to
+ // the new, incoming left-hand subsegment? If so, validate that expectation & act accordingly.
+
+ let predecessor = subsegmentation.pre;
+ let firstChopPoint = subsegmentation.baseChop;
+ if(this.lingeringSubsegmentations.length) {
+ // First: check if the newly-finished subsegment should be merged with the lingering ones.
+ const mergedPrecursors = mergeSubsegmentationAccumulations(this.lingeringSubsegmentations);
+
+ const tailUnion = mergeSubsegmentationAccumulations([...this.lingeringSubsegmentations, subsegmentation]);
+ console.log("Verifying linkage to pending merges: ");
+ console.log(this.lingeringSubsegmentations);
+ console.log("verification check:");
+ console.log(mergedPrecursors);
+ const precursorMergeSegmentation = new Segmentation(mergedPrecursors, subsegmentation.pre, tailUnion, subsegmentation.endOfPre);
+ // Sometimes the start of a harsh turn seems like it's part of the same thing for a moment, but as it continues,
+ // becomes something VERY different. Validate that we should still merge the left-hand with its predecessors.
+ if(!PathSegmenter.shouldLinkSubsegments(precursorMergeSegmentation)) {
+ // Emit as separate subsegment.
+ const finishedSegment = mergeSubsegmentationAccumulations(this.lingeringSubsegmentations);
+ this._protoSegments.push(finishedSegment);
+ this._protoSegmentSets.push(this.lingeringSubsegmentations.map((val) => val.pre));
+ this.lingeringSubsegmentations = [];
+
+ console.log("Proto-segments length: " + this._protoSegments.length);
+
+ console.log("Did not merge a lingering subsegment with the incoming one!");
+ } else {
+ predecessor = tailUnion;
+ firstChopPoint = this.lingeringSubsegmentations[0].baseChop;
+ console.log("Will merge in old subsegments!");
+ }
+ }
+
+ // Step 2: okay, predecessor handling complete. Now, do we think the newly-starting right-hand
+ // subsegment is likely to be 'linked' to the newly-completed left-hand subsegment?
+
+ console.log("Double-checking right-side split subsegment for xy/yx correlation with prior segment candidate(s)");
+ const fullUnion = subsegmentation.endpoint.deaccumulate(firstChopPoint);
+ const fullMergeSegmentation = new Segmentation(predecessor, subsegmentation.post, fullUnion, subsegmentation.endpoint);
+ console.log("segmentation-prevention check:")
+ console.log(fullMergeSegmentation);
+ // if(!force && PathSegmenter.shouldMergeSubsegments(subsegmentation.pre, subsegmentation.post, subsegmentation.union)) {
+ if(!finalize && PathSegmenter.shouldLinkSubsegments(fullMergeSegmentation)) {
+ this.lingeringSubsegmentations.push(subsegmentation);
+ } else {
+ // Merge all as a completed segment!
+ this._protoSegments.push(predecessor);
+ this._protoSegmentSets.push([...this.lingeringSubsegmentations.map((val) => val.pre), subsegmentation.pre]);
+ this.lingeringSubsegmentations = [];
+
+ console.log("Proto-segments length: " + this._protoSegments.length);
+ }
+ }
+
+ /**
+ * Used by the 'filter' step (phase 2 of segmentation) to determine whether or not
+ * two subsegments should be 'linked' as members of the same segment. (That is, if
+ * a user would likely consider the two subsegments as belonging to the "same arc
+ * of motion".)
+ * @param segmentation
+ * @returns
+ */
+ private static shouldLinkSubsegments(segmentation: Segmentation): boolean {
+ // // Known case #1:
+ // // Near-identical direction, but heavy speed difference.
+ // // Speed's still high enough to not be a 'wait'.
+ // // We may want to 'note' the higher speed; that may be relevant for flick detection.
+ // // Known case #2:
+ // // Low-speed pivot; angle change caught on very low velocity for both subsegments.
+ // // ... or should this be merged? Multiple mini-segments from 'wiggling' could just go ignored instead...
+
+ // This is kinda plain and arbitrary, but it seems to work "well enough" for now,
+ // at least at this stage of development. (Most testing was done with Chrome emulation of
+ // an iPhone SE.)
+
+ // .mean('v') < 0.08: the mean speed (taken timestamp to timestamp) does not exceed 0.08px/millisec.
+ if(segmentation.pre.mean('v') < 0.08 && segmentation.post.mean('v') > 0.08) {
+ console.log("desegmentation exception");
+ return;
+ }
+ if(segmentation.pre.mean('v') > 0.08 && segmentation.post.mean('v') < 0.08) {
+ console.log("desegmentation exception");
+ return;
+ }
+
+ console.log("Desegmentation under consideration: ");
+ console.log(segmentation);
+
+ const xF = segmentation.segReg('x', 'y');
+ const yF = segmentation.segReg('y', 'x');
+ console.log(`merger F-test (xy): F_(${xF.fDoF1}, ${xF.fDoF2}) = ${xF.fStat} @ ${xF.certaintyThreshold}`);
+ console.log(`merger F-test (yx): F_(${yF.fDoF1}, ${yF.fDoF2}) = ${yF.fStat} @ ${yF.certaintyThreshold}`);
+ console.log(`will remerge: ${segmentation.mergeMerited}`);
+
+ return segmentation.mergeMerited;
+ }
+ }
+}
\ No newline at end of file
diff --git a/common/web/gesture-recognizer/src/segment.ts b/common/web/gesture-recognizer/src/segment.ts
new file mode 100644
index 0000000000..4f52a7cb91
--- /dev/null
+++ b/common/web/gesture-recognizer/src/segment.ts
@@ -0,0 +1,5 @@
+namespace com.keyman.osk {
+ export class Segment {
+ // TODO: stuff.
+ }
+}
\ No newline at end of file
diff --git a/common/web/gesture-recognizer/src/tools/recorder/src/recorder.js b/common/web/gesture-recognizer/src/tools/recorder/src/recorder.js
index 6be45767d1..00f95e7e66 100644
--- a/common/web/gesture-recognizer/src/tools/recorder/src/recorder.js
+++ b/common/web/gesture-recognizer/src/tools/recorder/src/recorder.js
@@ -48,6 +48,7 @@ loadPromise.then((recognizer) => {
});
sequence.on('end', function() {
+ //let stats = sequence.touchpoints[0].path.segments[0].stats;
logElement.value = recorder.recordingsToJSON();
});
});
diff --git a/common/web/gesture-recognizer/src/trackedPath.ts b/common/web/gesture-recognizer/src/trackedPath.ts
index 9cfdc4a5ee..2398d07129 100644
--- a/common/web/gesture-recognizer/src/trackedPath.ts
+++ b/common/web/gesture-recognizer/src/trackedPath.ts
@@ -7,7 +7,7 @@ namespace com.keyman.osk {
export type JSONTrackedPath = {
coords: InputSample[]; // ensures type match with public class property.
wasCancelled?: boolean;
- // segments: Segment[];
+ //segments: Segment[];
}
interface EventMap {
@@ -36,6 +36,10 @@ namespace com.keyman.osk {
*/
export class TrackedPath extends EventEmitter {
private samples: InputSample[] = [];
+ private _segments: Segment[] = [];
+
+ private readonly segmenter: PathSegmenter;
+
private _isComplete: boolean = false;
private wasCancelled?: boolean;
@@ -56,10 +60,16 @@ namespace com.keyman.osk {
if(jsonObj) {
this.samples = [...jsonObj.coords.map((obj) => ({...obj} as InputSample))];
- // If we're reconstructing this from a JSON.parse, it's a previously-recorded, completed path.
+ // If we're reconstructing this from a JSON.parse, it's a previously-recorded,
+ // completed path.
this._isComplete = true;
this.wasCancelled = jsonObj.wasCancelled;
}
+
+ // Keep this as the _final_ statement in the constructor. `PathSegmenter` will
+ // need a reference to this instance, even if only via closure.
+ // (Most likely; not yet done.) Kinda awkward, but it's useful for compartmentalization.
+ this.segmenter = new PathSegmenter();
}
/**
@@ -80,6 +90,7 @@ namespace com.keyman.osk {
}
this.samples.push(sample);
+ this.segmenter.add(sample);
this.emit('step', sample);
}
@@ -93,6 +104,7 @@ namespace com.keyman.osk {
}
this.wasCancelled = cancel;
this._isComplete = true;
+ this.segmenter.close();
if(cancel) {
this.emit('invalidated');
@@ -110,6 +122,10 @@ namespace com.keyman.osk {
return this.samples;
}
+ public get segments(): readonly Segment[] {
+ return this._segments;
+ }
+
/**
* Creates a serialization-friendly version of this instance for use by
* `JSON.stringify`.