erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-09-16 04:37:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

Merge pull request #7058 from keymanapp/feat/web/touchpath-segmentation

Browse source 
feat(web): gesture touchpath segmentation core 🐵
This commit is contained in:
Joshua Horton 2022-10-11 08:10:03 +07:00 • committed by GitHub
commit c2e846524b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 1729 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

746
common/web/gesture-recognizer/src/cumulativePathStats.ts Normal file
View file

@ -0,0 +1,746 @@
namespace com.keyman.osk {
/**
* Denotes one dimension utilized by touchpath input coordinates - 'x' and y' for space,
* 't' for time.
*/
export type PathCoordAxis = 'x' | 'y' | 't';
/**
* Denotes a pair of dimensions utilized by touchpath input coordinates. The two axes
* (see `PathCoordAxis`) must be specified in alphabetical order.
*/
export type PathCoordAxisPair = 'tx' | 'ty' | 'xy';
/**
* Denotes one dimension or feature (velocity) that this class tracks statistics for.
*
* Sine and Cosine stats are currently excluded due to their necessary lack of statistical
* independence.
*/
type StatAxis = PathCoordAxis | 'v';
/**
* Acts as a subtraction operation with a built-in, adaptive "significance" check.
* If the result * 2^30 (~ * 10^9) is still smaller in magnitude than an operand, we
* assume it to be a floating-point error that should have been 0 and act accordingly,
* returning 0.
*
* For reference, (32-bit) floats have 23 bits of significand precision, while (64-bit)
* doubles have 52. Therefore, we'll still be more precise than baseline floats.
*/
function sigMinus(operand1: number, operand2: number) {
const diff = operand1 - operand2;
const logDiff = Math.log2(Math.abs(operand1)) - Math.log2(Math.abs(operand2));
// If an operand is 2^30 (or ~10^9) larger than the result of the difference, it's
// nigh-certainly a floating-point error at play.
return logDiff < 30 ? diff : 0;
}
/**
* As the name suggests, this class facilitates tracking of cumulative mathematical values, etc
* necessary to perform the statistical operations necessary for path segmentation.
*
* Instances of this class are immutable.
*/
export class CumulativePathStats {
/**
* Provides linear-regression statistics & fitting values based on the underlying `CumulativePathStats`
* object used to generate it. All operations are O(1).
*/
static readonly regression = class RegressionFromSums {
readonly independent: PathCoordAxis;
readonly dependent: PathCoordAxis;
readonly paired: PathCoordAxisPair;
readonly accumulator: CumulativePathStats;
/**
*
* @param mainStats The `CumulativePathStats` instance to base all regression data on.
* @param dependentAxis The 'output' axis / dimension; the axis whose behavior should be predicted based on
* existing data of its relationship with the independent axis.
* @param independentAxis The 'input' axis/dimension.
*/
constructor(mainStats: CumulativePathStats, dependentAxis: PathCoordAxis, independentAxis: PathCoordAxis) {
if(dependentAxis == independentAxis) {
throw "Two different axes must be specified for the regression object.";
}
this.accumulator = mainStats;
this.dependent = dependentAxis;
this.independent = independentAxis;
if(dependentAxis < independentAxis) {
this.paired = dependentAxis.concat(independentAxis) as PathCoordAxisPair;
} else {
this.paired = independentAxis.concat(dependentAxis) as PathCoordAxisPair;
}
}
/**
* The 'slope' of the 'slope-intercept' form of the line that best fits the relationship between
* this regression's selected axes.
*/
get slope(): number {
// The technical definition is the commented-out line, but the denominator component of both
// cancels out - it's 'more efficient' to use the following line as a result.
// this.accumulator.covariance(this.paired) / this.accumulator.variance(this.independent);
const val = this.accumulator.crossSum(this.paired) / this.accumulator.squaredSum(this.independent);
return val;
}
/**
* The 'intercept' of the 'slope-intercept' form of the line that best fits the relationship between
* this regression's selected axes.
*/
get intercept(): number {
// Performing a regression based on these pre-summed values means that our obtained intercept is in
// the mapped coordinate system.
const mappedIntercept = this.accumulator.mappedMean(this.dependent) - this.slope * this.accumulator.mappedMean(this.independent);
const val = mappedIntercept + this.accumulator.mappingConstant(this.dependent) -
this.slope * this.accumulator.mappingConstant(this.independent);
return val;
}
/**
* The total summed squared-distances of the best fitting line from actually-observed values;
* in other words, the "sum of the squared errors".
*
* Statistically, this is the portion of the dependent variable's variance (un-normalized)
* that is unexplained by this regression.
*/
get sumOfSquaredError(): number {
return this.accumulator.squaredSum(this.dependent) - this.sumOfSquaredModeled;
}
/**
* The portion of the dependent variable's variance that is successfully explained by this regression.
*/
get sumOfSquaredModeled(): number {
// If we have a perfectly straight vertical line, from the perspective of our independent axis,
// we get infinite slope. That's... not great for the math.
//
// Fortunately, it ALSO means that we can perfectly model the segment.
if(this.accumulator.squaredSum(this.independent) == 0) {
return this.accumulator.squaredSum(this.dependent);
} else {
return this.slope * this.accumulator.crossSum(this.paired);
}
}
/**
* A statistical term that signals how successful the regression is. Always has values on
* the interval [0, 1], with 1 being a perfect fit.
*/
get coefficientOfDetermination(): number {
if(this.accumulator.squaredSum(this.dependent) == 0 || this.accumulator.squaredSum(this.independent) == 0) {
return 1;
}
const acc = this.accumulator;
const num = acc.crossSum(this.paired) * acc.crossSum(this.paired);
const denom = acc.squaredSum(this.dependent) * acc.squaredSum(this.independent);
return num / denom;
}
/**
* Gets the value of the dependent axis that lies on the regression's fitted line
* for a specified independent axis value.
*
* @param value The input value to use for the independent axis's variable.
* @returns The predicted dependent axis value.
*/
predictFromValue(value: number) {
return this.slope * value + this.intercept;
}
}
/**
* Floating-point errors may result from cross-sum calculations, and they may be slightly larger than
* Number.EPSILON as the sums grow. (Taking the difference of cross-sums)
*/
private static readonly CANCELLATION_EPSILON = Math.sqrt(Number.EPSILON);
private rawLinearSums = {'x': 0, 'y': 0, 't': 0, 'v': 0};
private rawSquaredSums = {'x': 0, 'y': 0, 't': 0, 'v': 0};
// Would 'tv' (time vs velocity) be worth it to track? And possibly even do a regression for?
// If so, maybe throw that in.
private rawCrossSums = {'tx': 0, 'ty': 0, 'xy': 0};
private coordArcSum: number = 0;
private arcSampleCount: number = 0;
// These two are kept separate because of their extreme interconnectedness - after all,
// they actually represent a SINGLE (polar) axis - the angle.
//
// Sadly, there's no straightforward, well-founded way to use these to give a proper
// statistical sense of 'fit' or 'regression' here, _especially_ in regard to segmentation.
// Proper angle-[other] cross-sums are pretty much impossible, at least as efficiently
// as the others are handled [O(1)].
private cosLinearSum: number = 0;
private sinLinearSum: number = 0;
/**
* The base sample used to transpose all other received samples. Use of this helps
* avoid potential "catastrophic cancellation" effects that can occur when diffing
* two numbers far from the sample-space's mathematical origin.
*
* Refer to https://en.wikipedia.org/wiki/Catastrophic_cancellation.
*/
private baseSample?: InputSample;
/**
* The initial sample included by this instance's computed stats. Needed for
* the 'directness' properties.
*/
private initialSample?: InputSample;
private _lastSample?: InputSample;
private followingSample?: InputSample;
private _sampleCount = 0;
constructor();
constructor(sample: InputSample);
constructor(instance: CumulativePathStats);
constructor(obj?: InputSample | CumulativePathStats) {
if(!obj) {
return;
}
// Will worry about JSON form later.
if(obj instanceof CumulativePathStats) {
Object.assign(this, obj);
this.rawLinearSums = {...obj.rawLinearSums};
this.rawCrossSums = {...obj.rawCrossSums};
this.rawSquaredSums = {...obj.rawSquaredSums};
} else if(isAnInputSample(obj)) {
Object.assign(this, this.extend(obj));
} else {
throw "A constructor for this input pattern has not yet been implemented";
}
}
/**
* Statistically "observes" a new sample point on the touchpath, accumulating values
* useful for provision of relevant statistical properties.
* @param sample A newly-sampled point on the touchpath.
* @returns A new, separate instance for the cumulative properties up to the
* newly-sampled point.
*/
public extend(sample: InputSample): CumulativePathStats {
if(!this.initialSample) {
this.initialSample = sample;
this.baseSample = sample;
}
const result = new CumulativePathStats(this);
// Set _after_ deep-copying this for the result.
this.followingSample = sample;
// Helps prevent "catastrophic cancellation" issues from floating-point computation
// for these statistical properties and properties based upon them.
const x = sample.targetX - this.baseSample.targetX;
const y = sample.targetY - this.baseSample.targetY;
const t = sample.t - this.baseSample.t;
result.rawLinearSums.x += x;
result.rawLinearSums.y += y;
result.rawLinearSums.t += t;
result.rawCrossSums.tx += t * x;
result.rawCrossSums.ty += t * y;
result.rawCrossSums.xy += x * y;
result.rawSquaredSums.x += x * x;
result.rawSquaredSums.y += y * y;
result.rawSquaredSums.t += t * t;
if(this.lastSample) {
// arc length stuff!
const xDelta = sample.targetX - this.lastSample.targetX;
const yDelta = sample.targetY - this.lastSample.targetY;
const tDelta = sample.t - this.lastSample.t;
const coordArcDeltaSq = xDelta * xDelta + yDelta * yDelta;
const coordArcDelta = Math.sqrt(coordArcDeltaSq);
result.coordArcSum += coordArcDelta;
if(xDelta || yDelta) {
// We wish to measure angle clockwise from <0, -1> in the DOM. So, cos values should
// align with that axis, while sin values should align with the positive x-axis.
//
// This provides a mathematical 'transformation' to the axes used by `atan2` in the
// `angleMean` property.
result.cosLinearSum += -yDelta / coordArcDelta;
result.sinLinearSum += xDelta / coordArcDelta;
result.arcSampleCount += 1;
}
if(tDelta) {
result.rawLinearSums.v += coordArcDelta / tDelta;
result.rawSquaredSums.v += coordArcDeltaSq / (tDelta * tDelta);
}
}
result._lastSample = sample;
result.sampleCount = this.sampleCount + 1;
return result;
}
/**
* "De-accumulates" currently-accumulated values corresponding to the specified
* subset, which should represent an earlier, previously-observed part of the path.
* @param subsetStats The accumulated stats for the part of the path being removed
* from this instance's current accumulation.
* @returns
*/
public deaccumulate(subsetStats?: CumulativePathStats): CumulativePathStats {
// Possible addition: use `this.buildRenormalized` on the returned version
// if catastrophic cancellation effects (random, small floating point errors)
// are not sufficiently mitigated & handled by the measures currently in place.
//
// Even then, we'd need to apply such generated objects carefully - we can't
// re-merge the accumulated values or remap them to their old coordinate system
// afterward.`buildRenormalize`'s remapping maneuver is a one-way stats-abuse trick.
//
// Hint: we'd need to pay attention to the "lingering segments" aspects in which
// detected sub-segments might be "re-merged".
// - Whenever they're merged & cleared, we should be clear to recentralize
// the cumulative stats that follow. If any are still active, we can't
// recentralize.
const result = new CumulativePathStats(this);
// We actually WILL accept a `null` argument; makes some of the segmentation
// logic simpler.
if(!subsetStats) {
return result;
}
if(!subsetStats.followingSample || !subsetStats.lastSample) {
throw 'Invalid argument: stats missing necessary tracking variable.';
}
for(let dim in result.rawLinearSums) {
result.rawLinearSums[dim] -= subsetStats.rawLinearSums[dim];
}
for(let dimPair in result.rawCrossSums) {
result.rawCrossSums[dimPair] -= subsetStats.rawCrossSums[dimPair];
}
for(let dim in result.rawSquaredSums) {
result.rawSquaredSums[dim] -= subsetStats.rawSquaredSums[dim];
}
// arc length stuff!
if(subsetStats.followingSample && subsetStats.lastSample) {
const xDelta = subsetStats.followingSample.targetX - subsetStats.lastSample.targetX;
const yDelta = subsetStats.followingSample.targetY - subsetStats.lastSample.targetY;
const tDelta = subsetStats.followingSample.t - subsetStats.lastSample.t;
const coordArcDeltaSq = xDelta * xDelta + yDelta * yDelta;
const coordArcDelta = Math.sqrt(coordArcDeltaSq);
// Due to how arc length stuff gets segmented.
// There's the arc length within the prefix subset (operand 2 below) AND the part connecting it to the
// 'remaining' subset (operand 1 below) before the portion wholly within what remains (the result)
result.coordArcSum -= coordArcDelta;
result.coordArcSum -= subsetStats.coordArcSum;
result.cosLinearSum -= subsetStats.cosLinearSum;
result.sinLinearSum -= subsetStats.sinLinearSum;
result.arcSampleCount -= subsetStats.arcSampleCount;
if(tDelta) {
result.rawLinearSums.v -= coordArcDelta / tDelta;
result.rawSquaredSums.v -= coordArcDeltaSq / (tDelta * tDelta);
}
}
result.sampleCount -= subsetStats.sampleCount;
// NOTE: baseSample MUST REMAIN THE SAME. All math is based on the corresponding diff.
// Though... very long touchpoint interactions could start being affected by that "catastrophic
// cancellation" effect without further adjustment. (If it matters, we'll get to that later.)
// But _probably_ not; we don't go far beyond a couple of orders of magnitude from the origin in
// ANY case except the timestamp (.t) - and even then, not far from the baseSample's timestamp value.
// initialSample, though, we need to update b/c of the 'directness' properties.
result.initialSample = subsetStats.followingSample;
return result;
}
public get lastSample() {
return this._lastSample;
}
public get lastTimestamp(): number {
return this.lastSample?.t;
}
public get sampleCount() {
return this._sampleCount;
}
private set sampleCount(value: number) {
this._sampleCount = value;
}
/**
* In order to mitigate the accumulation of small floating-point errors during the
* various accumulations performed by this class, the domain of incoming values
* is remapped near to the origin via axis-specific mapping constants.
* @param dim
* @returns
*/
private mappingConstant(dim: StatAxis) {
if(!this.baseSample) {
return undefined;
}
if(dim == 't') {
return this.baseSample.t;
} else if(dim == 'x') {
return this.baseSample.targetX;
} else if(dim == 'y') {
return this.baseSample.targetY;
} else {
return 0;
}
}
/**
* Gets the statistical mean, utilizing the internal 'mapped' coordinate space.
* This is the version compatible with cross-sums and squared-sums.
* @param dim
* @returns
*/
private mappedMean(dim: StatAxis) {
return this.rawLinearSums[dim] / this.sampleCount;
}
/**
* Gets the statistical mean value of the samples observed during the represented
* interval on the specified axis.
* @param dim
* @returns
*/
public mean(dim: StatAxis) {
// This external-facing version needs to provide values in 'external'-friendly
// coordinate space.
return this.mappedMean(dim) + this.mappingConstant(dim);
}
/**
* Gets the sum of the squared distance from the mean seen in samples observed
* during the represented interval on the specified axis.
* @param dim
* @returns
*/
public squaredSum(dim: StatAxis) {
const x2 = this.rawSquaredSums[dim];
const x1 = this.rawLinearSums[dim];
return sigMinus(x2, x1 * x1 / this.sampleCount);
}
/**
* Gets the sum of the statistical 'cross' term "distance" away from the mean
* observed during the represented interval on the specified axis.
* @param dimPair
* @returns
*/
public crossSum(dimPair: PathCoordAxisPair) {
const dim1 = dimPair.charAt(0);
const dim2 = dimPair.charAt(1);
let orderedDims: string = dimPair;
const ab = this.rawCrossSums[orderedDims];
const a = this.rawLinearSums[dim1];
const b = this.rawLinearSums[dim2];
return sigMinus(ab, a * b / this.sampleCount);
}
/**
* Gets the unbiased covariance between the specified pair of axes for samples
* observed during the represented interval.
* @param dimPair
* @returns
*/
public covariance(dimPair: PathCoordAxisPair) {
return this.crossSum(dimPair) / (this.sampleCount - 1);
}
/**
* Gets the unbiased variance on the specified axis for samples observed
* during the represented interval.
*/
public variance(dim: StatAxis) {
return this.squaredSum(dim) / (this.sampleCount - 1);
}
/**
* Utilizing (and possibly abusing) statistical identities, this function produces
* an equivalent, but-recentered copy of this instance's statistical accumulations
* that will be less prone to catastrophic cancellation.
*
* In non-stats speak, the new instance will suffer smaller floating-point
* errors than the old instance whenever they do occur.
* @returns
*/
public buildRenormalized(): CumulativePathStats {
// Other (internal) notes: the internal mapping of the new instance will not
// match that of the old instance. This should not affect the practical
// results of any mapping to and from the external coordinate space, however.
let result = new CumulativePathStats(this);
let newBase: InputSample = {
targetX: this.mappedMean('x') + this.baseSample.targetX,
targetY: this.mappedMean('y') + this.baseSample.targetY,
t: this.mappedMean('t') + this.baseSample.t
};
result.baseSample = newBase;
for(const dimPair in result.rawCrossSums) {
result.rawCrossSums[dimPair] = this.crossSum(dimPair as PathCoordAxisPair);
}
for(const dim in result.rawSquaredSums) {
// 'v' does not need renormalization.
if(dim == 'v') {
break;
}
// The identity we're using to renormalize rawCrossSums and rawSquaredSums
// automatically guarantees a mean of 0 after the renormalization.
result.rawLinearSums[dim] = 0;
result.rawSquaredSums[dim] = this.squaredSum(dim as PathCoordAxis);
}
return result;
}
/**
* Provides a linear-regression perspective on two specified axes over the represented
* interval.
* @param dependent
* @param independent
* @returns
*/
public fitRegression(dependent: PathCoordAxis, independent: PathCoordAxis) {
return new CumulativePathStats.regression(this, dependent, independent);
}
/**
* Provides the direct Euclidean distance between the start and end points of the segment
* (or curve) of the interval represented by this instance.
*
* This will likely not match the actual pixel distance traveled.
*/
public get netDistance() {
// No issue with a net distance of 0 due to a single point.
if(!this.lastSample || !this.initialSample) {
return 0;
}
const xDelta = this.lastSample.targetX - this.initialSample.targetX;
const yDelta = this.lastSample.targetY - this.initialSample.targetY;
return Math.sqrt(xDelta * xDelta + yDelta * yDelta);
}
/**
* Gets the duration of the represented interval in milliseconds.
*/
public get duration() {
// no issue with a duration of zero from just one sample.
if(!this.lastSample || !this.initialSample) {
return 0;
}
return (this.lastSample.t - this.initialSample.t);
}
/**
* Returns the angle (in radians) traveled by the corresponding segment clockwise
* from the unit vector <0, -1> in the DOM (the unit "upward" direction).
*/
public get angle() {
if(this.sampleCount == 1 || !this.lastSample || !this.initialSample) {
return Number.NaN;
} else if(this.netDistance < 1) {
// < 1 px, thus sub-pixel, means we have nothing relevant enough to base an angle on.
return Number.NaN;
}
const xDelta = this.lastSample.targetX - this.initialSample.targetX;
const yDelta = this.lastSample.targetY - this.initialSample.targetY;
const yAngleDiff = Math.acos(-yDelta / this.netDistance);
return xDelta < 0 ? (2 * Math.PI - yAngleDiff) : yAngleDiff;
}
/**
* Returns the angle (in degrees) traveled by the corresponding segment clockwise
* from the unit vector <0, -1> in the DOM (the unit "upward" direction).
*/
public get angleInDegrees() {
return this.angle * 180 / Math.PI;
}
/**
* Returns the cardinal or intercardinal direction on the screen that most
* closely matches the direction of movement represented by the represented
* segment.
*
* @return A string one or two letters in length (e.g: 'n', 'sw'), or
`undefined` if not enough data to determine a direction.
*/
public get cardinalDirection() {
if(this.sampleCount == 1 || !this.lastSample || !this.initialSample) {
return undefined;
}
const buckets = ['n', 'ne', 'e', 'se', 's', 'sw', 'w', 'nw', 'n'];
// We could be 'more efficient' and use radians here instead, but this
// version helps a bit more with easy maintainability.
const bucketIndex = Math.ceil((this.angleInDegrees - 22.5)/45);
return buckets[bucketIndex];
}
/**
* Measured in pixels per second.
* @return a speed in pixels per millisecond, or `Number.NaN` if no data
*/
public get speed() {
return this.duration ? this.netDistance / this.duration : Number.NaN;
}
/**
* Returns the represented interval's 'mean angle' clockwise from the DOM's
* <0, -1> (the unit vector toward the top of the screen) in radians.
*
* Based upon the 'circular mean'. Refer to https://en.wikipedia.org/wiki/Circular_mean.
*
* Note that very slow-moving segments may be heavily affected by pixel aliasing
* effects; mouse and touch events usually do not provide sub-pixel resolution.
*/
public get angleMean() {
if(this.arcSampleCount == 0) {
return Number.NaN;
}
// Neato reference: https://rosettacode.org/wiki/Averages/Mean_angle
// But we don't actually need to divide by sample count; `atan2` handles that!
const sinMean = this.sinLinearSum;
const cosMean = this.cosLinearSum;
let angle = Math.atan2(sinMean, cosMean); // result: on the interval (-pi, pi]
// Convert to [0, 2*pi).
if(angle < 0) {
angle = angle + 2 * Math.PI;
}
return angle;
}
/**
* Provides the rSquared value needed internally for circular-statistic properties.
*
* Range: floating-point values on the interval [0, 1].
*/
private get angleRSquared() {
// Refer to https://en.wikipedia.org/wiki/Directional_statistics#Distribution_of_the_mean.
// We're computing the squared value of that page's R-bar stat.
//
// Now, why it's called that? ... good question. My best guess is that it's meant to
// correspond to linear regression's 'r' stat, which when squared serves as the
// coefficient of determination for the regression. Intuitively, that does seem to
// match what this represents - though for normal regressions, the c.o.d isn't normally
// used to compute deviation or variance!
const rSquaredBase = this.cosLinearSum * this.cosLinearSum + this.sinLinearSum * this.sinLinearSum;
return rSquaredBase / (this.arcSampleCount * this.arcSampleCount);
}
/**
* The **circular standard deviation** of the represented interval's angle observations.
*
* Refer to https://en.wikipedia.org/wiki/Directional_statistics#Standard_deviation.
*
* Note that very slow-moving segments may be heavily affected by pixel aliasing
* effects; mouse and touch events usually do not provide sub-pixel resolution.
* This can result in very high deviation values.
*
* In less-technical terms - the "stair-stepping" effect seen on high zoom levels means we don't
* get perfectly straight lines, and that can cause this value to be unexpectedly high.
*/
public get angleDeviation() {
if(this.arcSampleCount == 0) {
return Number.NaN;
}
return Math.sqrt(-Math.log(this.angleRSquared));
}
/**
* Provides the actual, pixel-based distance actually traveled by the represented segment.
* May not be an integer (because diagonals are a thing).
*/
public get rawDistance() {
return this.coordArcSum;
}
// Convert to a `toJSON` method for use during investigative debugging.
private toDebuggingJSON() {
// This `likelyState` value is extremely prototyped & just here for reviewer/tester convenience.
// It'll need to be developed a bit more fully, but follows my intuitions from development &
// testing.
let likelyType = 'unknown';
if(this.mean('v') < 0.08 && this.rawDistance < 12 && this.duration > 100) {
likelyType = 'hold';
} else if(this.mean('v') < 0.08 && this.rawDistance < 6) {
likelyType = 'hold';
}
if(this.mean('v') > 0.4 || (this.mean('v') > 0.2 && this.duration > 80) || this.netDistance > 20) {
likelyType = 'move';
}
return {
angle: this.angle,
cardinal: this.cardinalDirection,
likelyType: likelyType,
speedMean: this.mean('v'),
rawDistance: this.rawDistance,
duration: this.duration,
sampleCount: this.sampleCount,
angleMeanDegrees: this.angleMean * 180 / Math.PI,
angleDeviation: this.angleDeviation,
speedVariance: this.variance('v')
}
}
public toJSON() {
// We're not actually saving the JSON out to anything yet or loading/parsing it
// for any use beyond direct human interpretation, so... it's "okay" to
// leave like this for now.
return this.toDebuggingJSON();
}
}
}

4
common/web/gesture-recognizer/src/inputSample.ts
View file

@ -36,4 +36,8 @@ namespace com.keyman.osk {
}
export type InputSampleSequence = InputSample[];
export function isAnInputSample(obj: any): obj is InputSample {
return 'targetX' in obj && 'targetY' in obj && 't' in obj;
}
}

955
common/web/gesture-recognizer/src/pathSegmenter.ts Normal file
View file

@ -0,0 +1,955 @@
/// <reference path="cumulativePathStats.ts" />
/* FIXME: Too much `console`.
*
* There's an AWFUL LOT of `console.log`-ing in this file at present.
* It was definitely useful for prototyping & development of its (and
* of `cumulativePathStats`'s) code, but it's GOTTA be cleaned up
* in the next PR... if not sooner.
*/
namespace com.keyman.osk {
// Note: the only `export`-ed class from this file is `PathSegmenter`, hence the helper
// classes being placed within the same file. Not to say we _couldn't_ put them elsewhere.
// ------------
// Mostly used here to compare the sum-squared error components of segmented regressions
// to the sum-squared "modeled" components of their overall variance. Those are the two
// "independent random variables" we're examining. Variances (which are sums of squared
// values themselves) tend to be chi-squared distributed, fulfilling the conditions.
// That's stats-speak for "what this module does with it has a solid theoretical basis."
//
// We do NOT want to be computing this thing at run-time. (Just take one look at the
// equations found in that Wikipedia article!) Fortunately, it's very common in stats
// circles to just... use a lookup table for finding p-values for things following
// the f-distribution.
//
// For future non-stats-inclined maintainers:
// One could say that the p-value = the chance that what we have just "randomly" happened
// to occur without our expectations actually being met. It's a super-common
// stats term; they'd then say that if the p-value is sufficiently low, then we\
// "reject the null hypothesis" - the theory that it actually DID happen randomly - in
// favor of the high likelihood that we really are onto something.
/**
* Acts as a lookup table for null-hypothesis tests utilizing the F-distribution from the
* domain of statistics. Refer to https://en.wikipedia.org/wiki/F-distribution for details
* on its properties.
*
* This is only a partial implementation; only a couple of numerator degrees-of-freedom are
* worth our consideration, and there's only marginal gain to be found supporting more
* denominator degrees-of-freedom than are already supported here. Keeping it small also
* helps with maintenance and readability.
*/
class FDistribution {
/**
* Indexing: [threshold-index][num-2][denom-1].
*
* If numerator has only 1 degree of freedom, that means we don't really have
* segments so much as different averages between the two segments - there's
* no slope on either side. In other words, "just a bump". No segmentation
* there.
*
* 2 degrees of freedom: no movement on the axis on only one side.
*
* threshold-index:
* - p-value of .100: 0
* - p-value of .050: 1
*
* The f-statistic value must match or exceed its corresponding entry in
* the table below, based on the detected DoF (degrees of freedom) for the
* 'numerator' and 'denominator' components. Exceeding the threshold specified
* for the p = .100 section indicates less than a 10% probability of the statistic
* being tested having arisen by purely random chance. Failure to exceed even
* that implies an unacceptably high chance of being "convenient but meaningless" -
* the "null hypothesis", in stats speak.
*/
private static readonly table = [
// p = .100
[
// numerator: 2
[
// denom: 1-10
49.50, 9.00, 5.46, 4.32, 3.78, 3.46, 3.26, 3.11, 3.01, 2.92,
// denom: 11-20
2.86, 2.81, 2.76, 2.73, 2.70, 2.67, 2.64, 2.62, 2.61, 2.59
// we COULD continue with threshold-dropping after that,
// but I doubt we'll need it in practicality.
// At 30: 2.49
// limit -> infinity: 2.30
],
// numerator: 3
[
// denom: 1-10
53.59, 9.16, 5.39, 4.19, 3.62, 3.29, 3.07, 2.92, 2.81, 2.73,
// denom: 11-20
2.66, 2.61, 2.56, 2.52, 2.49, 2.46, 2.44, 2.42, 2.40, 2.38
// Likewise, from above.
// At 30: 2.28
// limit -> infinity: 2.08
]
],
// p = .050
[
// numerator: 2
[
// denom: 1-10
199.5, 19.0, 9.55, 6.94, 5.79, 5.14, 4.74, 4.46, 4.26, 4.10,
// denom: 10-20
3.98, 3.89, 3.81, 3.74, 3.68, 3.63, 3.59, 3.55, 3.52, 3.49
// At 30: 3.32
// limit -> infinity: 3.00
],
// numerator: 3
[
// denom: 1-10
215.7,19.16, 9.28, 6.59, 5.41, 4.76, 4.35, 4.07, 3.86, 3.71,
// denom: 10-20
3.59, 3.49, 3.41, 3.34, 3.29, 3.24, 3.20, 3.16, 3.13, 3.10
// At 30: 2.92
// limit -> infinity: 2.60
]
]
]
/**
* Determines a threshold tier for segmentation based on the segmented
* regression f-statistic.
* @param statistic
* @param numDoF
* @param denomDoF
*
* Returns 0.050 if the statistic indicates a p-value of 0.050 or lower.
* Failing that, returns 0.100 if a p-value of 0.100 or lower is indicated.
* Otherwise, returns 1.0 - we have no basis to "reject the null hypothesis".
*/
static thresholdTier(statistic: number, numDoF: number, denomDoF: number) {
// Former case: we'd never segment anyway
// Latter case: it's currently wrong to statistically test.
// The F-distribution is not defined for this case.
if(numDoF < 2 || denomDoF < 1) {
return 1;
}
const numIndex = (numDoF > 3 ? 3 : numDoF) - 2;
const denomIndex = (denomDoF > 20 ? 20 : denomDoF) - 1;
const tier2Threshold = FDistribution.table[1][numIndex][denomIndex];
if(statistic >= tier2Threshold) {
return 0.05;
}
const tier1Threshold = FDistribution.table[0][numIndex][denomIndex];
// If arising purely randomly isn't at least less than 10% likely,
// we'll categorically say it happened randomly with full certainty (1).
// Obviously not actually true, but it works for our thresholding.
return statistic >= tier1Threshold ? 0.10 : 1;
}
}
/**
* Represents the result of a segmentation of the search path and the related
* statistical accumulations needed to properly test the segmentation's
* validity.
*/
class Segmentation {
public static readonly SPLIT_CRITERION_THRESHOLD = 1.5;
/**
* The properties of the "left-hand" / earlier half of the time interval
* being segmented.
*/
readonly pre: CumulativePathStats;
/**
* The properties of the "right-hand" / later half of the time interval
* being segmented.
*/
readonly post: CumulativePathStats;
/**
* The properties of the full interval being segmented, before the split.
*/
readonly union: CumulativePathStats;
/**
* The full running accumulation for the ongoing touch path. May include
* accumulations from before the interval under examination.
*/
readonly endpoint: CumulativePathStats;
/**
* Provides segmented-regression statistics & fitting values on the two specified axes /
* dimensions based on the subsegments and their corresponding linear-regression
* statistics. All operations are O(1).
*/
static readonly segmentationComparison = class SegmentedRegression {
host: Segmentation;
readonly independent: PathCoordAxis;
readonly dependent: PathCoordAxis;
readonly paired: 'tx' | 'ty' | 'xy';
/**
* The linear regression for the underlying `pre` (earlier) subsegment.
*/
pre: typeof CumulativePathStats.regression.prototype;
/**
* The linear regression for the underlying `post` (later) subsegment.
*/
post: typeof CumulativePathStats.regression.prototype;
/**
* The linear, unsegmented regression for the underlying `union` (combined) subsegment.
*/
union: typeof CumulativePathStats.regression.prototype;
constructor(host: Segmentation, dependentAxis: PathCoordAxis, independentAxis: PathCoordAxis) {
if(dependentAxis == independentAxis) {
throw "Two different axes must be specified for the regression object.";
}
this.host = host;
this.dependent = dependentAxis;
this.independent = independentAxis;
if(dependentAxis < independentAxis) {
this.paired = dependentAxis.concat(independentAxis) as 'tx' | 'ty' | 'xy';
} else {
this.paired = independentAxis.concat(dependentAxis) as 'tx' | 'ty' | 'xy';
}
this.pre = this.host.pre .fitRegression(dependentAxis, independentAxis);
this.post = this.host.post .fitRegression(dependentAxis, independentAxis);
this.union = this.host.union.fitRegression(dependentAxis, independentAxis);
}
/**
* The total summed squared-distances of the best fitting line from actually-observed values
* for their corresponding subsegment; in other words, the "sum of the squared errors" when
* utilizing segmented regression.
*
* Statistically, this is the portion of the dependent variable's variance (un-normalized)
* that is unexplained even after dividing the interval into separate subsegments.
*/
get remainingSumSquaredError(): number {
return this.pre.sumOfSquaredError + this.post.sumOfSquaredError;
}
/**
* Provides the coordinate that belongs to BOTH subsegments, as it ends one and begins
* the other.
*/
private get splitPoint() {
let splitPoint = this.host.pre.lastSample;
let indep = splitPoint.t;
if(this.independent == 'x') {
indep = splitPoint.targetX;
} else if(this.independent == 'y') {
indep = splitPoint.targetY;
}
let dep = splitPoint.t;
if(this.dependent == 'x') {
dep = splitPoint.targetX;
} else if(this.dependent == 'y') {
dep = splitPoint.targetY;
}
return {
dep: dep,
indep: indep
};
}
/**
* The total summed squared-distances of the best fitting line from actually-observed values
* when **not** utilizing segmented regression. Double-counts the subsegment's common point, as
* it is counted within both subsegments' sum-of-squared error term.
*/
get unsegmentedSumSquaredError(): number {
// What was our remaining error for the unsegmented regression?
let baseSSE = this.union.sumOfSquaredError;
// We double-count the split point when segmenting, so we should add an
// extra copy of its residual.
const splitPoint = this.splitPoint;
const splitPointError = this.splitPoint.dep - this.union.predictFromValue(splitPoint.indep);
// This is the total sum-squared error to be handled by segmenting.
return baseSSE + splitPointError * splitPointError;
}
/**
* Gets the amount of (unnormalized) variance not explained by a standard regression on
* the full interval but succesfully explained by splitting the interval in twain via
* segmented regression.
*/
get sumSquaredGainFromSegmentation(): number {
return this.unsegmentedSumSquaredError - this.remainingSumSquaredError;
}
/**
* A statistical term that signals how successful the segmented regression is. Always
* has values on the interval [0, 1], with 1 being a perfect fit.
*/
get coefficientOfDetermination(): number {
if(this.host.union.squaredSum(this.dependent) == 0 || this.host.union.squaredSum(this.independent) == 0) {
return 1;
}
return 1 - this.remainingSumSquaredError / this.host.union.squaredSum(this.dependent);
}
/**
* Gets the raw statistic value needed to test for validity of segmentation based on the axes
* under examination. The higher the proportion of newly-explained variance to still-unexplained
* variance is, the more confident we can _formally_ be in our segmentation.
*/
get fStat(): number {
const val = this.sumSquaredGainFromSegmentation / this.remainingSumSquaredError;
// We're fine with Infinity. Just... not so much NaN.
return isNaN(val) ? 0 : val;
}
/**
* Gets the degrees-of-freedom to be used for the numerator DoF parameter of
* the F-distribution needed for validity testing.
*/
get fDoF1(): number {
let numDoF = 3;
// Cases where there clearly may as well be 'no slope' at all.
// This saves us a degree of freedom, which is VERY useful for segmenting
// the boundary between a 'hold' and a 'move'.
if(this.host.pre.squaredSum(this.dependent) < 1e-8) {
numDoF--;
}
if(this.host.post.squaredSum(this.dependent) < 1e-8) {
numDoF--;
}
return numDoF;
}
/**
* Gets the degrees-of-freedom to be used for the denominator DoF parameter of
* the F-distribution needed for validity testing.
*/
get fDoF2(): number {
return this.host.union.sampleCount - 2 - this.fDoF1;
}
/**
* States a statistically-founded level of confidence we may place in upholding
* the represented segmentation, as based on the regressions and axes under examination.
*
* If we cannot hold any confidence whatsoever (due to being unable to "reject the
* null hypothesis" from this specific regression), will return 0.
*/
get certaintyThreshold() {
return 1 - FDistribution.thresholdTier(this.fStat, this.fDoF1, this.fDoF2);
}
}
constructor(pre: CumulativePathStats,
post: CumulativePathStats,
union: CumulativePathStats,
cumulativeEndpoint: CumulativePathStats) {
this.pre = pre;
this.post = post;
this.union = union;
this.endpoint = cumulativeEndpoint;
}
/**
* Provides a segmented-regression perspective for the represented interval and proposed
* split point based upon the two specified axes.
* @param dependent
* @param independent
* @returns
*/
public segReg(dependentAxis: PathCoordAxis, independentAxis: PathCoordAxis) {
return new Segmentation.segmentationComparison(this, dependentAxis, independentAxis);
}
/**
* Defines our test for upholding a minimum of sub-segmentation based upon changes in the
* interval's coordinates over time. This may occur even with no change in spacial direction
* if there is significant enough change in _speed_, as that "curves the line" when one axis
* is time.
*/
get segmentationMerited(): boolean {
let totalThreshold = 0;
const xTest = new Segmentation.segmentationComparison(this, 'x', 't');
const yTest = new Segmentation.segmentationComparison(this, 'y', 't');
// Our testing thresholds are for p=0.05 and p=0.10, which correspond to certainties of 95% and
// 90% that our segmentation did not arrive from random chance based on the axis being tested.
const p05 = 0.95; // 1 - 0.05
const p10 = 0.9; // 1 - 0.10
totalThreshold += xTest.certaintyThreshold >= p05 ? 2 : (xTest.certaintyThreshold >= p10 ? 1 : 0) ;
totalThreshold += yTest.certaintyThreshold >= p05 ? 2 : (yTest.certaintyThreshold >= p10 ? 1 : 0) ;
return totalThreshold >= 2;
}
/**
* Defines our test for considering two (or more) sub-segments as part of the same overall
* segment. This seeks to 'link' geometrically-related subsegments - especially those that
* maintain the same direction but differ only in observed speed.
*/
get mergeMerited(): boolean {
// Because of caret-like motions (as in, in the '^' shape), we need to text for
// regression on both axes. One may have notably higher variance than the other.
//
// These tests ignore time, and therefore speed. Only the raw geometry of the motion
// matters for this test.
const xTest = new Segmentation.segmentationComparison(this, 'x', 'y');
const yTest = new Segmentation.segmentationComparison(this, 'y', 'x');
// If we don't get a p-value less than .100, then as far as x & y are concerned - and thus the user
// is concerned - it's the same segment. Speed may be different, but not the direction.
if(xTest.certaintyThreshold > 0) {
return false;
}
return yTest.certaintyThreshold == 0;
}
}
/**
* A specialized form of the `Segmentation` class that assists in its
* construction from the interval directly under examination by the
* segmentation algorithm.
*/
class PotentialSegmentation extends Segmentation {
/**
* The full running accumulation for the ongoing touch path until the point
* before `post` / the later subsegment. May include
* accumulations from before the interval under examination.
*/
readonly chopPoint: CumulativePathStats;
/**
* The full running accumulation for the ongoing touch path until the point
* at the end of `pre` / the earlier subsegment. May include
* accumulations from before the interval under examination, and will
* include accumulations from the first point represented by `post`.
*/
readonly endOfPre: CumulativePathStats;
/**
* The full running accumulation for the ongoing touch path until the point
* before `pre` / the earlier subsegment. Will only include
* accumulations from before the interval under examination.
*/
readonly baseChop: CumulativePathStats;
constructor(steppedStats: CumulativePathStats[],
choppedStats: CumulativePathStats,
splitIndex: number) {
const pre = steppedStats[splitIndex].deaccumulate(choppedStats);
// Keep stats value components based on the final point of the 'pre' segment.
const finalStats = steppedStats[steppedStats.length-1];
const post = finalStats.deaccumulate(steppedStats[splitIndex-1]);
const union = finalStats.deaccumulate(choppedStats);
super(pre, post, union, finalStats);
this.baseChop = choppedStats;
this.chopPoint = steppedStats[splitIndex-1];
this.endOfPre = steppedStats[splitIndex];
}
}
/**
* The core logic, algorithm, and manager for touchpath segmentation.
*/
export class PathSegmenter {
/**
* The minimum amount of time (in ms) to wait between sample repetitions
* once inputs stop arriving, so long as the path is still active.
*/
private readonly REPEAT_INTERVAL = 33;
/**
* The time-interval length (in ms) at the end of the path to consider
* when determining whether or not to trigger path segmentation.
*/
private readonly SLIDING_WINDOW_INTERVAL = 50;
/**
* Tracks the mathematical values used to provide path segment stats
* at each point on the path. Individual steps may be removed (and
* batched into `choppedStats`) once their respective points on the
* path have been fully processed.
*/
private steppedCumulativeStats: CumulativePathStats[];
/**
* Tracks all subsegments awaiting completion of their overall segment.
*/
private lingeringSubsegmentations: PotentialSegmentation[];
/**
* TODO: These directly represent path segments produced by the prototype
* algorithm. Just... the stats analysis of the path segment, without
* obvious / public members to relevant coordinates.
*
* In follow-up work, this should be used to build & to update
* cleaner, public-facing segment objects for the touch path.
*/
private _protoSegments: CumulativePathStats[] = [];
/**
* TODO: These directly represent the subsegments comprising their
* corresponding path segments (in `_protoSegments`) produced by the
* prototype algorithm. Just... the stats analysis of each subsegment.
*
* In follow-up work, these should be used internally to build & update
* the internals of TBD cleaner, public-facing segment objects for the
* touch path.
*/
private _protoSegmentSets: CumulativePathStats[][] = [];
/**
* Used to 'repeat' the most-recently observed incoming sample if no
* other replaces it before it triggers.
*
* A repeating sample indicates lack of motion, which is valuable
* information for stats-based segmentation.
*/
private repeatTimer: number | NodeJS.Timeout;
/**
* The timestamp of observation of the most recently observed sample's
* most recent repetition - even if it's only the first evaluation.
*/
private repeatTimestamp: number;
/**
* Represents the cumulative statistics of all points on the path
* that lie on already-fully-segmented parts of it.
*/
private choppedStats: CumulativePathStats = null;
constructor() {
this.steppedCumulativeStats = [];
this.lingeringSubsegmentations = [];
}
/**
* Appends a new coordinate to the touch path and also kick-starts a timer for replicating it
* should neither further inputs be received nor termination of the touchpoint be indicated.
* @param sample
*/
public add(sample: InputSample) {
// Set up the input-repeater (in case we don't get further feedback but remain active)
const repeater = (timeDelta: number) => {
this.observe(sample, timeDelta);
}
// If we previously set up an input-repeater, cancel it. We've got a more up-to-date
// coordinate on the touchpath now.
if(this.repeatTimer) {
// @ts-ignore
clearInterval(this.repeatTimer);
this.repeatTimer = null;
}
this.repeatTimer = setInterval(() => {
const timeDelta = Date.now() - this.repeatTimestamp;
repeater(timeDelta);
}, this.REPEAT_INTERVAL);
this.repeatTimestamp = Date.now();
repeater(0);
}
/**
* Used to finalize all segmentation for the touchpath whenever termination of the corresponding
* touchpoint has been terminated (mouse-up, touch-end).
*/
public close() {
// The Node clearTimeout & DOM clearTimeout appear to TS as overloads of each other,
// and their type definitions will conflict. A simple @ts-ignore will bypass this issue.
// @ts-ignore
clearInterval(this.repeatTimer); // Cancels the input-repeater.
this.repeatTimer = null;
// The way things are structured, finalization.pre = final segment. It's a bit hacky, but
// it _is_ happy, intentional 'fallout' from the implementation's design. `post` is effectively
// a single-point 'subsegment' and will go effectively unutilized by the phase 2 subsegment linker.
let finalization = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, this.steppedCumulativeStats.length-1);
console.log("! Finalization !");
this.processSubsegmentation(finalization, true);
// TODO: these are temporary statements to facilitate exploration, experimentation, & debugging.
// We should be providing output to the touchpath object (`.path.segments`).
// But... that'll be left for a follow-up PR.
console.log(this._protoSegments);
console.log(this._protoSegmentSets);
console.log(this._protoSegments.map((val) => (val.toJSON())));
}
/**
* Adds a statistic 'observation' of the state of the touchpath.
*
* This is either to be called directly upon reception of a new input-coordinate or
* upon replication of a prior input should the touchpath still be active without any
* indication of motion.
* @param sample
* @param timeDelta
*/
private observe(sample: InputSample, timeDelta: number) {
let cumulativeStats: CumulativePathStats;
if(this.steppedCumulativeStats.length) {
cumulativeStats = this.steppedCumulativeStats[this.steppedCumulativeStats.length-1];
} else {
cumulativeStats = new CumulativePathStats();
}
sample = {... sample};
sample.t += timeDelta;
const extendedStats = cumulativeStats.extend(sample);
this.steppedCumulativeStats.push(extendedStats);
this.performSubsegmentation();
}
private _debugLogSegmentationReport(candidateSplit: PotentialSegmentation) {
console.log("------------------------------------------------------------------");
console.log("Combined: ");
console.log(candidateSplit.union.toJSON());
console.log("Pre: ")
console.log(candidateSplit.pre.toJSON());
console.log("Post: ");
console.log(candidateSplit.post.toJSON());
console.log();
const xF = candidateSplit.segReg('x', 't');
const yF = candidateSplit.segReg('y', 't');
console.log(`x F-test: F_(${xF.fDoF1}, ${xF.fDoF2}) = ${xF.fStat} @ ${xF.certaintyThreshold}`);
console.log(`y F-test: F_(${yF.fDoF1}, ${yF.fDoF2}) = ${yF.fStat} @ ${yF.certaintyThreshold}`);
console.log();
console.log("Split object: ");
console.log(candidateSplit);
console.log("------------------------------------------------------------------");
// END: DO NOT RELEASE.
}
// The "reported via event" aspect mentioned below is necessary because this may be
// called via setTimeout callback on a held touchpoint. We need that `setTimeout`
// (which replicates the most recently-seen input coordinate) in order to have good
// sample-data for detecting the boundary between motion and lack thereof during
// segmentation.
/**
* This is the "main method" for touchpath segmentation. Should a new segment result
* from its analysis, it will be reported via event.
* @returns
*/
private performSubsegmentation() {
const cumulativeStats = this.steppedCumulativeStats[this.steppedCumulativeStats.length - 1];
const unsegmentedDuration = cumulativeStats.lastTimestamp - this.steppedCumulativeStats[0].lastTimestamp;
// STEP 1: Determine the range of the sliding time window for the most recent samples.
if(unsegmentedDuration < this.SLIDING_WINDOW_INTERVAL * 2) {
console.log("Interval too short for segmentation.");
return;
}
let splitPoint = 0;
// Do not consider the just-added `extendedStats` entry when building the sliding
// time window.
for(let i = this.steppedCumulativeStats.length-2; i >=0; i--) {
if(this.steppedCumulativeStats[i].lastTimestamp < cumulativeStats.lastTimestamp - this.SLIDING_WINDOW_INTERVAL) {
splitPoint = i+1;
break;
}
}
// We split the cumulative stats on a specific point, which then resides on the edge
// of both of the resulting intervals. This completes "step 1".
let candidateSplit = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint);
// STEP 2: given the proposed time window, do we have a basis for segmentation? And is there a better
// candidate split point nearby?
// NOTE: During initial development, I actually had a hard rule about not even searching if segmentation
// failed on the initially-constructed sliding window. Some analysis may be wise here, as it'd be nice
// to optimize away the 'need to search' if we can find hard & fast rules about when we'll never need
// to go looking. But... being too aggressive can cause nasty problems.
if(!candidateSplit.segmentationMerited) {
// // Debug logging statements:
const xTest = candidateSplit.segReg('x', 't');
const yTest = candidateSplit.segReg('y', 't');
console.log(`x F-test: F_(${xTest.fDoF1}, ${xTest.fDoF2}) = ${xTest.fStat} @ ${xTest.certaintyThreshold}`);
console.log(`y F-test: F_(${yTest.fDoF1}, ${yTest.fDoF2}) = ${yTest.fStat} @ ${yTest.certaintyThreshold}`);
console.log("candidate split: " );
console.log(candidateSplit);
}
// We either have the conditions to trigger segmentation or just became long enough to consider it.
// If we're only just long enough to consider it, there may be a better segmentation point to start with.
// Now... is there a better segmentation point?
class SplitSearchState {
readonly xTest: typeof Segmentation.segmentationComparison.prototype;
readonly yTest: typeof Segmentation.segmentationComparison.prototype;
readonly candidate: PotentialSegmentation;
constructor(candidate?: PotentialSegmentation) {
this.candidate = candidate;
if(candidate == null) {
return;
}
this.xTest = candidate.segReg('x', 't');
this.yTest = candidate.segReg('y', 't');
}
// The value for an 'objective function' to optimize in our search for a better candidate.
// The higher this is, the more we like its potential as the segmentation point.
get segRating() {
if(this.candidate) {
return Math.max(this.xTest.coefficientOfDetermination, this.yTest.coefficientOfDetermination);
} else {
return 0;
}
}
// But, if it turns out this potential point wouldn't actually result in segmentation, well...
// "abandon ship".
get segmentationMerited() {
return this.candidate?.segmentationMerited ?? false;
}
}
// Start: split-point search
/* We'll use our initial sliding window as a starting point. It's close to the active location
* of the touch, so we're unlikely to miss a proper segmentation point if it lies close.
* We'll find the local maximum (best split point for the local part of the path)
* via gradient descent.
*
* Note: binary search is a 'bad idea', as the touchpath is likely not following a
* monotonous path, mathematically speaking. (It'd be possible to overshoot a "peak" or
* "valley" quite easily.)
*/
let currentSplit = new SplitSearchState(candidateSplit);
let leftSplit = new SplitSearchState(new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint-1));
let rightCandidate: PotentialSegmentation = null;
if(splitPoint+1 < this.steppedCumulativeStats.length) {
rightCandidate = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint+1);
}
let rightSplit = new SplitSearchState(rightCandidate);
// Step 2a: detect which direction gives the best improvement in segmentation potential.
const criteria = [leftSplit.segRating, currentSplit.segRating, rightSplit.segRating];
let sortedCriteria = [...criteria].sort();
const delta = criteria.indexOf(sortedCriteria[2])-1; // -1 if 'left' is best, 1 if 'right' is best.
// Step 2b: we've found the direction: go searching!
if(delta != 0) {
// We can get better segmentation by shifting. Proceed in the optimal direction.
do {
const nextSplitIndex = splitPoint + delta;
if(nextSplitIndex >= this.steppedCumulativeStats.length || nextSplitIndex < 0) {
break;
}
let nextCandidate = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint + delta);
let nextSplit = new SplitSearchState(nextCandidate);
// Prevent overly-short intervals / over-segmentation.
if(nextCandidate.pre.duration * 1000 < this.SLIDING_WINDOW_INTERVAL / 2) {
break;
} else if(nextCandidate.post.duration * 1000 < this.SLIDING_WINDOW_INTERVAL / 2) {
break;
}
// Not an improvement? Guess we found the best spot.
if(nextSplit.segRating <= currentSplit.segRating) {
break;
} else if(!nextSplit.segmentationMerited && currentSplit.segmentationMerited) {
// Note: this can happen if segmentation is triggered due to divergence on both axes
// if one of them drops a threshold tier and the other fails to improve sufficiently.
console.warn("aborting split-point relocation due to no longer segmenting");
break;
} else {
splitPoint += delta;
currentSplit = nextSplit;
candidateSplit = nextCandidate;
}
// If we found a new best segmentation point, we then ask if we can get even better by shifting further.
} while(true);
}
// Step 2 complete.
// Step 3: evaluate our candidate split-point - do we sub-segment? If so, do so.
console.log("best split: ");
console.log(candidateSplit);
if(!candidateSplit.segmentationMerited) {
return;
}
// First phase of segmentation: complete!
// FIXME: DO NOT RELEASE.
// This is exploratory / diagnostic code assisting development of the path segmentation
// algorithm.
this._debugLogSegmentationReport(candidateSplit);
// END: DO NOT RELEASE.
this.steppedCumulativeStats = this.steppedCumulativeStats.slice(splitPoint);
this.choppedStats = candidateSplit.chopPoint;
// Step 4: process the implications.
// There are some cases where we want to prevent segmentation from fully happening.
// The next method will both handle that and the signal of any completed segments that
// may result.
this.processSubsegmentation(candidateSplit);
}
/**
* Given the results of the subsegmentation process, this method analyzes the results
* in order to produce and update the detected segments.
*
* TODO: Should produce actual `Segment` objects and trigger their emission
* them via EventEmitter mechanics.
* @param subsegmentation
* @param finalize Set to `true` for a terminating touchpath, indicating that
* no further subsegmentation will occur for this touchpath.
*/
private processSubsegmentation(subsegmentation: PotentialSegmentation, finalize?: boolean) {
finalize = !!finalize;
// This function makes a LOT of assumptions particular to this method and class.
// It is not safe to make this class-level, and it's especially unsafe to make it `public`.
const mergeSubsegmentationAccumulations = function(array: PotentialSegmentation[]) {
if(array.length == 1) {
return array[0].pre; // It's pre-calculated, so just use it.
} else {
const finalSubsegmentation = array[array.length-1];
return finalSubsegmentation.endOfPre.deaccumulate(array[0].baseChop);
}
}
// Step 1: did we previously note that any previous subsegments are likely to be 'linked' to
// the new, incoming left-hand subsegment? If so, validate that expectation & act accordingly.
let predecessor = subsegmentation.pre;
let firstChopPoint = subsegmentation.baseChop;
if(this.lingeringSubsegmentations.length) {
// First: check if the newly-finished subsegment should be merged with the lingering ones.
const mergedPrecursors = mergeSubsegmentationAccumulations(this.lingeringSubsegmentations);
const tailUnion = mergeSubsegmentationAccumulations([...this.lingeringSubsegmentations, subsegmentation]);
console.log("Verifying linkage to pending merges: ");
console.log(this.lingeringSubsegmentations);
console.log("verification check:");
console.log(mergedPrecursors);
const precursorMergeSegmentation = new Segmentation(mergedPrecursors, subsegmentation.pre, tailUnion, subsegmentation.endOfPre);
// Sometimes the start of a harsh turn seems like it's part of the same thing for a moment, but as it continues,
// becomes something VERY different. Validate that we should still merge the left-hand with its predecessors.
if(!PathSegmenter.shouldLinkSubsegments(precursorMergeSegmentation)) {
// Emit as separate subsegment.
const finishedSegment = mergeSubsegmentationAccumulations(this.lingeringSubsegmentations);
this._protoSegments.push(finishedSegment);
this._protoSegmentSets.push(this.lingeringSubsegmentations.map((val) => val.pre));
this.lingeringSubsegmentations = [];
console.log("Proto-segments length: " + this._protoSegments.length);
console.log("Did not merge a lingering subsegment with the incoming one!");
} else {
predecessor = tailUnion;
firstChopPoint = this.lingeringSubsegmentations[0].baseChop;
console.log("Will merge in old subsegments!");
}
}
// Step 2: okay, predecessor handling complete. Now, do we think the newly-starting right-hand
// subsegment is likely to be 'linked' to the newly-completed left-hand subsegment?
console.log("Double-checking right-side split subsegment for xy/yx correlation with prior segment candidate(s)");
const fullUnion = subsegmentation.endpoint.deaccumulate(firstChopPoint);
const fullMergeSegmentation = new Segmentation(predecessor, subsegmentation.post, fullUnion, subsegmentation.endpoint);
console.log("segmentation-prevention check:")
console.log(fullMergeSegmentation);
// if(!force && PathSegmenter.shouldMergeSubsegments(subsegmentation.pre, subsegmentation.post, subsegmentation.union)) {
if(!finalize && PathSegmenter.shouldLinkSubsegments(fullMergeSegmentation)) {
this.lingeringSubsegmentations.push(subsegmentation);
} else {
// Merge all as a completed segment!
this._protoSegments.push(predecessor);
this._protoSegmentSets.push([...this.lingeringSubsegmentations.map((val) => val.pre), subsegmentation.pre]);
this.lingeringSubsegmentations = [];
console.log("Proto-segments length: " + this._protoSegments.length);
}
}
/**
* Used by the 'filter' step (phase 2 of segmentation) to determine whether or not
* two subsegments should be 'linked' as members of the same segment. (That is, if
* a user would likely consider the two subsegments as belonging to the "same arc
* of motion".)
* @param segmentation
* @returns
*/
private static shouldLinkSubsegments(segmentation: Segmentation): boolean {
// // Known case #1:
// // Near-identical direction, but heavy speed difference.
// // Speed's still high enough to not be a 'wait'.
// // We may want to 'note' the higher speed; that may be relevant for flick detection.
// // Known case #2:
// // Low-speed pivot; angle change caught on very low velocity for both subsegments.
// // ... or should this be merged? Multiple mini-segments from 'wiggling' could just go ignored instead...
// This is kinda plain and arbitrary, but it seems to work "well enough" for now,
// at least at this stage of development. (Most testing was done with Chrome emulation of
// an iPhone SE.)
// .mean('v') < 0.08: the mean speed (taken timestamp to timestamp) does not exceed 0.08px/millisec.
if(segmentation.pre.mean('v') < 0.08 && segmentation.post.mean('v') > 0.08) {
console.log("desegmentation exception");
return;
}
if(segmentation.pre.mean('v') > 0.08 && segmentation.post.mean('v') < 0.08) {
console.log("desegmentation exception");
return;
}
console.log("Desegmentation under consideration: ");
console.log(segmentation);
const xF = segmentation.segReg('x', 'y');
const yF = segmentation.segReg('y', 'x');
console.log(`merger F-test (xy): F_(${xF.fDoF1}, ${xF.fDoF2}) = ${xF.fStat} @ ${xF.certaintyThreshold}`);
console.log(`merger F-test (yx): F_(${yF.fDoF1}, ${yF.fDoF2}) = ${yF.fStat} @ ${yF.certaintyThreshold}`);
console.log(`will remerge: ${segmentation.mergeMerited}`);
return segmentation.mergeMerited;
}
}
}

5
common/web/gesture-recognizer/src/segment.ts Normal file
View file

@ -0,0 +1,5 @@
namespace com.keyman.osk {
export class Segment {
// TODO: stuff.
}
}

1
common/web/gesture-recognizer/src/tools/recorder/src/recorder.js
View file

@ -48,6 +48,7 @@ loadPromise.then((recognizer) => {
});
sequence.on('end', function() {
//let stats = sequence.touchpoints[0].path.segments[0].stats;
logElement.value = recorder.recordingsToJSON();
});
});

20
common/web/gesture-recognizer/src/trackedPath.ts
View file

@ -7,7 +7,7 @@ namespace com.keyman.osk {
export type JSONTrackedPath = {
coords: InputSample[]; // ensures type match with public class property.
wasCancelled?: boolean;
// segments: Segment[];
//segments: Segment[];
}
interface EventMap {
@ -36,6 +36,10 @@ namespace com.keyman.osk {
*/
export class TrackedPath extends EventEmitter<EventMap> {
private samples: InputSample[] = [];
private _segments: Segment[] = [];
private readonly segmenter: PathSegmenter;
private _isComplete: boolean = false;
private wasCancelled?: boolean;
@ -56,10 +60,16 @@ namespace com.keyman.osk {
if(jsonObj) {
this.samples = [...jsonObj.coords.map((obj) => ({...obj} as InputSample))];
// If we're reconstructing this from a JSON.parse, it's a previously-recorded, completed path.
// If we're reconstructing this from a JSON.parse, it's a previously-recorded,
// completed path.
this._isComplete = true;
this.wasCancelled = jsonObj.wasCancelled;
}
// Keep this as the _final_ statement in the constructor. `PathSegmenter` will
// need a reference to this instance, even if only via closure.
// (Most likely; not yet done.) Kinda awkward, but it's useful for compartmentalization.
this.segmenter = new PathSegmenter();
}
/**
@ -80,6 +90,7 @@ namespace com.keyman.osk {
}
this.samples.push(sample);
this.segmenter.add(sample);
this.emit('step', sample);
}
@ -93,6 +104,7 @@ namespace com.keyman.osk {
}
this.wasCancelled = cancel;
this._isComplete = true;
this.segmenter.close();
if(cancel) {
this.emit('invalidated');
@ -110,6 +122,10 @@ namespace com.keyman.osk {
return this.samples;
}
public get segments(): readonly Segment[] {
return this._segments;
}
/**
* Creates a serialization-friendly version of this instance for use by
* `JSON.stringify`.