From a30ca642acd4aa99cec1de7bb16cccff2a5babc8 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 29 Jul 2022 11:16:54 +0700 Subject: [PATCH 01/22] feat(web): initial touchpath-stat calculation logic, exploration --- .../web/gesture-recognizer/src/inputSample.ts | 4 + common/web/gesture-recognizer/src/segment.ts | 40 +++ .../gesture-recognizer/src/segmentStats.ts | 337 ++++++++++++++++++ .../src/tools/recorder/src/recorder.js | 1 + .../web/gesture-recognizer/src/trackedPath.ts | 9 +- 5 files changed, 390 insertions(+), 1 deletion(-) create mode 100644 common/web/gesture-recognizer/src/segment.ts create mode 100644 common/web/gesture-recognizer/src/segmentStats.ts diff --git a/common/web/gesture-recognizer/src/inputSample.ts b/common/web/gesture-recognizer/src/inputSample.ts index cc2a0fa0fe..f0b5b39981 100644 --- a/common/web/gesture-recognizer/src/inputSample.ts +++ b/common/web/gesture-recognizer/src/inputSample.ts @@ -36,4 +36,8 @@ namespace com.keyman.osk { } export type InputSampleSequence = InputSample[]; + + export function isAnInputSample(obj: any): obj is InputSample { + return 'targetX' in obj && 'targetY' in obj && 't' in obj; + } } \ No newline at end of file diff --git a/common/web/gesture-recognizer/src/segment.ts b/common/web/gesture-recognizer/src/segment.ts new file mode 100644 index 0000000000..d242bb9f3f --- /dev/null +++ b/common/web/gesture-recognizer/src/segment.ts @@ -0,0 +1,40 @@ +/// + +namespace com.keyman.osk { + export class Segment { + // May be best to keep an array of these, one per sample. + // Can then diff the stats to determine better cut-offs. + // Though... the whole arc-dist aspect will need a mite more help. + // - chopping off from the end: ez-pz. Raw diff is great. + // - or, well, just use the appropriate one from mid-way. + // - chopping off from the beginning: need an extra sample reference. + // - .nextSample. + // + // The FINAL version, once resolved, may be published. + // But until resolved, we probably want to keep an array. + private _stats: SegmentStats[]; + + constructor() { + this._stats = []; + } + + public get stats(): readonly SegmentStats[] { + return this._stats; + } + + public add(sample: InputSample) { + let baseStats: SegmentStats; + if(this.stats.length > 0) { + baseStats = this.stats[this.stats.length-1]; + } else { + baseStats = new SegmentStats(); + } + + let extendedStats = baseStats.unionWith(sample); + + // FIXME: VERY temp logging. + console.log(extendedStats.toJSON()); + this._stats.push(extendedStats); + } + } +} \ No newline at end of file diff --git a/common/web/gesture-recognizer/src/segmentStats.ts b/common/web/gesture-recognizer/src/segmentStats.ts new file mode 100644 index 0000000000..dd175a5eb2 --- /dev/null +++ b/common/web/gesture-recognizer/src/segmentStats.ts @@ -0,0 +1,337 @@ +namespace com.keyman.osk { + export class SegmentStats { + private static TIME_DIST_WEIGHT = .033; // Effect: 33ms ~= 1px distance. + + private xLinearSum: number = 0; + private yLinearSum: number = 0; + private tLinearSum: number = 0; + + private xCentroidSum: number = 0; + private yCentroidSum: number = 0; + + private xQuadSum: number = 0; + private yQuadSum: number = 0; + private tQuadSum: number = 0; + + private xtCrossSum: number = 0; + private ytCrossSum: number = 0; + private xyCrossSum: number = 0; + + private coordArcSum: number = 0; + private spacetimeArcSum: number = 0; + + /** + * The base sample used to transpose all other received samples. Use of this helps + * avoid potential "catastrophic cancellation" effects that can occur when diffing two + * numbers far from the sample-space's mathematical origin. + * + * Refer to https://en.wikipedia.org/wiki/Catastrophic_cancellation. + */ + private baseSample?: InputSample; + + /** + * The initial sample included by this instance's computed stats. Needed for + * the 'directness' properties. + */ + private initialSample?: InputSample; + + private lastSample?: InputSample; + private followingSample?: InputSample; + private sampleCount = 0; + + constructor(); + constructor(sample: InputSample); + constructor(instance: SegmentStats); + constructor(obj?: InputSample | SegmentStats) { + if(!obj) { + return; + } + + // Will worry about JSON form later. + if(obj instanceof SegmentStats) { + Object.assign(this, obj); + } else if(isAnInputSample(obj)) { + Object.assign(this, this.unionWith(obj)); + } + } + + public unionWith(sample: InputSample): SegmentStats { + if(!this.initialSample) { + this.initialSample = sample; + this.baseSample = sample; + } else { + this.followingSample = sample; + } + const result = new SegmentStats(this); + + // Helps prevent "catastrophic cancellation" issues from floating-point computation + // for these statistical properties and properties based upon them. + const x = sample.targetX - this.baseSample.targetX; + const y = sample.targetY - this.baseSample.targetY; + const t = sample.t - this.baseSample.t; + + result.xLinearSum += x; + result.yLinearSum += y; + result.tLinearSum += t; + + result.xtCrossSum += x * t; + result.ytCrossSum += y * t; + result.xyCrossSum += x * y; + + result.xQuadSum += x * x; + result.yQuadSum += y * y; + result.tQuadSum += t * t; + + if(this.lastSample) { + // arc length stuff! + const xDelta = sample.targetX - this.lastSample.targetX; + const yDelta = sample.targetY - this.lastSample.targetY; + const tDeltaInSec = (sample.t - this.lastSample.t) / 1000; + const weightedTDelta = (sample.t - this.lastSample.t) * SegmentStats.TIME_DIST_WEIGHT; + + const coordArcSq = xDelta * xDelta + yDelta * yDelta; + + result.coordArcSum += Math.sqrt(coordArcSq); + result.spacetimeArcSum += Math.sqrt(coordArcSq + weightedTDelta * weightedTDelta); + + // Approximates weighting the time spent at each coord by splitting the time since + // last event evenly for both coordinates. Note: does NOT shift based upon .baseSample! + result.xCentroidSum += 0.5 * tDeltaInSec * (sample.targetX + this.lastSample.targetX); + result.yCentroidSum += 0.5 * tDeltaInSec * (sample.targetY + this.lastSample.targetY); + } + + result.lastSample = sample; + result.sampleCount = this.sampleCount + 1; + + return result; + } + + public withoutPrefixSubset(subsetStats: SegmentStats): SegmentStats { + const result = new SegmentStats(this); + + if(!subsetStats.followingSample || !subsetStats.lastSample) { + throw 'Invalid argument: stats missing necessary tracking variable.'; + } + + result.xLinearSum -= subsetStats.xLinearSum; + result.yLinearSum -= subsetStats.yLinearSum; + result.tLinearSum -= subsetStats.tLinearSum; + + result.xtCrossSum -= subsetStats.xtCrossSum; + result.ytCrossSum -= subsetStats.ytCrossSum; + result.xyCrossSum -= subsetStats.xyCrossSum; + + result.xQuadSum -= subsetStats.xQuadSum; + result.yQuadSum -= subsetStats.yQuadSum; + result.tQuadSum -= subsetStats.tQuadSum; + + // arc length stuff! + if(subsetStats.followingSample && subsetStats.lastSample) { + const xDelta = subsetStats.followingSample.targetX - subsetStats.lastSample.targetX; + const yDelta = subsetStats.followingSample.targetY - subsetStats.lastSample.targetY; + const tDelta = (subsetStats.followingSample.t - subsetStats.lastSample.t) * SegmentStats.TIME_DIST_WEIGHT; + + const coordArcSq = xDelta * xDelta + yDelta * yDelta; + + // Due to how arc length stuff gets segmented. + // There's the arc length within the prefix subset (operand 2 below) AND the part connecting it to the + // 'remaining' subset (operand 1 below) before the portion wholly within what remains (the result) + result.coordArcSum -= Math.sqrt(coordArcSq); + result.coordArcSum -= subsetStats.coordArcSum; + result.spacetimeArcSum -= Math.sqrt(coordArcSq + tDelta * tDelta); + result.spacetimeArcSum -= subsetStats.spacetimeArcSum; + + // Centroid sum management! + const tDeltaInMs = (subsetStats.followingSample.t - subsetStats.lastSample.t) / 1000; + // Same reasoning pattern as for the 'arc length stuff'. + result.xCentroidSum -= 0.5 * tDeltaInMs * (subsetStats.followingSample.targetX + subsetStats.lastSample.targetX) + result.xCentroidSum -= subsetStats.xCentroidSum; + result.yCentroidSum -= 0.5 * tDeltaInMs * (subsetStats.followingSample.targetY + subsetStats.lastSample.targetY) + result.yCentroidSum -= subsetStats.yCentroidSum; + } + + result.sampleCount -= subsetStats.sampleCount; + + // NOTE: baseSample MUST REMAIN THE SAME. All math is based on the corresponding diff. + // Though... very long touchpoint interactions could start being affected by that "catastrophic + // cancellation" effect without further adjustment. (If it matters, we'll get to that later.) + // But _probably_ not; we don't go far beyond a couple of orders of magnitude from the origin in + // ANY case except the timestamp (.t) - and even then, not far from the baseSample's timestamp value. + + // initialSample, though, we need to update b/c of the 'directness' properties. + result.initialSample = subsetStats.followingSample; + + return result; + } + + private get xSampleMean() { + return this.xLinearSum / this.sampleCount; + } + + private get ySampleMean() { + return this.yLinearSum / this.sampleCount; + } + + private get tSampleMean() { + return this.tLinearSum / this.sampleCount; + } + + public get centroid(): {x: number, y: number} { + if(this.sampleCount == 0) { + return undefined; + } else if(this.sampleCount == 1) { + return { + x: this.lastSample.targetX, + y: this.lastSample.targetY + }; + } else { + const coeff = 1 / (this.duration); // * (this.sampleCount-1)); + return { + x: this.xCentroidSum * coeff, + y: this.yCentroidSum * coeff + }; + } + } + + public get xtCovariance() { + return this.xtCrossSum / this.sampleCount - (this.xSampleMean * this.tSampleMean); + } + + public get ytCovariance() { + return this.ytCrossSum / this.sampleCount - (this.ySampleMean * this.tSampleMean); + } + + public get xyCovariance() { + return this.xyCrossSum / this.sampleCount - (this.xSampleMean * this.ySampleMean); + } + + public get xVariance() { + return this.xQuadSum / this.sampleCount - (this.xSampleMean * this.xSampleMean); + } + + public get yVariance() { + return this.yQuadSum / this.sampleCount - (this.ySampleMean * this.ySampleMean); + } + + public get tVariance() { + return this.tQuadSum / this.sampleCount - (this.tSampleMean * this.tSampleMean); + } + + public get xtCorrelation() { + if(this.xVariance == 0) { + return Number.NaN; + } + + return this.xtCovariance / (Math.sqrt(this.xVariance * this.tVariance)); + } + + public get ytCorrelation() { + if(this.yVariance == 0) { + return Number.NaN; + } + + return this.ytCovariance / (Math.sqrt(this.yVariance * this.tVariance)); + } + + public get xyCorrelation() { + if(this.xVariance == 0 || this.yVariance == 0) { + return Number.NaN; + } + + return this.xyCovariance / (Math.sqrt(this.xVariance * this.yVariance)); + } + + public get movementRatio() { + return this.coordArcSum / this.spacetimeArcSum; + } + + public get directDistance() { + // No issue with a net distance of 0 due to a single point. + if(!this.lastSample || !this.initialSample) { + return Number.NaN; + } + + const xDelta = this.lastSample.targetX - this.initialSample.targetX; + const yDelta = this.lastSample.targetY - this.initialSample.targetY; + + return Math.sqrt(xDelta * xDelta + yDelta * yDelta); + } + + public get directnessRatio() { + return this.directDistance / this.coordArcSum; + } + + public get duration() { + // no issue with a duration of zero from just one sample. + if(!this.lastSample || !this.initialSample) { + return Number.NaN; + } + return (this.lastSample.t - this.initialSample.t) * 0.001; + } + + /** + * Returns the angle (in radians) traveled by the corresponding segment clockwise + * from the unit vector <0, -1> in the DOM (the unit "upward" direction). + */ + public get angle() { + if(this.sampleCount == 1 || !this.lastSample || !this.initialSample) { + return Number.NaN; + } else if(this.directDistance < 1) { + // < 1 px, thus sub-pixel, means we have nothing relevant enough to base an angle on. + return Number.NaN; + } + + const xDelta = this.lastSample.targetX - this.initialSample.targetX; + const yDelta = this.lastSample.targetY - this.initialSample.targetY; + const yAngleDiff = Math.acos(-yDelta / this.directDistance); + + return xDelta < 0 ? (2 * Math.PI - yAngleDiff) : yAngleDiff; + } + + public get angleInDegrees() { + return this.angle * 180 / Math.PI; + } + + public get cardinalDirection() { + if(this.sampleCount == 1 || !this.lastSample || !this.initialSample) { + return undefined; + } + + const angle = this.angleInDegrees; + const buckets = ['n', 'ne', 'e', 'se', 's', 'sw', 'w', 'nw']; + + for(let threshold = 22.5, bucketIndex = 0; threshold < 360; threshold += 45, bucketIndex += 1) { + if(angle < threshold) { + return buckets[bucketIndex]; + } + } + + return 'n'; + } + + // px per s. + public get speed() { + // this.duration is already in seconds, not milliseconds. + return this.duration ? this.directDistance / this.duration : 0; + } + + public toJSON() { + return { + xtCorrelation: this.xtCorrelation, + ytCorrelation: this.ytCorrelation, + xyCorrelation: this.xyCorrelation, + directnessRatio: this.directnessRatio, + directDistance: this.directDistance, + movementRatio: this.movementRatio, + angle: this.angle, + speed: this.speed, + cardinalDirection: this.cardinalDirection, + centroid: this.centroid, + duration: this.duration, + // Probably doesn't need to be reported in the long run, but useful while we're still nailing down the math & such. + sampleMean: {x: this.baseSample.targetX + this.xSampleMean, y: this.baseSample.targetY + this.ySampleMean} + }; + } + } + +} \ No newline at end of file diff --git a/common/web/gesture-recognizer/src/tools/recorder/src/recorder.js b/common/web/gesture-recognizer/src/tools/recorder/src/recorder.js index 6be45767d1..4a030ebcc8 100644 --- a/common/web/gesture-recognizer/src/tools/recorder/src/recorder.js +++ b/common/web/gesture-recognizer/src/tools/recorder/src/recorder.js @@ -48,6 +48,7 @@ loadPromise.then((recognizer) => { }); sequence.on('end', function() { + let stats = sequence.touchpoints[0].path.segments[0].stats; logElement.value = recorder.recordingsToJSON(); }); }); diff --git a/common/web/gesture-recognizer/src/trackedPath.ts b/common/web/gesture-recognizer/src/trackedPath.ts index 9cfdc4a5ee..441b0b819c 100644 --- a/common/web/gesture-recognizer/src/trackedPath.ts +++ b/common/web/gesture-recognizer/src/trackedPath.ts @@ -7,7 +7,7 @@ namespace com.keyman.osk { export type JSONTrackedPath = { coords: InputSample[]; // ensures type match with public class property. wasCancelled?: boolean; - // segments: Segment[]; + //segments: Segment[]; } interface EventMap { @@ -36,6 +36,8 @@ namespace com.keyman.osk { */ export class TrackedPath extends EventEmitter { private samples: InputSample[] = []; + private _segments: Segment[] = [new Segment()]; + private _isComplete: boolean = false; private wasCancelled?: boolean; @@ -80,6 +82,7 @@ namespace com.keyman.osk { } this.samples.push(sample); + this.segments[0].add(sample); this.emit('step', sample); } @@ -110,6 +113,10 @@ namespace com.keyman.osk { return this.samples; } + public get segments(): readonly Segment[] { + return this._segments; + } + /** * Creates a serialization-friendly version of this instance for use by * `JSON.stringify`. From 58c24bee70686299ec67de0718981611e3010862 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 10 Aug 2022 09:31:15 +0700 Subject: [PATCH 02/22] feat(web): extreme rough-draft path segmentation --- .../{segmentStats.ts => pathSegmentStats.ts} | 171 ++++++++++++---- .../gesture-recognizer/src/pathSegmenter.ts | 183 ++++++++++++++++++ common/web/gesture-recognizer/src/segment.ts | 37 +--- .../src/tools/recorder/src/recorder.js | 2 +- .../web/gesture-recognizer/src/trackedPath.ts | 7 +- 5 files changed, 327 insertions(+), 73 deletions(-) rename common/web/gesture-recognizer/src/{segmentStats.ts => pathSegmentStats.ts} (62%) create mode 100644 common/web/gesture-recognizer/src/pathSegmenter.ts diff --git a/common/web/gesture-recognizer/src/segmentStats.ts b/common/web/gesture-recognizer/src/pathSegmentStats.ts similarity index 62% rename from common/web/gesture-recognizer/src/segmentStats.ts rename to common/web/gesture-recognizer/src/pathSegmentStats.ts index dd175a5eb2..07c00801a1 100644 --- a/common/web/gesture-recognizer/src/segmentStats.ts +++ b/common/web/gesture-recognizer/src/pathSegmentStats.ts @@ -1,5 +1,5 @@ namespace com.keyman.osk { - export class SegmentStats { + export class PathSegmentStats { private static TIME_DIST_WEIGHT = .033; // Effect: 33ms ~= 1px distance. private xLinearSum: number = 0; @@ -20,6 +20,16 @@ namespace com.keyman.osk { private coordArcSum: number = 0; private spacetimeArcSum: number = 0; + private speedLinearSum: number = 0; + private speedQuadSum: number = 0; + + private cosLinearSum: number = 0; + private sinLinearSum: number = 0; + private sinCosCrossSum: number = 0; + private cosQuadSum: number = 0; + private sinQuadSum: number = 0; + private arcSampleCount: number = 0; + /** * The base sample used to transpose all other received samples. Use of this helps * avoid potential "catastrophic cancellation" effects that can occur when diffing two @@ -35,34 +45,34 @@ namespace com.keyman.osk { */ private initialSample?: InputSample; - private lastSample?: InputSample; + public lastSample?: InputSample; private followingSample?: InputSample; private sampleCount = 0; constructor(); constructor(sample: InputSample); - constructor(instance: SegmentStats); - constructor(obj?: InputSample | SegmentStats) { + constructor(instance: PathSegmentStats); + constructor(obj?: InputSample | PathSegmentStats) { if(!obj) { return; } // Will worry about JSON form later. - if(obj instanceof SegmentStats) { + if(obj instanceof PathSegmentStats) { Object.assign(this, obj); } else if(isAnInputSample(obj)) { Object.assign(this, this.unionWith(obj)); } } - public unionWith(sample: InputSample): SegmentStats { + public unionWith(sample: InputSample): PathSegmentStats { if(!this.initialSample) { this.initialSample = sample; this.baseSample = sample; } else { this.followingSample = sample; } - const result = new SegmentStats(this); + const result = new PathSegmentStats(this); // Helps prevent "catastrophic cancellation" issues from floating-point computation // for these statistical properties and properties based upon them. @@ -86,18 +96,38 @@ namespace com.keyman.osk { // arc length stuff! const xDelta = sample.targetX - this.lastSample.targetX; const yDelta = sample.targetY - this.lastSample.targetY; - const tDeltaInSec = (sample.t - this.lastSample.t) / 1000; - const weightedTDelta = (sample.t - this.lastSample.t) * SegmentStats.TIME_DIST_WEIGHT; + const tDelta = sample.t - this.lastSample.t; + const tDeltaInSec = tDelta / 1000; + const weightedTDelta = tDelta * PathSegmentStats.TIME_DIST_WEIGHT; - const coordArcSq = xDelta * xDelta + yDelta * yDelta; + const coordArcDeltaSq = xDelta * xDelta + yDelta * yDelta; + const coordArcDelta = Math.sqrt(coordArcDeltaSq); - result.coordArcSum += Math.sqrt(coordArcSq); - result.spacetimeArcSum += Math.sqrt(coordArcSq + weightedTDelta * weightedTDelta); + result.coordArcSum += Math.sqrt(coordArcDeltaSq); + result.spacetimeArcSum += Math.sqrt(coordArcDeltaSq + weightedTDelta * weightedTDelta); // Approximates weighting the time spent at each coord by splitting the time since // last event evenly for both coordinates. Note: does NOT shift based upon .baseSample! result.xCentroidSum += 0.5 * tDeltaInSec * (sample.targetX + this.lastSample.targetX); result.yCentroidSum += 0.5 * tDeltaInSec * (sample.targetY + this.lastSample.targetY); + + if(xDelta || yDelta) { + const cos = -yDelta / coordArcDelta; // alignment with <0, -1> in the DOM + const sin = xDelta / coordArcDelta; // alignment with <1, 0> in the DOM + result.cosLinearSum += cos; + result.sinLinearSum += sin; + + result.sinCosCrossSum += sin * cos; + result.cosQuadSum += cos * cos; + result.sinQuadSum += sin * sin; + + result.arcSampleCount += 1; + } + + if(tDeltaInSec) { + result.speedLinearSum += Math.sqrt(coordArcDeltaSq) / tDeltaInSec; + result.speedQuadSum += coordArcDeltaSq / (tDeltaInSec * tDeltaInSec); + } } result.lastSample = sample; @@ -106,8 +136,8 @@ namespace com.keyman.osk { return result; } - public withoutPrefixSubset(subsetStats: SegmentStats): SegmentStats { - const result = new SegmentStats(this); + public withoutPrefixSubset(subsetStats: PathSegmentStats): PathSegmentStats { + const result = new PathSegmentStats(this); if(!subsetStats.followingSample || !subsetStats.lastSample) { throw 'Invalid argument: stats missing necessary tracking variable.'; @@ -129,7 +159,9 @@ namespace com.keyman.osk { if(subsetStats.followingSample && subsetStats.lastSample) { const xDelta = subsetStats.followingSample.targetX - subsetStats.lastSample.targetX; const yDelta = subsetStats.followingSample.targetY - subsetStats.lastSample.targetY; - const tDelta = (subsetStats.followingSample.t - subsetStats.lastSample.t) * SegmentStats.TIME_DIST_WEIGHT; + const tDelta = subsetStats.followingSample.t - subsetStats.lastSample.t; + const weightedTDelta = tDelta * PathSegmentStats.TIME_DIST_WEIGHT; + const tDeltaInSec = tDelta / 1000; const coordArcSq = xDelta * xDelta + yDelta * yDelta; @@ -138,16 +170,31 @@ namespace com.keyman.osk { // 'remaining' subset (operand 1 below) before the portion wholly within what remains (the result) result.coordArcSum -= Math.sqrt(coordArcSq); result.coordArcSum -= subsetStats.coordArcSum; - result.spacetimeArcSum -= Math.sqrt(coordArcSq + tDelta * tDelta); + result.spacetimeArcSum -= Math.sqrt(coordArcSq + weightedTDelta * weightedTDelta); result.spacetimeArcSum -= subsetStats.spacetimeArcSum; // Centroid sum management! - const tDeltaInMs = (subsetStats.followingSample.t - subsetStats.lastSample.t) / 1000; // Same reasoning pattern as for the 'arc length stuff'. - result.xCentroidSum -= 0.5 * tDeltaInMs * (subsetStats.followingSample.targetX + subsetStats.lastSample.targetX) + result.xCentroidSum -= 0.5 * tDeltaInSec * (subsetStats.followingSample.targetX + subsetStats.lastSample.targetX) result.xCentroidSum -= subsetStats.xCentroidSum; - result.yCentroidSum -= 0.5 * tDeltaInMs * (subsetStats.followingSample.targetY + subsetStats.lastSample.targetY) + result.yCentroidSum -= 0.5 * tDeltaInSec * (subsetStats.followingSample.targetY + subsetStats.lastSample.targetY) result.yCentroidSum -= subsetStats.yCentroidSum; + + result.cosLinearSum -= subsetStats.cosLinearSum; + result.sinLinearSum -= subsetStats.sinLinearSum; + result.sinCosCrossSum -= subsetStats.sinCosCrossSum; + result.sinQuadSum -= subsetStats.sinQuadSum; + result.cosQuadSum -= subsetStats.cosQuadSum; + + result.arcSampleCount -= subsetStats.arcSampleCount; + + result.speedLinearSum -= subsetStats.speedLinearSum; + result.speedQuadSum -= subsetStats.speedQuadSum; + + if(tDeltaInSec) { + result.speedLinearSum -= Math.sqrt(coordArcSq) / tDeltaInSec; + result.speedQuadSum -= coordArcSq / (tDeltaInSec * tDeltaInSec); + } } result.sampleCount -= subsetStats.sampleCount; @@ -312,25 +359,81 @@ namespace com.keyman.osk { // px per s. public get speed() { // this.duration is already in seconds, not milliseconds. - return this.duration ? this.directDistance / this.duration : 0; + return this.duration ? this.directDistance / this.duration : Number.NaN; + } + + // ... may not be "right". + public get speedMean() { + return this.speedLinearSum / (this.sampleCount-1); + } + + public get speedVariance() { + return this.speedQuadSum / (this.sampleCount-1) - (this.speedMean * this.speedMean); + } + + public get sinVariance() { + const sinMean = this.sinLinearSum / this.arcSampleCount; + return this.sinQuadSum / (this.arcSampleCount) - sinMean * sinMean; + } + + public get cosVariance() { + const cosMean = this.cosLinearSum / this.arcSampleCount; + return this.cosQuadSum / this.arcSampleCount - cosMean * cosMean; + } + + public get angleMean() { + if(this.arcSampleCount == 0) { + return Number.NaN; + } + // Neato reference: https://rosettacode.org/wiki/Averages/Mean_angle + // But we don't actually need to divide by sample count; `atan2` handles that! + const sinMean = this.sinLinearSum / this.arcSampleCount; + const cosMean = this.cosLinearSum / this.arcSampleCount; + + let angle = Math.atan2(sinMean, cosMean); // result: on the interval (-pi, pi] + // Convert to [0, 2*pi). + if(angle < 0) { + angle = angle + 2 * Math.PI; + } + + return angle; + } + + // Based on https://www.ebi.ac.uk/thornton-srv/software/PROCHECK/nmr_manual/man_cv.html + public get angleVariance() { + if(this.arcSampleCount == 0) { + return Number.NaN; + } + const rSquared = this.cosLinearSum * this.cosLinearSum + this.sinLinearSum * this.sinLinearSum; + return 1 - (rSquared / (this.arcSampleCount * this.arcSampleCount)); } public toJSON() { + // return { + // xtCorrelation: this.xtCorrelation, + // ytCorrelation: this.ytCorrelation, + // xyCorrelation: this.xyCorrelation, + // directnessRatio: this.directnessRatio, + // directDistance: this.directDistance, + // movementRatio: this.movementRatio, + // angle: this.angle, + // speed: this.speed, + // cardinalDirection: this.cardinalDirection, + // centroid: this.centroid, + // duration: this.duration, + // // Probably doesn't need to be reported in the long run, but useful while we're still nailing down the math & such. + // sampleMean: {x: this.baseSample.targetX + this.xSampleMean, y: this.baseSample.targetY + this.ySampleMean} + // }; return { - xtCorrelation: this.xtCorrelation, - ytCorrelation: this.ytCorrelation, - xyCorrelation: this.xyCorrelation, - directnessRatio: this.directnessRatio, - directDistance: this.directDistance, - movementRatio: this.movementRatio, - angle: this.angle, - speed: this.speed, - cardinalDirection: this.cardinalDirection, - centroid: this.centroid, - duration: this.duration, - // Probably doesn't need to be reported in the long run, but useful while we're still nailing down the math & such. - sampleMean: {x: this.baseSample.targetX + this.xSampleMean, y: this.baseSample.targetY + this.ySampleMean} - }; + angleMean: this.angleMean, + angleVariance: this.angleVariance, + speedMean: this.speedMean, + speedVariance: this.speedVariance, + coordArcSum: this.coordArcSum, + asTheBirdFlies: this.directDistance, + duration: this.duration, + sampleCount: this.sampleCount + } } } diff --git a/common/web/gesture-recognizer/src/pathSegmenter.ts b/common/web/gesture-recognizer/src/pathSegmenter.ts new file mode 100644 index 0000000000..0086870fa2 --- /dev/null +++ b/common/web/gesture-recognizer/src/pathSegmenter.ts @@ -0,0 +1,183 @@ +/// + +namespace com.keyman.osk { + export class PathSegmenter { + private readonly REPEAT_INTERVAL = 33; + private readonly SLIDING_WINDOW_INTERVAL = 50; + + // May be best to keep an array of these, one per sample. + // Can then diff the stats to determine better cut-offs. + // Though... the whole arc-dist aspect will need a mite more help. + // - chopping off from the end: ez-pz. Raw diff is great. + // - or, well, just use the appropriate one from mid-way. + // - chopping off from the beginning: need an extra sample reference. + // - .nextSample. + // + // The FINAL version, once resolved, may be published. + // But until resolved, we probably want to keep an array. + private _stats: PathSegmentStats[]; + + private _protoSegments: PathSegmentStats[] = []; + + private repeatTimer: number | NodeJS.Timeout; + private repeatTimestamp: number; + + private choppedStats: PathSegmentStats = null; + + constructor() { + this._stats = []; + } + + public get stats(): readonly PathSegmentStats[] { + return this._stats; + } + + public add(sample: InputSample) { + const repeater = (timeDelta: number) => { + this.observe(sample, timeDelta); + } + + if(this.repeatTimer) { + // @ts-ignore + clearInterval(this.repeatTimer); + this.repeatTimer = null; + } + + this.repeatTimer = setInterval(() => { + const timeDelta = Date.now() - this.repeatTimestamp; + repeater(timeDelta); + }, this.REPEAT_INTERVAL); + this.repeatTimestamp = Date.now(); + repeater(0); + } + + public close() { + // The Node clearTimeout & DOM clearTimeout appear to TS as overloads of each other, + // and their type definitions will conflict. A simple @ts-ignore will bypass this issue. + // @ts-ignore + clearInterval(this.repeatTimer); + this.repeatTimer = null; + + let intervalStats = this.stats[this.stats.length-1]; + if(this.choppedStats) { + intervalStats = intervalStats.withoutPrefixSubset(this.choppedStats); + } + this._protoSegments.push(intervalStats); + } + + private observe(sample: InputSample, timeDelta: number) { + let baseStats: PathSegmentStats; + if(this.stats.length) { + baseStats = this.stats[this.stats.length-1]; + } else { + baseStats = new PathSegmentStats(); + } + + sample = {... sample}; + sample.t += timeDelta; + const extendedStats = baseStats.unionWith(sample); + this._stats.push(extendedStats); + + let preWindowEnd = 0; + // Do not consider the just-added `extendedStats` entry. + for(let i = this.stats.length-2; i >=0 ; i--) { + if(this.stats[i].lastSample.t + this.SLIDING_WINDOW_INTERVAL < sample.t) { + preWindowEnd = i; + break; + } + } + + // Do not consider segmenting before at least two samples exist before the current + // sliding time window. (At least two samples are needed for 'over time' + // properties to have a chance at becoming 'defined'.) + //console.log(sample); + if(preWindowEnd > 0) { + const cumulativePreCandidate = this.stats[preWindowEnd+1]; + let preCandidate = cumulativePreCandidate; + if(this.choppedStats) { + preCandidate = preCandidate.withoutPrefixSubset(this.choppedStats); + } + let postCandidate = extendedStats.withoutPrefixSubset(this.stats[preWindowEnd]); + + let combined = extendedStats; + if(this.choppedStats) { + combined = combined.withoutPrefixSubset(this.choppedStats); + } + + // Run a comparison on various stats of the two. + + // FIXME: VERY temp logging. + if(preCandidate.duration * 1000 >= this.SLIDING_WINDOW_INTERVAL /* / 2000*/) { + let performSegmentation = false; + + let angleSplitVariance = preCandidate.angleVariance + postCandidate.angleVariance; + let angleVarianceRatio = combined.angleVariance / angleSplitVariance; + + const speedSplitVariance = preCandidate.speedVariance + postCandidate.speedVariance; + const speedVarianceRatio = combined.speedVariance / speedSplitVariance; + + /* + * Okay, so this isn't probably quite the most statistically well-founded approach, but... + * + * A "variance ratio" of 1 indicates something of a break-even point; exceeding that threshold + * means that the variations in value seen between the two potential segments are better + * explained as being from two separate segments than from a single segment. (Loosely speaking; + * it'd take some effort to cement the statistical basis here; this is more 'inspired by' what + * the values represent.) + * + * Of course... when it comes to speed, acceleration and such are factors. It'd be all + * too easy to split the slow and fast parts of an accelerating linear motion as two separate + * pieces. Requiring a higher degree of separation alleviates this - hence, the `/2` in the + * condition below. (That divisor's not the most statistically-based thing to do, but it + * works well here.) + * + * So to reach the threshold set below, these three conditions will work: + * - strong difference in angle between the segment candidates + * - moderate difference in both angle and speed between the segment candidates (equal levels) + * - very strong difference in speed between the segment candidates. + */ + if(angleVarianceRatio + speedVarianceRatio / 2 > 1.5) { + performSegmentation = true; + } + + // Hmm. Perhaps this should only serve as the "okay, let's segment" trigger... to then + // find the BEST segmentation. + + if(performSegmentation) { + console.log("------------------------------------------------------------------"); + } + + console.log("Angle variance ratio: " + angleVarianceRatio); + console.log("Speed variance ratio: " + speedVarianceRatio); + + if(performSegmentation) { + console.log("Combined: "); + console.log(combined.toJSON()); + console.log(combined); + console.log("Pre: ") + console.log(preCandidate.toJSON()); + console.log(preCandidate); + console.log("Post: "); + console.log(postCandidate.toJSON()); + console.log(postCandidate); + console.log("Angle variance ratio: " + angleVarianceRatio); + console.log("Speed variance ratio: " + speedVarianceRatio); + + console.log(); + + this._stats = this._stats.slice(preWindowEnd+1); + console.log("Dropped samples: " + (preWindowEnd+1)); + console.log("Remaining samples: " + this._stats.length); + + console.log("Prototype segment: "); + console.log(preCandidate); + this._protoSegments.push(preCandidate); + this.choppedStats = cumulativePreCandidate; + + console.log("------------------------------------------------------------------"); + } + } + } + } + } +} \ No newline at end of file diff --git a/common/web/gesture-recognizer/src/segment.ts b/common/web/gesture-recognizer/src/segment.ts index d242bb9f3f..4f52a7cb91 100644 --- a/common/web/gesture-recognizer/src/segment.ts +++ b/common/web/gesture-recognizer/src/segment.ts @@ -1,40 +1,5 @@ -/// - namespace com.keyman.osk { export class Segment { - // May be best to keep an array of these, one per sample. - // Can then diff the stats to determine better cut-offs. - // Though... the whole arc-dist aspect will need a mite more help. - // - chopping off from the end: ez-pz. Raw diff is great. - // - or, well, just use the appropriate one from mid-way. - // - chopping off from the beginning: need an extra sample reference. - // - .nextSample. - // - // The FINAL version, once resolved, may be published. - // But until resolved, we probably want to keep an array. - private _stats: SegmentStats[]; - - constructor() { - this._stats = []; - } - - public get stats(): readonly SegmentStats[] { - return this._stats; - } - - public add(sample: InputSample) { - let baseStats: SegmentStats; - if(this.stats.length > 0) { - baseStats = this.stats[this.stats.length-1]; - } else { - baseStats = new SegmentStats(); - } - - let extendedStats = baseStats.unionWith(sample); - - // FIXME: VERY temp logging. - console.log(extendedStats.toJSON()); - this._stats.push(extendedStats); - } + // TODO: stuff. } } \ No newline at end of file diff --git a/common/web/gesture-recognizer/src/tools/recorder/src/recorder.js b/common/web/gesture-recognizer/src/tools/recorder/src/recorder.js index 4a030ebcc8..00f95e7e66 100644 --- a/common/web/gesture-recognizer/src/tools/recorder/src/recorder.js +++ b/common/web/gesture-recognizer/src/tools/recorder/src/recorder.js @@ -48,7 +48,7 @@ loadPromise.then((recognizer) => { }); sequence.on('end', function() { - let stats = sequence.touchpoints[0].path.segments[0].stats; + //let stats = sequence.touchpoints[0].path.segments[0].stats; logElement.value = recorder.recordingsToJSON(); }); }); diff --git a/common/web/gesture-recognizer/src/trackedPath.ts b/common/web/gesture-recognizer/src/trackedPath.ts index 441b0b819c..6ac7f2df82 100644 --- a/common/web/gesture-recognizer/src/trackedPath.ts +++ b/common/web/gesture-recognizer/src/trackedPath.ts @@ -36,7 +36,9 @@ namespace com.keyman.osk { */ export class TrackedPath extends EventEmitter { private samples: InputSample[] = []; - private _segments: Segment[] = [new Segment()]; + private _segments: Segment[] = []; + + private segmenter = new PathSegmenter(); private _isComplete: boolean = false; private wasCancelled?: boolean; @@ -82,7 +84,7 @@ namespace com.keyman.osk { } this.samples.push(sample); - this.segments[0].add(sample); + this.segmenter.add(sample); this.emit('step', sample); } @@ -96,6 +98,7 @@ namespace com.keyman.osk { } this.wasCancelled = cancel; this._isComplete = true; + this.segmenter.close(); if(cancel) { this.emit('invalidated'); From 534b3087cd479fc85c0a964caab083457513a12b Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 10 Aug 2022 10:06:46 +0700 Subject: [PATCH 03/22] chore(web): moderate post-experimentation cleanup --- .../src/pathSegmentStats.ts | 171 ++++-------------- .../gesture-recognizer/src/pathSegmenter.ts | 23 ++- 2 files changed, 53 insertions(+), 141 deletions(-) diff --git a/common/web/gesture-recognizer/src/pathSegmentStats.ts b/common/web/gesture-recognizer/src/pathSegmentStats.ts index 07c00801a1..ae52fd6add 100644 --- a/common/web/gesture-recognizer/src/pathSegmentStats.ts +++ b/common/web/gesture-recognizer/src/pathSegmentStats.ts @@ -1,31 +1,15 @@ namespace com.keyman.osk { export class PathSegmentStats { - private static TIME_DIST_WEIGHT = .033; // Effect: 33ms ~= 1px distance. - - private xLinearSum: number = 0; - private yLinearSum: number = 0; - private tLinearSum: number = 0; - private xCentroidSum: number = 0; private yCentroidSum: number = 0; - private xQuadSum: number = 0; - private yQuadSum: number = 0; - private tQuadSum: number = 0; - - private xtCrossSum: number = 0; - private ytCrossSum: number = 0; - private xyCrossSum: number = 0; - private coordArcSum: number = 0; - private spacetimeArcSum: number = 0; private speedLinearSum: number = 0; private speedQuadSum: number = 0; private cosLinearSum: number = 0; private sinLinearSum: number = 0; - private sinCosCrossSum: number = 0; private cosQuadSum: number = 0; private sinQuadSum: number = 0; private arcSampleCount: number = 0; @@ -45,7 +29,7 @@ namespace com.keyman.osk { */ private initialSample?: InputSample; - public lastSample?: InputSample; + private lastSample?: InputSample; private followingSample?: InputSample; private sampleCount = 0; @@ -80,31 +64,17 @@ namespace com.keyman.osk { const y = sample.targetY - this.baseSample.targetY; const t = sample.t - this.baseSample.t; - result.xLinearSum += x; - result.yLinearSum += y; - result.tLinearSum += t; - - result.xtCrossSum += x * t; - result.ytCrossSum += y * t; - result.xyCrossSum += x * y; - - result.xQuadSum += x * x; - result.yQuadSum += y * y; - result.tQuadSum += t * t; - if(this.lastSample) { // arc length stuff! const xDelta = sample.targetX - this.lastSample.targetX; const yDelta = sample.targetY - this.lastSample.targetY; const tDelta = sample.t - this.lastSample.t; const tDeltaInSec = tDelta / 1000; - const weightedTDelta = tDelta * PathSegmentStats.TIME_DIST_WEIGHT; const coordArcDeltaSq = xDelta * xDelta + yDelta * yDelta; const coordArcDelta = Math.sqrt(coordArcDeltaSq); result.coordArcSum += Math.sqrt(coordArcDeltaSq); - result.spacetimeArcSum += Math.sqrt(coordArcDeltaSq + weightedTDelta * weightedTDelta); // Approximates weighting the time spent at each coord by splitting the time since // last event evenly for both coordinates. Note: does NOT shift based upon .baseSample! @@ -117,7 +87,6 @@ namespace com.keyman.osk { result.cosLinearSum += cos; result.sinLinearSum += sin; - result.sinCosCrossSum += sin * cos; result.cosQuadSum += cos * cos; result.sinQuadSum += sin * sin; @@ -143,24 +112,11 @@ namespace com.keyman.osk { throw 'Invalid argument: stats missing necessary tracking variable.'; } - result.xLinearSum -= subsetStats.xLinearSum; - result.yLinearSum -= subsetStats.yLinearSum; - result.tLinearSum -= subsetStats.tLinearSum; - - result.xtCrossSum -= subsetStats.xtCrossSum; - result.ytCrossSum -= subsetStats.ytCrossSum; - result.xyCrossSum -= subsetStats.xyCrossSum; - - result.xQuadSum -= subsetStats.xQuadSum; - result.yQuadSum -= subsetStats.yQuadSum; - result.tQuadSum -= subsetStats.tQuadSum; - // arc length stuff! if(subsetStats.followingSample && subsetStats.lastSample) { const xDelta = subsetStats.followingSample.targetX - subsetStats.lastSample.targetX; const yDelta = subsetStats.followingSample.targetY - subsetStats.lastSample.targetY; const tDelta = subsetStats.followingSample.t - subsetStats.lastSample.t; - const weightedTDelta = tDelta * PathSegmentStats.TIME_DIST_WEIGHT; const tDeltaInSec = tDelta / 1000; const coordArcSq = xDelta * xDelta + yDelta * yDelta; @@ -170,8 +126,6 @@ namespace com.keyman.osk { // 'remaining' subset (operand 1 below) before the portion wholly within what remains (the result) result.coordArcSum -= Math.sqrt(coordArcSq); result.coordArcSum -= subsetStats.coordArcSum; - result.spacetimeArcSum -= Math.sqrt(coordArcSq + weightedTDelta * weightedTDelta); - result.spacetimeArcSum -= subsetStats.spacetimeArcSum; // Centroid sum management! // Same reasoning pattern as for the 'arc length stuff'. @@ -182,7 +136,6 @@ namespace com.keyman.osk { result.cosLinearSum -= subsetStats.cosLinearSum; result.sinLinearSum -= subsetStats.sinLinearSum; - result.sinCosCrossSum -= subsetStats.sinCosCrossSum; result.sinQuadSum -= subsetStats.sinQuadSum; result.cosQuadSum -= subsetStats.cosQuadSum; @@ -211,16 +164,8 @@ namespace com.keyman.osk { return result; } - private get xSampleMean() { - return this.xLinearSum / this.sampleCount; - } - - private get ySampleMean() { - return this.yLinearSum / this.sampleCount; - } - - private get tSampleMean() { - return this.tLinearSum / this.sampleCount; + public get lastTimestamp(): number { + return this.lastSample?.t; } public get centroid(): {x: number, y: number} { @@ -240,58 +185,6 @@ namespace com.keyman.osk { } } - public get xtCovariance() { - return this.xtCrossSum / this.sampleCount - (this.xSampleMean * this.tSampleMean); - } - - public get ytCovariance() { - return this.ytCrossSum / this.sampleCount - (this.ySampleMean * this.tSampleMean); - } - - public get xyCovariance() { - return this.xyCrossSum / this.sampleCount - (this.xSampleMean * this.ySampleMean); - } - - public get xVariance() { - return this.xQuadSum / this.sampleCount - (this.xSampleMean * this.xSampleMean); - } - - public get yVariance() { - return this.yQuadSum / this.sampleCount - (this.ySampleMean * this.ySampleMean); - } - - public get tVariance() { - return this.tQuadSum / this.sampleCount - (this.tSampleMean * this.tSampleMean); - } - - public get xtCorrelation() { - if(this.xVariance == 0) { - return Number.NaN; - } - - return this.xtCovariance / (Math.sqrt(this.xVariance * this.tVariance)); - } - - public get ytCorrelation() { - if(this.yVariance == 0) { - return Number.NaN; - } - - return this.ytCovariance / (Math.sqrt(this.yVariance * this.tVariance)); - } - - public get xyCorrelation() { - if(this.xVariance == 0 || this.yVariance == 0) { - return Number.NaN; - } - - return this.xyCovariance / (Math.sqrt(this.xVariance * this.yVariance)); - } - - public get movementRatio() { - return this.coordArcSum / this.spacetimeArcSum; - } - public get directDistance() { // No issue with a net distance of 0 due to a single point. if(!this.lastSample || !this.initialSample) { @@ -304,10 +197,6 @@ namespace com.keyman.osk { return Math.sqrt(xDelta * xDelta + yDelta * yDelta); } - public get directnessRatio() { - return this.directDistance / this.coordArcSum; - } - public get duration() { // no issue with a duration of zero from just one sample. if(!this.lastSample || !this.initialSample) { @@ -381,14 +270,21 @@ namespace com.keyman.osk { return this.cosQuadSum / this.arcSampleCount - cosMean * cosMean; } + /** + * Returns the represented interval's 'mean angle' clockwise from the DOM's + * <0, -1> (the unit vector toward the top of the screen) in radians. + * + * Uses the 'circular mean'. Refer to https://en.wikipedia.org/wiki/Circular_mean. + */ public get angleMean() { if(this.arcSampleCount == 0) { return Number.NaN; } + // Neato reference: https://rosettacode.org/wiki/Averages/Mean_angle // But we don't actually need to divide by sample count; `atan2` handles that! - const sinMean = this.sinLinearSum / this.arcSampleCount; - const cosMean = this.cosLinearSum / this.arcSampleCount; + const sinMean = this.sinLinearSum; + const cosMean = this.cosLinearSum; let angle = Math.atan2(sinMean, cosMean); // result: on the interval (-pi, pi] // Convert to [0, 2*pi). @@ -399,38 +295,47 @@ namespace com.keyman.osk { return angle; } - // Based on https://www.ebi.ac.uk/thornton-srv/software/PROCHECK/nmr_manual/man_cv.html + /** + * The **circular variance** of the represented interval's angle observations. + * + * Refer to https://en.wikipedia.org/wiki/Directional_statistics#Variance. + */ public get angleVariance() { if(this.arcSampleCount == 0) { return Number.NaN; } + // https://www.ebi.ac.uk/thornton-srv/software/PROCHECK/nmr_manual/man_cv.html may be a useful + // reference for this tidbit. The Wikipedia article's more dense... not that this link isn't + // a bit dense itself. const rSquared = this.cosLinearSum * this.cosLinearSum + this.sinLinearSum * this.sinLinearSum; return 1 - (rSquared / (this.arcSampleCount * this.arcSampleCount)); } + /** + * The **circular standard deviation** of the represented interval's angle observations. + * + * Refer to https://en.wikipedia.org/wiki/Directional_statistics#Standard_deviation. + */ + public get angleDeviation() { + if(this.arcSampleCount == 0) { + return Number.NaN; + } + + // Excludes the divisor; we can add that in the following line. + // https://www.ebi.ac.uk/thornton-srv/software/PROCHECK/nmr_manual/man_cv.html may be a useful + // reference for this as well. + const rSquared = this.cosLinearSum * this.cosLinearSum + this.sinLinearSum * this.sinLinearSum; + return Math.sqrt(Math.log(this.arcSampleCount * this.arcSampleCount / rSquared)); + } + public toJSON() { - // return { - // xtCorrelation: this.xtCorrelation, - // ytCorrelation: this.ytCorrelation, - // xyCorrelation: this.xyCorrelation, - // directnessRatio: this.directnessRatio, - // directDistance: this.directDistance, - // movementRatio: this.movementRatio, - // angle: this.angle, - // speed: this.speed, - // cardinalDirection: this.cardinalDirection, - // centroid: this.centroid, - // duration: this.duration, - // // Probably doesn't need to be reported in the long run, but useful while we're still nailing down the math & such. - // sampleMean: {x: this.baseSample.targetX + this.xSampleMean, y: this.baseSample.targetY + this.ySampleMean} - // }; return { angleMean: this.angleMean, angleVariance: this.angleVariance, + angleDeviation: this.angleDeviation, speedMean: this.speedMean, speedVariance: this.speedVariance, coordArcSum: this.coordArcSum, - asTheBirdFlies: this.directDistance, duration: this.duration, sampleCount: this.sampleCount } diff --git a/common/web/gesture-recognizer/src/pathSegmenter.ts b/common/web/gesture-recognizer/src/pathSegmenter.ts index 0086870fa2..25149a1592 100644 --- a/common/web/gesture-recognizer/src/pathSegmenter.ts +++ b/common/web/gesture-recognizer/src/pathSegmenter.ts @@ -17,6 +17,10 @@ namespace com.keyman.osk { // But until resolved, we probably want to keep an array. private _stats: PathSegmentStats[]; + // Currently used as an in-development diagnostic assist... but these + // directly represent actual path segments as produced by the prototype + // algorithm. Just... the stats analysis of the path segment, without + // obvious / public members to relevant coordinates. private _protoSegments: PathSegmentStats[] = []; private repeatTimer: number | NodeJS.Timeout; @@ -81,14 +85,14 @@ namespace com.keyman.osk { let preWindowEnd = 0; // Do not consider the just-added `extendedStats` entry. for(let i = this.stats.length-2; i >=0 ; i--) { - if(this.stats[i].lastSample.t + this.SLIDING_WINDOW_INTERVAL < sample.t) { + if(this.stats[i].lastTimestamp + this.SLIDING_WINDOW_INTERVAL < sample.t) { preWindowEnd = i; break; } } // Do not consider segmenting before at least two samples exist before the current - // sliding time window. (At least two samples are needed for 'over time' + // sliding time window. (A minimum of two samples are needed for 'over time' // properties to have a chance at becoming 'defined'.) //console.log(sample); if(preWindowEnd > 0) { @@ -105,9 +109,7 @@ namespace com.keyman.osk { } // Run a comparison on various stats of the two. - - // FIXME: VERY temp logging. - if(preCandidate.duration * 1000 >= this.SLIDING_WINDOW_INTERVAL /* / 2000*/) { + if(preCandidate.duration * 1000 >= this.SLIDING_WINDOW_INTERVAL) { // sec vs millisec. let performSegmentation = false; let angleSplitVariance = preCandidate.angleVariance + postCandidate.angleVariance; @@ -143,6 +145,9 @@ namespace com.keyman.osk { // Hmm. Perhaps this should only serve as the "okay, let's segment" trigger... to then // find the BEST segmentation. + // FIXME: DO NOT RELEASE. + // This is exploratory / diagnostic code assisting development of the path segmentation + // algorithm. if(performSegmentation) { console.log("------------------------------------------------------------------"); } @@ -165,16 +170,18 @@ namespace com.keyman.osk { console.log(); - this._stats = this._stats.slice(preWindowEnd+1); + this._stats = this._stats.slice(preWindowEnd+1); // DO release this line. console.log("Dropped samples: " + (preWindowEnd+1)); console.log("Remaining samples: " + this._stats.length); console.log("Prototype segment: "); console.log(preCandidate); - this._protoSegments.push(preCandidate); - this.choppedStats = cumulativePreCandidate; console.log("------------------------------------------------------------------"); + // END: DO NOT RELEASE. + + this._protoSegments.push(preCandidate); + this.choppedStats = cumulativePreCandidate; } } } From 84fd363c3605e5149a5fc0266728f66d247401e6 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 11 Aug 2022 08:33:33 +0700 Subject: [PATCH 04/22] chore(web): cleanup, rename, docs for pathSegmentStats --- ...SegmentStats.ts => cumulativePathStats.ts} | 96 ++++++++++-------- .../gesture-recognizer/src/pathSegmenter.ts | 99 ++++++++++++------- .../web/gesture-recognizer/src/trackedPath.ts | 10 +- 3 files changed, 125 insertions(+), 80 deletions(-) rename common/web/gesture-recognizer/src/{pathSegmentStats.ts => cumulativePathStats.ts} (80%) diff --git a/common/web/gesture-recognizer/src/pathSegmentStats.ts b/common/web/gesture-recognizer/src/cumulativePathStats.ts similarity index 80% rename from common/web/gesture-recognizer/src/pathSegmentStats.ts rename to common/web/gesture-recognizer/src/cumulativePathStats.ts index ae52fd6add..2b0be2ddc7 100644 --- a/common/web/gesture-recognizer/src/pathSegmentStats.ts +++ b/common/web/gesture-recognizer/src/cumulativePathStats.ts @@ -1,5 +1,12 @@ namespace com.keyman.osk { - export class PathSegmentStats { + /** + * As the name suggests, this class exists to track cumulative mathematical values, etc + * necessary to provide statistical information. This information is used to facilitate + * path segmentation. + * + * Instances of this class are immutable. + */ + export class CumulativePathStats { private xCentroidSum: number = 0; private yCentroidSum: number = 0; @@ -10,8 +17,6 @@ namespace com.keyman.osk { private cosLinearSum: number = 0; private sinLinearSum: number = 0; - private cosQuadSum: number = 0; - private sinQuadSum: number = 0; private arcSampleCount: number = 0; /** @@ -35,28 +40,35 @@ namespace com.keyman.osk { constructor(); constructor(sample: InputSample); - constructor(instance: PathSegmentStats); - constructor(obj?: InputSample | PathSegmentStats) { + constructor(instance: CumulativePathStats); + constructor(obj?: InputSample | CumulativePathStats) { if(!obj) { return; } // Will worry about JSON form later. - if(obj instanceof PathSegmentStats) { + if(obj instanceof CumulativePathStats) { Object.assign(this, obj); } else if(isAnInputSample(obj)) { - Object.assign(this, this.unionWith(obj)); + Object.assign(this, this.extend(obj)); } } - public unionWith(sample: InputSample): PathSegmentStats { + /** + * Statistically "observes" a new sample point on the touchpath, accumulating values + * useful for provision of relevant statistical properties. + * @param sample A newly-sampled point on the touchpath. + * @returns A new, separate instance for the cumulative properties up to the + * newly-sampled point. + */ + public extend(sample: InputSample): CumulativePathStats { if(!this.initialSample) { this.initialSample = sample; this.baseSample = sample; } else { this.followingSample = sample; } - const result = new PathSegmentStats(this); + const result = new CumulativePathStats(this); // Helps prevent "catastrophic cancellation" issues from floating-point computation // for these statistical properties and properties based upon them. @@ -82,14 +94,13 @@ namespace com.keyman.osk { result.yCentroidSum += 0.5 * tDeltaInSec * (sample.targetY + this.lastSample.targetY); if(xDelta || yDelta) { - const cos = -yDelta / coordArcDelta; // alignment with <0, -1> in the DOM - const sin = xDelta / coordArcDelta; // alignment with <1, 0> in the DOM - result.cosLinearSum += cos; - result.sinLinearSum += sin; - - result.cosQuadSum += cos * cos; - result.sinQuadSum += sin * sin; - + // We wish to measure angle clockwise from <0, -1> in the DOM. So, cos values should + // align with that axis, while sin values should align with the positive x-axis. + // + // This provides a mathematical 'transformation' to the axes used by `atan2` in the + // `angleMean` property. + result.cosLinearSum += -yDelta / coordArcDelta; + result.sinLinearSum += xDelta / coordArcDelta; result.arcSampleCount += 1; } @@ -105,8 +116,15 @@ namespace com.keyman.osk { return result; } - public withoutPrefixSubset(subsetStats: PathSegmentStats): PathSegmentStats { - const result = new PathSegmentStats(this); + /** + * "De-accumulates" currently-accumulated values corresponding to the specified + * subset, which should represent an earlier, previously-observed part of the path. + * @param subsetStats The accumulated stats for the part of the path being removed + * from this instance's current accumulation. + * @returns + */ + public deaccumulate(subsetStats: CumulativePathStats): CumulativePathStats { + const result = new CumulativePathStats(this); if(!subsetStats.followingSample || !subsetStats.lastSample) { throw 'Invalid argument: stats missing necessary tracking variable.'; @@ -136,9 +154,6 @@ namespace com.keyman.osk { result.cosLinearSum -= subsetStats.cosLinearSum; result.sinLinearSum -= subsetStats.sinLinearSum; - result.sinQuadSum -= subsetStats.sinQuadSum; - result.cosQuadSum -= subsetStats.cosQuadSum; - result.arcSampleCount -= subsetStats.arcSampleCount; result.speedLinearSum -= subsetStats.speedLinearSum; @@ -260,16 +275,6 @@ namespace com.keyman.osk { return this.speedQuadSum / (this.sampleCount-1) - (this.speedMean * this.speedMean); } - public get sinVariance() { - const sinMean = this.sinLinearSum / this.arcSampleCount; - return this.sinQuadSum / (this.arcSampleCount) - sinMean * sinMean; - } - - public get cosVariance() { - const cosMean = this.cosLinearSum / this.arcSampleCount; - return this.cosQuadSum / this.arcSampleCount - cosMean * cosMean; - } - /** * Returns the represented interval's 'mean angle' clockwise from the DOM's * <0, -1> (the unit vector toward the top of the screen) in radians. @@ -295,6 +300,19 @@ namespace com.keyman.osk { return angle; } + /** + * Provides the rSquared value needed internally for circular-statistic properties. + * + * Range: floating-point values on the interval [0, 1]. + */ + private get angleRSquared() { + // https://www.ebi.ac.uk/thornton-srv/software/PROCHECK/nmr_manual/man_cv.html may be a useful + // reference for this tidbit. The Wikipedia article's more dense... not that this link isn't + // a bit dense itself. + const rSquaredBase = this.cosLinearSum * this.cosLinearSum + this.sinLinearSum * this.sinLinearSum; + return rSquaredBase / (this.arcSampleCount * this.arcSampleCount); + } + /** * The **circular variance** of the represented interval's angle observations. * @@ -304,11 +322,7 @@ namespace com.keyman.osk { if(this.arcSampleCount == 0) { return Number.NaN; } - // https://www.ebi.ac.uk/thornton-srv/software/PROCHECK/nmr_manual/man_cv.html may be a useful - // reference for this tidbit. The Wikipedia article's more dense... not that this link isn't - // a bit dense itself. - const rSquared = this.cosLinearSum * this.cosLinearSum + this.sinLinearSum * this.sinLinearSum; - return 1 - (rSquared / (this.arcSampleCount * this.arcSampleCount)); + return 1 - (this.angleRSquared); } /** @@ -321,13 +335,11 @@ namespace com.keyman.osk { return Number.NaN; } - // Excludes the divisor; we can add that in the following line. - // https://www.ebi.ac.uk/thornton-srv/software/PROCHECK/nmr_manual/man_cv.html may be a useful - // reference for this as well. - const rSquared = this.cosLinearSum * this.cosLinearSum + this.sinLinearSum * this.sinLinearSum; - return Math.sqrt(Math.log(this.arcSampleCount * this.arcSampleCount / rSquared)); + return Math.sqrt(-Math.log(this.angleRSquared)); } + // TODO: is this actually ideal? This was certainly useful for experimentation via interactive + // debugging, but it may not be the best thing long-term. public toJSON() { return { angleMean: this.angleMean, diff --git a/common/web/gesture-recognizer/src/pathSegmenter.ts b/common/web/gesture-recognizer/src/pathSegmenter.ts index 25149a1592..3ede92ae3e 100644 --- a/common/web/gesture-recognizer/src/pathSegmenter.ts +++ b/common/web/gesture-recognizer/src/pathSegmenter.ts @@ -1,39 +1,56 @@ -/// +/// namespace com.keyman.osk { export class PathSegmenter { + /** + * The minimum amount of time (in ms) to wait between sample repetitions + * once inputs stop arriving, so long as the path is still active. + */ private readonly REPEAT_INTERVAL = 33; + + /** + * The time-interval length (in ms) at the end of the path to consider + * when determining whether or not to trigger path segmentation. + */ private readonly SLIDING_WINDOW_INTERVAL = 50; - // May be best to keep an array of these, one per sample. - // Can then diff the stats to determine better cut-offs. - // Though... the whole arc-dist aspect will need a mite more help. - // - chopping off from the end: ez-pz. Raw diff is great. - // - or, well, just use the appropriate one from mid-way. - // - chopping off from the beginning: need an extra sample reference. - // - .nextSample. - // - // The FINAL version, once resolved, may be published. - // But until resolved, we probably want to keep an array. - private _stats: PathSegmentStats[]; + /** + * Tracks the mathematical values used to provide path segment stats + * at each point on the path. Individual steps may be removed (and + * batched into `choppedStats`) once their respective points on the + * path have been fully processed. + */ + private steppedCumulativeStats: CumulativePathStats[]; // Currently used as an in-development diagnostic assist... but these // directly represent actual path segments as produced by the prototype // algorithm. Just... the stats analysis of the path segment, without // obvious / public members to relevant coordinates. - private _protoSegments: PathSegmentStats[] = []; + private _protoSegments: CumulativePathStats[] = []; + /** + * Used to 'repeat' the most-recently observed incoming sample if no + * other replaces it before it triggers. + * + * A repeating sample indicates lack of motion, which is valuable + * information for stats-based segmentation. + */ private repeatTimer: number | NodeJS.Timeout; + + /** + * The timestamp of observation of the most recently observed sample's + * most recent repetition - even if it's only the first evaluation. + */ private repeatTimestamp: number; - private choppedStats: PathSegmentStats = null; + /** + * Represents the cumulative statistics of all points on the path + * that lie on already-fully-segmented parts of it. + */ + private choppedStats: CumulativePathStats = null; constructor() { - this._stats = []; - } - - public get stats(): readonly PathSegmentStats[] { - return this._stats; + this.steppedCumulativeStats = []; } public add(sample: InputSample) { @@ -62,30 +79,30 @@ namespace com.keyman.osk { clearInterval(this.repeatTimer); this.repeatTimer = null; - let intervalStats = this.stats[this.stats.length-1]; + let intervalStats = this.steppedCumulativeStats[this.steppedCumulativeStats.length-1]; if(this.choppedStats) { - intervalStats = intervalStats.withoutPrefixSubset(this.choppedStats); + intervalStats = intervalStats.deaccumulate(this.choppedStats); } this._protoSegments.push(intervalStats); } private observe(sample: InputSample, timeDelta: number) { - let baseStats: PathSegmentStats; - if(this.stats.length) { - baseStats = this.stats[this.stats.length-1]; + let cumulativeStats: CumulativePathStats; + if(this.steppedCumulativeStats.length) { + cumulativeStats = this.steppedCumulativeStats[this.steppedCumulativeStats.length-1]; } else { - baseStats = new PathSegmentStats(); + cumulativeStats = new CumulativePathStats(); } sample = {... sample}; sample.t += timeDelta; - const extendedStats = baseStats.unionWith(sample); - this._stats.push(extendedStats); + const extendedStats = cumulativeStats.extend(sample); + this.steppedCumulativeStats.push(extendedStats); let preWindowEnd = 0; // Do not consider the just-added `extendedStats` entry. - for(let i = this.stats.length-2; i >=0 ; i--) { - if(this.stats[i].lastTimestamp + this.SLIDING_WINDOW_INTERVAL < sample.t) { + for(let i = this.steppedCumulativeStats.length-2; i >=0 ; i--) { + if(this.steppedCumulativeStats[i].lastTimestamp + this.SLIDING_WINDOW_INTERVAL < sample.t) { preWindowEnd = i; break; } @@ -96,23 +113,33 @@ namespace com.keyman.osk { // properties to have a chance at becoming 'defined'.) //console.log(sample); if(preWindowEnd > 0) { - const cumulativePreCandidate = this.stats[preWindowEnd+1]; + // We split the cumulative stats on a specific point, which then resides on the edge + // of both of the resulting intervals. + const cumulativePreCandidate = this.steppedCumulativeStats[preWindowEnd+1]; let preCandidate = cumulativePreCandidate; if(this.choppedStats) { - preCandidate = preCandidate.withoutPrefixSubset(this.choppedStats); + preCandidate = preCandidate.deaccumulate(this.choppedStats); } - let postCandidate = extendedStats.withoutPrefixSubset(this.stats[preWindowEnd]); + let postCandidate = extendedStats.deaccumulate(this.steppedCumulativeStats[preWindowEnd]); let combined = extendedStats; if(this.choppedStats) { - combined = combined.withoutPrefixSubset(this.choppedStats); + combined = combined.deaccumulate(this.choppedStats); } // Run a comparison on various stats of the two. if(preCandidate.duration * 1000 >= this.SLIDING_WINDOW_INTERVAL) { // sec vs millisec. let performSegmentation = false; - let angleSplitVariance = preCandidate.angleVariance + postCandidate.angleVariance; + // Note: circular variance is defined separately from circular std. deviation. + // They _are_ close, though. But which is "best" to use here? + // Even if sourced differently, the best initial guess is to compare variance + // to variance. + // + // Also note that if there is no motion in either segmentation candidate, angleVariance + // is undefined, thus NaN, mathematically. Practically... no angle = no variance. + let angleSplitVariance = (isNaN(preCandidate.angleVariance) ? 0 : preCandidate.angleVariance) + + (isNaN(postCandidate.angleVariance) ? 0 : postCandidate.angleVariance); let angleVarianceRatio = combined.angleVariance / angleSplitVariance; const speedSplitVariance = preCandidate.speedVariance + postCandidate.speedVariance; @@ -170,9 +197,9 @@ namespace com.keyman.osk { console.log(); - this._stats = this._stats.slice(preWindowEnd+1); // DO release this line. + this.steppedCumulativeStats = this.steppedCumulativeStats.slice(preWindowEnd+1); // DO release this line. console.log("Dropped samples: " + (preWindowEnd+1)); - console.log("Remaining samples: " + this._stats.length); + console.log("Remaining samples: " + this.steppedCumulativeStats.length); console.log("Prototype segment: "); console.log(preCandidate); diff --git a/common/web/gesture-recognizer/src/trackedPath.ts b/common/web/gesture-recognizer/src/trackedPath.ts index 6ac7f2df82..2398d07129 100644 --- a/common/web/gesture-recognizer/src/trackedPath.ts +++ b/common/web/gesture-recognizer/src/trackedPath.ts @@ -38,7 +38,7 @@ namespace com.keyman.osk { private samples: InputSample[] = []; private _segments: Segment[] = []; - private segmenter = new PathSegmenter(); + private readonly segmenter: PathSegmenter; private _isComplete: boolean = false; private wasCancelled?: boolean; @@ -60,10 +60,16 @@ namespace com.keyman.osk { if(jsonObj) { this.samples = [...jsonObj.coords.map((obj) => ({...obj} as InputSample))]; - // If we're reconstructing this from a JSON.parse, it's a previously-recorded, completed path. + // If we're reconstructing this from a JSON.parse, it's a previously-recorded, + // completed path. this._isComplete = true; this.wasCancelled = jsonObj.wasCancelled; } + + // Keep this as the _final_ statement in the constructor. `PathSegmenter` will + // need a reference to this instance, even if only via closure. + // (Most likely; not yet done.) Kinda awkward, but it's useful for compartmentalization. + this.segmenter = new PathSegmenter(); } /** From af7df8a12354e9bab5d6df57e88d713af00b9b9b Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 11 Aug 2022 11:52:56 +0700 Subject: [PATCH 05/22] change(web): segmenter organization --- .../src/cumulativePathStats.ts | 8 +- .../gesture-recognizer/src/pathSegmenter.ts | 149 +++++++++++------- 2 files changed, 98 insertions(+), 59 deletions(-) diff --git a/common/web/gesture-recognizer/src/cumulativePathStats.ts b/common/web/gesture-recognizer/src/cumulativePathStats.ts index 2b0be2ddc7..3591dffe9f 100644 --- a/common/web/gesture-recognizer/src/cumulativePathStats.ts +++ b/common/web/gesture-recognizer/src/cumulativePathStats.ts @@ -123,9 +123,15 @@ namespace com.keyman.osk { * from this instance's current accumulation. * @returns */ - public deaccumulate(subsetStats: CumulativePathStats): CumulativePathStats { + public deaccumulate(subsetStats?: CumulativePathStats): CumulativePathStats { const result = new CumulativePathStats(this); + // We actually WILL accept a `null` argument; makes some of the segmentation + // logic simpler. + if(!subsetStats) { + return result; + } + if(!subsetStats.followingSample || !subsetStats.lastSample) { throw 'Invalid argument: stats missing necessary tracking variable.'; } diff --git a/common/web/gesture-recognizer/src/pathSegmenter.ts b/common/web/gesture-recognizer/src/pathSegmenter.ts index 3ede92ae3e..5db315f070 100644 --- a/common/web/gesture-recognizer/src/pathSegmenter.ts +++ b/common/web/gesture-recognizer/src/pathSegmenter.ts @@ -1,6 +1,44 @@ /// namespace com.keyman.osk { + class PotentialSegmentation { + readonly pre: CumulativePathStats; + readonly post: CumulativePathStats; + readonly union: CumulativePathStats; + readonly chopPoint: CumulativePathStats; + + constructor(steppedStats: CumulativePathStats[], + choppedStats: CumulativePathStats, + splitIndex: number) { + this.chopPoint = steppedStats[splitIndex]; + this.pre = steppedStats[splitIndex].deaccumulate(choppedStats); + + // Keep stats value components based on the final point of the 'pre' segment. + const finalStats = steppedStats[steppedStats.length-1]; + this.post = finalStats.deaccumulate(steppedStats[splitIndex-1]); + this.union = finalStats.deaccumulate(choppedStats); + } + + // Note: circular variance is defined separately from circular std. deviation. + // They _are_ close, though. But which is "best" to use here? + // Even if sourced differently, the best initial guess is to compare variance + // to variance. + // + // Also note that if there is no motion in either segmentation candidate, angleVariance + // is undefined, thus NaN, mathematically. Practically... no angle = no variance. + public get angleVarianceRatio() { + const preVariance = (isNaN(this.pre.angleVariance) ? 0 : this.pre.angleVariance); + const postVariance = (isNaN(this.post.angleVariance) ? 0 : this.post.angleVariance); + + return this.union.angleVariance / (preVariance + postVariance); + } + + public get speedVarianceRatio() { + const splitVariance = this.pre.speedVariance + this.post.speedVariance; + return this.union.speedVariance / splitVariance; + } + } + export class PathSegmenter { /** * The minimum amount of time (in ms) to wait between sample repetitions @@ -84,6 +122,9 @@ namespace com.keyman.osk { intervalStats = intervalStats.deaccumulate(this.choppedStats); } this._protoSegments.push(intervalStats); + + // FIXME: temporary statement to facilitate exploration, experimentation, & debugging + console.log(this._protoSegments); } private observe(sample: InputSample, timeDelta: number) { @@ -99,10 +140,47 @@ namespace com.keyman.osk { const extendedStats = cumulativeStats.extend(sample); this.steppedCumulativeStats.push(extendedStats); + this.attemptSegmentation(); + } + + private _debugLogSegmentationReport(candidateSplit: PotentialSegmentation) { + console.log("------------------------------------------------------------------"); + + console.log("Angle variance ratio: " + candidateSplit.angleVarianceRatio); + console.log("Speed variance ratio: " + candidateSplit.speedVarianceRatio); + + console.log("Combined: "); + console.log(candidateSplit.union.toJSON()); + console.log(candidateSplit.union); + console.log("Pre: ") + console.log(candidateSplit.pre.toJSON()); + console.log(candidateSplit.pre); + console.log("Post: "); + console.log(candidateSplit.post.toJSON()); + console.log(candidateSplit.post); + console.log("Angle variance ratio: " + candidateSplit.angleVarianceRatio); + console.log("Speed variance ratio: " + candidateSplit.speedVarianceRatio); + + console.log(); + + console.log("Prototype segment: "); + console.log(candidateSplit.pre); + + console.log("------------------------------------------------------------------"); + // END: DO NOT RELEASE. + } + + private attemptSegmentation() { + const extendedStats = this.steppedCumulativeStats[this.steppedCumulativeStats.length - 1]; let preWindowEnd = 0; // Do not consider the just-added `extendedStats` entry. - for(let i = this.steppedCumulativeStats.length-2; i >=0 ; i--) { - if(this.steppedCumulativeStats[i].lastTimestamp + this.SLIDING_WINDOW_INTERVAL < sample.t) { + // + // Note: even if we do reconsider the segmentation point... I don't think we + // should reconsider anything earlier than where this marker falls. + // + // If we didn't segment earlier before, why would we suddenly do so now? + for(let i = this.steppedCumulativeStats.length-2; i >=0; i--) { + if(this.steppedCumulativeStats[i].lastTimestamp + this.SLIDING_WINDOW_INTERVAL < extendedStats.lastTimestamp) { preWindowEnd = i; break; } @@ -111,40 +189,16 @@ namespace com.keyman.osk { // Do not consider segmenting before at least two samples exist before the current // sliding time window. (A minimum of two samples are needed for 'over time' // properties to have a chance at becoming 'defined'.) - //console.log(sample); if(preWindowEnd > 0) { // We split the cumulative stats on a specific point, which then resides on the edge // of both of the resulting intervals. - const cumulativePreCandidate = this.steppedCumulativeStats[preWindowEnd+1]; - let preCandidate = cumulativePreCandidate; - if(this.choppedStats) { - preCandidate = preCandidate.deaccumulate(this.choppedStats); - } - let postCandidate = extendedStats.deaccumulate(this.steppedCumulativeStats[preWindowEnd]); - - let combined = extendedStats; - if(this.choppedStats) { - combined = combined.deaccumulate(this.choppedStats); - } + let candidateSplit = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, preWindowEnd+1); // Run a comparison on various stats of the two. - if(preCandidate.duration * 1000 >= this.SLIDING_WINDOW_INTERVAL) { // sec vs millisec. + // Oh. And _there's_ why - if we didn't check before b/c minimum time req't. + if(candidateSplit.pre.duration * 1000 >= this.SLIDING_WINDOW_INTERVAL) { // sec vs millisec. let performSegmentation = false; - // Note: circular variance is defined separately from circular std. deviation. - // They _are_ close, though. But which is "best" to use here? - // Even if sourced differently, the best initial guess is to compare variance - // to variance. - // - // Also note that if there is no motion in either segmentation candidate, angleVariance - // is undefined, thus NaN, mathematically. Practically... no angle = no variance. - let angleSplitVariance = (isNaN(preCandidate.angleVariance) ? 0 : preCandidate.angleVariance) + - (isNaN(postCandidate.angleVariance) ? 0 : postCandidate.angleVariance); - let angleVarianceRatio = combined.angleVariance / angleSplitVariance; - - const speedSplitVariance = preCandidate.speedVariance + postCandidate.speedVariance; - const speedVarianceRatio = combined.speedVariance / speedSplitVariance; - /* * Okay, so this isn't probably quite the most statistically well-founded approach, but... * @@ -165,7 +219,7 @@ namespace com.keyman.osk { * - moderate difference in both angle and speed between the segment candidates (equal levels) * - very strong difference in speed between the segment candidates. */ - if(angleVarianceRatio + speedVarianceRatio / 2 > 1.5) { + if(candidateSplit.angleVarianceRatio + candidateSplit.speedVarianceRatio / 2 > 1.5) { performSegmentation = true; } @@ -176,39 +230,18 @@ namespace com.keyman.osk { // This is exploratory / diagnostic code assisting development of the path segmentation // algorithm. if(performSegmentation) { - console.log("------------------------------------------------------------------"); + this._debugLogSegmentationReport(candidateSplit); + } else { + console.log("Angle variance ratio: " + candidateSplit.angleVarianceRatio); + console.log("Speed variance ratio: " + candidateSplit.speedVarianceRatio); } - - console.log("Angle variance ratio: " + angleVarianceRatio); - console.log("Speed variance ratio: " + speedVarianceRatio); + // END: DO NOT RELEASE. if(performSegmentation) { - console.log("Combined: "); - console.log(combined.toJSON()); - console.log(combined); - console.log("Pre: ") - console.log(preCandidate.toJSON()); - console.log(preCandidate); - console.log("Post: "); - console.log(postCandidate.toJSON()); - console.log(postCandidate); - console.log("Angle variance ratio: " + angleVarianceRatio); - console.log("Speed variance ratio: " + speedVarianceRatio); - - console.log(); - this.steppedCumulativeStats = this.steppedCumulativeStats.slice(preWindowEnd+1); // DO release this line. - console.log("Dropped samples: " + (preWindowEnd+1)); - console.log("Remaining samples: " + this.steppedCumulativeStats.length); - console.log("Prototype segment: "); - console.log(preCandidate); - - console.log("------------------------------------------------------------------"); - // END: DO NOT RELEASE. - - this._protoSegments.push(preCandidate); - this.choppedStats = cumulativePreCandidate; + this._protoSegments.push(candidateSplit.pre); + this.choppedStats = candidateSplit.chopPoint; } } } From c445838980659afa66ea22a4fe9bfd741f76450a Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 11 Aug 2022 13:07:41 +0700 Subject: [PATCH 06/22] feat(web): allows shifting of segmentation point --- .../gesture-recognizer/src/pathSegmenter.ts | 163 +++++++++++------- 1 file changed, 103 insertions(+), 60 deletions(-) diff --git a/common/web/gesture-recognizer/src/pathSegmenter.ts b/common/web/gesture-recognizer/src/pathSegmenter.ts index 5db315f070..2fd01c1af4 100644 --- a/common/web/gesture-recognizer/src/pathSegmenter.ts +++ b/common/web/gesture-recognizer/src/pathSegmenter.ts @@ -2,6 +2,8 @@ namespace com.keyman.osk { class PotentialSegmentation { + public static readonly SPLIT_CRITERION_THRESHOLD = 1.5; + readonly pre: CumulativePathStats; readonly post: CumulativePathStats; readonly union: CumulativePathStats; @@ -37,6 +39,33 @@ namespace com.keyman.osk { const splitVariance = this.pre.speedVariance + this.post.speedVariance; return this.union.speedVariance / splitVariance; } + + public get splitCriterion() { + /* + * Okay, so this isn't probably quite the most statistically well-founded approach, but... + * + * A "variance ratio" of 1 indicates something of a break-even point; exceeding that threshold + * means that the variations in value seen between the two potential segments are better + * explained as being from two separate segments than from a single segment. (Loosely speaking; + * it'd take some effort to cement the statistical basis here; this is more 'inspired by' what + * the values represent.) + * + * Of course... when it comes to speed, acceleration and such are factors. It'd be all + * too easy to split the slow and fast parts of an accelerating linear motion as two separate + * pieces. Requiring a higher degree of separation alleviates this - hence, the `/2` in the + * condition below. (That divisor's not the most statistically-based thing to do, but it + * works well here.) + * + * So to reach the threshold set below, these three conditions will work: + * - strong difference in angle between the segment candidates + * - moderate difference in both angle and speed between the segment candidates (equal levels) + * - very strong difference in speed between the segment candidates. + */ + + const angleComponent = isNaN(this.angleVarianceRatio) ? 0 : this.angleVarianceRatio; + const speedComponent = isNaN(this.speedVarianceRatio) ? 0 : (this.speedVarianceRatio / 2); + return angleComponent + speedComponent; + } } export class PathSegmenter { @@ -171,8 +200,14 @@ namespace com.keyman.osk { } private attemptSegmentation() { - const extendedStats = this.steppedCumulativeStats[this.steppedCumulativeStats.length - 1]; - let preWindowEnd = 0; + const cumulativeStats = this.steppedCumulativeStats[this.steppedCumulativeStats.length - 1]; + const unsegmentedDuration = cumulativeStats.lastTimestamp - this.steppedCumulativeStats[0].lastTimestamp; + + if(unsegmentedDuration < this.SLIDING_WINDOW_INTERVAL * 2) { + return; + } + + let splitPoint = 0; // Do not consider the just-added `extendedStats` entry. // // Note: even if we do reconsider the segmentation point... I don't think we @@ -180,71 +215,79 @@ namespace com.keyman.osk { // // If we didn't segment earlier before, why would we suddenly do so now? for(let i = this.steppedCumulativeStats.length-2; i >=0; i--) { - if(this.steppedCumulativeStats[i].lastTimestamp + this.SLIDING_WINDOW_INTERVAL < extendedStats.lastTimestamp) { - preWindowEnd = i; + if(this.steppedCumulativeStats[i].lastTimestamp < cumulativeStats.lastTimestamp - this.SLIDING_WINDOW_INTERVAL) { + splitPoint = i+1; break; } } - // Do not consider segmenting before at least two samples exist before the current - // sliding time window. (A minimum of two samples are needed for 'over time' - // properties to have a chance at becoming 'defined'.) - if(preWindowEnd > 0) { - // We split the cumulative stats on a specific point, which then resides on the edge - // of both of the resulting intervals. - let candidateSplit = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, preWindowEnd+1); + // We split the cumulative stats on a specific point, which then resides on the edge + // of both of the resulting intervals. + let candidateSplit = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint); - // Run a comparison on various stats of the two. - // Oh. And _there's_ why - if we didn't check before b/c minimum time req't. - if(candidateSplit.pre.duration * 1000 >= this.SLIDING_WINDOW_INTERVAL) { // sec vs millisec. - let performSegmentation = false; + // Run a comparison on various stats of the two. + // Oh. And _there's_ why - if we didn't check before b/c minimum time req't. + let performSegmentation = false; - /* - * Okay, so this isn't probably quite the most statistically well-founded approach, but... - * - * A "variance ratio" of 1 indicates something of a break-even point; exceeding that threshold - * means that the variations in value seen between the two potential segments are better - * explained as being from two separate segments than from a single segment. (Loosely speaking; - * it'd take some effort to cement the statistical basis here; this is more 'inspired by' what - * the values represent.) - * - * Of course... when it comes to speed, acceleration and such are factors. It'd be all - * too easy to split the slow and fast parts of an accelerating linear motion as two separate - * pieces. Requiring a higher degree of separation alleviates this - hence, the `/2` in the - * condition below. (That divisor's not the most statistically-based thing to do, but it - * works well here.) - * - * So to reach the threshold set below, these three conditions will work: - * - strong difference in angle between the segment candidates - * - moderate difference in both angle and speed between the segment candidates (equal levels) - * - very strong difference in speed between the segment candidates. - */ - if(candidateSplit.angleVarianceRatio + candidateSplit.speedVarianceRatio / 2 > 1.5) { - performSegmentation = true; - } - - // Hmm. Perhaps this should only serve as the "okay, let's segment" trigger... to then - // find the BEST segmentation. - - // FIXME: DO NOT RELEASE. - // This is exploratory / diagnostic code assisting development of the path segmentation - // algorithm. - if(performSegmentation) { - this._debugLogSegmentationReport(candidateSplit); - } else { - console.log("Angle variance ratio: " + candidateSplit.angleVarianceRatio); - console.log("Speed variance ratio: " + candidateSplit.speedVarianceRatio); - } - // END: DO NOT RELEASE. - - if(performSegmentation) { - this.steppedCumulativeStats = this.steppedCumulativeStats.slice(preWindowEnd+1); // DO release this line. - - this._protoSegments.push(candidateSplit.pre); - this.choppedStats = candidateSplit.chopPoint; - } - } + const initialSplitCriterion = candidateSplit.splitCriterion; + if(initialSplitCriterion > PotentialSegmentation.SPLIT_CRITERION_THRESHOLD) { + performSegmentation = true; } + + if(!performSegmentation) { + // // Debug logging statements: + console.log("Angle variance ratio: " + candidateSplit.angleVarianceRatio); + console.log("Speed variance ratio: " + candidateSplit.speedVarianceRatio); + return; + } + + // We've met the conditions to trigger segmentation. Now... is there a better segmentation point? + // + + let currentSplitCriterion = initialSplitCriterion; + let leftCandidate = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint-1); + let rightCandidate = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint+1); + + const criteria = [leftCandidate.splitCriterion, candidateSplit.splitCriterion, rightCandidate.splitCriterion]; + let sortedCriteria = [...criteria].sort(); + + const delta = criteria.indexOf(sortedCriteria[2])-1; // -1 if 'left' is best, 1 if 'right' is best. + + if(delta != 0) { + // We can get better segmentation by shifting. Proceed in the optimal direction. + do { + let nextCandidate = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint + delta); + let nextSplitCriterion = nextCandidate.splitCriterion; + + // Prevent overly-short intervals / over-segmentation. + if(nextCandidate.pre.duration * 1000 < this.SLIDING_WINDOW_INTERVAL / 2) { + break; + } else if(nextCandidate.post.duration * 1000 < this.SLIDING_WINDOW_INTERVAL / 2) { + break; + } + + // Not an improvement? Guess we found the best spot. + if(nextSplitCriterion < currentSplitCriterion) { + break; + } else { + splitPoint += delta; + currentSplitCriterion = nextSplitCriterion; + candidateSplit = nextCandidate; + } + // If we found a new best segmentation point, we then ask if we can get even better by shifting further. + } while(true); + } + + // FIXME: DO NOT RELEASE. + // This is exploratory / diagnostic code assisting development of the path segmentation + // algorithm. + this._debugLogSegmentationReport(candidateSplit); + // END: DO NOT RELEASE. + + this.steppedCumulativeStats = this.steppedCumulativeStats.slice(splitPoint+1); // DO release this line. + + this._protoSegments.push(candidateSplit.pre); + this.choppedStats = candidateSplit.chopPoint; } } } \ No newline at end of file From a284a5092d9435e55b3ca864d1539c7e7aa72ab6 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 11 Aug 2022 15:10:54 +0700 Subject: [PATCH 07/22] feat(web): initial code toward subsegment filtering/recombination --- .../src/cumulativePathStats.ts | 31 ++++++-- .../gesture-recognizer/src/pathSegmenter.ts | 77 +++++++++++++++++-- 2 files changed, 95 insertions(+), 13 deletions(-) diff --git a/common/web/gesture-recognizer/src/cumulativePathStats.ts b/common/web/gesture-recognizer/src/cumulativePathStats.ts index 3591dffe9f..9bf237367f 100644 --- a/common/web/gesture-recognizer/src/cumulativePathStats.ts +++ b/common/web/gesture-recognizer/src/cumulativePathStats.ts @@ -206,7 +206,7 @@ namespace com.keyman.osk { } } - public get directDistance() { + public get netDistance() { // No issue with a net distance of 0 due to a single point. if(!this.lastSample || !this.initialSample) { return Number.NaN; @@ -233,14 +233,14 @@ namespace com.keyman.osk { public get angle() { if(this.sampleCount == 1 || !this.lastSample || !this.initialSample) { return Number.NaN; - } else if(this.directDistance < 1) { + } else if(this.netDistance < 1) { // < 1 px, thus sub-pixel, means we have nothing relevant enough to base an angle on. return Number.NaN; } const xDelta = this.lastSample.targetX - this.initialSample.targetX; const yDelta = this.lastSample.targetY - this.initialSample.targetY; - const yAngleDiff = Math.acos(-yDelta / this.directDistance); + const yAngleDiff = Math.acos(-yDelta / this.netDistance); return xDelta < 0 ? (2 * Math.PI - yAngleDiff) : yAngleDiff; } @@ -269,7 +269,7 @@ namespace com.keyman.osk { // px per s. public get speed() { // this.duration is already in seconds, not milliseconds. - return this.duration ? this.directDistance / this.duration : Number.NaN; + return this.duration ? this.netDistance / this.duration : Number.NaN; } // ... may not be "right". @@ -344,6 +344,25 @@ namespace com.keyman.osk { return Math.sqrt(-Math.log(this.angleRSquared)); } + public get rawDistance() { + return this.coordArcSum; + } + + public get maxEndpointDistanceFromCentroid() { + if(!this.initialSample || !this.lastSample) { + return 0; + } + + const centroid = this.centroid; + const startXDist = centroid.x - this.initialSample.targetX; + const startYDist = centroid.y - this.initialSample.targetY; + const endXDist = this.lastSample.targetX - centroid.x; + const endYDist = this.lastSample.targetY - centroid.y; + + return Math.sqrt(Math.max(startXDist * startXDist + startYDist * startYDist, + endXDist * endXDist + endYDist * endYDist)); + } + // TODO: is this actually ideal? This was certainly useful for experimentation via interactive // debugging, but it may not be the best thing long-term. public toJSON() { @@ -353,7 +372,9 @@ namespace com.keyman.osk { angleDeviation: this.angleDeviation, speedMean: this.speedMean, speedVariance: this.speedVariance, - coordArcSum: this.coordArcSum, + rawDistance: this.rawDistance, + netDistance: this.netDistance, + distanceFromCentroid: this.maxEndpointDistanceFromCentroid, duration: this.duration, sampleCount: this.sampleCount } diff --git a/common/web/gesture-recognizer/src/pathSegmenter.ts b/common/web/gesture-recognizer/src/pathSegmenter.ts index 2fd01c1af4..4cb475d6ca 100644 --- a/common/web/gesture-recognizer/src/pathSegmenter.ts +++ b/common/web/gesture-recognizer/src/pathSegmenter.ts @@ -8,11 +8,14 @@ namespace com.keyman.osk { readonly post: CumulativePathStats; readonly union: CumulativePathStats; readonly chopPoint: CumulativePathStats; + readonly baseChop: CumulativePathStats; constructor(steppedStats: CumulativePathStats[], choppedStats: CumulativePathStats, splitIndex: number) { + this.baseChop = choppedStats; this.chopPoint = steppedStats[splitIndex]; + this.pre = steppedStats[splitIndex].deaccumulate(choppedStats); // Keep stats value components based on the final point of the 'pre' segment. @@ -89,6 +92,8 @@ namespace com.keyman.osk { */ private steppedCumulativeStats: CumulativePathStats[]; + private lingeringSubsegmentations: PotentialSegmentation[]; + // Currently used as an in-development diagnostic assist... but these // directly represent actual path segments as produced by the prototype // algorithm. Just... the stats analysis of the path segment, without @@ -118,6 +123,7 @@ namespace com.keyman.osk { constructor() { this.steppedCumulativeStats = []; + this.lingeringSubsegmentations = []; } public add(sample: InputSample) { @@ -146,14 +152,16 @@ namespace com.keyman.osk { clearInterval(this.repeatTimer); this.repeatTimer = null; - let intervalStats = this.steppedCumulativeStats[this.steppedCumulativeStats.length-1]; - if(this.choppedStats) { - intervalStats = intervalStats.deaccumulate(this.choppedStats); - } - this._protoSegments.push(intervalStats); + // The way things are structured, finalization.pre = final segment. It's some happy + // 'fallout' from the implementation's design. + let finalization = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, this.steppedCumulativeStats.length-1); + + this.filterSubsegmentation(finalization, true); // forces out the final segment. + // Hacky, but "enough" for now. // FIXME: temporary statement to facilitate exploration, experimentation, & debugging console.log(this._protoSegments); + console.log(this._protoSegments.map((val) => (val.toJSON()))); } private observe(sample: InputSample, timeDelta: number) { @@ -278,16 +286,69 @@ namespace com.keyman.osk { } while(true); } + // First phase of segmentation: complete! + + // But... there are some cases where we want to prevent segmentation from fully happening. + // TODO: That. + // FIXME: DO NOT RELEASE. // This is exploratory / diagnostic code assisting development of the path segmentation // algorithm. this._debugLogSegmentationReport(candidateSplit); // END: DO NOT RELEASE. - this.steppedCumulativeStats = this.steppedCumulativeStats.slice(splitPoint+1); // DO release this line. - - this._protoSegments.push(candidateSplit.pre); + this.steppedCumulativeStats = this.steppedCumulativeStats.slice(splitPoint); this.choppedStats = candidateSplit.chopPoint; + + this.filterSubsegmentation(candidateSplit); + } + + // NOTE: This function, as well as code called by it, are still very much still in prototyping. + private filterSubsegmentation(subsegmentation: PotentialSegmentation, force?: boolean) { + force = !!force; + const mergeSubsegmentations = function(array: PotentialSegmentation[]) { + if(array.length == 1) { + return array[0].pre; // It's pre-calculated, so just use it. + } else { + const finalSubsegmentation = array[array.length-1]; + return finalSubsegmentation.chopPoint.deaccumulate(array[0].baseChop); + } + } + + if(this.lingeringSubsegmentations.length) { + // First: check if the newly-finished subsegment should be merged with the lingering ones. + const lastSubsegment = this.lingeringSubsegmentations[this.lingeringSubsegmentations.length-1]; + + if(!PathSegmenter.shouldMergeSubsegments(lastSubsegment.pre, subsegmentation.pre)) { + // Emit as separate subsegment. + const finishedSegment = mergeSubsegmentations(this.lingeringSubsegmentations); + this._protoSegments.push(finishedSegment); + this.lingeringSubsegmentations = []; + + // IN DEVELOPMENT: does this happen much? If so... maybe we need intervening checks to pre-filter even + // if not segmenting. + console.warn("Did not merge a lingering subsegment with the incoming one!"); + } else { + console.log("Will merge in old subsegments!"); + } + } + + if(!force && PathSegmenter.shouldMergeSubsegments(subsegmentation.pre, subsegmentation.post)) { + this.lingeringSubsegmentations.push(subsegmentation); + } else { + // Merge all as a completed segment! + const finishedSegment = mergeSubsegmentations([...this.lingeringSubsegmentations, subsegmentation]); + this._protoSegments.push(finishedSegment); + } + } + + private static shouldMergeSubsegments(segment1: CumulativePathStats, segment2: CumulativePathStats): boolean { + // Known case #1: + // Almost identical direction, but heavy speed variance. + // Speed's still high enough to not be a 'wait'. + // Known case #2: + // Low-speed pivot; angle change caught on very low velocity for both subsegments. + return false; } } } \ No newline at end of file From e2a0aa7e4996f5a34f77fd866d27645232ab4c2a Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Wed, 17 Aug 2022 10:12:08 +0700 Subject: [PATCH 08/22] feat(web): segmented regression to the rescue --- .../src/cumulativePathStats.ts | 202 +++++++- .../gesture-recognizer/src/pathSegmenter.ts | 489 +++++++++++++----- 2 files changed, 572 insertions(+), 119 deletions(-) diff --git a/common/web/gesture-recognizer/src/cumulativePathStats.ts b/common/web/gesture-recognizer/src/cumulativePathStats.ts index 9bf237367f..2dbeef5b86 100644 --- a/common/web/gesture-recognizer/src/cumulativePathStats.ts +++ b/common/web/gesture-recognizer/src/cumulativePathStats.ts @@ -7,9 +7,22 @@ namespace com.keyman.osk { * Instances of this class are immutable. */ export class CumulativePathStats { + private xLinearSum: number = 0; + private yLinearSum: number = 0; + private tLinearSum: number = 0; + private xCentroidSum: number = 0; private yCentroidSum: number = 0; + private xQuadSum: number = 0; + private yQuadSum: number = 0; + private tQuadSum: number = 0; + + private xtCrossSum: number = 0; + private ytCrossSum: number = 0; + // As x and y are both treated as dependent on t, we don't need their cross-sum + // with each other. + private coordArcSum: number = 0; private speedLinearSum: number = 0; @@ -65,17 +78,29 @@ namespace com.keyman.osk { if(!this.initialSample) { this.initialSample = sample; this.baseSample = sample; - } else { - this.followingSample = sample; } const result = new CumulativePathStats(this); + // Set _after_ deep-copying this for the result. + this.followingSample = sample; + // Helps prevent "catastrophic cancellation" issues from floating-point computation // for these statistical properties and properties based upon them. const x = sample.targetX - this.baseSample.targetX; const y = sample.targetY - this.baseSample.targetY; const t = sample.t - this.baseSample.t; + result.xLinearSum += x; + result.yLinearSum += y; + result.tLinearSum += t; + + result.xtCrossSum += x * t; + result.ytCrossSum += y * t; + + result.xQuadSum += x * x; + result.yQuadSum += y * y; + result.tQuadSum += t * t; + if(this.lastSample) { // arc length stuff! const xDelta = sample.targetX - this.lastSample.targetX; @@ -124,6 +149,18 @@ namespace com.keyman.osk { * @returns */ public deaccumulate(subsetStats?: CumulativePathStats): CumulativePathStats { + // Possible TODO: Because of the properties of statistical variance & mean... + // we could further prevent catastrophic cancellation by re-centering + // all the linear, cross, and quad sums. + // - mostly noteworthy for _long_ duration touches that wander long distances. + // - basically, for cases that'd cause the floating-point error to exceed our + // test thresholds. Re-centering would keep that error consistently below + // our thresholds. + // + // We could then take the new mean coordinates as a 'base sample'. + // Kinda has to be the new mean b/c of the stats identities we'd be abusing, + // but that's also the best catastrophic-cancellation prevention move we + // could take. So, this limitation's not really a negative. const result = new CumulativePathStats(this); // We actually WILL accept a `null` argument; makes some of the segmentation @@ -136,6 +173,17 @@ namespace com.keyman.osk { throw 'Invalid argument: stats missing necessary tracking variable.'; } + result.xLinearSum -= subsetStats.xLinearSum; + result.yLinearSum -= subsetStats.yLinearSum; + result.tLinearSum -= subsetStats.tLinearSum; + + result.xtCrossSum -= subsetStats.xtCrossSum; + result.ytCrossSum -= subsetStats.ytCrossSum; + + result.xQuadSum -= subsetStats.xQuadSum; + result.yQuadSum -= subsetStats.yQuadSum; + result.tQuadSum -= subsetStats.tQuadSum; + // arc length stuff! if(subsetStats.followingSample && subsetStats.lastSample) { const xDelta = subsetStats.followingSample.targetX - subsetStats.lastSample.targetX; @@ -189,6 +237,22 @@ namespace com.keyman.osk { return this.lastSample?.t; } + public get count() { + return this.sampleCount; + } + + private get xSampleMean() { + return this.xLinearSum / this.sampleCount; + } + + private get ySampleMean() { + return this.yLinearSum / this.sampleCount; + } + + private get tSampleMean() { + return this.tLinearSum / this.sampleCount; + } + public get centroid(): {x: number, y: number} { if(this.sampleCount == 0) { return undefined; @@ -206,6 +270,140 @@ namespace com.keyman.osk { } } + public get xtCovariance() { + return this.xtCrossSum / this.sampleCount - (this.xSampleMean * this.tSampleMean); + } + + public get ytCovariance() { + return this.ytCrossSum / this.sampleCount - (this.ySampleMean * this.tSampleMean); + } + + public get xVariance() { + return this.xQuadSum / this.sampleCount - (this.xSampleMean * this.xSampleMean); + } + + public get yVariance() { + return this.yQuadSum / this.sampleCount - (this.ySampleMean * this.ySampleMean); + } + + public get tVariance() { + return this.tQuadSum / this.sampleCount - (this.tSampleMean * this.tSampleMean); + } + + public get xRegressionSlope() { + return this.xtCovariance / this.tVariance; + } + + public get yRegressionSlope() { + return this.ytCovariance / this.tVariance; + } + + public get xRegressionIntercept() { + return (this.xSampleMean) - this.xRegressionSlope * this.tSampleMean; + } + + public get yRegressionIntercept() { + return (this.ySampleMean) - this.yRegressionSlope * this.tSampleMean; + } + + public get xRegressionSSE() { + return this.sampleCount * (this.xVariance - (this.xRegressionSlope * this.xtCovariance)); + } + + public get yRegressionSSE() { + return this.sampleCount * (this.yVariance - (this.yRegressionSlope * this.ytCovariance)); + } + + public get xRegressionModeledVariance() { + return this.xRegressionSlope * this.xtCovariance * this.sampleCount; + } + + public get yRegressionModeledVariance() { + return this.yRegressionSlope * this.ytCovariance * this.sampleCount; + } + + public get xRegressionCOD() { + // In truth, the proper answer is NaN (not defined). But for our purposes, + // it's a perfect fit, so we'll indicate "perfect fit". + if(this.xVariance == 0) { + return 1; + } + + return this.xtCovariance * this.xtCovariance / (this.xVariance * this.tVariance); + } + + public get yRegressionCOD() { + // In truth, the proper answer is NaN (not defined). But for our purposes, + // it's a perfect fit, so we'll indicate "perfect fit". + if(this.yVariance == 0) { + return 1; + } + + return this.ytCovariance * this.ytCovariance / (this.yVariance * this.tVariance); + } + + public get xRegressionFinalE() { + if(!this.lastSample || !this.baseSample) { + return undefined; + } + + const time = this.lastSample.t - this.baseSample.t; + const adjustedX = this.lastSample.targetX - this.baseSample.targetX; + const error = adjustedX - (this.xRegressionSlope * time + this.xRegressionIntercept); + return error;// * error; + } + + public get xRegressionFinalSE() { + return this.xRegressionFinalE * this.xRegressionFinalE; + } + + public get yRegressionFinalE() { + if(!this.lastSample || !this.baseSample) { + return undefined; + } + + const time = this.lastSample.t - this.baseSample.t; + const adjustedY = this.lastSample.targetY - this.baseSample.targetY; + const error = adjustedY - (this.yRegressionSlope * time + this.yRegressionIntercept); + return error;// * error; + } + + public get yRegressionFinalSE() { + return this.yRegressionFinalE * this.yRegressionFinalE; + } + + public get xRegressionInitialE() { + if(!this.initialSample || !this.baseSample) { + return undefined; + } + + const time = this.initialSample.t - this.baseSample.t; + const adjustedX = this.initialSample.targetX - this.baseSample.targetX; + const error = adjustedX - (this.xRegressionSlope * time + this.xRegressionIntercept); + return error;// * error; + } + + //public + + public get xRegressionInitialSE() { + return this.xRegressionInitialE * this.xRegressionInitialE; + } + + public get yRegressionInitialE() { + if(!this.initialSample || !this.baseSample) { + return undefined; + } + + const time = this.initialSample.t - this.baseSample.t; + const adjustedY = this.initialSample.targetY - this.baseSample.targetY; + const error = adjustedY - (this.yRegressionSlope * time + this.yRegressionIntercept); + return error;// * error; + } + + public get yRegressionInitialSE() { + return this.yRegressionInitialE * this.yRegressionInitialE; + } + public get netDistance() { // No issue with a net distance of 0 due to a single point. if(!this.lastSample || !this.initialSample) { diff --git a/common/web/gesture-recognizer/src/pathSegmenter.ts b/common/web/gesture-recognizer/src/pathSegmenter.ts index 4cb475d6ca..74b9681476 100644 --- a/common/web/gesture-recognizer/src/pathSegmenter.ts +++ b/common/web/gesture-recognizer/src/pathSegmenter.ts @@ -1,76 +1,285 @@ /// namespace com.keyman.osk { - class PotentialSegmentation { + // We do NOT want to be computing this thing at run-time. Fortunately, it's very common + // in stats circles to just... use a lookup table for finding p-values for things + // following the f-distribution. + class FDistribution { + /** + * Indexing: [threshold-index][num-2][denom-1]. + * + * If numerator has only 1 degree of freedom, that means we don't really have + * segments so much as different averages between the two segments - there's + * no slope on either side. In other words, "just a bump". No segmentation + * there. + * + * 2 degrees of freedom: no movement on the axis on only one side. + * + * threshold-index: + * - p-value of .100: 0 + * - p-value of .050: 1 + * + */ + private static readonly table = [ + // p = .100 + [ + // numerator: 2 + [ + // denom: 1-10 + 49.50, 9.00, 5.46, 4.32, 3.78, 3.46, 3.26, 3.11, 3.01, 2.92, + // denom: 11-20 + 2.86, 2.81, 2.76, 2.73, 2.70, 2.67, 2.64, 2.62, 2.61, 2.59 + // we COULD continue with threshold-dropping after that, + // but I doubt we'll need it in practicality. + // At 30: 2.49 + // limit -> infinity: 2.30 + ], + // numerator: 3 + [ + // denom: 1-10 + 53.59, 9.16, 5.39, 4.19, 3.62, 3.29, 3.07, 2.92, 2.81, 2.73, + // denom: 11-20 + 2.66, 2.61, 2.56, 2.52, 2.49, 2.46, 2.44, 2.42, 2.40, 2.38 + // Likewise, from above. + // At 30: 2.28 + // limit -> infinity: 2.08 + ] + ], + // p = .050 + [ + // numerator: 2 + [ + // denom: 1-10 + 199.5, 19.0, 9.55, 6.94, 5.79, 5.14, 4.74, 4.46, 4.26, 4.10, + // denom: 10-20 + 3.98, 3.89, 3.81, 3.74, 3.68, 3.63, 3.59, 3.55, 3.52, 3.49 + // At 30: 3.32 + // limit -> infinity: 3.00 + ], + // numerator: 3 + [ + // denom: 1-10 + 215.7,19.16, 9.28, 6.59, 5.41, 4.76, 4.35, 4.07, 3.86, 3.71, + // denom: 10-20 + 3.59, 3.49, 3.41, 3.34, 3.29, 3.24, 3.20, 3.16, 3.13, 3.10 + // At 30: 2.92 + // limit -> infinity: 2.60 + ] + ] + ] + + /** + * Determines a threshold tier for segmentation based on the segmented + * regression f-statistic. + * @param statistic + * @param numDoF + * @param denomDoF + * + * Tier 0: don't segment + * Tier 1: segment if the other axis also says to segment + * - p-value < 0.100 on the tested axis + * Tier 2: segment regardless of what the other axis says + * - p-value < 0.050 on the tested axis + */ + static thresholdTier(statistic: number, numDoF: number, denomDoF: number) { + // Former case: we'd never segment anyway + // Latter case: it's currently wrong to statistically test. + // The F-distribution is not defined for this case. + if(numDoF < 2 || denomDoF < 1) { + return 0; + } + + if(numDoF > 3) { + numDoF = 3; + } + + const numIndex = (numDoF > 3 ? 3 : numDoF) - 2; + const denomIndex = (denomDoF > 20 ? 20 : denomDoF) - 1; + + const tier2Threshold = FDistribution.table[1][numIndex][denomIndex]; + if(statistic > tier2Threshold) { + return 2; + } + + const tier1Threshold = FDistribution.table[0][numIndex][denomIndex]; + return statistic > tier1Threshold ? 1 : 0; + } + } + + class Segmentation { public static readonly SPLIT_CRITERION_THRESHOLD = 1.5; readonly pre: CumulativePathStats; readonly post: CumulativePathStats; readonly union: CumulativePathStats; + + constructor(pre: CumulativePathStats, post: CumulativePathStats, union: CumulativePathStats) { + this.pre = pre; + this.post = post; + this.union = union; + } + + private get xSegmentationSSE() { + // Technically double-counts the split point... but we're not preventing two separate predicted + // values AT the split point, each with their own error... + const summedSSE = this.pre.xRegressionSSE + this.post.xRegressionSSE/* - this.pre.xRegressionFinalSE*/; + + // 1e-8: catastrophic cancellation may cause small residuals due to floating-point arithmetic. + // Such residuals may be negative (same reason), but a true SSE value never will be. + return summedSSE > 1e-8 ? summedSSE : 0; + } + + private get ySegmentationSSE() { + // Technically double-counts the split point... but we're not preventing two separate predicted + // values AT the split point, each with their own error... + const summedSSE = this.pre.yRegressionSSE + this.post.yRegressionSSE/* - this.pre.yRegressionFinalSE*/; + + // 1e-8: catastrophic cancellation may cause small residuals due to floating-point arithmetic. + // Such residuals may be negative (same reason), but a true SSE value never will be. + return summedSSE > 1e-8 ? summedSSE: 0; + } + + private get xSegmentationModeledVariance() { + // Technically double-counts the split point... but we're not preventing two separate predicted + // values AT the split point. It's even trickier to adjust here than in the SSE properties, + // and it "balances out" (roughly) by existing in both components of the test statistic. + // + // Note: "balances out" is not a formal mathematical declaration. I'm playing things a bit + // loose here in the name of implementation clarity & simplicity. + const summedModeledVar = this.pre.xRegressionModeledVariance + this.post.xRegressionModeledVariance; + return summedModeledVar > 1e-8 ? summedModeledVar : 0; + } + + private get ySegmentationModeledVariance() { + // Technically double-counts the split point... but we're not preventing two separate predicted + // values AT the split point. It's even trickier to adjust here than in the SSE properties, + // and it "balances out" (roughly) by existing in both components of the test statistic. + // + // Note: "balances out" is not a formal mathematical declaration. I'm playing things a bit + // loose here in the name of implementation clarity & simplicity. + const summedModeledVar = this.pre.yRegressionModeledVariance + this.post.yRegressionModeledVariance; + return summedModeledVar > 1e-8 ? summedModeledVar : 0; + } + + public get xSegmentedRegressionCOD() { + if(!this.union.xVariance) { + return 1; + } + // The point @ the segmentation split point would be double-counted without the .xRegressionFinalSE part. + return 1 - this.xSegmentationSSE / (this.union.xVariance * this.union.count); + } + + public get ySegmentedRegressionCOD() { + if(!this.union.yVariance) { + return 1; + } + // The point @ the segmentation split point would be double-counted without the .yRegressionFinalSE part. + return 1 - this.ySegmentationSSE / (this.union.yVariance * this.union.count); + } + + public get xUnsegmentedRegressionCOD() { + return this.union.xRegressionCOD; + } + + public get yUnsegmentedRegressionCOD() { + return this.union.yRegressionCOD; + } + + /*private*/ get xFTestConfiguration() { + const fStat = this.xSegmentationModeledVariance / this.xSegmentationSSE; + let numDoF = 3; + // Cases where there clearly may as well be 'no slope' at all. + // This saves us a degree of freedom, which is VERY useful for segmenting + // the boundary between a 'hold' and a 'move'. + if(this.pre.xVariance * this.pre.count < 1e-8) { + numDoF--; + } + if(this.post.xVariance * this.post.count < 1e-8) { + numDoF--; + } + const denomDoF = this.union.count - 2 - numDoF; + + // So... kind of requires at least 6 observations to have a valid test. YAY. + return { + fStat: fStat, + numDoF: numDoF, + denomDoF: denomDoF + }; + } + + /*private*/ get yFTestConfiguration() { + const fStat = this.ySegmentationModeledVariance / this.ySegmentationSSE; + let numDoF = 3; + // Cases where there clearly may as well be 'no slope' at all. + // This saves us a degree of freedom, which is VERY useful for segmenting + // the boundary between a 'hold' and a 'move'. + if(this.pre.yVariance * this.pre.count < 1e-8) { + numDoF--; + } + if(this.post.yVariance * this.post.count < 1e-8) { + numDoF--; + } + const denomDoF = this.union.count - 2 - numDoF; + + // So... kind of requires at least 6 observations to have a valid test. YAY. + return { + fStat: fStat, + numDoF: numDoF, + denomDoF: denomDoF + }; + } + + get segmentationMerited(): boolean { + let totalThreshold = 0; + const xTestConfig = this.xFTestConfiguration; + const yTestConfig = this.yFTestConfiguration; + + totalThreshold += FDistribution.thresholdTier(xTestConfig.fStat, xTestConfig.numDoF, xTestConfig.denomDoF); + totalThreshold += FDistribution.thresholdTier(yTestConfig.fStat, yTestConfig.numDoF, yTestConfig.denomDoF); + + return totalThreshold >= 2; + } + } + + class PotentialSegmentation extends Segmentation { readonly chopPoint: CumulativePathStats; readonly baseChop: CumulativePathStats; constructor(steppedStats: CumulativePathStats[], choppedStats: CumulativePathStats, splitIndex: number) { - this.baseChop = choppedStats; - this.chopPoint = steppedStats[splitIndex]; - - this.pre = steppedStats[splitIndex].deaccumulate(choppedStats); + const pre = steppedStats[splitIndex].deaccumulate(choppedStats); // Keep stats value components based on the final point of the 'pre' segment. const finalStats = steppedStats[steppedStats.length-1]; - this.post = finalStats.deaccumulate(steppedStats[splitIndex-1]); - this.union = finalStats.deaccumulate(choppedStats); - } + const post = finalStats.deaccumulate(steppedStats[splitIndex-1]); + const union = finalStats.deaccumulate(choppedStats); - // Note: circular variance is defined separately from circular std. deviation. - // They _are_ close, though. But which is "best" to use here? - // Even if sourced differently, the best initial guess is to compare variance - // to variance. - // - // Also note that if there is no motion in either segmentation candidate, angleVariance - // is undefined, thus NaN, mathematically. Practically... no angle = no variance. - public get angleVarianceRatio() { - const preVariance = (isNaN(this.pre.angleVariance) ? 0 : this.pre.angleVariance); - const postVariance = (isNaN(this.post.angleVariance) ? 0 : this.post.angleVariance); - - return this.union.angleVariance / (preVariance + postVariance); - } - - public get speedVarianceRatio() { - const splitVariance = this.pre.speedVariance + this.post.speedVariance; - return this.union.speedVariance / splitVariance; - } - - public get splitCriterion() { - /* - * Okay, so this isn't probably quite the most statistically well-founded approach, but... - * - * A "variance ratio" of 1 indicates something of a break-even point; exceeding that threshold - * means that the variations in value seen between the two potential segments are better - * explained as being from two separate segments than from a single segment. (Loosely speaking; - * it'd take some effort to cement the statistical basis here; this is more 'inspired by' what - * the values represent.) - * - * Of course... when it comes to speed, acceleration and such are factors. It'd be all - * too easy to split the slow and fast parts of an accelerating linear motion as two separate - * pieces. Requiring a higher degree of separation alleviates this - hence, the `/2` in the - * condition below. (That divisor's not the most statistically-based thing to do, but it - * works well here.) - * - * So to reach the threshold set below, these three conditions will work: - * - strong difference in angle between the segment candidates - * - moderate difference in both angle and speed between the segment candidates (equal levels) - * - very strong difference in speed between the segment candidates. - */ - - const angleComponent = isNaN(this.angleVarianceRatio) ? 0 : this.angleVarianceRatio; - const speedComponent = isNaN(this.speedVarianceRatio) ? 0 : (this.speedVarianceRatio / 2); - return angleComponent + speedComponent; + super(pre, post, union); + this.baseChop = choppedStats; + this.chopPoint = steppedStats[splitIndex-1]; } } + /* FIXME: Note that this function is a temporary development stopgap and will likely shift + * as development continues for a few reasons: + * 1. In its current form, it'd be better to return a completed `Segment`; this is being used + * to finalize `Segment`s, after all. + * 2. Except... we'll actually want it for uncompleted `Segment`s too, for the tail member of + * the public `path.segments` array, which'll need UPDATING, not replacement. + * 3. In some cases, subsegmentation provides an advantage for constructing / updating `Segment`s. + * E.g: Flicks threshold based on top speed, and the faster subsegment's stats are far better + * for this than the combined interval's stats. + */ + const mergeSubsegmentations = function(array: PotentialSegmentation[]) { + if(array.length == 1) { + return array[0].pre; // It's pre-calculated, so just use it. + } else { + const finalSubsegmentation = array[array.length-1]; + return finalSubsegmentation.chopPoint.deaccumulate(array[0].baseChop); + } + } + export class PathSegmenter { /** * The minimum amount of time (in ms) to wait between sample repetitions @@ -120,6 +329,7 @@ namespace com.keyman.osk { * that lie on already-fully-segmented parts of it. */ private choppedStats: CumulativePathStats = null; + private lastIntervalDuration = 0; constructor() { this.steppedCumulativeStats = []; @@ -183,25 +393,27 @@ namespace com.keyman.osk { private _debugLogSegmentationReport(candidateSplit: PotentialSegmentation) { console.log("------------------------------------------------------------------"); - console.log("Angle variance ratio: " + candidateSplit.angleVarianceRatio); - console.log("Speed variance ratio: " + candidateSplit.speedVarianceRatio); + // console.log("Angle variance ratio: " + candidateSplit.angleVarianceRatio); + // console.log("Speed variance ratio: " + candidateSplit.speedVarianceRatio); console.log("Combined: "); console.log(candidateSplit.union.toJSON()); - console.log(candidateSplit.union); console.log("Pre: ") console.log(candidateSplit.pre.toJSON()); - console.log(candidateSplit.pre); console.log("Post: "); console.log(candidateSplit.post.toJSON()); - console.log(candidateSplit.post); - console.log("Angle variance ratio: " + candidateSplit.angleVarianceRatio); - console.log("Speed variance ratio: " + candidateSplit.speedVarianceRatio); console.log(); - console.log("Prototype segment: "); - console.log(candidateSplit.pre); + const xF = candidateSplit.xFTestConfiguration; + const yF = candidateSplit.yFTestConfiguration; + console.log(`x F-test: F_(${xF.numDoF}, ${xF.denomDoF}) = ${xF.fStat}`); + console.log(`y F-test: F_(${yF.numDoF}, ${yF.denomDoF}) = ${yF.fStat}`); + + console.log(); + + console.log("Split object: "); + console.log(candidateSplit); console.log("------------------------------------------------------------------"); // END: DO NOT RELEASE. @@ -233,57 +445,78 @@ namespace com.keyman.osk { // of both of the resulting intervals. let candidateSplit = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint); - // Run a comparison on various stats of the two. - // Oh. And _there's_ why - if we didn't check before b/c minimum time req't. - let performSegmentation = false; + const lastIntervalDuration = this.lastIntervalDuration; + this.lastIntervalDuration = unsegmentedDuration; - const initialSplitCriterion = candidateSplit.splitCriterion; - if(initialSplitCriterion > PotentialSegmentation.SPLIT_CRITERION_THRESHOLD) { - performSegmentation = true; - } - - if(!performSegmentation) { + if(!candidateSplit.segmentationMerited) { // // Debug logging statements: - console.log("Angle variance ratio: " + candidateSplit.angleVarianceRatio); - console.log("Speed variance ratio: " + candidateSplit.speedVarianceRatio); - return; + // console.log("Angle variance ratio: " + candidateSplit.angleVarianceRatio); + // console.log("Speed variance ratio: " + candidateSplit.speedVarianceRatio); + + const xF = candidateSplit.xFTestConfiguration; + const yF = candidateSplit.yFTestConfiguration; + console.log(`x F-test: F_(${xF.numDoF}, ${xF.denomDoF}) = ${xF.fStat}`); + console.log(`y F-test: F_(${yF.numDoF}, ${yF.denomDoF}) = ${yF.fStat}`); + + console.log("candidate split: " ); + console.log(candidateSplit); + // QUESTION: wait, what if we don't exit early? Does that help 'wait' detection? + // if(lastIntervalDuration >= this.SLIDING_WINDOW_INTERVAL * 2) { + // return; + // } } - // We've met the conditions to trigger segmentation. Now... is there a better segmentation point? + // We either have the conditions to trigger segmentation or just became long enough to consider it. + // If we're only just long enough to consider it, there may be a better segmentation point to start with. + // Now... is there a better segmentation point? // + // TODO: With the new segmented-regression pattern, this is where the coefficients of determination (COD) come in. - let currentSplitCriterion = initialSplitCriterion; - let leftCandidate = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint-1); - let rightCandidate = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint+1); + // let currentXCOD; + // let currentYCOD; + // let leftCandidate = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint-1); + // let rightCandidate: PotentialSegmentation = null; + // if(splitPoint+1 < this.steppedCumulativeStats.length) { + // rightCandidate = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint+1); + // } - const criteria = [leftCandidate.splitCriterion, candidateSplit.splitCriterion, rightCandidate.splitCriterion]; - let sortedCriteria = [...criteria].sort(); + // TODO: both x & y. + // const criteria = [leftCandidate.splitCriterion, candidateSplit.splitCriterion, rightCandidate?.splitCriterion ?? 0]; + // let sortedCriteria = [...criteria].sort(); - const delta = criteria.indexOf(sortedCriteria[2])-1; // -1 if 'left' is best, 1 if 'right' is best. + // TODO: if we're better on both axes on one side, let's start shifting. + // const delta = criteria.indexOf(sortedCriteria[2])-1; // -1 if 'left' is best, 1 if 'right' is best. - if(delta != 0) { - // We can get better segmentation by shifting. Proceed in the optimal direction. - do { - let nextCandidate = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint + delta); - let nextSplitCriterion = nextCandidate.splitCriterion; + // if(delta != 0) { + // // We can get better segmentation by shifting. Proceed in the optimal direction. + // do { + // let nextCandidate = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint + delta); - // Prevent overly-short intervals / over-segmentation. - if(nextCandidate.pre.duration * 1000 < this.SLIDING_WINDOW_INTERVAL / 2) { - break; - } else if(nextCandidate.post.duration * 1000 < this.SLIDING_WINDOW_INTERVAL / 2) { - break; - } + // // Prevent overly-short intervals / over-segmentation. + // if(nextCandidate.pre.duration * 1000 < this.SLIDING_WINDOW_INTERVAL / 2) { + // break; + // } else if(nextCandidate.post.duration * 1000 < this.SLIDING_WINDOW_INTERVAL / 2) { + // break; + // } - // Not an improvement? Guess we found the best spot. - if(nextSplitCriterion < currentSplitCriterion) { - break; - } else { - splitPoint += delta; - currentSplitCriterion = nextSplitCriterion; - candidateSplit = nextCandidate; - } - // If we found a new best segmentation point, we then ask if we can get even better by shifting further. - } while(true); + // // TODO: Determine if it's an improvement. + // // Not an improvement? Guess we found the best spot. + // if(nextSplitCriterion < currentSplitCriterion) { + // break; + // } else { + // splitPoint += delta; + // // TODO: update current 'bests' tracker variables (if still needed) + // candidateSplit = nextCandidate; + // } + // // If we found a new best segmentation point, we then ask if we can get even better by shifting further. + // } while(true); + // } + + // console.log("best split: "); + // console.log(candidateSplit); + + if(!candidateSplit.segmentationMerited) { + return; } // First phase of segmentation: complete! @@ -299,6 +532,7 @@ namespace com.keyman.osk { this.steppedCumulativeStats = this.steppedCumulativeStats.slice(splitPoint); this.choppedStats = candidateSplit.chopPoint; + this.lastIntervalDuration = candidateSplit.post.duration * 1000; this.filterSubsegmentation(candidateSplit); } @@ -306,20 +540,13 @@ namespace com.keyman.osk { // NOTE: This function, as well as code called by it, are still very much still in prototyping. private filterSubsegmentation(subsegmentation: PotentialSegmentation, force?: boolean) { force = !!force; - const mergeSubsegmentations = function(array: PotentialSegmentation[]) { - if(array.length == 1) { - return array[0].pre; // It's pre-calculated, so just use it. - } else { - const finalSubsegmentation = array[array.length-1]; - return finalSubsegmentation.chopPoint.deaccumulate(array[0].baseChop); - } - } if(this.lingeringSubsegmentations.length) { // First: check if the newly-finished subsegment should be merged with the lingering ones. const lastSubsegment = this.lingeringSubsegmentations[this.lingeringSubsegmentations.length-1]; - if(!PathSegmenter.shouldMergeSubsegments(lastSubsegment.pre, subsegmentation.pre)) { + const tailUnion = mergeSubsegmentations([lastSubsegment, subsegmentation]); + if(!PathSegmenter.shouldMergeSubsegments(lastSubsegment.pre, subsegmentation.pre, tailUnion)) { // Emit as separate subsegment. const finishedSegment = mergeSubsegmentations(this.lingeringSubsegmentations); this._protoSegments.push(finishedSegment); @@ -327,27 +554,55 @@ namespace com.keyman.osk { // IN DEVELOPMENT: does this happen much? If so... maybe we need intervening checks to pre-filter even // if not segmenting. + // So far... not _much_, but I've seen it a few times. console.warn("Did not merge a lingering subsegment with the incoming one!"); } else { console.log("Will merge in old subsegments!"); } } - if(!force && PathSegmenter.shouldMergeSubsegments(subsegmentation.pre, subsegmentation.post)) { + if(!force && PathSegmenter.shouldMergeSubsegments(subsegmentation.pre, subsegmentation.post, subsegmentation.union)) { this.lingeringSubsegmentations.push(subsegmentation); } else { // Merge all as a completed segment! const finishedSegment = mergeSubsegmentations([...this.lingeringSubsegmentations, subsegmentation]); + this.lingeringSubsegmentations = []; this._protoSegments.push(finishedSegment); } } - private static shouldMergeSubsegments(segment1: CumulativePathStats, segment2: CumulativePathStats): boolean { - // Known case #1: - // Almost identical direction, but heavy speed variance. - // Speed's still high enough to not be a 'wait'. - // Known case #2: - // Low-speed pivot; angle change caught on very low velocity for both subsegments. + private static shouldMergeSubsegments(segment1: CumulativePathStats, + segment2: CumulativePathStats, + combined: CumulativePathStats): boolean { + // // Known case #1: + // // Near-identical direction, but heavy speed difference. + // // Speed's still high enough to not be a 'wait'. + // // We may want to 'note' the higher speed; that may be relevant for flick detection. + // // Known case #2: + // // Low-speed pivot; angle change caught on very low velocity for both subsegments. + // // ... or should this be merged? Multiple mini-segments from 'wiggling' could just go ignored instead... + + // if(segment1.speedMean < 160) return; // SUPER TEMP: needs further work; avoids the "low-speed pivot" case. + + // const asSegmentation = new Segmentation(segment1, segment2, combined); + + // // If 'speed' is overwhelmingly the cause of segmentation, not angle, maybe don't segment. + // // At breakeven with minimum threshold, we'd have 0.375 vs (2.25 / 2) = 1.5. + // // (As 2.25 / 0.375 = 6.) + // // In practice, this usually happens when the speed ratio is super-high, so angle ratio + // // may still have significance! + // // May need experimental tweaking, but the base principle seems solid. + // if(asSegmentation.criterionCauseRatio > 6 && asSegmentation.union.angleDeviation < Math.PI / 8) { + // // ISSUE: _Heavy_ angle variance when taking an intercardinal slowly. (B/c 'jaggies'.) + // // + // // TODO: unless distance is super-small on one or the other; coming to a rest probably + // // should remain segmented. + + // // console.log("should merge"); + // // console.log(asSegmentation); + // return true; + // } + return false; } } From e4b0ab537b4130674dd69ed239f16c0f7ac5b191 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Thu, 18 Aug 2022 10:36:44 +0700 Subject: [PATCH 09/22] feat(web): better re-merge logic, fixes f-test issues --- .../src/cumulativePathStats.ts | 221 ++++++++---- .../gesture-recognizer/src/pathSegmenter.ts | 315 +++++++++++++++--- 2 files changed, 424 insertions(+), 112 deletions(-) diff --git a/common/web/gesture-recognizer/src/cumulativePathStats.ts b/common/web/gesture-recognizer/src/cumulativePathStats.ts index 2dbeef5b86..c14c30b41b 100644 --- a/common/web/gesture-recognizer/src/cumulativePathStats.ts +++ b/common/web/gesture-recognizer/src/cumulativePathStats.ts @@ -20,8 +20,10 @@ namespace com.keyman.osk { private xtCrossSum: number = 0; private ytCrossSum: number = 0; - // As x and y are both treated as dependent on t, we don't need their cross-sum - // with each other. + // While it's not used to _segment_, it's used within criteria referenced when + // recombining segments same-angle segments that were only split because of + // time-based (i.e, speed) differences. + private yxCrossSum: number = 0; private coordArcSum: number = 0; @@ -47,7 +49,7 @@ namespace com.keyman.osk { */ private initialSample?: InputSample; - private lastSample?: InputSample; + /*private*/ lastSample?: InputSample; private followingSample?: InputSample; private sampleCount = 0; @@ -96,6 +98,7 @@ namespace com.keyman.osk { result.xtCrossSum += x * t; result.ytCrossSum += y * t; + result.yxCrossSum += x * y; result.xQuadSum += x * x; result.yQuadSum += y * y; @@ -179,6 +182,7 @@ namespace com.keyman.osk { result.xtCrossSum -= subsetStats.xtCrossSum; result.ytCrossSum -= subsetStats.ytCrossSum; + result.yxCrossSum -= subsetStats.yxCrossSum; result.xQuadSum -= subsetStats.xQuadSum; result.yQuadSum -= subsetStats.yQuadSum; @@ -278,6 +282,10 @@ namespace com.keyman.osk { return this.ytCrossSum / this.sampleCount - (this.ySampleMean * this.tSampleMean); } + public get yxCovariance() { + return this.yxCrossSum / this.sampleCount - (this.xSampleMean * this.ySampleMean); + } + public get xVariance() { return this.xQuadSum / this.sampleCount - (this.xSampleMean * this.xSampleMean); } @@ -290,39 +298,72 @@ namespace com.keyman.osk { return this.tQuadSum / this.sampleCount - (this.tSampleMean * this.tSampleMean); } - public get xRegressionSlope() { + public get xtRegressionSlope() { return this.xtCovariance / this.tVariance; } - public get yRegressionSlope() { + public get ytRegressionSlope() { return this.ytCovariance / this.tVariance; } - public get xRegressionIntercept() { - return (this.xSampleMean) - this.xRegressionSlope * this.tSampleMean; + public get yxRegressionSlope() { // gets the 'a' of y=ax+b. + return this.yxCovariance / this.xVariance; } - public get yRegressionIntercept() { - return (this.ySampleMean) - this.yRegressionSlope * this.tSampleMean; + public get xyRegressionSlope() { + // xyCovariance and yxCovariance would be identical. + return this.yxCovariance / this.yVariance; } - public get xRegressionSSE() { - return this.sampleCount * (this.xVariance - (this.xRegressionSlope * this.xtCovariance)); + public get xtRegressionIntercept() { + return (this.xSampleMean) - this.xtRegressionSlope * this.tSampleMean; } - public get yRegressionSSE() { - return this.sampleCount * (this.yVariance - (this.yRegressionSlope * this.ytCovariance)); + public get ytRegressionIntercept() { + return (this.ySampleMean) - this.ytRegressionSlope * this.tSampleMean; } - public get xRegressionModeledVariance() { - return this.xRegressionSlope * this.xtCovariance * this.sampleCount; + public get yxRegressionIntercept() { + return (this.ySampleMean) - this.yxRegressionSlope * this.xSampleMean; } - public get yRegressionModeledVariance() { - return this.yRegressionSlope * this.ytCovariance * this.sampleCount; + public get xyRegressionIntercept() { + return (this.xSampleMean) - this.xyRegressionSlope * this.ySampleMean; } - public get xRegressionCOD() { + public get xtRegressionSSE() { + return this.sampleCount * (this.xVariance - (this.xtRegressionSlope * this.xtCovariance)); + } + + public get ytRegressionSSE() { + return this.sampleCount * (this.yVariance - (this.ytRegressionSlope * this.ytCovariance)); + } + + public get yxRegressionSSE() { + return this.sampleCount * (this.yVariance - (this.yxRegressionSlope * this.yxCovariance)); + } + + public get xyRegressionSSE() { + return this.sampleCount * (this.xVariance - (this.xyRegressionSlope * this.yxCovariance)); + } + + public get xtRegressionModeledVariance() { + return this.xtRegressionSlope * this.xtCovariance * this.sampleCount; + } + + public get ytRegressionModeledVariance() { + return this.ytRegressionSlope * this.ytCovariance * this.sampleCount; + } + + public get yxRegressionModeledVariance() { + return this.yxRegressionSlope * this.yxCovariance * this.sampleCount; + } + + public get xyRegressionModeledVariance() { + return this.xyRegressionSlope * this.yxCovariance * this.sampleCount; + } + + public get xtRegressionCOD() { // In truth, the proper answer is NaN (not defined). But for our purposes, // it's a perfect fit, so we'll indicate "perfect fit". if(this.xVariance == 0) { @@ -332,7 +373,7 @@ namespace com.keyman.osk { return this.xtCovariance * this.xtCovariance / (this.xVariance * this.tVariance); } - public get yRegressionCOD() { + public get ytRegressionCOD() { // In truth, the proper answer is NaN (not defined). But for our purposes, // it's a perfect fit, so we'll indicate "perfect fit". if(this.yVariance == 0) { @@ -342,68 +383,118 @@ namespace com.keyman.osk { return this.ytCovariance * this.ytCovariance / (this.yVariance * this.tVariance); } - public get xRegressionFinalE() { - if(!this.lastSample || !this.baseSample) { - return undefined; + public get yxRegressionCOD() { + // In truth, the proper answer is NaN (not defined). But for our purposes, + // it's a perfect fit, so we'll indicate "perfect fit". + if(this.yVariance == 0 || this.xVariance == 0) { + return 1; } - const time = this.lastSample.t - this.baseSample.t; - const adjustedX = this.lastSample.targetX - this.baseSample.targetX; - const error = adjustedX - (this.xRegressionSlope * time + this.xRegressionIntercept); - return error;// * error; + return this.yxCovariance * this.yxCovariance / (this.yVariance * this.xVariance); } - public get xRegressionFinalSE() { - return this.xRegressionFinalE * this.xRegressionFinalE; + public regressionXFitForT(t: number) { + const internalT = t - this.baseSample.t; + return this.xtRegressionIntercept + this.xtRegressionSlope * internalT + this.baseSample.targetX; } - public get yRegressionFinalE() { - if(!this.lastSample || !this.baseSample) { - return undefined; - } - - const time = this.lastSample.t - this.baseSample.t; - const adjustedY = this.lastSample.targetY - this.baseSample.targetY; - const error = adjustedY - (this.yRegressionSlope * time + this.yRegressionIntercept); - return error;// * error; + public regressionYFitForT(t: number) { + const internalT = t - this.baseSample.t; + return this.ytRegressionIntercept + this.ytRegressionSlope * internalT + this.baseSample.targetY; } - public get yRegressionFinalSE() { - return this.yRegressionFinalE * this.yRegressionFinalE; + public regressionXErrorForSampleByT(sample: InputSample) { + const fitX = this.regressionXFitForT(sample.t); + return sample.targetX - fitX; } - public get xRegressionInitialE() { - if(!this.initialSample || !this.baseSample) { - return undefined; - } - - const time = this.initialSample.t - this.baseSample.t; - const adjustedX = this.initialSample.targetX - this.baseSample.targetX; - const error = adjustedX - (this.xRegressionSlope * time + this.xRegressionIntercept); - return error;// * error; + public regressionYErrorForSampleByT(sample: InputSample) { + const fitY = this.regressionYFitForT(sample.t); + return sample.targetY - fitY; } - //public - - public get xRegressionInitialSE() { - return this.xRegressionInitialE * this.xRegressionInitialE; + public regressionXFitForY(y: number) { + const internalY = y - this.baseSample.targetY; + return this.xyRegressionIntercept + this.xyRegressionSlope * internalY + this.baseSample.targetX; } - public get yRegressionInitialE() { - if(!this.initialSample || !this.baseSample) { - return undefined; - } - - const time = this.initialSample.t - this.baseSample.t; - const adjustedY = this.initialSample.targetY - this.baseSample.targetY; - const error = adjustedY - (this.yRegressionSlope * time + this.yRegressionIntercept); - return error;// * error; + public regressionYFitForX(x: number) { + const internalX = x - this.baseSample.targetX; + return this.yxRegressionIntercept + this.yxRegressionSlope * internalX + this.baseSample.targetY; } - public get yRegressionInitialSE() { - return this.yRegressionInitialE * this.yRegressionInitialE; + public regressionXErrorForSampleByY(sample: InputSample) { + const fitX = this.regressionXFitForY(sample.targetY); + return sample.targetX - fitX; } + public regressionYErrorForSampleByX(sample: InputSample) { + const fitY = this.regressionYFitForX(sample.targetX); + return sample.targetY - fitY; + } + + // public get xtRegressionFinalE() { + // if(!this.lastSample || !this.baseSample) { + // return undefined; + // } + + // const time = this.lastSample.t - this.baseSample.t; + // const adjustedX = this.lastSample.targetX - this.baseSample.targetX; + // const error = adjustedX - (this.xtRegressionSlope * time + this.xtRegressionIntercept); + // return error;// * error; + // } + + // public get xtRegressionFinalSE() { + // return this.xtRegressionFinalE * this.xtRegressionFinalE; + // } + + // public get ytRegressionFinalE() { + // if(!this.lastSample || !this.baseSample) { + // return undefined; + // } + + // const time = this.lastSample.t - this.baseSample.t; + // const adjustedY = this.lastSample.targetY - this.baseSample.targetY; + // const error = adjustedY - (this.ytRegressionSlope * time + this.ytRegressionIntercept); + // return error;// * error; + // } + + // public get ytRegressionFinalSE() { + // return this.ytRegressionFinalE * this.ytRegressionFinalE; + // } + + // public get xtRegressionInitialE() { + // if(!this.initialSample || !this.baseSample) { + // return undefined; + // } + + // const time = this.initialSample.t - this.baseSample.t; + // const adjustedX = this.initialSample.targetX - this.baseSample.targetX; + // const error = adjustedX - (this.xtRegressionSlope * time + this.xtRegressionIntercept); + // return error;// * error; + // } + + // //public + + // public get xtRegressionInitialSE() { + // return this.xtRegressionInitialE * this.xtRegressionInitialE; + // } + + // public get ytRegressionInitialE() { + // if(!this.initialSample || !this.baseSample) { + // return undefined; + // } + + // const time = this.initialSample.t - this.baseSample.t; + // const adjustedY = this.initialSample.targetY - this.baseSample.targetY; + // const error = adjustedY - (this.ytRegressionSlope * time + this.ytRegressionIntercept); + // return error;// * error; + // } + + // public get ytRegressionInitialSE() { + // return this.ytRegressionInitialE * this.ytRegressionInitialE; + // } + public get netDistance() { // No issue with a net distance of 0 due to a single point. if(!this.lastSample || !this.initialSample) { @@ -566,13 +657,11 @@ namespace com.keyman.osk { public toJSON() { return { angleMean: this.angleMean, + angleMeanDegrees: this.angleMean * 180 / Math.PI, angleVariance: this.angleVariance, - angleDeviation: this.angleDeviation, speedMean: this.speedMean, speedVariance: this.speedVariance, rawDistance: this.rawDistance, - netDistance: this.netDistance, - distanceFromCentroid: this.maxEndpointDistanceFromCentroid, duration: this.duration, sampleCount: this.sampleCount } diff --git a/common/web/gesture-recognizer/src/pathSegmenter.ts b/common/web/gesture-recognizer/src/pathSegmenter.ts index 74b9681476..12e4413854 100644 --- a/common/web/gesture-recognizer/src/pathSegmenter.ts +++ b/common/web/gesture-recognizer/src/pathSegmenter.ts @@ -1,10 +1,26 @@ /// namespace com.keyman.osk { - // We do NOT want to be computing this thing at run-time. Fortunately, it's very common - // in stats circles to just... use a lookup table for finding p-values for things - // following the f-distribution. + + // https://en.wikipedia.org/wiki/F-distribution + // Mostly used here to compare the sum-squared error components of segmented regressions + // to the sum-squared "modeled" components of their overall variance. Those are the two + // "independent random variables" we're examining. Variances (which are sums of squared + // values themselves) tend to be chi-squared distributed, fulfilling the conditions. + // + // We do NOT want to be computing this thing at run-time. (Just take one look at the + // equations found in that Wikipedia article!) Fortunately, it's very common in stats + // circles to just... use a lookup table for finding p-values for things following + // the f-distribution. + // + // For future non-stats-inclined maintainers: + // One could say that the p-value = the chance that what we have just "randomly" happened + // to occur without our expectations actually being met. It's a super-common + // stats term; they'd then say that if the p-value is sufficiently low, then we\ + // "reject the null hypothesis" - the theory that it actually DID happen randomly - in + // favor of the high likelihood that we really are onto something. class FDistribution { + /** * Indexing: [threshold-index][num-2][denom-1]. * @@ -19,6 +35,9 @@ namespace com.keyman.osk { * - p-value of .100: 0 * - p-value of .050: 1 * + * The f-statistic value must match or exceed its corresponding entry in + * the table below, based on the detected DoF (degrees of freedom) for the + * 'numerator' and 'denominator' components. */ private static readonly table = [ // p = .100 @@ -119,74 +138,133 @@ namespace com.keyman.osk { this.union = union; } - private get xSegmentationSSE() { + private get xtSegmentationSSE() { // Technically double-counts the split point... but we're not preventing two separate predicted // values AT the split point, each with their own error... - const summedSSE = this.pre.xRegressionSSE + this.post.xRegressionSSE/* - this.pre.xRegressionFinalSE*/; + const summedSSE = this.pre.xtRegressionSSE + this.post.xtRegressionSSE/* - this.pre.xRegressionFinalSE*/; // 1e-8: catastrophic cancellation may cause small residuals due to floating-point arithmetic. // Such residuals may be negative (same reason), but a true SSE value never will be. return summedSSE > 1e-8 ? summedSSE : 0; } - private get ySegmentationSSE() { + private get ytSegmentationSSE() { // Technically double-counts the split point... but we're not preventing two separate predicted // values AT the split point, each with their own error... - const summedSSE = this.pre.yRegressionSSE + this.post.yRegressionSSE/* - this.pre.yRegressionFinalSE*/; + const summedSSE = this.pre.ytRegressionSSE + this.post.ytRegressionSSE/* - this.pre.yRegressionFinalSE*/; // 1e-8: catastrophic cancellation may cause small residuals due to floating-point arithmetic. // Such residuals may be negative (same reason), but a true SSE value never will be. return summedSSE > 1e-8 ? summedSSE: 0; } - private get xSegmentationModeledVariance() { + private get yxSegmentationSSE() { // Technically double-counts the split point... but we're not preventing two separate predicted - // values AT the split point. It's even trickier to adjust here than in the SSE properties, - // and it "balances out" (roughly) by existing in both components of the test statistic. - // - // Note: "balances out" is not a formal mathematical declaration. I'm playing things a bit - // loose here in the name of implementation clarity & simplicity. - const summedModeledVar = this.pre.xRegressionModeledVariance + this.post.xRegressionModeledVariance; - return summedModeledVar > 1e-8 ? summedModeledVar : 0; + // values AT the split point, each with their own error... + const summedSSE = this.pre.yxRegressionSSE + this.post.yxRegressionSSE/* - this.pre.yRegressionFinalSE*/; + + // 1e-8: catastrophic cancellation may cause small residuals due to floating-point arithmetic. + // Such residuals may be negative (same reason), but a true SSE value never will be. + return summedSSE > 1e-8 ? summedSSE: 0; } - private get ySegmentationModeledVariance() { + private get xySegmentationSSE() { // Technically double-counts the split point... but we're not preventing two separate predicted - // values AT the split point. It's even trickier to adjust here than in the SSE properties, - // and it "balances out" (roughly) by existing in both components of the test statistic. - // - // Note: "balances out" is not a formal mathematical declaration. I'm playing things a bit - // loose here in the name of implementation clarity & simplicity. - const summedModeledVar = this.pre.yRegressionModeledVariance + this.post.yRegressionModeledVariance; - return summedModeledVar > 1e-8 ? summedModeledVar : 0; - } + // values AT the split point, each with their own error... + const summedSSE = this.pre.xyRegressionSSE + this.post.xyRegressionSSE/* - this.pre.yRegressionFinalSE*/; - public get xSegmentedRegressionCOD() { + // 1e-8: catastrophic cancellation may cause small residuals due to floating-point arithmetic. + // Such residuals may be negative (same reason), but a true SSE value never will be. + return summedSSE > 1e-8 ? summedSSE: 0; + } + + // private get xtSegmentationModeledVariance() { + // // Technically double-counts the split point... but we're not preventing two separate predicted + // // values AT the split point. It's even trickier to adjust here than in the SSE properties, + // // and it "balances out" (roughly) by existing in both components of the test statistic. + // // + // // Note: "balances out" is not a formal mathematical declaration. I'm playing things a bit + // // loose here in the name of implementation clarity & simplicity. + // // const prePart = (this.xtSegmentedRegressionCOD - this.pre.xtRegressionCOD) * this.pre.xVariance * this.pre.count; + // // const postPart = (this.xtSegmentedRegressionCOD - this.post.xtRegressionCOD) * this.post.xVariance * this.post.count; + + // const summedModeledVar = this.pre.xtRegressionModeledVariance + this.post.xtRegressionModeledVariance; + // // const summedModeledVar = prePart + postPart; + // return summedModeledVar > 1e-8 ? summedModeledVar : 0; + // } + + // private get ytSegmentationModeledVariance() { + // // Technically double-counts the split point... but we're not preventing two separate predicted + // // values AT the split point. It's even trickier to adjust here than in the SSE properties, + // // and it "balances out" (roughly) by existing in both components of the test statistic. + // // + // // Note: "balances out" is not a formal mathematical declaration. I'm playing things a bit + // // loose here in the name of implementation clarity & simplicity. + // // const prePart = (this.ytSegmentedRegressionCOD - this.pre.ytRegressionCOD) * this.pre.yVariance * this.pre.count; + // // const postPart = (this.ytSegmentedRegressionCOD - this.post.ytRegressionCOD) * this.post.yVariance * this.post.count; + + // const summedModeledVar = this.pre.ytRegressionModeledVariance + this.post.ytRegressionModeledVariance; + // // const summedModeledVar = prePart + postPart; + // return summedModeledVar > 1e-8 ? summedModeledVar : 0; + // } + + // private get yxSegmentationModeledVariance() { + // // Technically double-counts the split point... but we're not preventing two separate predicted + // // values AT the split point. It's even trickier to adjust here than in the SSE properties, + // // and it "balances out" (roughly) by existing in both components of the test statistic. + // // + // // Note: "balances out" is not a formal mathematical declaration. I'm playing things a bit + // // loose here in the name of implementation clarity & simplicity. + // // const prePart = (this.ytSegmentedRegressionCOD - this.pre.ytRegressionCOD) * this.pre.yVariance * this.pre.count; + // // const postPart = (this.ytSegmentedRegressionCOD - this.post.ytRegressionCOD) * this.post.yVariance * this.post.count; + + // const summedModeledVar = this.pre.yxRegressionModeledVariance + this.post.yxRegressionModeledVariance; + // // const summedModeledVar = prePart + postPart; + // return summedModeledVar > 1e-8 ? summedModeledVar : 0; + // } + + public get xtSegmentedRegressionCOD() { if(!this.union.xVariance) { return 1; } // The point @ the segmentation split point would be double-counted without the .xRegressionFinalSE part. - return 1 - this.xSegmentationSSE / (this.union.xVariance * this.union.count); + return 1 - this.xtSegmentationSSE / (this.union.xVariance * this.union.count); } - public get ySegmentedRegressionCOD() { + public get ytSegmentedRegressionCOD() { if(!this.union.yVariance) { return 1; } // The point @ the segmentation split point would be double-counted without the .yRegressionFinalSE part. - return 1 - this.ySegmentationSSE / (this.union.yVariance * this.union.count); + return 1 - this.ytSegmentationSSE / (this.union.yVariance * this.union.count); } - public get xUnsegmentedRegressionCOD() { - return this.union.xRegressionCOD; + public get yxSegmentedRegressionCOD() { + if(!this.union.yVariance || !this.union.xVariance) { + return 1; + } + // The point @ the segmentation split point would be double-counted without the .yRegressionFinalSE part. + return 1 - this.yxSegmentationSSE / (this.union.yVariance * this.union.count); } - public get yUnsegmentedRegressionCOD() { - return this.union.yRegressionCOD; + public get xtUnsegmentedRegressionCOD() { + return this.union.xtRegressionCOD; } - /*private*/ get xFTestConfiguration() { - const fStat = this.xSegmentationModeledVariance / this.xSegmentationSSE; + public get ytUnsegmentedRegressionCOD() { + return this.union.ytRegressionCOD; + } + + public get yxUnsegmentedRegressionCOD() { + return this.union.yxRegressionCOD; + } + + /*private*/ get xtFTestConfiguration() { + // const fStat = this.xtSegmentationModeledVariance / this.xtSegmentationSSE; + // TODO: Is this right? Seems more reasonable than before, to say the least... + // But I've seen cases where the segmented version gives WORSE? + let doubledError = this.union.regressionXErrorForSampleByT(this.pre.lastSample); + const fStat = (this.union.xtRegressionSSE + doubledError * doubledError - this.xtSegmentationSSE) / this.xtSegmentationSSE; let numDoF = 3; // Cases where there clearly may as well be 'no slope' at all. // This saves us a degree of freedom, which is VERY useful for segmenting @@ -199,6 +277,17 @@ namespace com.keyman.osk { } const denomDoF = this.union.count - 2 - numDoF; + if(!isNaN(fStat) && fStat < 0) { + console.error("F-stat calculation should never be negative: xt!"); + console.error({ + fStat: fStat, + numDoF: numDoF, + denomDoF: denomDoF, + baseObject: this, + doubledErrorSq: doubledError * doubledError + }); + } + // So... kind of requires at least 6 observations to have a valid test. YAY. return { fStat: fStat, @@ -207,8 +296,12 @@ namespace com.keyman.osk { }; } - /*private*/ get yFTestConfiguration() { - const fStat = this.ySegmentationModeledVariance / this.ySegmentationSSE; + /*private*/ get ytFTestConfiguration() { + // const fStat = this.ytSegmentationModeledVariance / this.ytSegmentationSSE; + // TODO: Is this right? Seems more reasonable than before, to say the least... + // But I've seen cases where the segmented version gives WORSE? + let doubledError = this.union.regressionYErrorForSampleByT(this.pre.lastSample); + const fStat = (this.union.ytRegressionSSE + doubledError * doubledError - this.ytSegmentationSSE) / this.ytSegmentationSSE; let numDoF = 3; // Cases where there clearly may as well be 'no slope' at all. // This saves us a degree of freedom, which is VERY useful for segmenting @@ -221,6 +314,95 @@ namespace com.keyman.osk { } const denomDoF = this.union.count - 2 - numDoF; + if(!isNaN(fStat) && fStat < 0) { + console.error("F-stat calculation should never be negative: yt!"); + console.error({ + fStat: fStat, + numDoF: numDoF, + denomDoF: denomDoF, + baseObject: this, + doubledErrorSq: doubledError * doubledError + }); + } + + // So... kind of requires at least 6 observations to have a valid test. YAY. + return { + fStat: fStat, + numDoF: numDoF, + denomDoF: denomDoF + }; + } + + /*private*/ get yxFTestConfiguration() { + // const fStat = this.yxSegmentationModeledVariance / this.yxSegmentationSSE; + // TODO: Is this right? Seems more reasonable than before, to say the least... + // But I've seen cases where the segmented version gives WORSE? (for yx) + let doubledError = this.union.regressionYErrorForSampleByX(this.pre.lastSample); + const fStat = (this.union.yxRegressionSSE + doubledError * doubledError - this.yxSegmentationSSE) / this.yxSegmentationSSE; + let numDoF = 3; + // TODO: Do these cases actually make sense for the xy-case? + // Cases where there clearly may as well be 'no slope' at all. + // This saves us a degree of freedom, which is VERY useful for segmenting + // the boundary between a 'hold' and a 'move'. + if(this.pre.yVariance * this.pre.count < 1e-8) { + numDoF--; + } + if(this.post.yVariance * this.post.count < 1e-8) { + numDoF--; + } + // TODO: proper handling of the non-variant x case? + let denomDoF = this.union.count - 2 - numDoF; + + if(!isNaN(fStat) && fStat < 0) { + console.error("F-stat calculation should never be negative: yx!"); + console.error({ + fStat: fStat, + numDoF: numDoF, + denomDoF: denomDoF, + baseObject: this, + doubledErrorSq: doubledError * doubledError + }); + } + + // So... kind of requires at least 6 observations to have a valid test. YAY. + return { + fStat: fStat, + numDoF: numDoF, + denomDoF: denomDoF + }; + } + + /*private*/ get xyFTestConfiguration() { + // const fStat = this.yxSegmentationModeledVariance / this.yxSegmentationSSE; + // TODO: Is this right? Seems more reasonable than before, to say the least... + let doubledError = this.union.regressionXErrorForSampleByY(this.pre.lastSample); + // But I've seen cases where the segmented version gives WORSE? (for yx) + const fStat = (this.union.xyRegressionSSE + doubledError * doubledError - this.xySegmentationSSE) / this.xySegmentationSSE; + let numDoF = 3; + // TODO: Do these cases actually make sense for the xy-case? + // Cases where there clearly may as well be 'no slope' at all. + // This saves us a degree of freedom, which is VERY useful for segmenting + // the boundary between a 'hold' and a 'move'. + if(this.pre.xVariance * this.pre.count < 1e-8) { + numDoF--; + } + if(this.post.xVariance * this.post.count < 1e-8) { + numDoF--; + } + // TODO: proper handling of the non-variant x case? + let denomDoF = this.union.count - 2 - numDoF; + + if(!isNaN(fStat) && fStat < 0) { + console.error("F-stat calculation should never be negative: yx!"); + console.error({ + fStat: fStat, + numDoF: numDoF, + denomDoF: denomDoF, + baseObject: this, + doubledErrorSq: doubledError * doubledError + }); + } + // So... kind of requires at least 6 observations to have a valid test. YAY. return { fStat: fStat, @@ -231,18 +413,34 @@ namespace com.keyman.osk { get segmentationMerited(): boolean { let totalThreshold = 0; - const xTestConfig = this.xFTestConfiguration; - const yTestConfig = this.yFTestConfiguration; + const xTestConfig = this.xtFTestConfiguration; + const yTestConfig = this.ytFTestConfiguration; totalThreshold += FDistribution.thresholdTier(xTestConfig.fStat, xTestConfig.numDoF, xTestConfig.denomDoF); totalThreshold += FDistribution.thresholdTier(yTestConfig.fStat, yTestConfig.numDoF, yTestConfig.denomDoF); return totalThreshold >= 2; } + + get mergeMerited(): boolean { + // Because of caret-like motions, we need to text for regression on both axes. + // I think? Or does it really make any sort of difference? + const xTestConfig = this.xyFTestConfiguration; + const yTestConfig = this.yxFTestConfiguration; + + // If we don't get a p-value less than .100, then as far as x & y are concerned - and thus the user + // is concerned - it's the same segment. Speed may be different, but not the direction. + if(FDistribution.thresholdTier(yTestConfig.fStat, yTestConfig.numDoF, yTestConfig.denomDoF) != 0) { + return false; + } + + return FDistribution.thresholdTier(xTestConfig.fStat, xTestConfig.numDoF, xTestConfig.denomDoF) == 0; + } } class PotentialSegmentation extends Segmentation { readonly chopPoint: CumulativePathStats; + readonly endOfPre: CumulativePathStats; readonly baseChop: CumulativePathStats; constructor(steppedStats: CumulativePathStats[], @@ -258,6 +456,7 @@ namespace com.keyman.osk { super(pre, post, union); this.baseChop = choppedStats; this.chopPoint = steppedStats[splitIndex-1]; + this.endOfPre = steppedStats[splitIndex]; } } @@ -276,7 +475,7 @@ namespace com.keyman.osk { return array[0].pre; // It's pre-calculated, so just use it. } else { const finalSubsegmentation = array[array.length-1]; - return finalSubsegmentation.chopPoint.deaccumulate(array[0].baseChop); + return finalSubsegmentation.endOfPre.deaccumulate(array[0].baseChop); } } @@ -308,6 +507,7 @@ namespace com.keyman.osk { // algorithm. Just... the stats analysis of the path segment, without // obvious / public members to relevant coordinates. private _protoSegments: CumulativePathStats[] = []; + private _protoSegmentSets: CumulativePathStats[][] = []; /** * Used to 'repeat' the most-recently observed incoming sample if no @@ -366,11 +566,13 @@ namespace com.keyman.osk { // 'fallout' from the implementation's design. let finalization = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, this.steppedCumulativeStats.length-1); + console.log("! Finalization !"); this.filterSubsegmentation(finalization, true); // forces out the final segment. // Hacky, but "enough" for now. // FIXME: temporary statement to facilitate exploration, experimentation, & debugging console.log(this._protoSegments); + console.log(this._protoSegmentSets); console.log(this._protoSegments.map((val) => (val.toJSON()))); } @@ -405,8 +607,8 @@ namespace com.keyman.osk { console.log(); - const xF = candidateSplit.xFTestConfiguration; - const yF = candidateSplit.yFTestConfiguration; + const xF = candidateSplit.xtFTestConfiguration; + const yF = candidateSplit.ytFTestConfiguration; console.log(`x F-test: F_(${xF.numDoF}, ${xF.denomDoF}) = ${xF.fStat}`); console.log(`y F-test: F_(${yF.numDoF}, ${yF.denomDoF}) = ${yF.fStat}`); @@ -453,8 +655,8 @@ namespace com.keyman.osk { // console.log("Angle variance ratio: " + candidateSplit.angleVarianceRatio); // console.log("Speed variance ratio: " + candidateSplit.speedVarianceRatio); - const xF = candidateSplit.xFTestConfiguration; - const yF = candidateSplit.yFTestConfiguration; + const xF = candidateSplit.xtFTestConfiguration; + const yF = candidateSplit.ytFTestConfiguration; console.log(`x F-test: F_(${xF.numDoF}, ${xF.denomDoF}) = ${xF.fStat}`); console.log(`y F-test: F_(${yF.numDoF}, ${yF.denomDoF}) = ${yF.fStat}`); @@ -546,10 +748,13 @@ namespace com.keyman.osk { const lastSubsegment = this.lingeringSubsegmentations[this.lingeringSubsegmentations.length-1]; const tailUnion = mergeSubsegmentations([lastSubsegment, subsegmentation]); + console.log("Lingering segment(s) considered for linking: "); + console.log(this.lingeringSubsegmentations); if(!PathSegmenter.shouldMergeSubsegments(lastSubsegment.pre, subsegmentation.pre, tailUnion)) { // Emit as separate subsegment. const finishedSegment = mergeSubsegmentations(this.lingeringSubsegmentations); this._protoSegments.push(finishedSegment); + this._protoSegmentSets.push(this.lingeringSubsegmentations.map((val) => val.pre)); this.lingeringSubsegmentations = []; // IN DEVELOPMENT: does this happen much? If so... maybe we need intervening checks to pre-filter even @@ -561,6 +766,7 @@ namespace com.keyman.osk { } } + console.log("Double-checking newly split subsegments for xy/yx correlation"); if(!force && PathSegmenter.shouldMergeSubsegments(subsegmentation.pre, subsegmentation.post, subsegmentation.union)) { this.lingeringSubsegmentations.push(subsegmentation); } else { @@ -568,6 +774,7 @@ namespace com.keyman.osk { const finishedSegment = mergeSubsegmentations([...this.lingeringSubsegmentations, subsegmentation]); this.lingeringSubsegmentations = []; this._protoSegments.push(finishedSegment); + this._protoSegmentSets.push([...this.lingeringSubsegmentations.map((val) => val.pre), subsegmentation.pre]); } } @@ -582,9 +789,16 @@ namespace com.keyman.osk { // // Low-speed pivot; angle change caught on very low velocity for both subsegments. // // ... or should this be merged? Multiple mini-segments from 'wiggling' could just go ignored instead... - // if(segment1.speedMean < 160) return; // SUPER TEMP: needs further work; avoids the "low-speed pivot" case. + if(segment1.speedMean < 80 && segment2.speedMean > 80) { + console.log("desegmentation exception"); + return; // SUPER TEMP: needs further work; avoids the "low-speed pivot" case. + } + if(segment1.speedMean > 80 && segment2.speedMean < 80) { + console.log("desegmentation exception"); + return; + } - // const asSegmentation = new Segmentation(segment1, segment2, combined); + const asSegmentation = new Segmentation(segment1, segment2, combined); // // If 'speed' is overwhelmingly the cause of segmentation, not angle, maybe don't segment. // // At breakeven with minimum threshold, we'd have 0.375 vs (2.25 / 2) = 1.5. @@ -603,7 +817,16 @@ namespace com.keyman.osk { // return true; // } - return false; + console.log("Desegmentation under consideration: "); + console.log(asSegmentation); + console.log(`yx CoDs: 1 = ${segment1.yxRegressionCOD}, 2 = ${segment2.yxRegressionCOD} vs 1+2 = ${combined.yxRegressionCOD}`); + const yxFConfig = asSegmentation.yxFTestConfiguration; + const xyFConfig = asSegmentation.xyFTestConfiguration; + console.log(`merger F-test (yx): F_(${yxFConfig.numDoF}, ${yxFConfig.denomDoF}) = ${yxFConfig.fStat}`); + console.log(`merger F-test (xy): F_(${xyFConfig.numDoF}, ${xyFConfig.denomDoF}) = ${xyFConfig.fStat}`); + console.log(`will remerge: ${asSegmentation.mergeMerited}`); + + return asSegmentation.mergeMerited; } } } \ No newline at end of file From 313302e2322520710dc40aeec1bef6fd13d2a723 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 19 Aug 2022 07:18:35 +0700 Subject: [PATCH 10/22] refactor(web): DRYs out regression-related patterns --- .../src/cumulativePathStats.ts | 419 ++++++-------- .../gesture-recognizer/src/pathSegmenter.ts | 513 +++++------------- 2 files changed, 310 insertions(+), 622 deletions(-) diff --git a/common/web/gesture-recognizer/src/cumulativePathStats.ts b/common/web/gesture-recognizer/src/cumulativePathStats.ts index c14c30b41b..39860b15af 100644 --- a/common/web/gesture-recognizer/src/cumulativePathStats.ts +++ b/common/web/gesture-recognizer/src/cumulativePathStats.ts @@ -7,23 +7,84 @@ namespace com.keyman.osk { * Instances of this class are immutable. */ export class CumulativePathStats { - private xLinearSum: number = 0; - private yLinearSum: number = 0; - private tLinearSum: number = 0; + // So... class-level "inner classes" are possible in TS... if defined via assignment to a field. + static readonly regression = class RegressionFromSums { + readonly independent: 'x' | 'y' | 't'; + readonly dependent: 'x' | 'y' | 't'; + readonly paired: 'tx' | 'ty' | 'xy'; + + readonly accumulator: CumulativePathStats; + + constructor(mainStats: CumulativePathStats, dependentAxis: 'x' | 'y' | 't', independentAxis: 'x' | 'y' | 't') { + if(dependentAxis == independentAxis) { + throw "Two different axes must be specified for the regression object."; + } + + this.accumulator = mainStats; + + this.dependent = dependentAxis; + this.independent = independentAxis; + + if(dependentAxis < independentAxis) { + this.paired = dependentAxis.concat(independentAxis) as 'tx' | 'ty' | 'xy'; + } else { + this.paired = independentAxis.concat(dependentAxis) as 'tx' | 'ty' | 'xy'; + } + } + + get slope(): number { + // The technical definition is the commented-out line, but the denominator component of both + // cancels out - it's 'more efficient' to use the following line as a result. + + // this.accumulator.covariance(this.paired) / this.accumulator.variance(this.independent); + const val = this.accumulator.crossSum(this.paired) / this.accumulator.squaredSum(this.independent); + return val; + } + + get intercept(): number { + // Performing a regression based on these pre-summed values means that our obtained intercept is in + // the mapped coordinate system. + const mappedIntercept = this.accumulator.mappedMean(this.dependent) - this.slope * this.accumulator.mappedMean(this.independent); + + const val = mappedIntercept + this.accumulator.mappingConstant(this.dependent) - + this.slope * this.accumulator.mappingConstant(this.independent); + + return val; + } + + get sumOfSquaredError(): number { + return this.accumulator.squaredSum(this.dependent) - this.sumOfSquaredModeled; + } + + get sumOfSquaredModeled(): number { + return this.slope * this.accumulator.crossSum(this.paired); + } + + get coefficientOfDetermination(): number { + if(this.accumulator.squaredSum(this.dependent) == 0 || this.accumulator.squaredSum(this.independent) == 0) { + return 1; + } + + const acc = this.accumulator; + const num = acc.crossSum(this.paired) * acc.crossSum(this.paired); + const denom = acc.squaredSum(this.dependent) * acc.squaredSum(this.independent); + + return num / denom; + } + + predictFromValue(value: number) { + return this.slope * value + this.intercept; + } + } + + private rawLinearSums: {'x': number, 'y': number, 't': number} = {'x': 0, 'y': 0, 't': 0}; private xCentroidSum: number = 0; private yCentroidSum: number = 0; - private xQuadSum: number = 0; - private yQuadSum: number = 0; - private tQuadSum: number = 0; + private rawSquaredSums: {'x': number, 'y': number, 't': number} = {'x': 0, 'y': 0, 't': 0}; - private xtCrossSum: number = 0; - private ytCrossSum: number = 0; - // While it's not used to _segment_, it's used within criteria referenced when - // recombining segments same-angle segments that were only split because of - // time-based (i.e, speed) differences. - private yxCrossSum: number = 0; + private rawCrossSums: {'tx': number, 'ty': number, 'xy': number} = {'tx': 0, 'ty': 0, 'xy': 0}; private coordArcSum: number = 0; @@ -64,6 +125,10 @@ namespace com.keyman.osk { // Will worry about JSON form later. if(obj instanceof CumulativePathStats) { Object.assign(this, obj); + + this.rawLinearSums = {...obj.rawLinearSums}; + this.rawCrossSums = {...obj.rawCrossSums}; + this.rawSquaredSums = {...obj.rawSquaredSums}; } else if(isAnInputSample(obj)) { Object.assign(this, this.extend(obj)); } @@ -92,17 +157,17 @@ namespace com.keyman.osk { const y = sample.targetY - this.baseSample.targetY; const t = sample.t - this.baseSample.t; - result.xLinearSum += x; - result.yLinearSum += y; - result.tLinearSum += t; + result.rawLinearSums['x'] += x; + result.rawLinearSums['y'] += y; + result.rawLinearSums['t'] += t; - result.xtCrossSum += x * t; - result.ytCrossSum += y * t; - result.yxCrossSum += x * y; + result.rawCrossSums['tx'] += t * x; + result.rawCrossSums['ty'] += t * y; + result.rawCrossSums['xy'] += x * y; - result.xQuadSum += x * x; - result.yQuadSum += y * y; - result.tQuadSum += t * t; + result.rawSquaredSums['x'] += x * x; + result.rawSquaredSums['y'] += y * y; + result.rawSquaredSums['t'] += t * t; if(this.lastSample) { // arc length stuff! @@ -176,17 +241,17 @@ namespace com.keyman.osk { throw 'Invalid argument: stats missing necessary tracking variable.'; } - result.xLinearSum -= subsetStats.xLinearSum; - result.yLinearSum -= subsetStats.yLinearSum; - result.tLinearSum -= subsetStats.tLinearSum; + for(let dim in result.rawLinearSums) { + result.rawLinearSums[dim] -= subsetStats.rawLinearSums[dim]; + } - result.xtCrossSum -= subsetStats.xtCrossSum; - result.ytCrossSum -= subsetStats.ytCrossSum; - result.yxCrossSum -= subsetStats.yxCrossSum; + for(let dimPair in result.rawCrossSums) { + result.rawCrossSums[dimPair] -= subsetStats.rawCrossSums[dimPair]; + } - result.xQuadSum -= subsetStats.xQuadSum; - result.yQuadSum -= subsetStats.yQuadSum; - result.tQuadSum -= subsetStats.tQuadSum; + for(let dim in result.rawSquaredSums) { + result.rawSquaredSums[dim] -= subsetStats.rawSquaredSums[dim]; + } // arc length stuff! if(subsetStats.followingSample && subsetStats.lastSample) { @@ -245,16 +310,22 @@ namespace com.keyman.osk { return this.sampleCount; } - private get xSampleMean() { - return this.xLinearSum / this.sampleCount; + private mappingConstant(dim: 'x' | 'y' | 't') { + if(!this.baseSample) { + return undefined; + } + + if(dim == 't') { + return this.baseSample.t; + } else if(dim == 'x') { + return this.baseSample.targetX; + } else { + return this.baseSample.targetY; + } } - private get ySampleMean() { - return this.yLinearSum / this.sampleCount; - } - - private get tSampleMean() { - return this.tLinearSum / this.sampleCount; + private mappedMean(dim: 'x' | 'y' | 't') { + return this.rawLinearSums[dim] / this.sampleCount; } public get centroid(): {x: number, y: number} { @@ -274,227 +345,72 @@ namespace com.keyman.osk { } } - public get xtCovariance() { - return this.xtCrossSum / this.sampleCount - (this.xSampleMean * this.tSampleMean); + public squaredSum(dim: 'x' | 'y' | 't') { + const x2 = this.rawSquaredSums[dim]; + const x1 = this.rawLinearSums[dim]; + + const val = x2 - x1 * x1 / this.sampleCount; + + return val > 1e-8 ? val : 0; } - public get ytCovariance() { - return this.ytCrossSum / this.sampleCount - (this.ySampleMean * this.tSampleMean); - } + public crossSum(dimPair: 'tx' | 'ty' | 'xy') { + const dim1 = dimPair.charAt(0); + const dim2 = dimPair.charAt(1); - public get yxCovariance() { - return this.yxCrossSum / this.sampleCount - (this.xSampleMean * this.ySampleMean); - } - - public get xVariance() { - return this.xQuadSum / this.sampleCount - (this.xSampleMean * this.xSampleMean); - } - - public get yVariance() { - return this.yQuadSum / this.sampleCount - (this.ySampleMean * this.ySampleMean); - } - - public get tVariance() { - return this.tQuadSum / this.sampleCount - (this.tSampleMean * this.tSampleMean); - } - - public get xtRegressionSlope() { - return this.xtCovariance / this.tVariance; - } - - public get ytRegressionSlope() { - return this.ytCovariance / this.tVariance; - } - - public get yxRegressionSlope() { // gets the 'a' of y=ax+b. - return this.yxCovariance / this.xVariance; - } - - public get xyRegressionSlope() { - // xyCovariance and yxCovariance would be identical. - return this.yxCovariance / this.yVariance; - } - - public get xtRegressionIntercept() { - return (this.xSampleMean) - this.xtRegressionSlope * this.tSampleMean; - } - - public get ytRegressionIntercept() { - return (this.ySampleMean) - this.ytRegressionSlope * this.tSampleMean; - } - - public get yxRegressionIntercept() { - return (this.ySampleMean) - this.yxRegressionSlope * this.xSampleMean; - } - - public get xyRegressionIntercept() { - return (this.xSampleMean) - this.xyRegressionSlope * this.ySampleMean; - } - - public get xtRegressionSSE() { - return this.sampleCount * (this.xVariance - (this.xtRegressionSlope * this.xtCovariance)); - } - - public get ytRegressionSSE() { - return this.sampleCount * (this.yVariance - (this.ytRegressionSlope * this.ytCovariance)); - } - - public get yxRegressionSSE() { - return this.sampleCount * (this.yVariance - (this.yxRegressionSlope * this.yxCovariance)); - } - - public get xyRegressionSSE() { - return this.sampleCount * (this.xVariance - (this.xyRegressionSlope * this.yxCovariance)); - } - - public get xtRegressionModeledVariance() { - return this.xtRegressionSlope * this.xtCovariance * this.sampleCount; - } - - public get ytRegressionModeledVariance() { - return this.ytRegressionSlope * this.ytCovariance * this.sampleCount; - } - - public get yxRegressionModeledVariance() { - return this.yxRegressionSlope * this.yxCovariance * this.sampleCount; - } - - public get xyRegressionModeledVariance() { - return this.xyRegressionSlope * this.yxCovariance * this.sampleCount; - } - - public get xtRegressionCOD() { - // In truth, the proper answer is NaN (not defined). But for our purposes, - // it's a perfect fit, so we'll indicate "perfect fit". - if(this.xVariance == 0) { - return 1; + let orderedDims: string = dimPair; + if(dim2 < dim1) { + orderedDims = dim2.concat(dim1); } - return this.xtCovariance * this.xtCovariance / (this.xVariance * this.tVariance); + const ab = this.rawCrossSums[orderedDims]; + const a = this.rawLinearSums[dim1]; + const b = this.rawLinearSums[dim2]; + + const val = ab - a * b / this.sampleCount; + + return val > 1e-8 ? val : 0; } - public get ytRegressionCOD() { - // In truth, the proper answer is NaN (not defined). But for our purposes, - // it's a perfect fit, so we'll indicate "perfect fit". - if(this.yVariance == 0) { - return 1; + public covariance(dimPair: 'tx' | 'ty' | 'xy') { + return this.crossSum(dimPair) / (this.sampleCount - 1); + } + + public variance(dim: 'x' | 'y' | 't') { + return this.squaredSum(dim) / (this.sampleCount - 1); + } + + public buildRenormalized(): CumulativePathStats { + let result = new CumulativePathStats(this); + + // By abusing the statistical identities for calculating various expressions + // related to regression, we can re-center our mapped coordinate system on + // our current mean. We shouldn't do so too frequently, but this should help + // moderate effects from catastrophic cancellation. + + let newBase: InputSample = { + targetX: this.mappedMean['x'] + this.baseSample.targetX, + targetY: this.mappedMean['y'] + this.baseSample.targetY, + t: this.mappedMean['t'] + this.baseSample.t + }; + + result.baseSample = newBase; + + for(const dimPair in result.rawCrossSums) { + result.rawCrossSums[dimPair] = this.crossSum(dimPair as 'tx' | 'ty' | 'xy'); } - return this.ytCovariance * this.ytCovariance / (this.yVariance * this.tVariance); - } - - public get yxRegressionCOD() { - // In truth, the proper answer is NaN (not defined). But for our purposes, - // it's a perfect fit, so we'll indicate "perfect fit". - if(this.yVariance == 0 || this.xVariance == 0) { - return 1; + for(const dim in result.rawSquaredSums) { + result.rawSquaredSums[dim] = this.squaredSum(dim as 'x' | 'y' | 't'); } - return this.yxCovariance * this.yxCovariance / (this.yVariance * this.xVariance); + return result; } - public regressionXFitForT(t: number) { - const internalT = t - this.baseSample.t; - return this.xtRegressionIntercept + this.xtRegressionSlope * internalT + this.baseSample.targetX; + public fitRegression(dependent: 'x' | 'y' | 't', independent: 'x' | 'y' | 't') { + return new CumulativePathStats.regression(this, dependent, independent); } - public regressionYFitForT(t: number) { - const internalT = t - this.baseSample.t; - return this.ytRegressionIntercept + this.ytRegressionSlope * internalT + this.baseSample.targetY; - } - - public regressionXErrorForSampleByT(sample: InputSample) { - const fitX = this.regressionXFitForT(sample.t); - return sample.targetX - fitX; - } - - public regressionYErrorForSampleByT(sample: InputSample) { - const fitY = this.regressionYFitForT(sample.t); - return sample.targetY - fitY; - } - - public regressionXFitForY(y: number) { - const internalY = y - this.baseSample.targetY; - return this.xyRegressionIntercept + this.xyRegressionSlope * internalY + this.baseSample.targetX; - } - - public regressionYFitForX(x: number) { - const internalX = x - this.baseSample.targetX; - return this.yxRegressionIntercept + this.yxRegressionSlope * internalX + this.baseSample.targetY; - } - - public regressionXErrorForSampleByY(sample: InputSample) { - const fitX = this.regressionXFitForY(sample.targetY); - return sample.targetX - fitX; - } - - public regressionYErrorForSampleByX(sample: InputSample) { - const fitY = this.regressionYFitForX(sample.targetX); - return sample.targetY - fitY; - } - - // public get xtRegressionFinalE() { - // if(!this.lastSample || !this.baseSample) { - // return undefined; - // } - - // const time = this.lastSample.t - this.baseSample.t; - // const adjustedX = this.lastSample.targetX - this.baseSample.targetX; - // const error = adjustedX - (this.xtRegressionSlope * time + this.xtRegressionIntercept); - // return error;// * error; - // } - - // public get xtRegressionFinalSE() { - // return this.xtRegressionFinalE * this.xtRegressionFinalE; - // } - - // public get ytRegressionFinalE() { - // if(!this.lastSample || !this.baseSample) { - // return undefined; - // } - - // const time = this.lastSample.t - this.baseSample.t; - // const adjustedY = this.lastSample.targetY - this.baseSample.targetY; - // const error = adjustedY - (this.ytRegressionSlope * time + this.ytRegressionIntercept); - // return error;// * error; - // } - - // public get ytRegressionFinalSE() { - // return this.ytRegressionFinalE * this.ytRegressionFinalE; - // } - - // public get xtRegressionInitialE() { - // if(!this.initialSample || !this.baseSample) { - // return undefined; - // } - - // const time = this.initialSample.t - this.baseSample.t; - // const adjustedX = this.initialSample.targetX - this.baseSample.targetX; - // const error = adjustedX - (this.xtRegressionSlope * time + this.xtRegressionIntercept); - // return error;// * error; - // } - - // //public - - // public get xtRegressionInitialSE() { - // return this.xtRegressionInitialE * this.xtRegressionInitialE; - // } - - // public get ytRegressionInitialE() { - // if(!this.initialSample || !this.baseSample) { - // return undefined; - // } - - // const time = this.initialSample.t - this.baseSample.t; - // const adjustedY = this.initialSample.targetY - this.baseSample.targetY; - // const error = adjustedY - (this.ytRegressionSlope * time + this.ytRegressionIntercept); - // return error;// * error; - // } - - // public get ytRegressionInitialSE() { - // return this.ytRegressionInitialE * this.ytRegressionInitialE; - // } - public get netDistance() { // No issue with a net distance of 0 due to a single point. if(!this.lastSample || !this.initialSample) { @@ -637,21 +553,6 @@ namespace com.keyman.osk { return this.coordArcSum; } - public get maxEndpointDistanceFromCentroid() { - if(!this.initialSample || !this.lastSample) { - return 0; - } - - const centroid = this.centroid; - const startXDist = centroid.x - this.initialSample.targetX; - const startYDist = centroid.y - this.initialSample.targetY; - const endXDist = this.lastSample.targetX - centroid.x; - const endYDist = this.lastSample.targetY - centroid.y; - - return Math.sqrt(Math.max(startXDist * startXDist + startYDist * startYDist, - endXDist * endXDist + endYDist * endYDist)); - } - // TODO: is this actually ideal? This was certainly useful for experimentation via interactive // debugging, but it may not be the best thing long-term. public toJSON() { diff --git a/common/web/gesture-recognizer/src/pathSegmenter.ts b/common/web/gesture-recognizer/src/pathSegmenter.ts index 12e4413854..18bab16f2a 100644 --- a/common/web/gesture-recognizer/src/pathSegmenter.ts +++ b/common/web/gesture-recognizer/src/pathSegmenter.ts @@ -105,7 +105,7 @@ namespace com.keyman.osk { // Latter case: it's currently wrong to statistically test. // The F-distribution is not defined for this case. if(numDoF < 2 || denomDoF < 1) { - return 0; + return 1; } if(numDoF > 3) { @@ -116,12 +116,15 @@ namespace com.keyman.osk { const denomIndex = (denomDoF > 20 ? 20 : denomDoF) - 1; const tier2Threshold = FDistribution.table[1][numIndex][denomIndex]; - if(statistic > tier2Threshold) { - return 2; + if(statistic >= tier2Threshold) { + return 0.05; } const tier1Threshold = FDistribution.table[0][numIndex][denomIndex]; - return statistic > tier1Threshold ? 1 : 0; + // If arising purely randomly isn't at least less than 10% likely, + // we'll categorically say it happened randomly with full certainty (1). + // Obviously not actually true, but it works for our thresholding. + return statistic >= tier1Threshold ? 0.10 : 1; } } @@ -132,292 +135,140 @@ namespace com.keyman.osk { readonly post: CumulativePathStats; readonly union: CumulativePathStats; + private static readonly segmentationComparison = class SegmentedRegression { + host: Segmentation; + readonly independent: 'x' | 'y' | 't'; + readonly dependent: 'x' | 'y' | 't'; + readonly paired: 'tx' | 'ty' | 'xy'; + + pre: typeof CumulativePathStats.regression.prototype; + post: typeof CumulativePathStats.regression.prototype; + union: typeof CumulativePathStats.regression.prototype; + + constructor(host: Segmentation, dependentAxis: 'x' | 'y' | 't', independentAxis: 'x' | 'y' | 't') { + if(dependentAxis == independentAxis) { + throw "Two different axes must be specified for the regression object."; + } + + this.host = host; + + this.dependent = dependentAxis; + this.independent = independentAxis; + + if(dependentAxis < independentAxis) { + this.paired = dependentAxis.concat(independentAxis) as 'tx' | 'ty' | 'xy'; + } else { + this.paired = independentAxis.concat(dependentAxis) as 'tx' | 'ty' | 'xy'; + } + + this.pre = this.host.pre .fitRegression(dependentAxis, independentAxis); + this.post = this.host.post .fitRegression(dependentAxis, independentAxis); + this.union = this.host.union.fitRegression(dependentAxis, independentAxis); + } + + get remainingSumSquaredError(): number { + return this.pre.sumOfSquaredError + this.post.sumOfSquaredError; + } + + private get splitPoint() { + let splitPoint = this.host.pre.lastSample; + + let indep = splitPoint.t; + if(this.independent == 'x') { + indep = splitPoint.targetX; + } else if(this.independent == 'y') { + indep = splitPoint.targetY; + } + + let dep = splitPoint.t; + if(this.dependent == 'x') { + dep = splitPoint.targetX; + } else if(this.dependent == 'y') { + dep = splitPoint.targetY; + } + + return { + dep: dep, + indep: indep + }; + } + + get unsegmentedSumSquaredError(): number { + // What was our remaining error for the unsegmented regression? + let baseSSE = this.union.sumOfSquaredError; + + // We double-count the split point when segmenting, so we should add an + // extra copy of its residual. + const splitPoint = this.splitPoint; + const splitPointError = this.splitPoint.dep - this.union.predictFromValue(splitPoint.indep); + + // This is the total sum-squared error to be handled by segmenting. + return baseSSE + splitPointError * splitPointError; + } + + get sumSquaredGainFromSegmentation(): number { + return this.unsegmentedSumSquaredError - this.remainingSumSquaredError; + } + + get coefficientOfDetermination(): number { + if(this.host.union.squaredSum(this.dependent) == 0 || this.host.union.squaredSum(this.independent) == 0) { + return 1; + } + + return 1 - this.remainingSumSquaredError / this.host.union.squaredSum(this.dependent); + } + + get fStat(): number { + const val = this.sumSquaredGainFromSegmentation / this.remainingSumSquaredError; + + // We're fine with Infinity. Just... not so much NaN. + return isNaN(val) ? 0 : val; + } + + get fDoF1(): number { + let numDoF = 3; + // Cases where there clearly may as well be 'no slope' at all. + // This saves us a degree of freedom, which is VERY useful for segmenting + // the boundary between a 'hold' and a 'move'. + if(this.host.pre.squaredSum(this.dependent) < 1e-8) { + numDoF--; + } + if(this.host.post.squaredSum(this.dependent) < 1e-8) { + numDoF--; + } + + return numDoF; + } + + get fDoF2(): number { + return this.host.union.count - 2 - this.fDoF1; + } + + get certaintyThreshold() { + return 1 - FDistribution.thresholdTier(this.fStat, this.fDoF1, this.fDoF2); + } + } + constructor(pre: CumulativePathStats, post: CumulativePathStats, union: CumulativePathStats) { this.pre = pre; this.post = post; this.union = union; } - private get xtSegmentationSSE() { - // Technically double-counts the split point... but we're not preventing two separate predicted - // values AT the split point, each with their own error... - const summedSSE = this.pre.xtRegressionSSE + this.post.xtRegressionSSE/* - this.pre.xRegressionFinalSE*/; - - // 1e-8: catastrophic cancellation may cause small residuals due to floating-point arithmetic. - // Such residuals may be negative (same reason), but a true SSE value never will be. - return summedSSE > 1e-8 ? summedSSE : 0; - } - - private get ytSegmentationSSE() { - // Technically double-counts the split point... but we're not preventing two separate predicted - // values AT the split point, each with their own error... - const summedSSE = this.pre.ytRegressionSSE + this.post.ytRegressionSSE/* - this.pre.yRegressionFinalSE*/; - - // 1e-8: catastrophic cancellation may cause small residuals due to floating-point arithmetic. - // Such residuals may be negative (same reason), but a true SSE value never will be. - return summedSSE > 1e-8 ? summedSSE: 0; - } - - private get yxSegmentationSSE() { - // Technically double-counts the split point... but we're not preventing two separate predicted - // values AT the split point, each with their own error... - const summedSSE = this.pre.yxRegressionSSE + this.post.yxRegressionSSE/* - this.pre.yRegressionFinalSE*/; - - // 1e-8: catastrophic cancellation may cause small residuals due to floating-point arithmetic. - // Such residuals may be negative (same reason), but a true SSE value never will be. - return summedSSE > 1e-8 ? summedSSE: 0; - } - - private get xySegmentationSSE() { - // Technically double-counts the split point... but we're not preventing two separate predicted - // values AT the split point, each with their own error... - const summedSSE = this.pre.xyRegressionSSE + this.post.xyRegressionSSE/* - this.pre.yRegressionFinalSE*/; - - // 1e-8: catastrophic cancellation may cause small residuals due to floating-point arithmetic. - // Such residuals may be negative (same reason), but a true SSE value never will be. - return summedSSE > 1e-8 ? summedSSE: 0; - } - - // private get xtSegmentationModeledVariance() { - // // Technically double-counts the split point... but we're not preventing two separate predicted - // // values AT the split point. It's even trickier to adjust here than in the SSE properties, - // // and it "balances out" (roughly) by existing in both components of the test statistic. - // // - // // Note: "balances out" is not a formal mathematical declaration. I'm playing things a bit - // // loose here in the name of implementation clarity & simplicity. - // // const prePart = (this.xtSegmentedRegressionCOD - this.pre.xtRegressionCOD) * this.pre.xVariance * this.pre.count; - // // const postPart = (this.xtSegmentedRegressionCOD - this.post.xtRegressionCOD) * this.post.xVariance * this.post.count; - - // const summedModeledVar = this.pre.xtRegressionModeledVariance + this.post.xtRegressionModeledVariance; - // // const summedModeledVar = prePart + postPart; - // return summedModeledVar > 1e-8 ? summedModeledVar : 0; - // } - - // private get ytSegmentationModeledVariance() { - // // Technically double-counts the split point... but we're not preventing two separate predicted - // // values AT the split point. It's even trickier to adjust here than in the SSE properties, - // // and it "balances out" (roughly) by existing in both components of the test statistic. - // // - // // Note: "balances out" is not a formal mathematical declaration. I'm playing things a bit - // // loose here in the name of implementation clarity & simplicity. - // // const prePart = (this.ytSegmentedRegressionCOD - this.pre.ytRegressionCOD) * this.pre.yVariance * this.pre.count; - // // const postPart = (this.ytSegmentedRegressionCOD - this.post.ytRegressionCOD) * this.post.yVariance * this.post.count; - - // const summedModeledVar = this.pre.ytRegressionModeledVariance + this.post.ytRegressionModeledVariance; - // // const summedModeledVar = prePart + postPart; - // return summedModeledVar > 1e-8 ? summedModeledVar : 0; - // } - - // private get yxSegmentationModeledVariance() { - // // Technically double-counts the split point... but we're not preventing two separate predicted - // // values AT the split point. It's even trickier to adjust here than in the SSE properties, - // // and it "balances out" (roughly) by existing in both components of the test statistic. - // // - // // Note: "balances out" is not a formal mathematical declaration. I'm playing things a bit - // // loose here in the name of implementation clarity & simplicity. - // // const prePart = (this.ytSegmentedRegressionCOD - this.pre.ytRegressionCOD) * this.pre.yVariance * this.pre.count; - // // const postPart = (this.ytSegmentedRegressionCOD - this.post.ytRegressionCOD) * this.post.yVariance * this.post.count; - - // const summedModeledVar = this.pre.yxRegressionModeledVariance + this.post.yxRegressionModeledVariance; - // // const summedModeledVar = prePart + postPart; - // return summedModeledVar > 1e-8 ? summedModeledVar : 0; - // } - - public get xtSegmentedRegressionCOD() { - if(!this.union.xVariance) { - return 1; - } - // The point @ the segmentation split point would be double-counted without the .xRegressionFinalSE part. - return 1 - this.xtSegmentationSSE / (this.union.xVariance * this.union.count); - } - - public get ytSegmentedRegressionCOD() { - if(!this.union.yVariance) { - return 1; - } - // The point @ the segmentation split point would be double-counted without the .yRegressionFinalSE part. - return 1 - this.ytSegmentationSSE / (this.union.yVariance * this.union.count); - } - - public get yxSegmentedRegressionCOD() { - if(!this.union.yVariance || !this.union.xVariance) { - return 1; - } - // The point @ the segmentation split point would be double-counted without the .yRegressionFinalSE part. - return 1 - this.yxSegmentationSSE / (this.union.yVariance * this.union.count); - } - - public get xtUnsegmentedRegressionCOD() { - return this.union.xtRegressionCOD; - } - - public get ytUnsegmentedRegressionCOD() { - return this.union.ytRegressionCOD; - } - - public get yxUnsegmentedRegressionCOD() { - return this.union.yxRegressionCOD; - } - - /*private*/ get xtFTestConfiguration() { - // const fStat = this.xtSegmentationModeledVariance / this.xtSegmentationSSE; - // TODO: Is this right? Seems more reasonable than before, to say the least... - // But I've seen cases where the segmented version gives WORSE? - let doubledError = this.union.regressionXErrorForSampleByT(this.pre.lastSample); - const fStat = (this.union.xtRegressionSSE + doubledError * doubledError - this.xtSegmentationSSE) / this.xtSegmentationSSE; - let numDoF = 3; - // Cases where there clearly may as well be 'no slope' at all. - // This saves us a degree of freedom, which is VERY useful for segmenting - // the boundary between a 'hold' and a 'move'. - if(this.pre.xVariance * this.pre.count < 1e-8) { - numDoF--; - } - if(this.post.xVariance * this.post.count < 1e-8) { - numDoF--; - } - const denomDoF = this.union.count - 2 - numDoF; - - if(!isNaN(fStat) && fStat < 0) { - console.error("F-stat calculation should never be negative: xt!"); - console.error({ - fStat: fStat, - numDoF: numDoF, - denomDoF: denomDoF, - baseObject: this, - doubledErrorSq: doubledError * doubledError - }); - } - - // So... kind of requires at least 6 observations to have a valid test. YAY. - return { - fStat: fStat, - numDoF: numDoF, - denomDoF: denomDoF - }; - } - - /*private*/ get ytFTestConfiguration() { - // const fStat = this.ytSegmentationModeledVariance / this.ytSegmentationSSE; - // TODO: Is this right? Seems more reasonable than before, to say the least... - // But I've seen cases where the segmented version gives WORSE? - let doubledError = this.union.regressionYErrorForSampleByT(this.pre.lastSample); - const fStat = (this.union.ytRegressionSSE + doubledError * doubledError - this.ytSegmentationSSE) / this.ytSegmentationSSE; - let numDoF = 3; - // Cases where there clearly may as well be 'no slope' at all. - // This saves us a degree of freedom, which is VERY useful for segmenting - // the boundary between a 'hold' and a 'move'. - if(this.pre.yVariance * this.pre.count < 1e-8) { - numDoF--; - } - if(this.post.yVariance * this.post.count < 1e-8) { - numDoF--; - } - const denomDoF = this.union.count - 2 - numDoF; - - if(!isNaN(fStat) && fStat < 0) { - console.error("F-stat calculation should never be negative: yt!"); - console.error({ - fStat: fStat, - numDoF: numDoF, - denomDoF: denomDoF, - baseObject: this, - doubledErrorSq: doubledError * doubledError - }); - } - - // So... kind of requires at least 6 observations to have a valid test. YAY. - return { - fStat: fStat, - numDoF: numDoF, - denomDoF: denomDoF - }; - } - - /*private*/ get yxFTestConfiguration() { - // const fStat = this.yxSegmentationModeledVariance / this.yxSegmentationSSE; - // TODO: Is this right? Seems more reasonable than before, to say the least... - // But I've seen cases where the segmented version gives WORSE? (for yx) - let doubledError = this.union.regressionYErrorForSampleByX(this.pre.lastSample); - const fStat = (this.union.yxRegressionSSE + doubledError * doubledError - this.yxSegmentationSSE) / this.yxSegmentationSSE; - let numDoF = 3; - // TODO: Do these cases actually make sense for the xy-case? - // Cases where there clearly may as well be 'no slope' at all. - // This saves us a degree of freedom, which is VERY useful for segmenting - // the boundary between a 'hold' and a 'move'. - if(this.pre.yVariance * this.pre.count < 1e-8) { - numDoF--; - } - if(this.post.yVariance * this.post.count < 1e-8) { - numDoF--; - } - // TODO: proper handling of the non-variant x case? - let denomDoF = this.union.count - 2 - numDoF; - - if(!isNaN(fStat) && fStat < 0) { - console.error("F-stat calculation should never be negative: yx!"); - console.error({ - fStat: fStat, - numDoF: numDoF, - denomDoF: denomDoF, - baseObject: this, - doubledErrorSq: doubledError * doubledError - }); - } - - // So... kind of requires at least 6 observations to have a valid test. YAY. - return { - fStat: fStat, - numDoF: numDoF, - denomDoF: denomDoF - }; - } - - /*private*/ get xyFTestConfiguration() { - // const fStat = this.yxSegmentationModeledVariance / this.yxSegmentationSSE; - // TODO: Is this right? Seems more reasonable than before, to say the least... - let doubledError = this.union.regressionXErrorForSampleByY(this.pre.lastSample); - // But I've seen cases where the segmented version gives WORSE? (for yx) - const fStat = (this.union.xyRegressionSSE + doubledError * doubledError - this.xySegmentationSSE) / this.xySegmentationSSE; - let numDoF = 3; - // TODO: Do these cases actually make sense for the xy-case? - // Cases where there clearly may as well be 'no slope' at all. - // This saves us a degree of freedom, which is VERY useful for segmenting - // the boundary between a 'hold' and a 'move'. - if(this.pre.xVariance * this.pre.count < 1e-8) { - numDoF--; - } - if(this.post.xVariance * this.post.count < 1e-8) { - numDoF--; - } - // TODO: proper handling of the non-variant x case? - let denomDoF = this.union.count - 2 - numDoF; - - if(!isNaN(fStat) && fStat < 0) { - console.error("F-stat calculation should never be negative: yx!"); - console.error({ - fStat: fStat, - numDoF: numDoF, - denomDoF: denomDoF, - baseObject: this, - doubledErrorSq: doubledError * doubledError - }); - } - - // So... kind of requires at least 6 observations to have a valid test. YAY. - return { - fStat: fStat, - numDoF: numDoF, - denomDoF: denomDoF - }; + public segReg(dependentAxis: 'x' | 'y' | 't', independentAxis: 'x' | 'y' | 't') { + return new Segmentation.segmentationComparison(this, dependentAxis, independentAxis); } get segmentationMerited(): boolean { let totalThreshold = 0; - const xTestConfig = this.xtFTestConfiguration; - const yTestConfig = this.ytFTestConfiguration; + const xTest = new Segmentation.segmentationComparison(this, 'x', 't'); + const yTest = new Segmentation.segmentationComparison(this, 'y', 't'); - totalThreshold += FDistribution.thresholdTier(xTestConfig.fStat, xTestConfig.numDoF, xTestConfig.denomDoF); - totalThreshold += FDistribution.thresholdTier(yTestConfig.fStat, yTestConfig.numDoF, yTestConfig.denomDoF); + // const xTestConfig = this.xtFTestConfiguration; + // const yTestConfig = this.ytFTestConfiguration; + + totalThreshold += xTest.certaintyThreshold >= 0.95 ? 2 : (xTest.certaintyThreshold >= 0.90 ? 1 : 0) ; + totalThreshold += yTest.certaintyThreshold >= 0.95 ? 2 : (yTest.certaintyThreshold >= 0.90 ? 1 : 0) ; return totalThreshold >= 2; } @@ -425,16 +276,18 @@ namespace com.keyman.osk { get mergeMerited(): boolean { // Because of caret-like motions, we need to text for regression on both axes. // I think? Or does it really make any sort of difference? - const xTestConfig = this.xyFTestConfiguration; - const yTestConfig = this.yxFTestConfiguration; + const xTest = new Segmentation.segmentationComparison(this, 'x', 'y'); + const yTest = new Segmentation.segmentationComparison(this, 'y', 'x'); + // const xTestConfig = this.xyFTestConfiguration; + // const yTestConfig = this.yxFTestConfiguration; // If we don't get a p-value less than .100, then as far as x & y are concerned - and thus the user // is concerned - it's the same segment. Speed may be different, but not the direction. - if(FDistribution.thresholdTier(yTestConfig.fStat, yTestConfig.numDoF, yTestConfig.denomDoF) != 0) { + if(xTest.certaintyThreshold > 0) { return false; } - return FDistribution.thresholdTier(xTestConfig.fStat, xTestConfig.numDoF, xTestConfig.denomDoF) == 0; + return yTest.certaintyThreshold == 0; } } @@ -607,10 +460,10 @@ namespace com.keyman.osk { console.log(); - const xF = candidateSplit.xtFTestConfiguration; - const yF = candidateSplit.ytFTestConfiguration; - console.log(`x F-test: F_(${xF.numDoF}, ${xF.denomDoF}) = ${xF.fStat}`); - console.log(`y F-test: F_(${yF.numDoF}, ${yF.denomDoF}) = ${yF.fStat}`); + const xF = candidateSplit.segReg('x', 't'); + const yF = candidateSplit.segReg('y', 't'); + console.log(`x F-test: F_(${xF.fDoF1}, ${xF.fDoF2}) = ${xF.fStat} @ ${xF.certaintyThreshold}`); + console.log(`y F-test: F_(${yF.fDoF1}, ${yF.fDoF2}) = ${yF.fStat} @ ${yF.certaintyThreshold}`); console.log(); @@ -655,10 +508,10 @@ namespace com.keyman.osk { // console.log("Angle variance ratio: " + candidateSplit.angleVarianceRatio); // console.log("Speed variance ratio: " + candidateSplit.speedVarianceRatio); - const xF = candidateSplit.xtFTestConfiguration; - const yF = candidateSplit.ytFTestConfiguration; - console.log(`x F-test: F_(${xF.numDoF}, ${xF.denomDoF}) = ${xF.fStat}`); - console.log(`y F-test: F_(${yF.numDoF}, ${yF.denomDoF}) = ${yF.fStat}`); + const xF = candidateSplit.segReg('x', 't'); + const yF = candidateSplit.segReg('y', 't'); + console.log(`x F-test: F_(${xF.fDoF1}, ${xF.fDoF2}) = ${xF.fStat} @ ${xF.certaintyThreshold}`); + console.log(`y F-test: F_(${yF.fDoF1}, ${yF.fDoF2}) = ${yF.fStat} @ ${yF.certaintyThreshold}`); console.log("candidate split: " ); console.log(candidateSplit); @@ -668,55 +521,6 @@ namespace com.keyman.osk { // } } - // We either have the conditions to trigger segmentation or just became long enough to consider it. - // If we're only just long enough to consider it, there may be a better segmentation point to start with. - // Now... is there a better segmentation point? - // - // TODO: With the new segmented-regression pattern, this is where the coefficients of determination (COD) come in. - - // let currentXCOD; - // let currentYCOD; - // let leftCandidate = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint-1); - // let rightCandidate: PotentialSegmentation = null; - // if(splitPoint+1 < this.steppedCumulativeStats.length) { - // rightCandidate = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint+1); - // } - - // TODO: both x & y. - // const criteria = [leftCandidate.splitCriterion, candidateSplit.splitCriterion, rightCandidate?.splitCriterion ?? 0]; - // let sortedCriteria = [...criteria].sort(); - - // TODO: if we're better on both axes on one side, let's start shifting. - // const delta = criteria.indexOf(sortedCriteria[2])-1; // -1 if 'left' is best, 1 if 'right' is best. - - // if(delta != 0) { - // // We can get better segmentation by shifting. Proceed in the optimal direction. - // do { - // let nextCandidate = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint + delta); - - // // Prevent overly-short intervals / over-segmentation. - // if(nextCandidate.pre.duration * 1000 < this.SLIDING_WINDOW_INTERVAL / 2) { - // break; - // } else if(nextCandidate.post.duration * 1000 < this.SLIDING_WINDOW_INTERVAL / 2) { - // break; - // } - - // // TODO: Determine if it's an improvement. - // // Not an improvement? Guess we found the best spot. - // if(nextSplitCriterion < currentSplitCriterion) { - // break; - // } else { - // splitPoint += delta; - // // TODO: update current 'bests' tracker variables (if still needed) - // candidateSplit = nextCandidate; - // } - // // If we found a new best segmentation point, we then ask if we can get even better by shifting further. - // } while(true); - // } - - // console.log("best split: "); - // console.log(candidateSplit); - if(!candidateSplit.segmentationMerited) { return; } @@ -772,9 +576,9 @@ namespace com.keyman.osk { } else { // Merge all as a completed segment! const finishedSegment = mergeSubsegmentations([...this.lingeringSubsegmentations, subsegmentation]); - this.lingeringSubsegmentations = []; this._protoSegments.push(finishedSegment); this._protoSegmentSets.push([...this.lingeringSubsegmentations.map((val) => val.pre), subsegmentation.pre]); + this.lingeringSubsegmentations = []; } } @@ -800,30 +604,13 @@ namespace com.keyman.osk { const asSegmentation = new Segmentation(segment1, segment2, combined); - // // If 'speed' is overwhelmingly the cause of segmentation, not angle, maybe don't segment. - // // At breakeven with minimum threshold, we'd have 0.375 vs (2.25 / 2) = 1.5. - // // (As 2.25 / 0.375 = 6.) - // // In practice, this usually happens when the speed ratio is super-high, so angle ratio - // // may still have significance! - // // May need experimental tweaking, but the base principle seems solid. - // if(asSegmentation.criterionCauseRatio > 6 && asSegmentation.union.angleDeviation < Math.PI / 8) { - // // ISSUE: _Heavy_ angle variance when taking an intercardinal slowly. (B/c 'jaggies'.) - // // - // // TODO: unless distance is super-small on one or the other; coming to a rest probably - // // should remain segmented. - - // // console.log("should merge"); - // // console.log(asSegmentation); - // return true; - // } - console.log("Desegmentation under consideration: "); console.log(asSegmentation); - console.log(`yx CoDs: 1 = ${segment1.yxRegressionCOD}, 2 = ${segment2.yxRegressionCOD} vs 1+2 = ${combined.yxRegressionCOD}`); - const yxFConfig = asSegmentation.yxFTestConfiguration; - const xyFConfig = asSegmentation.xyFTestConfiguration; - console.log(`merger F-test (yx): F_(${yxFConfig.numDoF}, ${yxFConfig.denomDoF}) = ${yxFConfig.fStat}`); - console.log(`merger F-test (xy): F_(${xyFConfig.numDoF}, ${xyFConfig.denomDoF}) = ${xyFConfig.fStat}`); + + const xF = asSegmentation.segReg('x', 'y'); + const yF = asSegmentation.segReg('y', 'x'); + console.log(`merger F-test (xy): F_(${xF.fDoF1}, ${xF.fDoF2}) = ${xF.fStat} @ ${xF.certaintyThreshold}`); + console.log(`merger F-test (yx): F_(${yF.fDoF1}, ${yF.fDoF2}) = ${yF.fStat} @ ${yF.certaintyThreshold}`); console.log(`will remerge: ${asSegmentation.mergeMerited}`); return asSegmentation.mergeMerited; From cf31ca1c0f55538518d4ca6ce24a600457ded1a6 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 19 Aug 2022 07:19:38 +0700 Subject: [PATCH 11/22] feat(web): restores segment-split optimization --- .../gesture-recognizer/src/pathSegmenter.ts | 91 ++++++++++++++++++- 1 file changed, 88 insertions(+), 3 deletions(-) diff --git a/common/web/gesture-recognizer/src/pathSegmenter.ts b/common/web/gesture-recognizer/src/pathSegmenter.ts index 18bab16f2a..2f2f509848 100644 --- a/common/web/gesture-recognizer/src/pathSegmenter.ts +++ b/common/web/gesture-recognizer/src/pathSegmenter.ts @@ -135,7 +135,7 @@ namespace com.keyman.osk { readonly post: CumulativePathStats; readonly union: CumulativePathStats; - private static readonly segmentationComparison = class SegmentedRegression { + static readonly segmentationComparison = class SegmentedRegression { host: Segmentation; readonly independent: 'x' | 'y' | 't'; readonly dependent: 'x' | 'y' | 't'; @@ -479,6 +479,7 @@ namespace com.keyman.osk { const unsegmentedDuration = cumulativeStats.lastTimestamp - this.steppedCumulativeStats[0].lastTimestamp; if(unsegmentedDuration < this.SLIDING_WINDOW_INTERVAL * 2) { + console.log("Interval too short for segmentation."); return; } @@ -503,13 +504,13 @@ namespace com.keyman.osk { const lastIntervalDuration = this.lastIntervalDuration; this.lastIntervalDuration = unsegmentedDuration; + const xF = candidateSplit.segReg('x', 't'); + const yF = candidateSplit.segReg('y', 't'); if(!candidateSplit.segmentationMerited) { // // Debug logging statements: // console.log("Angle variance ratio: " + candidateSplit.angleVarianceRatio); // console.log("Speed variance ratio: " + candidateSplit.speedVarianceRatio); - const xF = candidateSplit.segReg('x', 't'); - const yF = candidateSplit.segReg('y', 't'); console.log(`x F-test: F_(${xF.fDoF1}, ${xF.fDoF2}) = ${xF.fStat} @ ${xF.certaintyThreshold}`); console.log(`y F-test: F_(${yF.fDoF1}, ${yF.fDoF2}) = ${yF.fStat} @ ${yF.certaintyThreshold}`); @@ -521,6 +522,90 @@ namespace com.keyman.osk { // } } + // We either have the conditions to trigger segmentation or just became long enough to consider it. + // If we're only just long enough to consider it, there may be a better segmentation point to start with. + // Now... is there a better segmentation point? + + class SplitSearchState { + readonly xTest: typeof Segmentation.segmentationComparison.prototype; + readonly yTest: typeof Segmentation.segmentationComparison.prototype; + readonly candidate: PotentialSegmentation; + + constructor(candidate?: PotentialSegmentation) { + this.candidate = candidate; + + if(candidate == null) { + return; + } + + this.xTest = candidate.segReg('x', 't'); + this.yTest = candidate.segReg('y', 't'); + } + + get segRating() { + if(this.candidate) { + return Math.max(this.xTest.coefficientOfDetermination, this.yTest.coefficientOfDetermination); + } else { + return 0; + } + } + + get segmentationMerited() { + return this.candidate?.segmentationMerited ?? false; + } + } + + let currentSplit = new SplitSearchState(candidateSplit); + + let leftSplit = new SplitSearchState(new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint-1)); + let rightCandidate: PotentialSegmentation = null; + if(splitPoint+1 < this.steppedCumulativeStats.length) { + rightCandidate = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint+1); + } + let rightSplit = new SplitSearchState(rightCandidate); + + const criteria = [leftSplit.segRating, currentSplit.segRating, rightSplit.segRating]; + let sortedCriteria = [...criteria].sort(); + + // TODO: if we're better on both axes on one side, let's start shifting. + const delta = criteria.indexOf(sortedCriteria[2])-1; // -1 if 'left' is best, 1 if 'right' is best. + + if(delta != 0) { + // We can get better segmentation by shifting. Proceed in the optimal direction. + do { + const nextSplitIndex = splitPoint + delta; + if(nextSplitIndex >= this.steppedCumulativeStats.length || nextSplitIndex < 0) { + break; + } + + let nextCandidate = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint + delta); + let nextSplit = new SplitSearchState(nextCandidate); + + // Prevent overly-short intervals / over-segmentation. + if(nextCandidate.pre.duration * 1000 < this.SLIDING_WINDOW_INTERVAL / 2) { + break; + } else if(nextCandidate.post.duration * 1000 < this.SLIDING_WINDOW_INTERVAL / 2) { + break; + } + + // Not an improvement? Guess we found the best spot. + if(nextSplit.segRating <= currentSplit.segRating) { + break; + } else if(!nextSplit.segmentationMerited && currentSplit.segmentationMerited) { + console.warn("aborting split-point relocation due to no longer segmenting"); + break; + } else { + splitPoint += delta; + currentSplit = nextSplit; + candidateSplit = nextCandidate; + } + // If we found a new best segmentation point, we then ask if we can get even better by shifting further. + } while(true); + } + + console.log("best split: "); + console.log(candidateSplit); + if(!candidateSplit.segmentationMerited) { return; } From aa35f945987eb52cf2336805cc37e5b25587f3c6 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 19 Aug 2022 09:24:02 +0700 Subject: [PATCH 12/22] fix(web): forgot to ignore sign on a signficance check --- common/web/gesture-recognizer/src/cumulativePathStats.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/web/gesture-recognizer/src/cumulativePathStats.ts b/common/web/gesture-recognizer/src/cumulativePathStats.ts index 39860b15af..bb923054eb 100644 --- a/common/web/gesture-recognizer/src/cumulativePathStats.ts +++ b/common/web/gesture-recognizer/src/cumulativePathStats.ts @@ -369,7 +369,8 @@ namespace com.keyman.osk { const val = ab - a * b / this.sampleCount; - return val > 1e-8 ? val : 0; + // Don't forget - cross-sums can be negative! + return Math.abs(val) > 1e-8 ? val : 0; } public covariance(dimPair: 'tx' | 'ty' | 'xy') { From c0437036ab942b18a05866ccea803db580db4769 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 19 Aug 2022 10:02:47 +0700 Subject: [PATCH 13/22] fix(web): handling of f-test when a segment has infinite slope --- .../web/gesture-recognizer/src/cumulativePathStats.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/common/web/gesture-recognizer/src/cumulativePathStats.ts b/common/web/gesture-recognizer/src/cumulativePathStats.ts index bb923054eb..6ed22c58fa 100644 --- a/common/web/gesture-recognizer/src/cumulativePathStats.ts +++ b/common/web/gesture-recognizer/src/cumulativePathStats.ts @@ -57,7 +57,15 @@ namespace com.keyman.osk { } get sumOfSquaredModeled(): number { - return this.slope * this.accumulator.crossSum(this.paired); + // If we have a perfectly straight vertical line, from the perspective of our independent axis, + // we get infinite slope. That's... not great for the math. + // + // Fortunately, it ALSO means that we can perfectly model the segment. + if(this.accumulator.squaredSum(this.independent) == 0) { + return this.accumulator.squaredSum(this.dependent); + } else { + return this.slope * this.accumulator.crossSum(this.paired); + } } get coefficientOfDetermination(): number { From 58c15c3eec5fc8c96fa3d6dafaae4238aea5342f Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 19 Aug 2022 10:03:38 +0700 Subject: [PATCH 14/22] feat(web): segment merge checks now use full merge set, not only latest --- .../gesture-recognizer/src/pathSegmenter.ts | 109 +++++++++++------- 1 file changed, 67 insertions(+), 42 deletions(-) diff --git a/common/web/gesture-recognizer/src/pathSegmenter.ts b/common/web/gesture-recognizer/src/pathSegmenter.ts index 2f2f509848..15088134ac 100644 --- a/common/web/gesture-recognizer/src/pathSegmenter.ts +++ b/common/web/gesture-recognizer/src/pathSegmenter.ts @@ -134,6 +134,7 @@ namespace com.keyman.osk { readonly pre: CumulativePathStats; readonly post: CumulativePathStats; readonly union: CumulativePathStats; + readonly endpoint: CumulativePathStats; static readonly segmentationComparison = class SegmentedRegression { host: Segmentation; @@ -249,10 +250,14 @@ namespace com.keyman.osk { } } - constructor(pre: CumulativePathStats, post: CumulativePathStats, union: CumulativePathStats) { + constructor(pre: CumulativePathStats, + post: CumulativePathStats, + union: CumulativePathStats, + cumulativeEndpoint: CumulativePathStats) { this.pre = pre; this.post = post; this.union = union; + this.endpoint = cumulativeEndpoint; } public segReg(dependentAxis: 'x' | 'y' | 't', independentAxis: 'x' | 'y' | 't') { @@ -306,7 +311,7 @@ namespace com.keyman.osk { const post = finalStats.deaccumulate(steppedStats[splitIndex-1]); const union = finalStats.deaccumulate(choppedStats); - super(pre, post, union); + super(pre, post, union, finalStats); this.baseChop = choppedStats; this.chopPoint = steppedStats[splitIndex-1]; this.endOfPre = steppedStats[splitIndex]; @@ -314,23 +319,28 @@ namespace com.keyman.osk { } /* FIXME: Note that this function is a temporary development stopgap and will likely shift - * as development continues for a few reasons: - * 1. In its current form, it'd be better to return a completed `Segment`; this is being used - * to finalize `Segment`s, after all. - * 2. Except... we'll actually want it for uncompleted `Segment`s too, for the tail member of - * the public `path.segments` array, which'll need UPDATING, not replacement. - * 3. In some cases, subsegmentation provides an advantage for constructing / updating `Segment`s. - * E.g: Flicks threshold based on top speed, and the faster subsegment's stats are far better - * for this than the combined interval's stats. - */ - const mergeSubsegmentations = function(array: PotentialSegmentation[]) { - if(array.length == 1) { - return array[0].pre; // It's pre-calculated, so just use it. - } else { - const finalSubsegmentation = array[array.length-1]; - return finalSubsegmentation.endOfPre.deaccumulate(array[0].baseChop); - } + * as development continues for a few reasons: + * 1. In its current form, it'd be better to return a completed `Segment`; this is being used + * to finalize `Segment`s, after all. + * 2. Except... we'll actually want it for uncompleted `Segment`s too, for the tail member of + * the public `path.segments` array, which'll need UPDATING, not replacement. + * 3. In some cases, subsegmentation provides an advantage for constructing / updating `Segment`s. + * E.g: Flicks threshold based on top speed, and the faster subsegment's stats are far better + * for this than the combined interval's stats. + * + * Also worth note: makes a LOT of assumptions about how it will be used; namely, that + * the members of the array originally were contiguous and are in their original + * segmentation order. This holds for current use cases, but this is why the func + * will not be exported. + */ + const mergeSubsegmentations = function(array: PotentialSegmentation[]) { + if(array.length == 1) { + return array[0].pre; // It's pre-calculated, so just use it. + } else { + const finalSubsegmentation = array[array.length-1]; + return finalSubsegmentation.endOfPre.deaccumulate(array[0].baseChop); } + } export class PathSegmenter { /** @@ -592,6 +602,8 @@ namespace com.keyman.osk { if(nextSplit.segRating <= currentSplit.segRating) { break; } else if(!nextSplit.segmentationMerited && currentSplit.segmentationMerited) { + // Note: this can happen if segmentation is triggered due to divergence on both axes + // if one of them drops a threshold tier and the other fails to improve sufficiently. console.warn("aborting split-point relocation due to no longer segmenting"); break; } else { @@ -632,44 +644,59 @@ namespace com.keyman.osk { private filterSubsegmentation(subsegmentation: PotentialSegmentation, force?: boolean) { force = !!force; + let predecessor = subsegmentation.pre; + let firstChopPoint = subsegmentation.baseChop; if(this.lingeringSubsegmentations.length) { // First: check if the newly-finished subsegment should be merged with the lingering ones. - const lastSubsegment = this.lingeringSubsegmentations[this.lingeringSubsegmentations.length-1]; + // const lastSubsegment = this.lingeringSubsegmentations[this.lingeringSubsegmentations.length-1]; + const mergedPrecursors = mergeSubsegmentations(this.lingeringSubsegmentations); - const tailUnion = mergeSubsegmentations([lastSubsegment, subsegmentation]); - console.log("Lingering segment(s) considered for linking: "); + // const tailUnion = mergeSubsegmentations([lastSubsegment, subsegmentation]); + const tailUnion = mergeSubsegmentations([...this.lingeringSubsegmentations, subsegmentation]); + console.log("Verifying linkage to pending merges: "); console.log(this.lingeringSubsegmentations); - if(!PathSegmenter.shouldMergeSubsegments(lastSubsegment.pre, subsegmentation.pre, tailUnion)) { + console.log("verification check:"); + console.log(mergedPrecursors); + const precursorMergeSegmentation = new Segmentation(mergedPrecursors, subsegmentation.pre, tailUnion, subsegmentation.endOfPre); + // Sometimes the start of a harsh turn seems like it's part of the same thing for a moment, but as it continues, + // becomes something VERY different. Validate that we should still merge the left-hand with its predecessors. + if(!PathSegmenter.shouldMergeSubsegments(precursorMergeSegmentation)) { // Emit as separate subsegment. const finishedSegment = mergeSubsegmentations(this.lingeringSubsegmentations); this._protoSegments.push(finishedSegment); this._protoSegmentSets.push(this.lingeringSubsegmentations.map((val) => val.pre)); this.lingeringSubsegmentations = []; - // IN DEVELOPMENT: does this happen much? If so... maybe we need intervening checks to pre-filter even - // if not segmenting. - // So far... not _much_, but I've seen it a few times. - console.warn("Did not merge a lingering subsegment with the incoming one!"); + console.log("Proto-segments length: " + this._protoSegments.length); + + console.log("Did not merge a lingering subsegment with the incoming one!"); } else { + predecessor = tailUnion; + firstChopPoint = this.lingeringSubsegmentations[0].baseChop; console.log("Will merge in old subsegments!"); } } - console.log("Double-checking newly split subsegments for xy/yx correlation"); - if(!force && PathSegmenter.shouldMergeSubsegments(subsegmentation.pre, subsegmentation.post, subsegmentation.union)) { + console.log("Double-checking right-side split subsegment for xy/yx correlation with prior segment candidate(s)"); + const fullUnion = subsegmentation.endpoint.deaccumulate(firstChopPoint); + const fullMergeSegmentation = new Segmentation(predecessor, subsegmentation.post, fullUnion, subsegmentation.endpoint); + console.log("segmentation-prevention check:") + console.log(fullMergeSegmentation); + // if(!force && PathSegmenter.shouldMergeSubsegments(subsegmentation.pre, subsegmentation.post, subsegmentation.union)) { + if(!force && PathSegmenter.shouldMergeSubsegments(fullMergeSegmentation)) { this.lingeringSubsegmentations.push(subsegmentation); } else { // Merge all as a completed segment! - const finishedSegment = mergeSubsegmentations([...this.lingeringSubsegmentations, subsegmentation]); - this._protoSegments.push(finishedSegment); + // const finishedSegment = mergeSubsegmentations([...this.lingeringSubsegmentations, subsegmentation]); + this._protoSegments.push(predecessor); this._protoSegmentSets.push([...this.lingeringSubsegmentations.map((val) => val.pre), subsegmentation.pre]); this.lingeringSubsegmentations = []; + + console.log("Proto-segments length: " + this._protoSegments.length); } } - private static shouldMergeSubsegments(segment1: CumulativePathStats, - segment2: CumulativePathStats, - combined: CumulativePathStats): boolean { + private static shouldMergeSubsegments(segmentation: Segmentation): boolean { // // Known case #1: // // Near-identical direction, but heavy speed difference. // // Speed's still high enough to not be a 'wait'. @@ -678,27 +705,25 @@ namespace com.keyman.osk { // // Low-speed pivot; angle change caught on very low velocity for both subsegments. // // ... or should this be merged? Multiple mini-segments from 'wiggling' could just go ignored instead... - if(segment1.speedMean < 80 && segment2.speedMean > 80) { + if(segmentation.pre.speedMean < 80 && segmentation.post.speedMean > 80) { console.log("desegmentation exception"); return; // SUPER TEMP: needs further work; avoids the "low-speed pivot" case. } - if(segment1.speedMean > 80 && segment2.speedMean < 80) { + if(segmentation.pre.speedMean > 80 && segmentation.post.speedMean < 80) { console.log("desegmentation exception"); return; } - const asSegmentation = new Segmentation(segment1, segment2, combined); - console.log("Desegmentation under consideration: "); - console.log(asSegmentation); + console.log(segmentation); - const xF = asSegmentation.segReg('x', 'y'); - const yF = asSegmentation.segReg('y', 'x'); + const xF = segmentation.segReg('x', 'y'); + const yF = segmentation.segReg('y', 'x'); console.log(`merger F-test (xy): F_(${xF.fDoF1}, ${xF.fDoF2}) = ${xF.fStat} @ ${xF.certaintyThreshold}`); console.log(`merger F-test (yx): F_(${yF.fDoF1}, ${yF.fDoF2}) = ${yF.fStat} @ ${yF.certaintyThreshold}`); - console.log(`will remerge: ${asSegmentation.mergeMerited}`); + console.log(`will remerge: ${segmentation.mergeMerited}`); - return asSegmentation.mergeMerited; + return segmentation.mergeMerited; } } } \ No newline at end of file From cde550a3eb32038bdcfe00a657176ec32616ebb1 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 19 Aug 2022 13:52:56 +0700 Subject: [PATCH 15/22] change(web): lots of docs, some cleanup --- .../src/cumulativePathStats.ts | 320 +++++++++++------ .../gesture-recognizer/src/pathSegmenter.ts | 324 ++++++++++++++---- 2 files changed, 471 insertions(+), 173 deletions(-) diff --git a/common/web/gesture-recognizer/src/cumulativePathStats.ts b/common/web/gesture-recognizer/src/cumulativePathStats.ts index 6ed22c58fa..e43b42d225 100644 --- a/common/web/gesture-recognizer/src/cumulativePathStats.ts +++ b/common/web/gesture-recognizer/src/cumulativePathStats.ts @@ -1,13 +1,16 @@ namespace com.keyman.osk { /** - * As the name suggests, this class exists to track cumulative mathematical values, etc - * necessary to provide statistical information. This information is used to facilitate - * path segmentation. + * As the name suggests, this class facilitates tracking of cumulative mathematical values, etc + * necessary to perform the statistical operations necessary for path segmentation. * * Instances of this class are immutable. */ export class CumulativePathStats { // So... class-level "inner classes" are possible in TS... if defined via assignment to a field. + /** + * Provides linear-regression statistics & fitting values based on the underlying `CumulativePathStats` + * object used to generate it. All operations are O(1). + */ static readonly regression = class RegressionFromSums { readonly independent: 'x' | 'y' | 't'; readonly dependent: 'x' | 'y' | 't'; @@ -15,6 +18,13 @@ namespace com.keyman.osk { readonly accumulator: CumulativePathStats; + /** + * + * @param mainStats The `CumulativePathStats` instance to base all regression data on. + * @param dependentAxis The 'output' axis / dimension; the axis whose behavior should be predicted based on + * existing data of its relationship with the independent axis. + * @param independentAxis The 'input' axis/dimension. + */ constructor(mainStats: CumulativePathStats, dependentAxis: 'x' | 'y' | 't', independentAxis: 'x' | 'y' | 't') { if(dependentAxis == independentAxis) { throw "Two different axes must be specified for the regression object."; @@ -32,6 +42,10 @@ namespace com.keyman.osk { } } + /** + * The 'slope' of the 'slope-intercept' form of the line that best fits the relationship between + * this regression's selected axes. + */ get slope(): number { // The technical definition is the commented-out line, but the denominator component of both // cancels out - it's 'more efficient' to use the following line as a result. @@ -41,6 +55,10 @@ namespace com.keyman.osk { return val; } + /** + * The 'intercept' of the 'slope-intercept' form of the line that best fits the relationship between + * this regression's selected axes. + */ get intercept(): number { // Performing a regression based on these pre-summed values means that our obtained intercept is in // the mapped coordinate system. @@ -52,10 +70,20 @@ namespace com.keyman.osk { return val; } + /** + * The total summed squared-distances of the best fitting line from actually-observed values; + * in other words, the "sum of the squared errors". + * + * Statistically, this is the portion of the dependent variable's variance (un-normalized) + * that is unexplained by this regression. + */ get sumOfSquaredError(): number { return this.accumulator.squaredSum(this.dependent) - this.sumOfSquaredModeled; } + /** + * The portion of the dependent variable's variance that is successfully explained by this regression. + */ get sumOfSquaredModeled(): number { // If we have a perfectly straight vertical line, from the perspective of our independent axis, // we get infinite slope. That's... not great for the math. @@ -68,6 +96,10 @@ namespace com.keyman.osk { } } + /** + * A statistical term that signals how successful the regression is. Always has values on + * the interval [0, 1], with 1 being a perfect fit. + */ get coefficientOfDetermination(): number { if(this.accumulator.squaredSum(this.dependent) == 0 || this.accumulator.squaredSum(this.independent) == 0) { return 1; @@ -80,33 +112,41 @@ namespace com.keyman.osk { return num / denom; } + /** + * Gets the value of the dependent axis that lies on the regression's fitted line + * for a specified independent axis value. + * + * @param value The input value to use for the independent axis's variable. + * @returns The predicted dependent axis value. + */ predictFromValue(value: number) { return this.slope * value + this.intercept; } } - private rawLinearSums: {'x': number, 'y': number, 't': number} = {'x': 0, 'y': 0, 't': 0}; - - private xCentroidSum: number = 0; - private yCentroidSum: number = 0; - - private rawSquaredSums: {'x': number, 'y': number, 't': number} = {'x': 0, 'y': 0, 't': 0}; - + private rawLinearSums: {'x': number, 'y': number, 't': number, 'v': number} = {'x': 0, 'y': 0, 't': 0, 'v': 0}; + private rawSquaredSums: {'x': number, 'y': number, 't': number, 'v': number} = {'x': 0, 'y': 0, 't': 0, 'v': 0}; + // Would 'tv' (time vs velocity) be worth it to track? And possibly even do a regression for? + // If so, maybe throw that in. private rawCrossSums: {'tx': number, 'ty': number, 'xy': number} = {'tx': 0, 'ty': 0, 'xy': 0}; private coordArcSum: number = 0; + private arcSampleCount: number = 0; - private speedLinearSum: number = 0; - private speedQuadSum: number = 0; - + // These two are kept separate because of their extreme interconnectedness - after all, + // they actually represent a SINGLE (polar) axis - the angle. + // + // Sadly, there's no straightforward, well-founded way to use these to give a proper + // statistical sense of 'fit' or 'regression' here, _especially_ in regard to segmentation. + // Proper angle-[other] cross-sums are pretty much impossible, at least as efficiently + // as the others are handled [O(1)]. private cosLinearSum: number = 0; private sinLinearSum: number = 0; - private arcSampleCount: number = 0; /** * The base sample used to transpose all other received samples. Use of this helps - * avoid potential "catastrophic cancellation" effects that can occur when diffing two - * numbers far from the sample-space's mathematical origin. + * avoid potential "catastrophic cancellation" effects that can occur when diffing + * two numbers far from the sample-space's mathematical origin. * * Refer to https://en.wikipedia.org/wiki/Catastrophic_cancellation. */ @@ -118,9 +158,9 @@ namespace com.keyman.osk { */ private initialSample?: InputSample; - /*private*/ lastSample?: InputSample; + private _lastSample?: InputSample; private followingSample?: InputSample; - private sampleCount = 0; + private _sampleCount = 0; constructor(); constructor(sample: InputSample); @@ -189,11 +229,6 @@ namespace com.keyman.osk { result.coordArcSum += Math.sqrt(coordArcDeltaSq); - // Approximates weighting the time spent at each coord by splitting the time since - // last event evenly for both coordinates. Note: does NOT shift based upon .baseSample! - result.xCentroidSum += 0.5 * tDeltaInSec * (sample.targetX + this.lastSample.targetX); - result.yCentroidSum += 0.5 * tDeltaInSec * (sample.targetY + this.lastSample.targetY); - if(xDelta || yDelta) { // We wish to measure angle clockwise from <0, -1> in the DOM. So, cos values should // align with that axis, while sin values should align with the positive x-axis. @@ -206,12 +241,12 @@ namespace com.keyman.osk { } if(tDeltaInSec) { - result.speedLinearSum += Math.sqrt(coordArcDeltaSq) / tDeltaInSec; - result.speedQuadSum += coordArcDeltaSq / (tDeltaInSec * tDeltaInSec); + result.rawLinearSums['v'] += Math.sqrt(coordArcDeltaSq) / tDeltaInSec; + result.rawSquaredSums['v'] += coordArcDeltaSq / (tDeltaInSec * tDeltaInSec); } } - result.lastSample = sample; + result._lastSample = sample; result.sampleCount = this.sampleCount + 1; return result; @@ -225,18 +260,20 @@ namespace com.keyman.osk { * @returns */ public deaccumulate(subsetStats?: CumulativePathStats): CumulativePathStats { - // Possible TODO: Because of the properties of statistical variance & mean... - // we could further prevent catastrophic cancellation by re-centering - // all the linear, cross, and quad sums. - // - mostly noteworthy for _long_ duration touches that wander long distances. - // - basically, for cases that'd cause the floating-point error to exceed our - // test thresholds. Re-centering would keep that error consistently below - // our thresholds. + // Possible addition: use `this.buildRenormalized` on the returned version + // if catastrophic cancellation effects (random, small floating point errors) + // are not sufficiently mitigated & handled by the measures currently in place. // - // We could then take the new mean coordinates as a 'base sample'. - // Kinda has to be the new mean b/c of the stats identities we'd be abusing, - // but that's also the best catastrophic-cancellation prevention move we - // could take. So, this limitation's not really a negative. + // Even then, we'd need to apply such generated objects carefully - we can't + // re-merge the accumulated values or remap them to their old coordinate system + // afterward.`buildRenormalize`'s remapping maneuver is a one-way stats-abuse trick. + // + // Hint: we'd need to pay attention to the "lingering segments" aspects in which + // detected sub-segments might be "re-merged". + // - Whenever they're merged & cleared, we should be clear to recentralize + // the cumulative stats that follow. If any are still active, we can't + // recentralize. + const result = new CumulativePathStats(this); // We actually WILL accept a `null` argument; makes some of the segmentation @@ -276,23 +313,13 @@ namespace com.keyman.osk { result.coordArcSum -= Math.sqrt(coordArcSq); result.coordArcSum -= subsetStats.coordArcSum; - // Centroid sum management! - // Same reasoning pattern as for the 'arc length stuff'. - result.xCentroidSum -= 0.5 * tDeltaInSec * (subsetStats.followingSample.targetX + subsetStats.lastSample.targetX) - result.xCentroidSum -= subsetStats.xCentroidSum; - result.yCentroidSum -= 0.5 * tDeltaInSec * (subsetStats.followingSample.targetY + subsetStats.lastSample.targetY) - result.yCentroidSum -= subsetStats.yCentroidSum; - result.cosLinearSum -= subsetStats.cosLinearSum; result.sinLinearSum -= subsetStats.sinLinearSum; result.arcSampleCount -= subsetStats.arcSampleCount; - result.speedLinearSum -= subsetStats.speedLinearSum; - result.speedQuadSum -= subsetStats.speedQuadSum; - if(tDeltaInSec) { - result.speedLinearSum -= Math.sqrt(coordArcSq) / tDeltaInSec; - result.speedQuadSum -= coordArcSq / (tDeltaInSec * tDeltaInSec); + result.rawLinearSums['v'] -= Math.sqrt(coordArcSq) / tDeltaInSec; + result.rawSquaredSums['v'] -= coordArcSq / (tDeltaInSec * tDeltaInSec); } } @@ -310,15 +337,30 @@ namespace com.keyman.osk { return result; } + public get lastSample() { + return this._lastSample; + } + public get lastTimestamp(): number { return this.lastSample?.t; } - public get count() { - return this.sampleCount; + public get sampleCount() { + return this._sampleCount; } - private mappingConstant(dim: 'x' | 'y' | 't') { + private set sampleCount(value: number) { + this._sampleCount = value; + } + + /** + * In order to mitigate the accumulation of small floating-point errors during the + * various accumulations performed by this class, the domain of incoming values + * is remapped near to the origin via axis-specific mapping constants. + * @param dim + * @returns + */ + private mappingConstant(dim: 'x' | 'y' | 't' | 'v') { if(!this.baseSample) { return undefined; } @@ -327,33 +369,42 @@ namespace com.keyman.osk { return this.baseSample.t; } else if(dim == 'x') { return this.baseSample.targetX; - } else { + } else if(dim == 'y') { return this.baseSample.targetY; + } else { + return 0; } } - private mappedMean(dim: 'x' | 'y' | 't') { + /** + * Gets the statistical mean, utilizing the internal 'mapped' coordinate space. + * This is the version compatible with cross-sums and squared-sums. + * @param dim + * @returns + */ + private mappedMean(dim: 'x' | 'y' | 't' |'v') { return this.rawLinearSums[dim] / this.sampleCount; } - public get centroid(): {x: number, y: number} { - if(this.sampleCount == 0) { - return undefined; - } else if(this.sampleCount == 1) { - return { - x: this.lastSample.targetX, - y: this.lastSample.targetY - }; - } else { - const coeff = 1 / (this.duration); // * (this.sampleCount-1)); - return { - x: this.xCentroidSum * coeff, - y: this.yCentroidSum * coeff - }; - } + /** + * Gets the statistical mean value of the samples observed during the represented + * interval on the specified axis. + * @param dim + * @returns + */ + public mean(dim: 'x' | 'y' | 't' | 'v') { + // This external-facing version needs to provide values in 'external'-friendly + // coordinate space. + return this.mappedMean(dim) + this.mappingConstant(dim); } - public squaredSum(dim: 'x' | 'y' | 't') { + /** + * Gets the sum of the squared distance from the mean seen in samples observed + * during the represented interval on the specified axis. + * @param dim + * @returns + */ + public squaredSum(dim: 'x' | 'y' | 't' | 'v') { const x2 = this.rawSquaredSums[dim]; const x1 = this.rawLinearSums[dim]; @@ -362,6 +413,12 @@ namespace com.keyman.osk { return val > 1e-8 ? val : 0; } + /** + * Gets the sum of the statistical 'cross' term "distance" away from the mean + * observed during the represented interval on the specified axis. + * @param dimPair + * @returns + */ public crossSum(dimPair: 'tx' | 'ty' | 'xy') { const dim1 = dimPair.charAt(0); const dim2 = dimPair.charAt(1); @@ -381,22 +438,39 @@ namespace com.keyman.osk { return Math.abs(val) > 1e-8 ? val : 0; } + /** + * Gets the unbiased covariance between the specified pair of axes for samples + * observed during the represented interval. + * @param dimPair + * @returns + */ public covariance(dimPair: 'tx' | 'ty' | 'xy') { return this.crossSum(dimPair) / (this.sampleCount - 1); } - public variance(dim: 'x' | 'y' | 't') { + /** + * Gets the unbiased variance on the specified axis for samples observed + * during the represented interval. + */ + public variance(dim: 'x' | 'y' | 't' | 'v') { return this.squaredSum(dim) / (this.sampleCount - 1); } + /** + * Utilizing (and possibly abusing) statistical identities, this function produces + * an equivalent, but-recentered copy of this instance's statistical accumulations + * that will be less prone to catastrophic cancellation. + * + * In non-stats speak, the new instance will suffer smaller floating-point + * errors than the old instance whenever they do occur. + * @returns + */ public buildRenormalized(): CumulativePathStats { + // Other (internal) notes: the internal mapping of the new instance will not + // match that of the old instance. This should not affect the practical + // results of any mapping to and from the external coordinate space, however. let result = new CumulativePathStats(this); - // By abusing the statistical identities for calculating various expressions - // related to regression, we can re-center our mapped coordinate system on - // our current mean. We shouldn't do so too frequently, but this should help - // moderate effects from catastrophic cancellation. - let newBase: InputSample = { targetX: this.mappedMean['x'] + this.baseSample.targetX, targetY: this.mappedMean['y'] + this.baseSample.targetY, @@ -416,10 +490,23 @@ namespace com.keyman.osk { return result; } + /** + * Provides a linear-regression perspective on two specified axes over the represented + * interval. + * @param dependent + * @param independent + * @returns + */ public fitRegression(dependent: 'x' | 'y' | 't', independent: 'x' | 'y' | 't') { return new CumulativePathStats.regression(this, dependent, independent); } + /** + * Provides the direct Euclidean distance between the start and end points of the segment + * (or curve) of the interval represented by this instance. + * + * This will likely not match the actual pixel distance traveled. + */ public get netDistance() { // No issue with a net distance of 0 due to a single point. if(!this.lastSample || !this.initialSample) { @@ -432,6 +519,11 @@ namespace com.keyman.osk { return Math.sqrt(xDelta * xDelta + yDelta * yDelta); } + /** + * Gets the duration of the represented interval, in seconds. + * + * Note: input samples provide their timestamps in milliseconds. + */ public get duration() { // no issue with a duration of zero from just one sample. if(!this.lastSample || !this.initialSample) { @@ -459,10 +551,21 @@ namespace com.keyman.osk { return xDelta < 0 ? (2 * Math.PI - yAngleDiff) : yAngleDiff; } + /** + * Returns the angle (in degrees) traveled by the corresponding segment clockwise + * from the unit vector <0, -1> in the DOM (the unit "upward" direction). + */ public get angleInDegrees() { return this.angle * 180 / Math.PI; } + /** + * Returns the cardinal or intercardinal direction on the screen that most + * closely matches the direction of movement represented by the represented + * segment. + * + * @return A string one or two letters in length. (e.g: 'n', 'sw') + */ public get cardinalDirection() { if(this.sampleCount == 1 || !this.lastSample || !this.initialSample) { return undefined; @@ -480,26 +583,22 @@ namespace com.keyman.osk { return 'n'; } - // px per s. + /** + * Measured in pixels per second. + */ public get speed() { // this.duration is already in seconds, not milliseconds. return this.duration ? this.netDistance / this.duration : Number.NaN; } - // ... may not be "right". - public get speedMean() { - return this.speedLinearSum / (this.sampleCount-1); - } - - public get speedVariance() { - return this.speedQuadSum / (this.sampleCount-1) - (this.speedMean * this.speedMean); - } - /** * Returns the represented interval's 'mean angle' clockwise from the DOM's * <0, -1> (the unit vector toward the top of the screen) in radians. * - * Uses the 'circular mean'. Refer to https://en.wikipedia.org/wiki/Circular_mean. + * Based upon the 'circular mean'. Refer to https://en.wikipedia.org/wiki/Circular_mean. + * + * Note that very slow-moving segments may be heavily affected by pixel aliasing + * effects; mouse and touch events usually do not provide sub-pixel resolution. */ public get angleMean() { if(this.arcSampleCount == 0) { @@ -533,22 +632,17 @@ namespace com.keyman.osk { return rSquaredBase / (this.arcSampleCount * this.arcSampleCount); } - /** - * The **circular variance** of the represented interval's angle observations. - * - * Refer to https://en.wikipedia.org/wiki/Directional_statistics#Variance. - */ - public get angleVariance() { - if(this.arcSampleCount == 0) { - return Number.NaN; - } - return 1 - (this.angleRSquared); - } - /** * The **circular standard deviation** of the represented interval's angle observations. * * Refer to https://en.wikipedia.org/wiki/Directional_statistics#Standard_deviation. + * + * Note that very slow-moving segments may be heavily affected by pixel aliasing + * effects; mouse and touch events usually do not provide sub-pixel resolution. + * This can result in very high deviation values. + * + * In less-technical terms - the "stair-stepping" effect seen on high zoom levels means we don't + * get perfectly straight lines, and that can cause this value to be unexpectedly high. */ public get angleDeviation() { if(this.arcSampleCount == 0) { @@ -558,24 +652,34 @@ namespace com.keyman.osk { return Math.sqrt(-Math.log(this.angleRSquared)); } + /** + * Provides the actual, pixel-based distance actually traveled by the represented segment. + * May not be an integer (because diagonals are a thing). + */ public get rawDistance() { return this.coordArcSum; } - // TODO: is this actually ideal? This was certainly useful for experimentation via interactive - // debugging, but it may not be the best thing long-term. - public toJSON() { + // Convert to a `toJSON` method for use during investigative debugging. + private toDebuggingJSON() { return { - angleMean: this.angleMean, - angleMeanDegrees: this.angleMean * 180 / Math.PI, - angleVariance: this.angleVariance, - speedMean: this.speedMean, - speedVariance: this.speedVariance, + angle: this.angle, + speedMean: this.mean('v'), rawDistance: this.rawDistance, duration: this.duration, - sampleCount: this.sampleCount + sampleCount: this.sampleCount, + angleMeanDegrees: this.angleMean * 180 / Math.PI, + angleDeviation: this.angleDeviation, + speedVariance: this.variance('v') } } + + public toJSON() { + // We're not actually saving the JSON out to anything yet or loading/parsing it + // for any use beyond direct human interpretation, so... it's "okay" to + // leave like this for now. + return this.toDebuggingJSON(); + } } } \ No newline at end of file diff --git a/common/web/gesture-recognizer/src/pathSegmenter.ts b/common/web/gesture-recognizer/src/pathSegmenter.ts index 15088134ac..5b34331913 100644 --- a/common/web/gesture-recognizer/src/pathSegmenter.ts +++ b/common/web/gesture-recognizer/src/pathSegmenter.ts @@ -2,11 +2,11 @@ namespace com.keyman.osk { - // https://en.wikipedia.org/wiki/F-distribution // Mostly used here to compare the sum-squared error components of segmented regressions // to the sum-squared "modeled" components of their overall variance. Those are the two // "independent random variables" we're examining. Variances (which are sums of squared // values themselves) tend to be chi-squared distributed, fulfilling the conditions. + // That's stats-speak for "what this module does with it has a solid theoretical basis." // // We do NOT want to be computing this thing at run-time. (Just take one look at the // equations found in that Wikipedia article!) Fortunately, it's very common in stats @@ -19,6 +19,17 @@ namespace com.keyman.osk { // stats term; they'd then say that if the p-value is sufficiently low, then we\ // "reject the null hypothesis" - the theory that it actually DID happen randomly - in // favor of the high likelihood that we really are onto something. + + /** + * Acts as a lookup table for null-hypothesis tests utilizing the F-distribution from the + * domain of statistics. Refer to https://en.wikipedia.org/wiki/F-distribution for details + * on its properties. + * + * This is only a partial implementation; only a couple of numerator degrees-of-freedom are + * worth our consideration, and there's only marginal gain to be found supporting more + * denominator degrees-of-freedom than are already supported here. Keeping it small also + * helps with maintenance and readability. + */ class FDistribution { /** @@ -37,7 +48,11 @@ namespace com.keyman.osk { * * The f-statistic value must match or exceed its corresponding entry in * the table below, based on the detected DoF (degrees of freedom) for the - * 'numerator' and 'denominator' components. + * 'numerator' and 'denominator' components. Exceeding the threshold specified + * for the p = .100 section indicates less than a 10% probability of the statistic + * being tested having arisen by purely random chance. Failure to exceed even + * that implies an unacceptably high chance of being "convenient but meaningless" - + * the "null hypothesis", in stats speak. */ private static readonly table = [ // p = .100 @@ -94,11 +109,9 @@ namespace com.keyman.osk { * @param numDoF * @param denomDoF * - * Tier 0: don't segment - * Tier 1: segment if the other axis also says to segment - * - p-value < 0.100 on the tested axis - * Tier 2: segment regardless of what the other axis says - * - p-value < 0.050 on the tested axis + * Returns 0.050 if the statistic indicates a p-value of 0.050 or lower. + * Failing that, returns 0.100 if a p-value of 0.100 or lower is indicated. + * Otherwise, returns 1.0 - we have no basis to "reject the null hypothesis". */ static thresholdTier(statistic: number, numDoF: number, denomDoF: number) { // Former case: we'd never segment anyway @@ -128,22 +141,61 @@ namespace com.keyman.osk { } } + /** + * Represents the result of a segmentation of the search path and the related + * statistical accumulations needed to properly test the segmentation's + * validity. + */ class Segmentation { public static readonly SPLIT_CRITERION_THRESHOLD = 1.5; + /** + * The properties of the "left-hand" / earlier half of the time interval + * being segmented. + */ readonly pre: CumulativePathStats; + + /** + * The properties of the "right-hand" / later half of the time interval + * being segmented. + */ readonly post: CumulativePathStats; + + /** + * The properties of the full interval being segmented, before the split. + */ readonly union: CumulativePathStats; + + /** + * The full running accumulation for the ongoing touch path. May include + * accumulations from before the interval under examination. + */ readonly endpoint: CumulativePathStats; + /** + * Provides segmented-regression statistics & fitting values on the two specified axes / + * dimensions based on the subsegments and their corresponding linear-regression + * statistics. All operations are O(1). + */ static readonly segmentationComparison = class SegmentedRegression { host: Segmentation; readonly independent: 'x' | 'y' | 't'; readonly dependent: 'x' | 'y' | 't'; readonly paired: 'tx' | 'ty' | 'xy'; + /** + * The linear regression for the underlying `pre` (earlier) subsegment. + */ pre: typeof CumulativePathStats.regression.prototype; + + /** + * The linear regression for the underlying `post` (later) subsegment. + */ post: typeof CumulativePathStats.regression.prototype; + + /** + * The linear, unsegmented regression for the underlying `union` (combined) subsegment. + */ union: typeof CumulativePathStats.regression.prototype; constructor(host: Segmentation, dependentAxis: 'x' | 'y' | 't', independentAxis: 'x' | 'y' | 't') { @@ -167,10 +219,22 @@ namespace com.keyman.osk { this.union = this.host.union.fitRegression(dependentAxis, independentAxis); } + /** + * The total summed squared-distances of the best fitting line from actually-observed values + * for their corresponding subsegment; in other words, the "sum of the squared errors" when + * utilizing segmented regression. + * + * Statistically, this is the portion of the dependent variable's variance (un-normalized) + * that is unexplained even after dividing the interval into separate subsegments. + */ get remainingSumSquaredError(): number { return this.pre.sumOfSquaredError + this.post.sumOfSquaredError; } + /** + * Provides the coordinate that belongs to BOTH subsegments, as it ends one and begins + * the other. + */ private get splitPoint() { let splitPoint = this.host.pre.lastSample; @@ -194,6 +258,11 @@ namespace com.keyman.osk { }; } + /** + * The total summed squared-distances of the best fitting line from actually-observed values + * when **not** utilizing segmented regression. Double-counts the subsegment's common point, as + * it is counted within both subsegments' sum-of-squared error term. + */ get unsegmentedSumSquaredError(): number { // What was our remaining error for the unsegmented regression? let baseSSE = this.union.sumOfSquaredError; @@ -207,10 +276,19 @@ namespace com.keyman.osk { return baseSSE + splitPointError * splitPointError; } + /** + * Gets the amount of (unnormalized) variance not explained by a standard regression on + * the full interval but succesfully explained by splitting the interval in twain via + * segmented regression. + */ get sumSquaredGainFromSegmentation(): number { return this.unsegmentedSumSquaredError - this.remainingSumSquaredError; } + /** + * A statistical term that signals how successful the segmented regression is. Always + * has values on the interval [0, 1], with 1 being a perfect fit. + */ get coefficientOfDetermination(): number { if(this.host.union.squaredSum(this.dependent) == 0 || this.host.union.squaredSum(this.independent) == 0) { return 1; @@ -219,6 +297,11 @@ namespace com.keyman.osk { return 1 - this.remainingSumSquaredError / this.host.union.squaredSum(this.dependent); } + /** + * Gets the raw statistic value needed to test for validity of segmentation based on the axes + * under examination. The higher the proportion of newly-explained variance to still-unexplained + * variance is, the more confident we can _formally_ be in our segmentation. + */ get fStat(): number { const val = this.sumSquaredGainFromSegmentation / this.remainingSumSquaredError; @@ -226,6 +309,10 @@ namespace com.keyman.osk { return isNaN(val) ? 0 : val; } + /** + * Gets the degrees-of-freedom to be used for the numerator DoF parameter of + * the F-distribution needed for validity testing. + */ get fDoF1(): number { let numDoF = 3; // Cases where there clearly may as well be 'no slope' at all. @@ -241,10 +328,21 @@ namespace com.keyman.osk { return numDoF; } + /** + * Gets the degrees-of-freedom to be used for the denominator DoF parameter of + * the F-distribution needed for validity testing. + */ get fDoF2(): number { - return this.host.union.count - 2 - this.fDoF1; + return this.host.union.sampleCount - 2 - this.fDoF1; } + /** + * States a statistically-founded level of confidence we may place in upholding + * the represented segmentation, as based on the regressions and axes under examination. + * + * If we cannot hold any confidence whatsoever (due to being unable to "reject the + * null hypothesis" from this specific regression), will return 0. + */ get certaintyThreshold() { return 1 - FDistribution.thresholdTier(this.fStat, this.fDoF1, this.fDoF2); } @@ -260,10 +358,23 @@ namespace com.keyman.osk { this.endpoint = cumulativeEndpoint; } + /** + * Provides a segmented-regression perspective for the represented interval and proposed + * split point based upon the two specified axes. + * @param dependent + * @param independent + * @returns + */ public segReg(dependentAxis: 'x' | 'y' | 't', independentAxis: 'x' | 'y' | 't') { return new Segmentation.segmentationComparison(this, dependentAxis, independentAxis); } + /** + * Defines our test for upholding a minimum of sub-segmentation based upon changes in the + * interval's coordinates over time. This may occur even with no change in spacial direction + * if there is significant enough change in _speed_, as that "curves the line" when one axis + * is time. + */ get segmentationMerited(): boolean { let totalThreshold = 0; const xTest = new Segmentation.segmentationComparison(this, 'x', 't'); @@ -278,13 +389,19 @@ namespace com.keyman.osk { return totalThreshold >= 2; } + /** + * Defines our test for considering two (or more) sub-segments as part of the same overall + * segment. This seeks to 'link' geometrically-related subsegments - especially those that + * maintain the same direction but differ only in observed speed. + */ get mergeMerited(): boolean { - // Because of caret-like motions, we need to text for regression on both axes. - // I think? Or does it really make any sort of difference? + // Because of caret-like motions (as in, in the '^' shape), we need to text for + // regression on both axes. One may have notably higher variance than the other. + // + // These tests ignore time, and therefore speed. Only the raw geometry of the motion + // matters for this test. const xTest = new Segmentation.segmentationComparison(this, 'x', 'y'); const yTest = new Segmentation.segmentationComparison(this, 'y', 'x'); - // const xTestConfig = this.xyFTestConfiguration; - // const yTestConfig = this.yxFTestConfiguration; // If we don't get a p-value less than .100, then as far as x & y are concerned - and thus the user // is concerned - it's the same segment. Speed may be different, but not the direction. @@ -296,9 +413,32 @@ namespace com.keyman.osk { } } + /** + * A specialized form of the `Segmentation` class that assists in its + * construction from the interval directly under examination by the + * segmentation algorithm. + */ class PotentialSegmentation extends Segmentation { + /** + * The full running accumulation for the ongoing touch path until the point + * before `post` / the later subsegment. May include + * accumulations from before the interval under examination. + */ readonly chopPoint: CumulativePathStats; + + /** + * The full running accumulation for the ongoing touch path until the point + * at the end of `pre` / the earlier subsegment. May include + * accumulations from before the interval under examination, and will + * include accumulations from the first point represented by `post`. + */ readonly endOfPre: CumulativePathStats; + + /** + * The full running accumulation for the ongoing touch path until the point + * before `pre` / the earlier subsegment. Will only include + * accumulations from before the interval under examination. + */ readonly baseChop: CumulativePathStats; constructor(steppedStats: CumulativePathStats[], @@ -318,30 +458,9 @@ namespace com.keyman.osk { } } - /* FIXME: Note that this function is a temporary development stopgap and will likely shift - * as development continues for a few reasons: - * 1. In its current form, it'd be better to return a completed `Segment`; this is being used - * to finalize `Segment`s, after all. - * 2. Except... we'll actually want it for uncompleted `Segment`s too, for the tail member of - * the public `path.segments` array, which'll need UPDATING, not replacement. - * 3. In some cases, subsegmentation provides an advantage for constructing / updating `Segment`s. - * E.g: Flicks threshold based on top speed, and the faster subsegment's stats are far better - * for this than the combined interval's stats. - * - * Also worth note: makes a LOT of assumptions about how it will be used; namely, that - * the members of the array originally were contiguous and are in their original - * segmentation order. This holds for current use cases, but this is why the func - * will not be exported. + /** + * The core logic, algorithm, and manager for touchpath segmentation. */ - const mergeSubsegmentations = function(array: PotentialSegmentation[]) { - if(array.length == 1) { - return array[0].pre; // It's pre-calculated, so just use it. - } else { - const finalSubsegmentation = array[array.length-1]; - return finalSubsegmentation.endOfPre.deaccumulate(array[0].baseChop); - } - } - export class PathSegmenter { /** * The minimum amount of time (in ms) to wait between sample repetitions @@ -363,13 +482,30 @@ namespace com.keyman.osk { */ private steppedCumulativeStats: CumulativePathStats[]; + /** + * Tracks all subsegments awaiting completion of their overall segment. + */ private lingeringSubsegmentations: PotentialSegmentation[]; - // Currently used as an in-development diagnostic assist... but these - // directly represent actual path segments as produced by the prototype - // algorithm. Just... the stats analysis of the path segment, without - // obvious / public members to relevant coordinates. + /** + * TODO: These directly represent path segments produced by the prototype + * algorithm. Just... the stats analysis of the path segment, without + * obvious / public members to relevant coordinates. + * + * In follow-up work, this should be used to build & to update + * cleaner, public-facing segment objects for the touch path. + */ private _protoSegments: CumulativePathStats[] = []; + + /** + * TODO: These directly represent the subsegments comprising their + * corresponding path segments (in `_protoSegments`) produced by the + * prototype algorithm. Just... the stats analysis of each subsegment. + * + * In follow-up work, these should be used internally to build & update + * the internals of TBD cleaner, public-facing segment objects for the + * touch path. + */ private _protoSegmentSets: CumulativePathStats[][] = []; /** @@ -392,13 +528,17 @@ namespace com.keyman.osk { * that lie on already-fully-segmented parts of it. */ private choppedStats: CumulativePathStats = null; - private lastIntervalDuration = 0; constructor() { this.steppedCumulativeStats = []; this.lingeringSubsegmentations = []; } + /** + * Appends a new coordinate to the touch path and also kick-starts a timer for replicating it + * should neither further inputs be received nor termination of the touchpoint be indicated. + * @param sample + */ public add(sample: InputSample) { const repeater = (timeDelta: number) => { this.observe(sample, timeDelta); @@ -418,6 +558,10 @@ namespace com.keyman.osk { repeater(0); } + /** + * Used to finalize all segmentation for the touchpath whenever termination of the corresponding + * touchpoint has been terminated (mouse-up, touch-end). + */ public close() { // The Node clearTimeout & DOM clearTimeout appear to TS as overloads of each other, // and their type definitions will conflict. A simple @ts-ignore will bypass this issue. @@ -433,12 +577,23 @@ namespace com.keyman.osk { this.filterSubsegmentation(finalization, true); // forces out the final segment. // Hacky, but "enough" for now. - // FIXME: temporary statement to facilitate exploration, experimentation, & debugging + // TODO: these are temporary statements to facilitate exploration, experimentation, & debugging. + // We should be providing output to the touchpath object (`.path.segments`). + // But... that'll be left for a follow-up PR. console.log(this._protoSegments); console.log(this._protoSegmentSets); console.log(this._protoSegments.map((val) => (val.toJSON()))); } + /** + * Adds a statistic 'observation' of the state of the touchpath. + * + * This is either to be called directly upon reception of a new input-coordinate or + * upon replication of a prior input should the touchpath still be active without any + * indication of motion. + * @param sample + * @param timeDelta + */ private observe(sample: InputSample, timeDelta: number) { let cumulativeStats: CumulativePathStats; if(this.steppedCumulativeStats.length) { @@ -452,7 +607,7 @@ namespace com.keyman.osk { const extendedStats = cumulativeStats.extend(sample); this.steppedCumulativeStats.push(extendedStats); - this.attemptSegmentation(); + this.performSubsegmentation(); } private _debugLogSegmentationReport(candidateSplit: PotentialSegmentation) { @@ -484,7 +639,17 @@ namespace com.keyman.osk { // END: DO NOT RELEASE. } - private attemptSegmentation() { + // The "reported via event" aspect mentioned below is necessary because this may be + // called via setTimeout callback on a held touchpoint. We need that `setTimeout` + // (which replicates the most recently-seen input coordinate) in order to have good + // sample-data for detecting the boundary between motion and lack thereof during + // segmentation. + /** + * This is the "main method" for touchpath segmentation. Should a new segment result + * from its analysis, it will be reported via event. + * @returns + */ + private performSubsegmentation() { const cumulativeStats = this.steppedCumulativeStats[this.steppedCumulativeStats.length - 1]; const unsegmentedDuration = cumulativeStats.lastTimestamp - this.steppedCumulativeStats[0].lastTimestamp; @@ -511,9 +676,6 @@ namespace com.keyman.osk { // of both of the resulting intervals. let candidateSplit = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint); - const lastIntervalDuration = this.lastIntervalDuration; - this.lastIntervalDuration = unsegmentedDuration; - const xF = candidateSplit.segReg('x', 't'); const yF = candidateSplit.segReg('y', 't'); if(!candidateSplit.segmentationMerited) { @@ -535,7 +697,6 @@ namespace com.keyman.osk { // We either have the conditions to trigger segmentation or just became long enough to consider it. // If we're only just long enough to consider it, there may be a better segmentation point to start with. // Now... is there a better segmentation point? - class SplitSearchState { readonly xTest: typeof Segmentation.segmentationComparison.prototype; readonly yTest: typeof Segmentation.segmentationComparison.prototype; @@ -552,6 +713,8 @@ namespace com.keyman.osk { this.yTest = candidate.segReg('y', 't'); } + // The value for an 'objective function' to optimize in our search for a better candidate. + // The higher this is, the more we like its potential as the segmentation point. get segRating() { if(this.candidate) { return Math.max(this.xTest.coefficientOfDetermination, this.yTest.coefficientOfDetermination); @@ -560,6 +723,8 @@ namespace com.keyman.osk { } } + // But, if it turns out this potential point wouldn't actually result in segmentation, well... + // "abandon ship". get segmentationMerited() { return this.candidate?.segmentationMerited ?? false; } @@ -577,7 +742,6 @@ namespace com.keyman.osk { const criteria = [leftSplit.segRating, currentSplit.segRating, rightSplit.segRating]; let sortedCriteria = [...criteria].sort(); - // TODO: if we're better on both axes on one side, let's start shifting. const delta = criteria.indexOf(sortedCriteria[2])-1; // -1 if 'left' is best, 1 if 'right' is best. if(delta != 0) { @@ -624,9 +788,6 @@ namespace com.keyman.osk { // First phase of segmentation: complete! - // But... there are some cases where we want to prevent segmentation from fully happening. - // TODO: That. - // FIXME: DO NOT RELEASE. // This is exploratory / diagnostic code assisting development of the path segmentation // algorithm. @@ -635,24 +796,45 @@ namespace com.keyman.osk { this.steppedCumulativeStats = this.steppedCumulativeStats.slice(splitPoint); this.choppedStats = candidateSplit.chopPoint; - this.lastIntervalDuration = candidateSplit.post.duration * 1000; + + // There are some cases where we want to prevent segmentation from fully happening. + // The next method will both handle that and the signal of any completed segments that + // may result. this.filterSubsegmentation(candidateSplit); } - // NOTE: This function, as well as code called by it, are still very much still in prototyping. - private filterSubsegmentation(subsegmentation: PotentialSegmentation, force?: boolean) { - force = !!force; + /** + * Given the results of the subsegmentation process, this method analyzes the results + * in order to produce and update the detected segments. + * + * TODO: Should produce actual `Segment` objects and trigger their emission + * them via EventEmitter mechanics. + * @param subsegmentation + * @param finalize Set to `true` for a terminating touchpath, indicating that + * no further subsegmentation will occur for this touchpath. + */ + private filterSubsegmentation(subsegmentation: PotentialSegmentation, finalize?: boolean) { + finalize = !!finalize; + + // This function makes a LOT of assumptions particular to this method and class. + // It is not safe to make this class-level, and it's especially unsafe to make it `public`. + const mergeSubsegmentationAccumulations = function(array: PotentialSegmentation[]) { + if(array.length == 1) { + return array[0].pre; // It's pre-calculated, so just use it. + } else { + const finalSubsegmentation = array[array.length-1]; + return finalSubsegmentation.endOfPre.deaccumulate(array[0].baseChop); + } + } let predecessor = subsegmentation.pre; let firstChopPoint = subsegmentation.baseChop; if(this.lingeringSubsegmentations.length) { // First: check if the newly-finished subsegment should be merged with the lingering ones. - // const lastSubsegment = this.lingeringSubsegmentations[this.lingeringSubsegmentations.length-1]; - const mergedPrecursors = mergeSubsegmentations(this.lingeringSubsegmentations); + const mergedPrecursors = mergeSubsegmentationAccumulations(this.lingeringSubsegmentations); - // const tailUnion = mergeSubsegmentations([lastSubsegment, subsegmentation]); - const tailUnion = mergeSubsegmentations([...this.lingeringSubsegmentations, subsegmentation]); + const tailUnion = mergeSubsegmentationAccumulations([...this.lingeringSubsegmentations, subsegmentation]); console.log("Verifying linkage to pending merges: "); console.log(this.lingeringSubsegmentations); console.log("verification check:"); @@ -662,7 +844,7 @@ namespace com.keyman.osk { // becomes something VERY different. Validate that we should still merge the left-hand with its predecessors. if(!PathSegmenter.shouldMergeSubsegments(precursorMergeSegmentation)) { // Emit as separate subsegment. - const finishedSegment = mergeSubsegmentations(this.lingeringSubsegmentations); + const finishedSegment = mergeSubsegmentationAccumulations(this.lingeringSubsegmentations); this._protoSegments.push(finishedSegment); this._protoSegmentSets.push(this.lingeringSubsegmentations.map((val) => val.pre)); this.lingeringSubsegmentations = []; @@ -683,11 +865,10 @@ namespace com.keyman.osk { console.log("segmentation-prevention check:") console.log(fullMergeSegmentation); // if(!force && PathSegmenter.shouldMergeSubsegments(subsegmentation.pre, subsegmentation.post, subsegmentation.union)) { - if(!force && PathSegmenter.shouldMergeSubsegments(fullMergeSegmentation)) { + if(!finalize && PathSegmenter.shouldMergeSubsegments(fullMergeSegmentation)) { this.lingeringSubsegmentations.push(subsegmentation); } else { // Merge all as a completed segment! - // const finishedSegment = mergeSubsegmentations([...this.lingeringSubsegmentations, subsegmentation]); this._protoSegments.push(predecessor); this._protoSegmentSets.push([...this.lingeringSubsegmentations.map((val) => val.pre), subsegmentation.pre]); this.lingeringSubsegmentations = []; @@ -696,6 +877,14 @@ namespace com.keyman.osk { } } + /** + * Used by the 'filter' step (phase 2 of segmentation) to determine whether or not + * two subsegments should be 'linked' as members of the same segment. (That is, if + * a user would likely consider the two subsegments as belonging to the "same arc + * of motion".) + * @param segmentation + * @returns + */ private static shouldMergeSubsegments(segmentation: Segmentation): boolean { // // Known case #1: // // Near-identical direction, but heavy speed difference. @@ -705,11 +894,16 @@ namespace com.keyman.osk { // // Low-speed pivot; angle change caught on very low velocity for both subsegments. // // ... or should this be merged? Multiple mini-segments from 'wiggling' could just go ignored instead... - if(segmentation.pre.speedMean < 80 && segmentation.post.speedMean > 80) { + // This is kinda plain and arbitrary, but it seems to work "well enough" for now, + // at least at this stage of development. (Most testing was done with Chrome emulation of + // an iPhone SE.) + + // .mean('v') < 80: the mean speed (taken timestamp to timestamp) does not exceed 80px/sec. + if(segmentation.pre.mean('v') < 80 && segmentation.post.mean('v') > 80) { console.log("desegmentation exception"); - return; // SUPER TEMP: needs further work; avoids the "low-speed pivot" case. + return; } - if(segmentation.pre.speedMean > 80 && segmentation.post.speedMean < 80) { + if(segmentation.pre.mean('v') > 80 && segmentation.post.mean('v') < 80) { console.log("desegmentation exception"); return; } From c425a52905df284646b74de586d78d34fbcedae6 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 19 Aug 2022 14:17:39 +0700 Subject: [PATCH 16/22] docs(web): more docs & some new-code renames --- .../gesture-recognizer/src/pathSegmenter.ts | 96 +++++++++++++------ 1 file changed, 65 insertions(+), 31 deletions(-) diff --git a/common/web/gesture-recognizer/src/pathSegmenter.ts b/common/web/gesture-recognizer/src/pathSegmenter.ts index 5b34331913..84fdd66937 100644 --- a/common/web/gesture-recognizer/src/pathSegmenter.ts +++ b/common/web/gesture-recognizer/src/pathSegmenter.ts @@ -1,6 +1,18 @@ /// +/* FIXME: Too much `console`. + * + * There's an AWFUL LOT of `console.log`-ing in this file at present. + * It was definitely useful for prototyping & development of its (and + * of `cumulativePathStats`'s) code, but it's GOTTA be cleaned up + * in the next PR... if not sooner. + */ + namespace com.keyman.osk { + // Note: the only `export`-ed class from this file is `PathSegmenter`, hence the helper + // classes being placed within the same file. Not to say we _couldn't_ put them elsewhere. + + // ------------ // Mostly used here to compare the sum-squared error components of segmented regressions // to the sum-squared "modeled" components of their overall variance. Those are the two @@ -540,10 +552,13 @@ namespace com.keyman.osk { * @param sample */ public add(sample: InputSample) { + // Set up the input-repeater (in case we don't get further feedback but remain active) const repeater = (timeDelta: number) => { this.observe(sample, timeDelta); } + // If we previously set up an input-repeater, cancel it. We've got a more up-to-date + // coordinate on the touchpath now. if(this.repeatTimer) { // @ts-ignore clearInterval(this.repeatTimer); @@ -566,16 +581,16 @@ namespace com.keyman.osk { // The Node clearTimeout & DOM clearTimeout appear to TS as overloads of each other, // and their type definitions will conflict. A simple @ts-ignore will bypass this issue. // @ts-ignore - clearInterval(this.repeatTimer); + clearInterval(this.repeatTimer); // Cancels the input-repeater. this.repeatTimer = null; - // The way things are structured, finalization.pre = final segment. It's some happy - // 'fallout' from the implementation's design. + // The way things are structured, finalization.pre = final segment. It's a bit hacky, but + // it _is_ happy, intentional 'fallout' from the implementation's design. `post` is effectively + // a single-point 'subsegment' and will go effectively unutilized by the phase 2 subsegment linker. let finalization = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, this.steppedCumulativeStats.length-1); console.log("! Finalization !"); - this.filterSubsegmentation(finalization, true); // forces out the final segment. - // Hacky, but "enough" for now. + this.processSubsegmentation(finalization, true); // TODO: these are temporary statements to facilitate exploration, experimentation, & debugging. // We should be providing output to the touchpath object (`.path.segments`). @@ -613,9 +628,6 @@ namespace com.keyman.osk { private _debugLogSegmentationReport(candidateSplit: PotentialSegmentation) { console.log("------------------------------------------------------------------"); - // console.log("Angle variance ratio: " + candidateSplit.angleVarianceRatio); - // console.log("Speed variance ratio: " + candidateSplit.speedVarianceRatio); - console.log("Combined: "); console.log(candidateSplit.union.toJSON()); console.log("Pre: ") @@ -653,18 +665,16 @@ namespace com.keyman.osk { const cumulativeStats = this.steppedCumulativeStats[this.steppedCumulativeStats.length - 1]; const unsegmentedDuration = cumulativeStats.lastTimestamp - this.steppedCumulativeStats[0].lastTimestamp; + // STEP 1: Determine the range of the sliding time window for the most recent samples. + if(unsegmentedDuration < this.SLIDING_WINDOW_INTERVAL * 2) { console.log("Interval too short for segmentation."); return; } let splitPoint = 0; - // Do not consider the just-added `extendedStats` entry. - // - // Note: even if we do reconsider the segmentation point... I don't think we - // should reconsider anything earlier than where this marker falls. - // - // If we didn't segment earlier before, why would we suddenly do so now? + // Do not consider the just-added `extendedStats` entry when building the sliding + // time window. for(let i = this.steppedCumulativeStats.length-2; i >=0; i--) { if(this.steppedCumulativeStats[i].lastTimestamp < cumulativeStats.lastTimestamp - this.SLIDING_WINDOW_INTERVAL) { splitPoint = i+1; @@ -672,26 +682,28 @@ namespace com.keyman.osk { } } + // We split the cumulative stats on a specific point, which then resides on the edge - // of both of the resulting intervals. + // of both of the resulting intervals. This completes "step 1". let candidateSplit = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint); - const xF = candidateSplit.segReg('x', 't'); - const yF = candidateSplit.segReg('y', 't'); + // STEP 2: given the proposed time window, do we have a basis for segmentation? And is there a better + // candidate split point nearby? + + // NOTE: During initial development, I actually had a hard rule about not even searching if segmentation + // failed on the initially-constructed sliding window. Some analysis may be wise here, as it'd be nice + // to optimize away the 'need to search' if we can find hard & fast rules about when we'll never need + // to go looking. But... being too aggressive can cause nasty problems. + if(!candidateSplit.segmentationMerited) { // // Debug logging statements: - // console.log("Angle variance ratio: " + candidateSplit.angleVarianceRatio); - // console.log("Speed variance ratio: " + candidateSplit.speedVarianceRatio); - - console.log(`x F-test: F_(${xF.fDoF1}, ${xF.fDoF2}) = ${xF.fStat} @ ${xF.certaintyThreshold}`); - console.log(`y F-test: F_(${yF.fDoF1}, ${yF.fDoF2}) = ${yF.fStat} @ ${yF.certaintyThreshold}`); + const xTest = candidateSplit.segReg('x', 't'); + const yTest = candidateSplit.segReg('y', 't'); + console.log(`x F-test: F_(${xTest.fDoF1}, ${xTest.fDoF2}) = ${xTest.fStat} @ ${xTest.certaintyThreshold}`); + console.log(`y F-test: F_(${yTest.fDoF1}, ${yTest.fDoF2}) = ${yTest.fStat} @ ${yTest.certaintyThreshold}`); console.log("candidate split: " ); console.log(candidateSplit); - // QUESTION: wait, what if we don't exit early? Does that help 'wait' detection? - // if(lastIntervalDuration >= this.SLIDING_WINDOW_INTERVAL * 2) { - // return; - // } } // We either have the conditions to trigger segmentation or just became long enough to consider it. @@ -730,6 +742,17 @@ namespace com.keyman.osk { } } + // Start: split-point search + + /* We'll use our initial sliding window as a starting point. It's close to the active location + * of the touch, so we're unlikely to miss a proper segmentation point if it lies close. + * We'll find the local maximum (best split point for the local part of the path) + * via gradient descent. + * + * Note: binary search is a 'bad idea', as the touchpath is likely not following a + * monotonous path, mathematically speaking. (It'd be possible to overshoot a "peak" or + * "valley" quite easily.) + */ let currentSplit = new SplitSearchState(candidateSplit); let leftSplit = new SplitSearchState(new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint-1)); @@ -739,11 +762,13 @@ namespace com.keyman.osk { } let rightSplit = new SplitSearchState(rightCandidate); + // Step 2a: detect which direction gives the best improvement in segmentation potential. const criteria = [leftSplit.segRating, currentSplit.segRating, rightSplit.segRating]; let sortedCriteria = [...criteria].sort(); const delta = criteria.indexOf(sortedCriteria[2])-1; // -1 if 'left' is best, 1 if 'right' is best. + // Step 2b: we've found the direction: go searching! if(delta != 0) { // We can get better segmentation by shifting. Proceed in the optimal direction. do { @@ -778,7 +803,9 @@ namespace com.keyman.osk { // If we found a new best segmentation point, we then ask if we can get even better by shifting further. } while(true); } + // Step 2 complete. + // Step 3: evaluate our candidate split-point - do we sub-segment? If so, do so. console.log("best split: "); console.log(candidateSplit); @@ -797,11 +824,12 @@ namespace com.keyman.osk { this.steppedCumulativeStats = this.steppedCumulativeStats.slice(splitPoint); this.choppedStats = candidateSplit.chopPoint; + // Step 4: process the implications. // There are some cases where we want to prevent segmentation from fully happening. // The next method will both handle that and the signal of any completed segments that // may result. - this.filterSubsegmentation(candidateSplit); + this.processSubsegmentation(candidateSplit); } /** @@ -814,7 +842,7 @@ namespace com.keyman.osk { * @param finalize Set to `true` for a terminating touchpath, indicating that * no further subsegmentation will occur for this touchpath. */ - private filterSubsegmentation(subsegmentation: PotentialSegmentation, finalize?: boolean) { + private processSubsegmentation(subsegmentation: PotentialSegmentation, finalize?: boolean) { finalize = !!finalize; // This function makes a LOT of assumptions particular to this method and class. @@ -828,6 +856,9 @@ namespace com.keyman.osk { } } + // Step 1: did we previously note that any previous subsegments are likely to be 'linked' to + // the new, incoming left-hand subsegment? If so, validate that expectation & act accordingly. + let predecessor = subsegmentation.pre; let firstChopPoint = subsegmentation.baseChop; if(this.lingeringSubsegmentations.length) { @@ -842,7 +873,7 @@ namespace com.keyman.osk { const precursorMergeSegmentation = new Segmentation(mergedPrecursors, subsegmentation.pre, tailUnion, subsegmentation.endOfPre); // Sometimes the start of a harsh turn seems like it's part of the same thing for a moment, but as it continues, // becomes something VERY different. Validate that we should still merge the left-hand with its predecessors. - if(!PathSegmenter.shouldMergeSubsegments(precursorMergeSegmentation)) { + if(!PathSegmenter.shouldLinkSubsegments(precursorMergeSegmentation)) { // Emit as separate subsegment. const finishedSegment = mergeSubsegmentationAccumulations(this.lingeringSubsegmentations); this._protoSegments.push(finishedSegment); @@ -859,13 +890,16 @@ namespace com.keyman.osk { } } + // Step 2: okay, predecessor handling complete. Now, do we think the newly-starting right-hand + // subsegment is likely to be 'linked' to the newly-completed left-hand subsegment? + console.log("Double-checking right-side split subsegment for xy/yx correlation with prior segment candidate(s)"); const fullUnion = subsegmentation.endpoint.deaccumulate(firstChopPoint); const fullMergeSegmentation = new Segmentation(predecessor, subsegmentation.post, fullUnion, subsegmentation.endpoint); console.log("segmentation-prevention check:") console.log(fullMergeSegmentation); // if(!force && PathSegmenter.shouldMergeSubsegments(subsegmentation.pre, subsegmentation.post, subsegmentation.union)) { - if(!finalize && PathSegmenter.shouldMergeSubsegments(fullMergeSegmentation)) { + if(!finalize && PathSegmenter.shouldLinkSubsegments(fullMergeSegmentation)) { this.lingeringSubsegmentations.push(subsegmentation); } else { // Merge all as a completed segment! @@ -885,7 +919,7 @@ namespace com.keyman.osk { * @param segmentation * @returns */ - private static shouldMergeSubsegments(segmentation: Segmentation): boolean { + private static shouldLinkSubsegments(segmentation: Segmentation): boolean { // // Known case #1: // // Near-identical direction, but heavy speed difference. // // Speed's still high enough to not be a 'wait'. From d823b8fffa40df2766258170702024cd35b11f03 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 19 Aug 2022 14:42:03 +0700 Subject: [PATCH 17/22] feat(web): fun labeling for in-PR demo --- .../src/cumulativePathStats.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/common/web/gesture-recognizer/src/cumulativePathStats.ts b/common/web/gesture-recognizer/src/cumulativePathStats.ts index e43b42d225..935b4e9841 100644 --- a/common/web/gesture-recognizer/src/cumulativePathStats.ts +++ b/common/web/gesture-recognizer/src/cumulativePathStats.ts @@ -662,8 +662,25 @@ namespace com.keyman.osk { // Convert to a `toJSON` method for use during investigative debugging. private toDebuggingJSON() { + // This `likelyState` value is extremely prototyped & just here for reviewer/tester convenience. + // It'll need to be developed a bit more fully, but follows my intuitions from development & + // testing. + let likelyState = 'unknown'; + + if(this.mean('v') < 80 && this.rawDistance < 12 && this.duration > 0.1) { + likelyState = 'hold'; + } else if(this.mean('v') < 80 && this.rawDistance < 6) { + likelyState = 'hold'; + } + + if(this.mean('v') > 400 || (this.mean('v') > 200 && this.duration > 0.1) || this.netDistance > 20) { + likelyState = 'move'; + } + return { angle: this.angle, + cardinal: this.cardinalDirection, + likelyType: likelyState, speedMean: this.mean('v'), rawDistance: this.rawDistance, duration: this.duration, From af2c803d6b2db43c3c01012b9f591fe212314dff Mon Sep 17 00:00:00 2001 From: Joshua Horton Date: Mon, 22 Aug 2022 09:46:41 +0700 Subject: [PATCH 18/22] chore(web): Apply suggestions from code review Co-authored-by: Marc Durdin --- common/web/gesture-recognizer/src/cumulativePathStats.ts | 8 +++++--- common/web/gesture-recognizer/src/pathSegmenter.ts | 5 +---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/common/web/gesture-recognizer/src/cumulativePathStats.ts b/common/web/gesture-recognizer/src/cumulativePathStats.ts index 935b4e9841..ebd4072458 100644 --- a/common/web/gesture-recognizer/src/cumulativePathStats.ts +++ b/common/web/gesture-recognizer/src/cumulativePathStats.ts @@ -227,7 +227,7 @@ namespace com.keyman.osk { const coordArcDeltaSq = xDelta * xDelta + yDelta * yDelta; const coordArcDelta = Math.sqrt(coordArcDeltaSq); - result.coordArcSum += Math.sqrt(coordArcDeltaSq); + result.coordArcSum += coordArcDelta; if(xDelta || yDelta) { // We wish to measure angle clockwise from <0, -1> in the DOM. So, cos values should @@ -241,7 +241,7 @@ namespace com.keyman.osk { } if(tDeltaInSec) { - result.rawLinearSums['v'] += Math.sqrt(coordArcDeltaSq) / tDeltaInSec; + result.rawLinearSums['v'] += coordArcDelta / tDeltaInSec; result.rawSquaredSums['v'] += coordArcDeltaSq / (tDeltaInSec * tDeltaInSec); } } @@ -564,7 +564,8 @@ namespace com.keyman.osk { * closely matches the direction of movement represented by the represented * segment. * - * @return A string one or two letters in length. (e.g: 'n', 'sw') + * @return A string one or two letters in length (e.g: 'n', 'sw'), or + `undefined` if not enough data to determine a direction. */ public get cardinalDirection() { if(this.sampleCount == 1 || !this.lastSample || !this.initialSample) { @@ -585,6 +586,7 @@ namespace com.keyman.osk { /** * Measured in pixels per second. + * @return a speed in pixels per second, or `Number.NaN` if no data */ public get speed() { // this.duration is already in seconds, not milliseconds. diff --git a/common/web/gesture-recognizer/src/pathSegmenter.ts b/common/web/gesture-recognizer/src/pathSegmenter.ts index 84fdd66937..9afee040dd 100644 --- a/common/web/gesture-recognizer/src/pathSegmenter.ts +++ b/common/web/gesture-recognizer/src/pathSegmenter.ts @@ -392,9 +392,6 @@ namespace com.keyman.osk { const xTest = new Segmentation.segmentationComparison(this, 'x', 't'); const yTest = new Segmentation.segmentationComparison(this, 'y', 't'); - // const xTestConfig = this.xtFTestConfiguration; - // const yTestConfig = this.ytFTestConfiguration; - totalThreshold += xTest.certaintyThreshold >= 0.95 ? 2 : (xTest.certaintyThreshold >= 0.90 ? 1 : 0) ; totalThreshold += yTest.certaintyThreshold >= 0.95 ? 2 : (yTest.certaintyThreshold >= 0.90 ? 1 : 0) ; @@ -407,7 +404,7 @@ namespace com.keyman.osk { * maintain the same direction but differ only in observed speed. */ get mergeMerited(): boolean { - // Because of caret-like motions (as in, in the '^' shape), we need to text for + // Because of caret-like motions (as in, in the '^' shape), we need to test for // regression on both axes. One may have notably higher variance than the other. // // These tests ignore time, and therefore speed. Only the raw geometry of the motion From 1d4de3f2a0144dec12f6e3465ceb8a56ffd1e0eb Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 22 Aug 2022 10:17:35 +0700 Subject: [PATCH 19/22] chore(web): adjustments per PR review --- .../src/cumulativePathStats.ts | 112 +++++++++++------- .../gesture-recognizer/src/pathSegmenter.ts | 16 ++- 2 files changed, 76 insertions(+), 52 deletions(-) diff --git a/common/web/gesture-recognizer/src/cumulativePathStats.ts b/common/web/gesture-recognizer/src/cumulativePathStats.ts index ebd4072458..088a371870 100644 --- a/common/web/gesture-recognizer/src/cumulativePathStats.ts +++ b/common/web/gesture-recognizer/src/cumulativePathStats.ts @@ -1,4 +1,24 @@ namespace com.keyman.osk { + /** + * Denotes one dimension utilized by touchpath input coordinates - 'x' and y' for space, + * 't' for time. + */ + export type PathCoordAxis = 'x' | 'y' | 't'; + + /** + * Denotes a pair of dimensions utilized by touchpath input coordinates. The two axes + * (see `PathCoordAxis`) must be specified in alphabetical order. + */ + export type PathCoordAxisPair = 'tx' | 'ty' | 'xy'; + + /** + * Denotes one dimension or feature (velocity) that this class tracks statistics for. + * + * Sine and Cosine stats are currently excluded due to their necessary lack of statistical + * independence. + */ + type StatAxis = PathCoordAxis | 'v'; + /** * As the name suggests, this class facilitates tracking of cumulative mathematical values, etc * necessary to perform the statistical operations necessary for path segmentation. @@ -6,15 +26,14 @@ namespace com.keyman.osk { * Instances of this class are immutable. */ export class CumulativePathStats { - // So... class-level "inner classes" are possible in TS... if defined via assignment to a field. /** * Provides linear-regression statistics & fitting values based on the underlying `CumulativePathStats` * object used to generate it. All operations are O(1). */ static readonly regression = class RegressionFromSums { - readonly independent: 'x' | 'y' | 't'; - readonly dependent: 'x' | 'y' | 't'; - readonly paired: 'tx' | 'ty' | 'xy'; + readonly independent: PathCoordAxis; + readonly dependent: PathCoordAxis; + readonly paired: PathCoordAxisPair; readonly accumulator: CumulativePathStats; @@ -25,7 +44,7 @@ namespace com.keyman.osk { * existing data of its relationship with the independent axis. * @param independentAxis The 'input' axis/dimension. */ - constructor(mainStats: CumulativePathStats, dependentAxis: 'x' | 'y' | 't', independentAxis: 'x' | 'y' | 't') { + constructor(mainStats: CumulativePathStats, dependentAxis: PathCoordAxis, independentAxis: PathCoordAxis) { if(dependentAxis == independentAxis) { throw "Two different axes must be specified for the regression object."; } @@ -36,9 +55,9 @@ namespace com.keyman.osk { this.independent = independentAxis; if(dependentAxis < independentAxis) { - this.paired = dependentAxis.concat(independentAxis) as 'tx' | 'ty' | 'xy'; + this.paired = dependentAxis.concat(independentAxis) as PathCoordAxisPair; } else { - this.paired = independentAxis.concat(dependentAxis) as 'tx' | 'ty' | 'xy'; + this.paired = independentAxis.concat(dependentAxis) as PathCoordAxisPair; } } @@ -124,6 +143,12 @@ namespace com.keyman.osk { } } + /** + * Floating-point errors may result from cross-sum calculations, and they may be slightly larger than + * Number.EPSILON as the sums grow. (Taking the difference of cross-sums) + */ + private static readonly CANCELLATION_EPSILON = Math.sqrt(Number.EPSILON); + private rawLinearSums: {'x': number, 'y': number, 't': number, 'v': number} = {'x': 0, 'y': 0, 't': 0, 'v': 0}; private rawSquaredSums: {'x': number, 'y': number, 't': number, 'v': number} = {'x': 0, 'y': 0, 't': 0, 'v': 0}; // Would 'tv' (time vs velocity) be worth it to track? And possibly even do a regression for? @@ -179,6 +204,8 @@ namespace com.keyman.osk { this.rawSquaredSums = {...obj.rawSquaredSums}; } else if(isAnInputSample(obj)) { Object.assign(this, this.extend(obj)); + } else { + throw "A constructor for this input pattern has not yet been implemented"; } } @@ -305,12 +332,13 @@ namespace com.keyman.osk { const tDelta = subsetStats.followingSample.t - subsetStats.lastSample.t; const tDeltaInSec = tDelta / 1000; - const coordArcSq = xDelta * xDelta + yDelta * yDelta; + const coordArcDeltaSq = xDelta * xDelta + yDelta * yDelta; + const coordArcDelta = Math.sqrt(coordArcDeltaSq); // Due to how arc length stuff gets segmented. // There's the arc length within the prefix subset (operand 2 below) AND the part connecting it to the // 'remaining' subset (operand 1 below) before the portion wholly within what remains (the result) - result.coordArcSum -= Math.sqrt(coordArcSq); + result.coordArcSum -= coordArcDelta; result.coordArcSum -= subsetStats.coordArcSum; result.cosLinearSum -= subsetStats.cosLinearSum; @@ -318,8 +346,8 @@ namespace com.keyman.osk { result.arcSampleCount -= subsetStats.arcSampleCount; if(tDeltaInSec) { - result.rawLinearSums['v'] -= Math.sqrt(coordArcSq) / tDeltaInSec; - result.rawSquaredSums['v'] -= coordArcSq / (tDeltaInSec * tDeltaInSec); + result.rawLinearSums['v'] -= coordArcDelta / tDeltaInSec; + result.rawSquaredSums['v'] -= coordArcDeltaSq / (tDeltaInSec * tDeltaInSec); } } @@ -360,7 +388,7 @@ namespace com.keyman.osk { * @param dim * @returns */ - private mappingConstant(dim: 'x' | 'y' | 't' | 'v') { + private mappingConstant(dim: StatAxis) { if(!this.baseSample) { return undefined; } @@ -382,7 +410,7 @@ namespace com.keyman.osk { * @param dim * @returns */ - private mappedMean(dim: 'x' | 'y' | 't' |'v') { + private mappedMean(dim: StatAxis) { return this.rawLinearSums[dim] / this.sampleCount; } @@ -392,7 +420,7 @@ namespace com.keyman.osk { * @param dim * @returns */ - public mean(dim: 'x' | 'y' | 't' | 'v') { + public mean(dim: StatAxis) { // This external-facing version needs to provide values in 'external'-friendly // coordinate space. return this.mappedMean(dim) + this.mappingConstant(dim); @@ -404,7 +432,7 @@ namespace com.keyman.osk { * @param dim * @returns */ - public squaredSum(dim: 'x' | 'y' | 't' | 'v') { + public squaredSum(dim: StatAxis) { const x2 = this.rawSquaredSums[dim]; const x1 = this.rawLinearSums[dim]; @@ -419,14 +447,11 @@ namespace com.keyman.osk { * @param dimPair * @returns */ - public crossSum(dimPair: 'tx' | 'ty' | 'xy') { + public crossSum(dimPair: PathCoordAxisPair) { const dim1 = dimPair.charAt(0); const dim2 = dimPair.charAt(1); let orderedDims: string = dimPair; - if(dim2 < dim1) { - orderedDims = dim2.concat(dim1); - } const ab = this.rawCrossSums[orderedDims]; const a = this.rawLinearSums[dim1]; @@ -444,7 +469,7 @@ namespace com.keyman.osk { * @param dimPair * @returns */ - public covariance(dimPair: 'tx' | 'ty' | 'xy') { + public covariance(dimPair: PathCoordAxisPair) { return this.crossSum(dimPair) / (this.sampleCount - 1); } @@ -452,7 +477,7 @@ namespace com.keyman.osk { * Gets the unbiased variance on the specified axis for samples observed * during the represented interval. */ - public variance(dim: 'x' | 'y' | 't' | 'v') { + public variance(dim: StatAxis) { return this.squaredSum(dim) / (this.sampleCount - 1); } @@ -480,11 +505,11 @@ namespace com.keyman.osk { result.baseSample = newBase; for(const dimPair in result.rawCrossSums) { - result.rawCrossSums[dimPair] = this.crossSum(dimPair as 'tx' | 'ty' | 'xy'); + result.rawCrossSums[dimPair] = this.crossSum(dimPair as PathCoordAxisPair); } for(const dim in result.rawSquaredSums) { - result.rawSquaredSums[dim] = this.squaredSum(dim as 'x' | 'y' | 't'); + result.rawSquaredSums[dim] = this.squaredSum(dim as PathCoordAxis); } return result; @@ -497,7 +522,7 @@ namespace com.keyman.osk { * @param independent * @returns */ - public fitRegression(dependent: 'x' | 'y' | 't', independent: 'x' | 'y' | 't') { + public fitRegression(dependent: PathCoordAxis, independent: PathCoordAxis) { return new CumulativePathStats.regression(this, dependent, independent); } @@ -510,7 +535,7 @@ namespace com.keyman.osk { public get netDistance() { // No issue with a net distance of 0 due to a single point. if(!this.lastSample || !this.initialSample) { - return Number.NaN; + return 0; } const xDelta = this.lastSample.targetX - this.initialSample.targetX; @@ -527,7 +552,7 @@ namespace com.keyman.osk { public get duration() { // no issue with a duration of zero from just one sample. if(!this.lastSample || !this.initialSample) { - return Number.NaN; + return 0; } return (this.lastSample.t - this.initialSample.t) * 0.001; } @@ -572,16 +597,12 @@ namespace com.keyman.osk { return undefined; } - const angle = this.angleInDegrees; - const buckets = ['n', 'ne', 'e', 'se', 's', 'sw', 'w', 'nw']; + const buckets = ['n', 'ne', 'e', 'se', 's', 'sw', 'w', 'nw', 'n']; - for(let threshold = 22.5, bucketIndex = 0; threshold < 360; threshold += 45, bucketIndex += 1) { - if(angle < threshold) { - return buckets[bucketIndex]; - } - } - - return 'n'; + // We could be 'more efficient' and use radians here instead, but this + // version helps a bit more with easy maintainability. + const bucketIndex = Math.ceil((this.angleInDegrees - 22.5)/45); + return buckets[bucketIndex]; } /** @@ -627,9 +648,14 @@ namespace com.keyman.osk { * Range: floating-point values on the interval [0, 1]. */ private get angleRSquared() { - // https://www.ebi.ac.uk/thornton-srv/software/PROCHECK/nmr_manual/man_cv.html may be a useful - // reference for this tidbit. The Wikipedia article's more dense... not that this link isn't - // a bit dense itself. + // Refer to https://en.wikipedia.org/wiki/Directional_statistics#Distribution_of_the_mean. + // We're computing the squared value of that page's R-bar stat. + // + // Now, why it's called that? ... good question. My best guess is that it's meant to + // correspond to linear regression's 'r' stat, which when squared serves as the + // coefficient of determination for the regression. Intuitively, that does seem to + // match what this represents - though for normal regressions, the c.o.d isn't normally + // used to compute deviation or variance! const rSquaredBase = this.cosLinearSum * this.cosLinearSum + this.sinLinearSum * this.sinLinearSum; return rSquaredBase / (this.arcSampleCount * this.arcSampleCount); } @@ -667,22 +693,22 @@ namespace com.keyman.osk { // This `likelyState` value is extremely prototyped & just here for reviewer/tester convenience. // It'll need to be developed a bit more fully, but follows my intuitions from development & // testing. - let likelyState = 'unknown'; + let likelyType = 'unknown'; if(this.mean('v') < 80 && this.rawDistance < 12 && this.duration > 0.1) { - likelyState = 'hold'; + likelyType = 'hold'; } else if(this.mean('v') < 80 && this.rawDistance < 6) { - likelyState = 'hold'; + likelyType = 'hold'; } if(this.mean('v') > 400 || (this.mean('v') > 200 && this.duration > 0.1) || this.netDistance > 20) { - likelyState = 'move'; + likelyType = 'move'; } return { angle: this.angle, cardinal: this.cardinalDirection, - likelyType: likelyState, + likelyType: likelyType, speedMean: this.mean('v'), rawDistance: this.rawDistance, duration: this.duration, diff --git a/common/web/gesture-recognizer/src/pathSegmenter.ts b/common/web/gesture-recognizer/src/pathSegmenter.ts index 9afee040dd..9a9357a24e 100644 --- a/common/web/gesture-recognizer/src/pathSegmenter.ts +++ b/common/web/gesture-recognizer/src/pathSegmenter.ts @@ -133,10 +133,6 @@ namespace com.keyman.osk { return 1; } - if(numDoF > 3) { - numDoF = 3; - } - const numIndex = (numDoF > 3 ? 3 : numDoF) - 2; const denomIndex = (denomDoF > 20 ? 20 : denomDoF) - 1; @@ -191,8 +187,8 @@ namespace com.keyman.osk { */ static readonly segmentationComparison = class SegmentedRegression { host: Segmentation; - readonly independent: 'x' | 'y' | 't'; - readonly dependent: 'x' | 'y' | 't'; + readonly independent: PathCoordAxis; + readonly dependent: PathCoordAxis; readonly paired: 'tx' | 'ty' | 'xy'; /** @@ -210,7 +206,7 @@ namespace com.keyman.osk { */ union: typeof CumulativePathStats.regression.prototype; - constructor(host: Segmentation, dependentAxis: 'x' | 'y' | 't', independentAxis: 'x' | 'y' | 't') { + constructor(host: Segmentation, dependentAxis: PathCoordAxis, independentAxis: PathCoordAxis) { if(dependentAxis == independentAxis) { throw "Two different axes must be specified for the regression object."; } @@ -377,7 +373,7 @@ namespace com.keyman.osk { * @param independent * @returns */ - public segReg(dependentAxis: 'x' | 'y' | 't', independentAxis: 'x' | 'y' | 't') { + public segReg(dependentAxis: PathCoordAxis, independentAxis: PathCoordAxis) { return new Segmentation.segmentationComparison(this, dependentAxis, independentAxis); } @@ -392,6 +388,8 @@ namespace com.keyman.osk { const xTest = new Segmentation.segmentationComparison(this, 'x', 't'); const yTest = new Segmentation.segmentationComparison(this, 'y', 't'); + // Our testing thresholds are for p=0.05 and p=0.10, which correspond to certainties of 95% and + // 90% that our segmentation did not arrive from random chance based on the axis being tested. totalThreshold += xTest.certaintyThreshold >= 0.95 ? 2 : (xTest.certaintyThreshold >= 0.90 ? 1 : 0) ; totalThreshold += yTest.certaintyThreshold >= 0.95 ? 2 : (yTest.certaintyThreshold >= 0.90 ? 1 : 0) ; @@ -404,7 +402,7 @@ namespace com.keyman.osk { * maintain the same direction but differ only in observed speed. */ get mergeMerited(): boolean { - // Because of caret-like motions (as in, in the '^' shape), we need to test for + // Because of caret-like motions (as in, in the '^' shape), we need to text for // regression on both axes. One may have notably higher variance than the other. // // These tests ignore time, and therefore speed. Only the raw geometry of the motion From b352c9dda762bcc01539fd778bb24173c6444a15 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 22 Aug 2022 10:26:51 +0700 Subject: [PATCH 20/22] change(web): speed now in millisec --- .../src/cumulativePathStats.ts | 29 ++++++++----------- .../gesture-recognizer/src/pathSegmenter.ts | 6 ++-- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/common/web/gesture-recognizer/src/cumulativePathStats.ts b/common/web/gesture-recognizer/src/cumulativePathStats.ts index 088a371870..9cc91298c9 100644 --- a/common/web/gesture-recognizer/src/cumulativePathStats.ts +++ b/common/web/gesture-recognizer/src/cumulativePathStats.ts @@ -249,7 +249,6 @@ namespace com.keyman.osk { const xDelta = sample.targetX - this.lastSample.targetX; const yDelta = sample.targetY - this.lastSample.targetY; const tDelta = sample.t - this.lastSample.t; - const tDeltaInSec = tDelta / 1000; const coordArcDeltaSq = xDelta * xDelta + yDelta * yDelta; const coordArcDelta = Math.sqrt(coordArcDeltaSq); @@ -267,9 +266,9 @@ namespace com.keyman.osk { result.arcSampleCount += 1; } - if(tDeltaInSec) { - result.rawLinearSums['v'] += coordArcDelta / tDeltaInSec; - result.rawSquaredSums['v'] += coordArcDeltaSq / (tDeltaInSec * tDeltaInSec); + if(tDelta) { + result.rawLinearSums['v'] += coordArcDelta / tDelta; + result.rawSquaredSums['v'] += coordArcDeltaSq / (tDelta * tDelta); } } @@ -330,7 +329,6 @@ namespace com.keyman.osk { const xDelta = subsetStats.followingSample.targetX - subsetStats.lastSample.targetX; const yDelta = subsetStats.followingSample.targetY - subsetStats.lastSample.targetY; const tDelta = subsetStats.followingSample.t - subsetStats.lastSample.t; - const tDeltaInSec = tDelta / 1000; const coordArcDeltaSq = xDelta * xDelta + yDelta * yDelta; const coordArcDelta = Math.sqrt(coordArcDeltaSq); @@ -345,9 +343,9 @@ namespace com.keyman.osk { result.sinLinearSum -= subsetStats.sinLinearSum; result.arcSampleCount -= subsetStats.arcSampleCount; - if(tDeltaInSec) { - result.rawLinearSums['v'] -= coordArcDelta / tDeltaInSec; - result.rawSquaredSums['v'] -= coordArcDeltaSq / (tDeltaInSec * tDeltaInSec); + if(tDelta) { + result.rawLinearSums['v'] -= coordArcDelta / tDelta; + result.rawSquaredSums['v'] -= coordArcDeltaSq / (tDelta * tDelta); } } @@ -545,16 +543,14 @@ namespace com.keyman.osk { } /** - * Gets the duration of the represented interval, in seconds. - * - * Note: input samples provide their timestamps in milliseconds. + * Gets the duration of the represented interval in milliseconds. */ public get duration() { // no issue with a duration of zero from just one sample. if(!this.lastSample || !this.initialSample) { return 0; } - return (this.lastSample.t - this.initialSample.t) * 0.001; + return (this.lastSample.t - this.initialSample.t); } /** @@ -607,10 +603,9 @@ namespace com.keyman.osk { /** * Measured in pixels per second. - * @return a speed in pixels per second, or `Number.NaN` if no data + * @return a speed in pixels per millisecond, or `Number.NaN` if no data */ public get speed() { - // this.duration is already in seconds, not milliseconds. return this.duration ? this.netDistance / this.duration : Number.NaN; } @@ -695,13 +690,13 @@ namespace com.keyman.osk { // testing. let likelyType = 'unknown'; - if(this.mean('v') < 80 && this.rawDistance < 12 && this.duration > 0.1) { + if(this.mean('v') < 0.08 && this.rawDistance < 12 && this.duration > 100) { likelyType = 'hold'; - } else if(this.mean('v') < 80 && this.rawDistance < 6) { + } else if(this.mean('v') < 0.08 && this.rawDistance < 6) { likelyType = 'hold'; } - if(this.mean('v') > 400 || (this.mean('v') > 200 && this.duration > 0.1) || this.netDistance > 20) { + if(this.mean('v') > 0.4 || (this.mean('v') > 0.2 && this.duration > 80) || this.netDistance > 20) { likelyType = 'move'; } diff --git a/common/web/gesture-recognizer/src/pathSegmenter.ts b/common/web/gesture-recognizer/src/pathSegmenter.ts index 9a9357a24e..d55e6c0a3f 100644 --- a/common/web/gesture-recognizer/src/pathSegmenter.ts +++ b/common/web/gesture-recognizer/src/pathSegmenter.ts @@ -927,12 +927,12 @@ namespace com.keyman.osk { // at least at this stage of development. (Most testing was done with Chrome emulation of // an iPhone SE.) - // .mean('v') < 80: the mean speed (taken timestamp to timestamp) does not exceed 80px/sec. - if(segmentation.pre.mean('v') < 80 && segmentation.post.mean('v') > 80) { + // .mean('v') < 0.08: the mean speed (taken timestamp to timestamp) does not exceed 0.08px/millisec. + if(segmentation.pre.mean('v') < 0.08 && segmentation.post.mean('v') > 0.08) { console.log("desegmentation exception"); return; } - if(segmentation.pre.mean('v') > 80 && segmentation.post.mean('v') < 80) { + if(segmentation.pre.mean('v') > 0.08 && segmentation.post.mean('v') < 0.08) { console.log("desegmentation exception"); return; } From 8ce9fa80703af456e349cf18f946834e1a1cecc9 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 22 Aug 2022 11:28:51 +0700 Subject: [PATCH 21/22] chore(web): more post-review tweaks --- .../src/cumulativePathStats.ts | 38 +++++++++---------- .../gesture-recognizer/src/pathSegmenter.ts | 7 +++- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/common/web/gesture-recognizer/src/cumulativePathStats.ts b/common/web/gesture-recognizer/src/cumulativePathStats.ts index 9cc91298c9..d7ae3dae7a 100644 --- a/common/web/gesture-recognizer/src/cumulativePathStats.ts +++ b/common/web/gesture-recognizer/src/cumulativePathStats.ts @@ -149,11 +149,11 @@ namespace com.keyman.osk { */ private static readonly CANCELLATION_EPSILON = Math.sqrt(Number.EPSILON); - private rawLinearSums: {'x': number, 'y': number, 't': number, 'v': number} = {'x': 0, 'y': 0, 't': 0, 'v': 0}; - private rawSquaredSums: {'x': number, 'y': number, 't': number, 'v': number} = {'x': 0, 'y': 0, 't': 0, 'v': 0}; + private rawLinearSums = {'x': 0, 'y': 0, 't': 0, 'v': 0}; + private rawSquaredSums = {'x': 0, 'y': 0, 't': 0, 'v': 0}; // Would 'tv' (time vs velocity) be worth it to track? And possibly even do a regression for? // If so, maybe throw that in. - private rawCrossSums: {'tx': number, 'ty': number, 'xy': number} = {'tx': 0, 'ty': 0, 'xy': 0}; + private rawCrossSums = {'tx': 0, 'ty': 0, 'xy': 0}; private coordArcSum: number = 0; private arcSampleCount: number = 0; @@ -232,17 +232,17 @@ namespace com.keyman.osk { const y = sample.targetY - this.baseSample.targetY; const t = sample.t - this.baseSample.t; - result.rawLinearSums['x'] += x; - result.rawLinearSums['y'] += y; - result.rawLinearSums['t'] += t; + result.rawLinearSums.x += x; + result.rawLinearSums.y += y; + result.rawLinearSums.t += t; - result.rawCrossSums['tx'] += t * x; - result.rawCrossSums['ty'] += t * y; - result.rawCrossSums['xy'] += x * y; + result.rawCrossSums.tx += t * x; + result.rawCrossSums.ty += t * y; + result.rawCrossSums.xy += x * y; - result.rawSquaredSums['x'] += x * x; - result.rawSquaredSums['y'] += y * y; - result.rawSquaredSums['t'] += t * t; + result.rawSquaredSums.x += x * x; + result.rawSquaredSums.y += y * y; + result.rawSquaredSums.t += t * t; if(this.lastSample) { // arc length stuff! @@ -267,8 +267,8 @@ namespace com.keyman.osk { } if(tDelta) { - result.rawLinearSums['v'] += coordArcDelta / tDelta; - result.rawSquaredSums['v'] += coordArcDeltaSq / (tDelta * tDelta); + result.rawLinearSums.v += coordArcDelta / tDelta; + result.rawSquaredSums.v += coordArcDeltaSq / (tDelta * tDelta); } } @@ -344,8 +344,8 @@ namespace com.keyman.osk { result.arcSampleCount -= subsetStats.arcSampleCount; if(tDelta) { - result.rawLinearSums['v'] -= coordArcDelta / tDelta; - result.rawSquaredSums['v'] -= coordArcDeltaSq / (tDelta * tDelta); + result.rawLinearSums.v -= coordArcDelta / tDelta; + result.rawSquaredSums.v -= coordArcDeltaSq / (tDelta * tDelta); } } @@ -495,9 +495,9 @@ namespace com.keyman.osk { let result = new CumulativePathStats(this); let newBase: InputSample = { - targetX: this.mappedMean['x'] + this.baseSample.targetX, - targetY: this.mappedMean['y'] + this.baseSample.targetY, - t: this.mappedMean['t'] + this.baseSample.t + targetX: this.mappedMean('x') + this.baseSample.targetX, + targetY: this.mappedMean('y') + this.baseSample.targetY, + t: this.mappedMean('t') + this.baseSample.t }; result.baseSample = newBase; diff --git a/common/web/gesture-recognizer/src/pathSegmenter.ts b/common/web/gesture-recognizer/src/pathSegmenter.ts index d55e6c0a3f..40c8b58a27 100644 --- a/common/web/gesture-recognizer/src/pathSegmenter.ts +++ b/common/web/gesture-recognizer/src/pathSegmenter.ts @@ -390,8 +390,11 @@ namespace com.keyman.osk { // Our testing thresholds are for p=0.05 and p=0.10, which correspond to certainties of 95% and // 90% that our segmentation did not arrive from random chance based on the axis being tested. - totalThreshold += xTest.certaintyThreshold >= 0.95 ? 2 : (xTest.certaintyThreshold >= 0.90 ? 1 : 0) ; - totalThreshold += yTest.certaintyThreshold >= 0.95 ? 2 : (yTest.certaintyThreshold >= 0.90 ? 1 : 0) ; + const p05 = 0.95; // 1 - 0.05 + const p10 = 0.9; // 1 - 0.10 + + totalThreshold += xTest.certaintyThreshold >= p05 ? 2 : (xTest.certaintyThreshold >= p10 ? 1 : 0) ; + totalThreshold += yTest.certaintyThreshold >= p05 ? 2 : (yTest.certaintyThreshold >= p10 ? 1 : 0) ; return totalThreshold >= 2; } From 0cd74165646c7b58e265725c04d248475ede7cb2 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 22 Aug 2022 12:14:16 +0700 Subject: [PATCH 22/22] refactor(web): sigMinus, for cata-cancel check on stat-sum operations --- .../src/cumulativePathStats.ts | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/common/web/gesture-recognizer/src/cumulativePathStats.ts b/common/web/gesture-recognizer/src/cumulativePathStats.ts index d7ae3dae7a..e148012d22 100644 --- a/common/web/gesture-recognizer/src/cumulativePathStats.ts +++ b/common/web/gesture-recognizer/src/cumulativePathStats.ts @@ -19,6 +19,24 @@ namespace com.keyman.osk { */ type StatAxis = PathCoordAxis | 'v'; + /** + * Acts as a subtraction operation with a built-in, adaptive "significance" check. + * If the result * 2^30 (~ * 10^9) is still smaller in magnitude than an operand, we + * assume it to be a floating-point error that should have been 0 and act accordingly, + * returning 0. + * + * For reference, (32-bit) floats have 23 bits of significand precision, while (64-bit) + * doubles have 52. Therefore, we'll still be more precise than baseline floats. + */ + function sigMinus(operand1: number, operand2: number) { + const diff = operand1 - operand2; + + const logDiff = Math.log2(Math.abs(operand1)) - Math.log2(Math.abs(operand2)); + // If an operand is 2^30 (or ~10^9) larger than the result of the difference, it's + // nigh-certainly a floating-point error at play. + return logDiff < 30 ? diff : 0; + } + /** * As the name suggests, this class facilitates tracking of cumulative mathematical values, etc * necessary to perform the statistical operations necessary for path segmentation. @@ -434,9 +452,7 @@ namespace com.keyman.osk { const x2 = this.rawSquaredSums[dim]; const x1 = this.rawLinearSums[dim]; - const val = x2 - x1 * x1 / this.sampleCount; - - return val > 1e-8 ? val : 0; + return sigMinus(x2, x1 * x1 / this.sampleCount); } /** @@ -455,10 +471,7 @@ namespace com.keyman.osk { const a = this.rawLinearSums[dim1]; const b = this.rawLinearSums[dim2]; - const val = ab - a * b / this.sampleCount; - - // Don't forget - cross-sums can be negative! - return Math.abs(val) > 1e-8 ? val : 0; + return sigMinus(ab, a * b / this.sampleCount); } /** @@ -507,6 +520,14 @@ namespace com.keyman.osk { } for(const dim in result.rawSquaredSums) { + // 'v' does not need renormalization. + if(dim == 'v') { + break; + } + + // The identity we're using to renormalize rawCrossSums and rawSquaredSums + // automatically guarantees a mean of 0 after the renormalization. + result.rawLinearSums[dim] = 0; result.rawSquaredSums[dim] = this.squaredSum(dim as PathCoordAxis); }