diff --git a/common/web/gesture-recognizer/src/inputSample.ts b/common/web/gesture-recognizer/src/inputSample.ts index cc2a0fa0fe..f0b5b39981 100644 --- a/common/web/gesture-recognizer/src/inputSample.ts +++ b/common/web/gesture-recognizer/src/inputSample.ts @@ -36,4 +36,8 @@ namespace com.keyman.osk { } export type InputSampleSequence = InputSample[]; + + export function isAnInputSample(obj: any): obj is InputSample { + return 'targetX' in obj && 'targetY' in obj && 't' in obj; + } } \ No newline at end of file diff --git a/common/web/gesture-recognizer/src/segment.ts b/common/web/gesture-recognizer/src/segment.ts new file mode 100644 index 0000000000..d242bb9f3f --- /dev/null +++ b/common/web/gesture-recognizer/src/segment.ts @@ -0,0 +1,40 @@ +/// + +namespace com.keyman.osk { + export class Segment { + // May be best to keep an array of these, one per sample. + // Can then diff the stats to determine better cut-offs. + // Though... the whole arc-dist aspect will need a mite more help. + // - chopping off from the end: ez-pz. Raw diff is great. + // - or, well, just use the appropriate one from mid-way. + // - chopping off from the beginning: need an extra sample reference. + // - .nextSample. + // + // The FINAL version, once resolved, may be published. + // But until resolved, we probably want to keep an array. + private _stats: SegmentStats[]; + + constructor() { + this._stats = []; + } + + public get stats(): readonly SegmentStats[] { + return this._stats; + } + + public add(sample: InputSample) { + let baseStats: SegmentStats; + if(this.stats.length > 0) { + baseStats = this.stats[this.stats.length-1]; + } else { + baseStats = new SegmentStats(); + } + + let extendedStats = baseStats.unionWith(sample); + + // FIXME: VERY temp logging. + console.log(extendedStats.toJSON()); + this._stats.push(extendedStats); + } + } +} \ No newline at end of file diff --git a/common/web/gesture-recognizer/src/segmentStats.ts b/common/web/gesture-recognizer/src/segmentStats.ts new file mode 100644 index 0000000000..dd175a5eb2 --- /dev/null +++ b/common/web/gesture-recognizer/src/segmentStats.ts @@ -0,0 +1,337 @@ +namespace com.keyman.osk { + export class SegmentStats { + private static TIME_DIST_WEIGHT = .033; // Effect: 33ms ~= 1px distance. + + private xLinearSum: number = 0; + private yLinearSum: number = 0; + private tLinearSum: number = 0; + + private xCentroidSum: number = 0; + private yCentroidSum: number = 0; + + private xQuadSum: number = 0; + private yQuadSum: number = 0; + private tQuadSum: number = 0; + + private xtCrossSum: number = 0; + private ytCrossSum: number = 0; + private xyCrossSum: number = 0; + + private coordArcSum: number = 0; + private spacetimeArcSum: number = 0; + + /** + * The base sample used to transpose all other received samples. Use of this helps + * avoid potential "catastrophic cancellation" effects that can occur when diffing two + * numbers far from the sample-space's mathematical origin. + * + * Refer to https://en.wikipedia.org/wiki/Catastrophic_cancellation. + */ + private baseSample?: InputSample; + + /** + * The initial sample included by this instance's computed stats. Needed for + * the 'directness' properties. + */ + private initialSample?: InputSample; + + private lastSample?: InputSample; + private followingSample?: InputSample; + private sampleCount = 0; + + constructor(); + constructor(sample: InputSample); + constructor(instance: SegmentStats); + constructor(obj?: InputSample | SegmentStats) { + if(!obj) { + return; + } + + // Will worry about JSON form later. + if(obj instanceof SegmentStats) { + Object.assign(this, obj); + } else if(isAnInputSample(obj)) { + Object.assign(this, this.unionWith(obj)); + } + } + + public unionWith(sample: InputSample): SegmentStats { + if(!this.initialSample) { + this.initialSample = sample; + this.baseSample = sample; + } else { + this.followingSample = sample; + } + const result = new SegmentStats(this); + + // Helps prevent "catastrophic cancellation" issues from floating-point computation + // for these statistical properties and properties based upon them. + const x = sample.targetX - this.baseSample.targetX; + const y = sample.targetY - this.baseSample.targetY; + const t = sample.t - this.baseSample.t; + + result.xLinearSum += x; + result.yLinearSum += y; + result.tLinearSum += t; + + result.xtCrossSum += x * t; + result.ytCrossSum += y * t; + result.xyCrossSum += x * y; + + result.xQuadSum += x * x; + result.yQuadSum += y * y; + result.tQuadSum += t * t; + + if(this.lastSample) { + // arc length stuff! + const xDelta = sample.targetX - this.lastSample.targetX; + const yDelta = sample.targetY - this.lastSample.targetY; + const tDeltaInSec = (sample.t - this.lastSample.t) / 1000; + const weightedTDelta = (sample.t - this.lastSample.t) * SegmentStats.TIME_DIST_WEIGHT; + + const coordArcSq = xDelta * xDelta + yDelta * yDelta; + + result.coordArcSum += Math.sqrt(coordArcSq); + result.spacetimeArcSum += Math.sqrt(coordArcSq + weightedTDelta * weightedTDelta); + + // Approximates weighting the time spent at each coord by splitting the time since + // last event evenly for both coordinates. Note: does NOT shift based upon .baseSample! + result.xCentroidSum += 0.5 * tDeltaInSec * (sample.targetX + this.lastSample.targetX); + result.yCentroidSum += 0.5 * tDeltaInSec * (sample.targetY + this.lastSample.targetY); + } + + result.lastSample = sample; + result.sampleCount = this.sampleCount + 1; + + return result; + } + + public withoutPrefixSubset(subsetStats: SegmentStats): SegmentStats { + const result = new SegmentStats(this); + + if(!subsetStats.followingSample || !subsetStats.lastSample) { + throw 'Invalid argument: stats missing necessary tracking variable.'; + } + + result.xLinearSum -= subsetStats.xLinearSum; + result.yLinearSum -= subsetStats.yLinearSum; + result.tLinearSum -= subsetStats.tLinearSum; + + result.xtCrossSum -= subsetStats.xtCrossSum; + result.ytCrossSum -= subsetStats.ytCrossSum; + result.xyCrossSum -= subsetStats.xyCrossSum; + + result.xQuadSum -= subsetStats.xQuadSum; + result.yQuadSum -= subsetStats.yQuadSum; + result.tQuadSum -= subsetStats.tQuadSum; + + // arc length stuff! + if(subsetStats.followingSample && subsetStats.lastSample) { + const xDelta = subsetStats.followingSample.targetX - subsetStats.lastSample.targetX; + const yDelta = subsetStats.followingSample.targetY - subsetStats.lastSample.targetY; + const tDelta = (subsetStats.followingSample.t - subsetStats.lastSample.t) * SegmentStats.TIME_DIST_WEIGHT; + + const coordArcSq = xDelta * xDelta + yDelta * yDelta; + + // Due to how arc length stuff gets segmented. + // There's the arc length within the prefix subset (operand 2 below) AND the part connecting it to the + // 'remaining' subset (operand 1 below) before the portion wholly within what remains (the result) + result.coordArcSum -= Math.sqrt(coordArcSq); + result.coordArcSum -= subsetStats.coordArcSum; + result.spacetimeArcSum -= Math.sqrt(coordArcSq + tDelta * tDelta); + result.spacetimeArcSum -= subsetStats.spacetimeArcSum; + + // Centroid sum management! + const tDeltaInMs = (subsetStats.followingSample.t - subsetStats.lastSample.t) / 1000; + // Same reasoning pattern as for the 'arc length stuff'. + result.xCentroidSum -= 0.5 * tDeltaInMs * (subsetStats.followingSample.targetX + subsetStats.lastSample.targetX) + result.xCentroidSum -= subsetStats.xCentroidSum; + result.yCentroidSum -= 0.5 * tDeltaInMs * (subsetStats.followingSample.targetY + subsetStats.lastSample.targetY) + result.yCentroidSum -= subsetStats.yCentroidSum; + } + + result.sampleCount -= subsetStats.sampleCount; + + // NOTE: baseSample MUST REMAIN THE SAME. All math is based on the corresponding diff. + // Though... very long touchpoint interactions could start being affected by that "catastrophic + // cancellation" effect without further adjustment. (If it matters, we'll get to that later.) + // But _probably_ not; we don't go far beyond a couple of orders of magnitude from the origin in + // ANY case except the timestamp (.t) - and even then, not far from the baseSample's timestamp value. + + // initialSample, though, we need to update b/c of the 'directness' properties. + result.initialSample = subsetStats.followingSample; + + return result; + } + + private get xSampleMean() { + return this.xLinearSum / this.sampleCount; + } + + private get ySampleMean() { + return this.yLinearSum / this.sampleCount; + } + + private get tSampleMean() { + return this.tLinearSum / this.sampleCount; + } + + public get centroid(): {x: number, y: number} { + if(this.sampleCount == 0) { + return undefined; + } else if(this.sampleCount == 1) { + return { + x: this.lastSample.targetX, + y: this.lastSample.targetY + }; + } else { + const coeff = 1 / (this.duration); // * (this.sampleCount-1)); + return { + x: this.xCentroidSum * coeff, + y: this.yCentroidSum * coeff + }; + } + } + + public get xtCovariance() { + return this.xtCrossSum / this.sampleCount - (this.xSampleMean * this.tSampleMean); + } + + public get ytCovariance() { + return this.ytCrossSum / this.sampleCount - (this.ySampleMean * this.tSampleMean); + } + + public get xyCovariance() { + return this.xyCrossSum / this.sampleCount - (this.xSampleMean * this.ySampleMean); + } + + public get xVariance() { + return this.xQuadSum / this.sampleCount - (this.xSampleMean * this.xSampleMean); + } + + public get yVariance() { + return this.yQuadSum / this.sampleCount - (this.ySampleMean * this.ySampleMean); + } + + public get tVariance() { + return this.tQuadSum / this.sampleCount - (this.tSampleMean * this.tSampleMean); + } + + public get xtCorrelation() { + if(this.xVariance == 0) { + return Number.NaN; + } + + return this.xtCovariance / (Math.sqrt(this.xVariance * this.tVariance)); + } + + public get ytCorrelation() { + if(this.yVariance == 0) { + return Number.NaN; + } + + return this.ytCovariance / (Math.sqrt(this.yVariance * this.tVariance)); + } + + public get xyCorrelation() { + if(this.xVariance == 0 || this.yVariance == 0) { + return Number.NaN; + } + + return this.xyCovariance / (Math.sqrt(this.xVariance * this.yVariance)); + } + + public get movementRatio() { + return this.coordArcSum / this.spacetimeArcSum; + } + + public get directDistance() { + // No issue with a net distance of 0 due to a single point. + if(!this.lastSample || !this.initialSample) { + return Number.NaN; + } + + const xDelta = this.lastSample.targetX - this.initialSample.targetX; + const yDelta = this.lastSample.targetY - this.initialSample.targetY; + + return Math.sqrt(xDelta * xDelta + yDelta * yDelta); + } + + public get directnessRatio() { + return this.directDistance / this.coordArcSum; + } + + public get duration() { + // no issue with a duration of zero from just one sample. + if(!this.lastSample || !this.initialSample) { + return Number.NaN; + } + return (this.lastSample.t - this.initialSample.t) * 0.001; + } + + /** + * Returns the angle (in radians) traveled by the corresponding segment clockwise + * from the unit vector <0, -1> in the DOM (the unit "upward" direction). + */ + public get angle() { + if(this.sampleCount == 1 || !this.lastSample || !this.initialSample) { + return Number.NaN; + } else if(this.directDistance < 1) { + // < 1 px, thus sub-pixel, means we have nothing relevant enough to base an angle on. + return Number.NaN; + } + + const xDelta = this.lastSample.targetX - this.initialSample.targetX; + const yDelta = this.lastSample.targetY - this.initialSample.targetY; + const yAngleDiff = Math.acos(-yDelta / this.directDistance); + + return xDelta < 0 ? (2 * Math.PI - yAngleDiff) : yAngleDiff; + } + + public get angleInDegrees() { + return this.angle * 180 / Math.PI; + } + + public get cardinalDirection() { + if(this.sampleCount == 1 || !this.lastSample || !this.initialSample) { + return undefined; + } + + const angle = this.angleInDegrees; + const buckets = ['n', 'ne', 'e', 'se', 's', 'sw', 'w', 'nw']; + + for(let threshold = 22.5, bucketIndex = 0; threshold < 360; threshold += 45, bucketIndex += 1) { + if(angle < threshold) { + return buckets[bucketIndex]; + } + } + + return 'n'; + } + + // px per s. + public get speed() { + // this.duration is already in seconds, not milliseconds. + return this.duration ? this.directDistance / this.duration : 0; + } + + public toJSON() { + return { + xtCorrelation: this.xtCorrelation, + ytCorrelation: this.ytCorrelation, + xyCorrelation: this.xyCorrelation, + directnessRatio: this.directnessRatio, + directDistance: this.directDistance, + movementRatio: this.movementRatio, + angle: this.angle, + speed: this.speed, + cardinalDirection: this.cardinalDirection, + centroid: this.centroid, + duration: this.duration, + // Probably doesn't need to be reported in the long run, but useful while we're still nailing down the math & such. + sampleMean: {x: this.baseSample.targetX + this.xSampleMean, y: this.baseSample.targetY + this.ySampleMean} + }; + } + } + +} \ No newline at end of file diff --git a/common/web/gesture-recognizer/src/tools/recorder/src/recorder.js b/common/web/gesture-recognizer/src/tools/recorder/src/recorder.js index 6be45767d1..4a030ebcc8 100644 --- a/common/web/gesture-recognizer/src/tools/recorder/src/recorder.js +++ b/common/web/gesture-recognizer/src/tools/recorder/src/recorder.js @@ -48,6 +48,7 @@ loadPromise.then((recognizer) => { }); sequence.on('end', function() { + let stats = sequence.touchpoints[0].path.segments[0].stats; logElement.value = recorder.recordingsToJSON(); }); }); diff --git a/common/web/gesture-recognizer/src/trackedPath.ts b/common/web/gesture-recognizer/src/trackedPath.ts index 9cfdc4a5ee..441b0b819c 100644 --- a/common/web/gesture-recognizer/src/trackedPath.ts +++ b/common/web/gesture-recognizer/src/trackedPath.ts @@ -7,7 +7,7 @@ namespace com.keyman.osk { export type JSONTrackedPath = { coords: InputSample[]; // ensures type match with public class property. wasCancelled?: boolean; - // segments: Segment[]; + //segments: Segment[]; } interface EventMap { @@ -36,6 +36,8 @@ namespace com.keyman.osk { */ export class TrackedPath extends EventEmitter { private samples: InputSample[] = []; + private _segments: Segment[] = [new Segment()]; + private _isComplete: boolean = false; private wasCancelled?: boolean; @@ -80,6 +82,7 @@ namespace com.keyman.osk { } this.samples.push(sample); + this.segments[0].add(sample); this.emit('step', sample); } @@ -110,6 +113,10 @@ namespace com.keyman.osk { return this.samples; } + public get segments(): readonly Segment[] { + return this._segments; + } + /** * Creates a serialization-friendly version of this instance for use by * `JSON.stringify`.