From cde550a3eb32038bdcfe00a657176ec32616ebb1 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Fri, 19 Aug 2022 13:52:56 +0700 Subject: [PATCH] change(web): lots of docs, some cleanup --- .../src/cumulativePathStats.ts | 320 +++++++++++------ .../gesture-recognizer/src/pathSegmenter.ts | 324 ++++++++++++++---- 2 files changed, 471 insertions(+), 173 deletions(-) diff --git a/common/web/gesture-recognizer/src/cumulativePathStats.ts b/common/web/gesture-recognizer/src/cumulativePathStats.ts index 6ed22c58fa..e43b42d225 100644 --- a/common/web/gesture-recognizer/src/cumulativePathStats.ts +++ b/common/web/gesture-recognizer/src/cumulativePathStats.ts @@ -1,13 +1,16 @@ namespace com.keyman.osk { /** - * As the name suggests, this class exists to track cumulative mathematical values, etc - * necessary to provide statistical information. This information is used to facilitate - * path segmentation. + * As the name suggests, this class facilitates tracking of cumulative mathematical values, etc + * necessary to perform the statistical operations necessary for path segmentation. * * Instances of this class are immutable. */ export class CumulativePathStats { // So... class-level "inner classes" are possible in TS... if defined via assignment to a field. + /** + * Provides linear-regression statistics & fitting values based on the underlying `CumulativePathStats` + * object used to generate it. All operations are O(1). + */ static readonly regression = class RegressionFromSums { readonly independent: 'x' | 'y' | 't'; readonly dependent: 'x' | 'y' | 't'; @@ -15,6 +18,13 @@ namespace com.keyman.osk { readonly accumulator: CumulativePathStats; + /** + * + * @param mainStats The `CumulativePathStats` instance to base all regression data on. + * @param dependentAxis The 'output' axis / dimension; the axis whose behavior should be predicted based on + * existing data of its relationship with the independent axis. + * @param independentAxis The 'input' axis/dimension. + */ constructor(mainStats: CumulativePathStats, dependentAxis: 'x' | 'y' | 't', independentAxis: 'x' | 'y' | 't') { if(dependentAxis == independentAxis) { throw "Two different axes must be specified for the regression object."; @@ -32,6 +42,10 @@ namespace com.keyman.osk { } } + /** + * The 'slope' of the 'slope-intercept' form of the line that best fits the relationship between + * this regression's selected axes. + */ get slope(): number { // The technical definition is the commented-out line, but the denominator component of both // cancels out - it's 'more efficient' to use the following line as a result. @@ -41,6 +55,10 @@ namespace com.keyman.osk { return val; } + /** + * The 'intercept' of the 'slope-intercept' form of the line that best fits the relationship between + * this regression's selected axes. + */ get intercept(): number { // Performing a regression based on these pre-summed values means that our obtained intercept is in // the mapped coordinate system. @@ -52,10 +70,20 @@ namespace com.keyman.osk { return val; } + /** + * The total summed squared-distances of the best fitting line from actually-observed values; + * in other words, the "sum of the squared errors". + * + * Statistically, this is the portion of the dependent variable's variance (un-normalized) + * that is unexplained by this regression. + */ get sumOfSquaredError(): number { return this.accumulator.squaredSum(this.dependent) - this.sumOfSquaredModeled; } + /** + * The portion of the dependent variable's variance that is successfully explained by this regression. + */ get sumOfSquaredModeled(): number { // If we have a perfectly straight vertical line, from the perspective of our independent axis, // we get infinite slope. That's... not great for the math. @@ -68,6 +96,10 @@ namespace com.keyman.osk { } } + /** + * A statistical term that signals how successful the regression is. Always has values on + * the interval [0, 1], with 1 being a perfect fit. + */ get coefficientOfDetermination(): number { if(this.accumulator.squaredSum(this.dependent) == 0 || this.accumulator.squaredSum(this.independent) == 0) { return 1; @@ -80,33 +112,41 @@ namespace com.keyman.osk { return num / denom; } + /** + * Gets the value of the dependent axis that lies on the regression's fitted line + * for a specified independent axis value. + * + * @param value The input value to use for the independent axis's variable. + * @returns The predicted dependent axis value. + */ predictFromValue(value: number) { return this.slope * value + this.intercept; } } - private rawLinearSums: {'x': number, 'y': number, 't': number} = {'x': 0, 'y': 0, 't': 0}; - - private xCentroidSum: number = 0; - private yCentroidSum: number = 0; - - private rawSquaredSums: {'x': number, 'y': number, 't': number} = {'x': 0, 'y': 0, 't': 0}; - + private rawLinearSums: {'x': number, 'y': number, 't': number, 'v': number} = {'x': 0, 'y': 0, 't': 0, 'v': 0}; + private rawSquaredSums: {'x': number, 'y': number, 't': number, 'v': number} = {'x': 0, 'y': 0, 't': 0, 'v': 0}; + // Would 'tv' (time vs velocity) be worth it to track? And possibly even do a regression for? + // If so, maybe throw that in. private rawCrossSums: {'tx': number, 'ty': number, 'xy': number} = {'tx': 0, 'ty': 0, 'xy': 0}; private coordArcSum: number = 0; + private arcSampleCount: number = 0; - private speedLinearSum: number = 0; - private speedQuadSum: number = 0; - + // These two are kept separate because of their extreme interconnectedness - after all, + // they actually represent a SINGLE (polar) axis - the angle. + // + // Sadly, there's no straightforward, well-founded way to use these to give a proper + // statistical sense of 'fit' or 'regression' here, _especially_ in regard to segmentation. + // Proper angle-[other] cross-sums are pretty much impossible, at least as efficiently + // as the others are handled [O(1)]. private cosLinearSum: number = 0; private sinLinearSum: number = 0; - private arcSampleCount: number = 0; /** * The base sample used to transpose all other received samples. Use of this helps - * avoid potential "catastrophic cancellation" effects that can occur when diffing two - * numbers far from the sample-space's mathematical origin. + * avoid potential "catastrophic cancellation" effects that can occur when diffing + * two numbers far from the sample-space's mathematical origin. * * Refer to https://en.wikipedia.org/wiki/Catastrophic_cancellation. */ @@ -118,9 +158,9 @@ namespace com.keyman.osk { */ private initialSample?: InputSample; - /*private*/ lastSample?: InputSample; + private _lastSample?: InputSample; private followingSample?: InputSample; - private sampleCount = 0; + private _sampleCount = 0; constructor(); constructor(sample: InputSample); @@ -189,11 +229,6 @@ namespace com.keyman.osk { result.coordArcSum += Math.sqrt(coordArcDeltaSq); - // Approximates weighting the time spent at each coord by splitting the time since - // last event evenly for both coordinates. Note: does NOT shift based upon .baseSample! - result.xCentroidSum += 0.5 * tDeltaInSec * (sample.targetX + this.lastSample.targetX); - result.yCentroidSum += 0.5 * tDeltaInSec * (sample.targetY + this.lastSample.targetY); - if(xDelta || yDelta) { // We wish to measure angle clockwise from <0, -1> in the DOM. So, cos values should // align with that axis, while sin values should align with the positive x-axis. @@ -206,12 +241,12 @@ namespace com.keyman.osk { } if(tDeltaInSec) { - result.speedLinearSum += Math.sqrt(coordArcDeltaSq) / tDeltaInSec; - result.speedQuadSum += coordArcDeltaSq / (tDeltaInSec * tDeltaInSec); + result.rawLinearSums['v'] += Math.sqrt(coordArcDeltaSq) / tDeltaInSec; + result.rawSquaredSums['v'] += coordArcDeltaSq / (tDeltaInSec * tDeltaInSec); } } - result.lastSample = sample; + result._lastSample = sample; result.sampleCount = this.sampleCount + 1; return result; @@ -225,18 +260,20 @@ namespace com.keyman.osk { * @returns */ public deaccumulate(subsetStats?: CumulativePathStats): CumulativePathStats { - // Possible TODO: Because of the properties of statistical variance & mean... - // we could further prevent catastrophic cancellation by re-centering - // all the linear, cross, and quad sums. - // - mostly noteworthy for _long_ duration touches that wander long distances. - // - basically, for cases that'd cause the floating-point error to exceed our - // test thresholds. Re-centering would keep that error consistently below - // our thresholds. + // Possible addition: use `this.buildRenormalized` on the returned version + // if catastrophic cancellation effects (random, small floating point errors) + // are not sufficiently mitigated & handled by the measures currently in place. // - // We could then take the new mean coordinates as a 'base sample'. - // Kinda has to be the new mean b/c of the stats identities we'd be abusing, - // but that's also the best catastrophic-cancellation prevention move we - // could take. So, this limitation's not really a negative. + // Even then, we'd need to apply such generated objects carefully - we can't + // re-merge the accumulated values or remap them to their old coordinate system + // afterward.`buildRenormalize`'s remapping maneuver is a one-way stats-abuse trick. + // + // Hint: we'd need to pay attention to the "lingering segments" aspects in which + // detected sub-segments might be "re-merged". + // - Whenever they're merged & cleared, we should be clear to recentralize + // the cumulative stats that follow. If any are still active, we can't + // recentralize. + const result = new CumulativePathStats(this); // We actually WILL accept a `null` argument; makes some of the segmentation @@ -276,23 +313,13 @@ namespace com.keyman.osk { result.coordArcSum -= Math.sqrt(coordArcSq); result.coordArcSum -= subsetStats.coordArcSum; - // Centroid sum management! - // Same reasoning pattern as for the 'arc length stuff'. - result.xCentroidSum -= 0.5 * tDeltaInSec * (subsetStats.followingSample.targetX + subsetStats.lastSample.targetX) - result.xCentroidSum -= subsetStats.xCentroidSum; - result.yCentroidSum -= 0.5 * tDeltaInSec * (subsetStats.followingSample.targetY + subsetStats.lastSample.targetY) - result.yCentroidSum -= subsetStats.yCentroidSum; - result.cosLinearSum -= subsetStats.cosLinearSum; result.sinLinearSum -= subsetStats.sinLinearSum; result.arcSampleCount -= subsetStats.arcSampleCount; - result.speedLinearSum -= subsetStats.speedLinearSum; - result.speedQuadSum -= subsetStats.speedQuadSum; - if(tDeltaInSec) { - result.speedLinearSum -= Math.sqrt(coordArcSq) / tDeltaInSec; - result.speedQuadSum -= coordArcSq / (tDeltaInSec * tDeltaInSec); + result.rawLinearSums['v'] -= Math.sqrt(coordArcSq) / tDeltaInSec; + result.rawSquaredSums['v'] -= coordArcSq / (tDeltaInSec * tDeltaInSec); } } @@ -310,15 +337,30 @@ namespace com.keyman.osk { return result; } + public get lastSample() { + return this._lastSample; + } + public get lastTimestamp(): number { return this.lastSample?.t; } - public get count() { - return this.sampleCount; + public get sampleCount() { + return this._sampleCount; } - private mappingConstant(dim: 'x' | 'y' | 't') { + private set sampleCount(value: number) { + this._sampleCount = value; + } + + /** + * In order to mitigate the accumulation of small floating-point errors during the + * various accumulations performed by this class, the domain of incoming values + * is remapped near to the origin via axis-specific mapping constants. + * @param dim + * @returns + */ + private mappingConstant(dim: 'x' | 'y' | 't' | 'v') { if(!this.baseSample) { return undefined; } @@ -327,33 +369,42 @@ namespace com.keyman.osk { return this.baseSample.t; } else if(dim == 'x') { return this.baseSample.targetX; - } else { + } else if(dim == 'y') { return this.baseSample.targetY; + } else { + return 0; } } - private mappedMean(dim: 'x' | 'y' | 't') { + /** + * Gets the statistical mean, utilizing the internal 'mapped' coordinate space. + * This is the version compatible with cross-sums and squared-sums. + * @param dim + * @returns + */ + private mappedMean(dim: 'x' | 'y' | 't' |'v') { return this.rawLinearSums[dim] / this.sampleCount; } - public get centroid(): {x: number, y: number} { - if(this.sampleCount == 0) { - return undefined; - } else if(this.sampleCount == 1) { - return { - x: this.lastSample.targetX, - y: this.lastSample.targetY - }; - } else { - const coeff = 1 / (this.duration); // * (this.sampleCount-1)); - return { - x: this.xCentroidSum * coeff, - y: this.yCentroidSum * coeff - }; - } + /** + * Gets the statistical mean value of the samples observed during the represented + * interval on the specified axis. + * @param dim + * @returns + */ + public mean(dim: 'x' | 'y' | 't' | 'v') { + // This external-facing version needs to provide values in 'external'-friendly + // coordinate space. + return this.mappedMean(dim) + this.mappingConstant(dim); } - public squaredSum(dim: 'x' | 'y' | 't') { + /** + * Gets the sum of the squared distance from the mean seen in samples observed + * during the represented interval on the specified axis. + * @param dim + * @returns + */ + public squaredSum(dim: 'x' | 'y' | 't' | 'v') { const x2 = this.rawSquaredSums[dim]; const x1 = this.rawLinearSums[dim]; @@ -362,6 +413,12 @@ namespace com.keyman.osk { return val > 1e-8 ? val : 0; } + /** + * Gets the sum of the statistical 'cross' term "distance" away from the mean + * observed during the represented interval on the specified axis. + * @param dimPair + * @returns + */ public crossSum(dimPair: 'tx' | 'ty' | 'xy') { const dim1 = dimPair.charAt(0); const dim2 = dimPair.charAt(1); @@ -381,22 +438,39 @@ namespace com.keyman.osk { return Math.abs(val) > 1e-8 ? val : 0; } + /** + * Gets the unbiased covariance between the specified pair of axes for samples + * observed during the represented interval. + * @param dimPair + * @returns + */ public covariance(dimPair: 'tx' | 'ty' | 'xy') { return this.crossSum(dimPair) / (this.sampleCount - 1); } - public variance(dim: 'x' | 'y' | 't') { + /** + * Gets the unbiased variance on the specified axis for samples observed + * during the represented interval. + */ + public variance(dim: 'x' | 'y' | 't' | 'v') { return this.squaredSum(dim) / (this.sampleCount - 1); } + /** + * Utilizing (and possibly abusing) statistical identities, this function produces + * an equivalent, but-recentered copy of this instance's statistical accumulations + * that will be less prone to catastrophic cancellation. + * + * In non-stats speak, the new instance will suffer smaller floating-point + * errors than the old instance whenever they do occur. + * @returns + */ public buildRenormalized(): CumulativePathStats { + // Other (internal) notes: the internal mapping of the new instance will not + // match that of the old instance. This should not affect the practical + // results of any mapping to and from the external coordinate space, however. let result = new CumulativePathStats(this); - // By abusing the statistical identities for calculating various expressions - // related to regression, we can re-center our mapped coordinate system on - // our current mean. We shouldn't do so too frequently, but this should help - // moderate effects from catastrophic cancellation. - let newBase: InputSample = { targetX: this.mappedMean['x'] + this.baseSample.targetX, targetY: this.mappedMean['y'] + this.baseSample.targetY, @@ -416,10 +490,23 @@ namespace com.keyman.osk { return result; } + /** + * Provides a linear-regression perspective on two specified axes over the represented + * interval. + * @param dependent + * @param independent + * @returns + */ public fitRegression(dependent: 'x' | 'y' | 't', independent: 'x' | 'y' | 't') { return new CumulativePathStats.regression(this, dependent, independent); } + /** + * Provides the direct Euclidean distance between the start and end points of the segment + * (or curve) of the interval represented by this instance. + * + * This will likely not match the actual pixel distance traveled. + */ public get netDistance() { // No issue with a net distance of 0 due to a single point. if(!this.lastSample || !this.initialSample) { @@ -432,6 +519,11 @@ namespace com.keyman.osk { return Math.sqrt(xDelta * xDelta + yDelta * yDelta); } + /** + * Gets the duration of the represented interval, in seconds. + * + * Note: input samples provide their timestamps in milliseconds. + */ public get duration() { // no issue with a duration of zero from just one sample. if(!this.lastSample || !this.initialSample) { @@ -459,10 +551,21 @@ namespace com.keyman.osk { return xDelta < 0 ? (2 * Math.PI - yAngleDiff) : yAngleDiff; } + /** + * Returns the angle (in degrees) traveled by the corresponding segment clockwise + * from the unit vector <0, -1> in the DOM (the unit "upward" direction). + */ public get angleInDegrees() { return this.angle * 180 / Math.PI; } + /** + * Returns the cardinal or intercardinal direction on the screen that most + * closely matches the direction of movement represented by the represented + * segment. + * + * @return A string one or two letters in length. (e.g: 'n', 'sw') + */ public get cardinalDirection() { if(this.sampleCount == 1 || !this.lastSample || !this.initialSample) { return undefined; @@ -480,26 +583,22 @@ namespace com.keyman.osk { return 'n'; } - // px per s. + /** + * Measured in pixels per second. + */ public get speed() { // this.duration is already in seconds, not milliseconds. return this.duration ? this.netDistance / this.duration : Number.NaN; } - // ... may not be "right". - public get speedMean() { - return this.speedLinearSum / (this.sampleCount-1); - } - - public get speedVariance() { - return this.speedQuadSum / (this.sampleCount-1) - (this.speedMean * this.speedMean); - } - /** * Returns the represented interval's 'mean angle' clockwise from the DOM's * <0, -1> (the unit vector toward the top of the screen) in radians. * - * Uses the 'circular mean'. Refer to https://en.wikipedia.org/wiki/Circular_mean. + * Based upon the 'circular mean'. Refer to https://en.wikipedia.org/wiki/Circular_mean. + * + * Note that very slow-moving segments may be heavily affected by pixel aliasing + * effects; mouse and touch events usually do not provide sub-pixel resolution. */ public get angleMean() { if(this.arcSampleCount == 0) { @@ -533,22 +632,17 @@ namespace com.keyman.osk { return rSquaredBase / (this.arcSampleCount * this.arcSampleCount); } - /** - * The **circular variance** of the represented interval's angle observations. - * - * Refer to https://en.wikipedia.org/wiki/Directional_statistics#Variance. - */ - public get angleVariance() { - if(this.arcSampleCount == 0) { - return Number.NaN; - } - return 1 - (this.angleRSquared); - } - /** * The **circular standard deviation** of the represented interval's angle observations. * * Refer to https://en.wikipedia.org/wiki/Directional_statistics#Standard_deviation. + * + * Note that very slow-moving segments may be heavily affected by pixel aliasing + * effects; mouse and touch events usually do not provide sub-pixel resolution. + * This can result in very high deviation values. + * + * In less-technical terms - the "stair-stepping" effect seen on high zoom levels means we don't + * get perfectly straight lines, and that can cause this value to be unexpectedly high. */ public get angleDeviation() { if(this.arcSampleCount == 0) { @@ -558,24 +652,34 @@ namespace com.keyman.osk { return Math.sqrt(-Math.log(this.angleRSquared)); } + /** + * Provides the actual, pixel-based distance actually traveled by the represented segment. + * May not be an integer (because diagonals are a thing). + */ public get rawDistance() { return this.coordArcSum; } - // TODO: is this actually ideal? This was certainly useful for experimentation via interactive - // debugging, but it may not be the best thing long-term. - public toJSON() { + // Convert to a `toJSON` method for use during investigative debugging. + private toDebuggingJSON() { return { - angleMean: this.angleMean, - angleMeanDegrees: this.angleMean * 180 / Math.PI, - angleVariance: this.angleVariance, - speedMean: this.speedMean, - speedVariance: this.speedVariance, + angle: this.angle, + speedMean: this.mean('v'), rawDistance: this.rawDistance, duration: this.duration, - sampleCount: this.sampleCount + sampleCount: this.sampleCount, + angleMeanDegrees: this.angleMean * 180 / Math.PI, + angleDeviation: this.angleDeviation, + speedVariance: this.variance('v') } } + + public toJSON() { + // We're not actually saving the JSON out to anything yet or loading/parsing it + // for any use beyond direct human interpretation, so... it's "okay" to + // leave like this for now. + return this.toDebuggingJSON(); + } } } \ No newline at end of file diff --git a/common/web/gesture-recognizer/src/pathSegmenter.ts b/common/web/gesture-recognizer/src/pathSegmenter.ts index 15088134ac..5b34331913 100644 --- a/common/web/gesture-recognizer/src/pathSegmenter.ts +++ b/common/web/gesture-recognizer/src/pathSegmenter.ts @@ -2,11 +2,11 @@ namespace com.keyman.osk { - // https://en.wikipedia.org/wiki/F-distribution // Mostly used here to compare the sum-squared error components of segmented regressions // to the sum-squared "modeled" components of their overall variance. Those are the two // "independent random variables" we're examining. Variances (which are sums of squared // values themselves) tend to be chi-squared distributed, fulfilling the conditions. + // That's stats-speak for "what this module does with it has a solid theoretical basis." // // We do NOT want to be computing this thing at run-time. (Just take one look at the // equations found in that Wikipedia article!) Fortunately, it's very common in stats @@ -19,6 +19,17 @@ namespace com.keyman.osk { // stats term; they'd then say that if the p-value is sufficiently low, then we\ // "reject the null hypothesis" - the theory that it actually DID happen randomly - in // favor of the high likelihood that we really are onto something. + + /** + * Acts as a lookup table for null-hypothesis tests utilizing the F-distribution from the + * domain of statistics. Refer to https://en.wikipedia.org/wiki/F-distribution for details + * on its properties. + * + * This is only a partial implementation; only a couple of numerator degrees-of-freedom are + * worth our consideration, and there's only marginal gain to be found supporting more + * denominator degrees-of-freedom than are already supported here. Keeping it small also + * helps with maintenance and readability. + */ class FDistribution { /** @@ -37,7 +48,11 @@ namespace com.keyman.osk { * * The f-statistic value must match or exceed its corresponding entry in * the table below, based on the detected DoF (degrees of freedom) for the - * 'numerator' and 'denominator' components. + * 'numerator' and 'denominator' components. Exceeding the threshold specified + * for the p = .100 section indicates less than a 10% probability of the statistic + * being tested having arisen by purely random chance. Failure to exceed even + * that implies an unacceptably high chance of being "convenient but meaningless" - + * the "null hypothesis", in stats speak. */ private static readonly table = [ // p = .100 @@ -94,11 +109,9 @@ namespace com.keyman.osk { * @param numDoF * @param denomDoF * - * Tier 0: don't segment - * Tier 1: segment if the other axis also says to segment - * - p-value < 0.100 on the tested axis - * Tier 2: segment regardless of what the other axis says - * - p-value < 0.050 on the tested axis + * Returns 0.050 if the statistic indicates a p-value of 0.050 or lower. + * Failing that, returns 0.100 if a p-value of 0.100 or lower is indicated. + * Otherwise, returns 1.0 - we have no basis to "reject the null hypothesis". */ static thresholdTier(statistic: number, numDoF: number, denomDoF: number) { // Former case: we'd never segment anyway @@ -128,22 +141,61 @@ namespace com.keyman.osk { } } + /** + * Represents the result of a segmentation of the search path and the related + * statistical accumulations needed to properly test the segmentation's + * validity. + */ class Segmentation { public static readonly SPLIT_CRITERION_THRESHOLD = 1.5; + /** + * The properties of the "left-hand" / earlier half of the time interval + * being segmented. + */ readonly pre: CumulativePathStats; + + /** + * The properties of the "right-hand" / later half of the time interval + * being segmented. + */ readonly post: CumulativePathStats; + + /** + * The properties of the full interval being segmented, before the split. + */ readonly union: CumulativePathStats; + + /** + * The full running accumulation for the ongoing touch path. May include + * accumulations from before the interval under examination. + */ readonly endpoint: CumulativePathStats; + /** + * Provides segmented-regression statistics & fitting values on the two specified axes / + * dimensions based on the subsegments and their corresponding linear-regression + * statistics. All operations are O(1). + */ static readonly segmentationComparison = class SegmentedRegression { host: Segmentation; readonly independent: 'x' | 'y' | 't'; readonly dependent: 'x' | 'y' | 't'; readonly paired: 'tx' | 'ty' | 'xy'; + /** + * The linear regression for the underlying `pre` (earlier) subsegment. + */ pre: typeof CumulativePathStats.regression.prototype; + + /** + * The linear regression for the underlying `post` (later) subsegment. + */ post: typeof CumulativePathStats.regression.prototype; + + /** + * The linear, unsegmented regression for the underlying `union` (combined) subsegment. + */ union: typeof CumulativePathStats.regression.prototype; constructor(host: Segmentation, dependentAxis: 'x' | 'y' | 't', independentAxis: 'x' | 'y' | 't') { @@ -167,10 +219,22 @@ namespace com.keyman.osk { this.union = this.host.union.fitRegression(dependentAxis, independentAxis); } + /** + * The total summed squared-distances of the best fitting line from actually-observed values + * for their corresponding subsegment; in other words, the "sum of the squared errors" when + * utilizing segmented regression. + * + * Statistically, this is the portion of the dependent variable's variance (un-normalized) + * that is unexplained even after dividing the interval into separate subsegments. + */ get remainingSumSquaredError(): number { return this.pre.sumOfSquaredError + this.post.sumOfSquaredError; } + /** + * Provides the coordinate that belongs to BOTH subsegments, as it ends one and begins + * the other. + */ private get splitPoint() { let splitPoint = this.host.pre.lastSample; @@ -194,6 +258,11 @@ namespace com.keyman.osk { }; } + /** + * The total summed squared-distances of the best fitting line from actually-observed values + * when **not** utilizing segmented regression. Double-counts the subsegment's common point, as + * it is counted within both subsegments' sum-of-squared error term. + */ get unsegmentedSumSquaredError(): number { // What was our remaining error for the unsegmented regression? let baseSSE = this.union.sumOfSquaredError; @@ -207,10 +276,19 @@ namespace com.keyman.osk { return baseSSE + splitPointError * splitPointError; } + /** + * Gets the amount of (unnormalized) variance not explained by a standard regression on + * the full interval but succesfully explained by splitting the interval in twain via + * segmented regression. + */ get sumSquaredGainFromSegmentation(): number { return this.unsegmentedSumSquaredError - this.remainingSumSquaredError; } + /** + * A statistical term that signals how successful the segmented regression is. Always + * has values on the interval [0, 1], with 1 being a perfect fit. + */ get coefficientOfDetermination(): number { if(this.host.union.squaredSum(this.dependent) == 0 || this.host.union.squaredSum(this.independent) == 0) { return 1; @@ -219,6 +297,11 @@ namespace com.keyman.osk { return 1 - this.remainingSumSquaredError / this.host.union.squaredSum(this.dependent); } + /** + * Gets the raw statistic value needed to test for validity of segmentation based on the axes + * under examination. The higher the proportion of newly-explained variance to still-unexplained + * variance is, the more confident we can _formally_ be in our segmentation. + */ get fStat(): number { const val = this.sumSquaredGainFromSegmentation / this.remainingSumSquaredError; @@ -226,6 +309,10 @@ namespace com.keyman.osk { return isNaN(val) ? 0 : val; } + /** + * Gets the degrees-of-freedom to be used for the numerator DoF parameter of + * the F-distribution needed for validity testing. + */ get fDoF1(): number { let numDoF = 3; // Cases where there clearly may as well be 'no slope' at all. @@ -241,10 +328,21 @@ namespace com.keyman.osk { return numDoF; } + /** + * Gets the degrees-of-freedom to be used for the denominator DoF parameter of + * the F-distribution needed for validity testing. + */ get fDoF2(): number { - return this.host.union.count - 2 - this.fDoF1; + return this.host.union.sampleCount - 2 - this.fDoF1; } + /** + * States a statistically-founded level of confidence we may place in upholding + * the represented segmentation, as based on the regressions and axes under examination. + * + * If we cannot hold any confidence whatsoever (due to being unable to "reject the + * null hypothesis" from this specific regression), will return 0. + */ get certaintyThreshold() { return 1 - FDistribution.thresholdTier(this.fStat, this.fDoF1, this.fDoF2); } @@ -260,10 +358,23 @@ namespace com.keyman.osk { this.endpoint = cumulativeEndpoint; } + /** + * Provides a segmented-regression perspective for the represented interval and proposed + * split point based upon the two specified axes. + * @param dependent + * @param independent + * @returns + */ public segReg(dependentAxis: 'x' | 'y' | 't', independentAxis: 'x' | 'y' | 't') { return new Segmentation.segmentationComparison(this, dependentAxis, independentAxis); } + /** + * Defines our test for upholding a minimum of sub-segmentation based upon changes in the + * interval's coordinates over time. This may occur even with no change in spacial direction + * if there is significant enough change in _speed_, as that "curves the line" when one axis + * is time. + */ get segmentationMerited(): boolean { let totalThreshold = 0; const xTest = new Segmentation.segmentationComparison(this, 'x', 't'); @@ -278,13 +389,19 @@ namespace com.keyman.osk { return totalThreshold >= 2; } + /** + * Defines our test for considering two (or more) sub-segments as part of the same overall + * segment. This seeks to 'link' geometrically-related subsegments - especially those that + * maintain the same direction but differ only in observed speed. + */ get mergeMerited(): boolean { - // Because of caret-like motions, we need to text for regression on both axes. - // I think? Or does it really make any sort of difference? + // Because of caret-like motions (as in, in the '^' shape), we need to text for + // regression on both axes. One may have notably higher variance than the other. + // + // These tests ignore time, and therefore speed. Only the raw geometry of the motion + // matters for this test. const xTest = new Segmentation.segmentationComparison(this, 'x', 'y'); const yTest = new Segmentation.segmentationComparison(this, 'y', 'x'); - // const xTestConfig = this.xyFTestConfiguration; - // const yTestConfig = this.yxFTestConfiguration; // If we don't get a p-value less than .100, then as far as x & y are concerned - and thus the user // is concerned - it's the same segment. Speed may be different, but not the direction. @@ -296,9 +413,32 @@ namespace com.keyman.osk { } } + /** + * A specialized form of the `Segmentation` class that assists in its + * construction from the interval directly under examination by the + * segmentation algorithm. + */ class PotentialSegmentation extends Segmentation { + /** + * The full running accumulation for the ongoing touch path until the point + * before `post` / the later subsegment. May include + * accumulations from before the interval under examination. + */ readonly chopPoint: CumulativePathStats; + + /** + * The full running accumulation for the ongoing touch path until the point + * at the end of `pre` / the earlier subsegment. May include + * accumulations from before the interval under examination, and will + * include accumulations from the first point represented by `post`. + */ readonly endOfPre: CumulativePathStats; + + /** + * The full running accumulation for the ongoing touch path until the point + * before `pre` / the earlier subsegment. Will only include + * accumulations from before the interval under examination. + */ readonly baseChop: CumulativePathStats; constructor(steppedStats: CumulativePathStats[], @@ -318,30 +458,9 @@ namespace com.keyman.osk { } } - /* FIXME: Note that this function is a temporary development stopgap and will likely shift - * as development continues for a few reasons: - * 1. In its current form, it'd be better to return a completed `Segment`; this is being used - * to finalize `Segment`s, after all. - * 2. Except... we'll actually want it for uncompleted `Segment`s too, for the tail member of - * the public `path.segments` array, which'll need UPDATING, not replacement. - * 3. In some cases, subsegmentation provides an advantage for constructing / updating `Segment`s. - * E.g: Flicks threshold based on top speed, and the faster subsegment's stats are far better - * for this than the combined interval's stats. - * - * Also worth note: makes a LOT of assumptions about how it will be used; namely, that - * the members of the array originally were contiguous and are in their original - * segmentation order. This holds for current use cases, but this is why the func - * will not be exported. + /** + * The core logic, algorithm, and manager for touchpath segmentation. */ - const mergeSubsegmentations = function(array: PotentialSegmentation[]) { - if(array.length == 1) { - return array[0].pre; // It's pre-calculated, so just use it. - } else { - const finalSubsegmentation = array[array.length-1]; - return finalSubsegmentation.endOfPre.deaccumulate(array[0].baseChop); - } - } - export class PathSegmenter { /** * The minimum amount of time (in ms) to wait between sample repetitions @@ -363,13 +482,30 @@ namespace com.keyman.osk { */ private steppedCumulativeStats: CumulativePathStats[]; + /** + * Tracks all subsegments awaiting completion of their overall segment. + */ private lingeringSubsegmentations: PotentialSegmentation[]; - // Currently used as an in-development diagnostic assist... but these - // directly represent actual path segments as produced by the prototype - // algorithm. Just... the stats analysis of the path segment, without - // obvious / public members to relevant coordinates. + /** + * TODO: These directly represent path segments produced by the prototype + * algorithm. Just... the stats analysis of the path segment, without + * obvious / public members to relevant coordinates. + * + * In follow-up work, this should be used to build & to update + * cleaner, public-facing segment objects for the touch path. + */ private _protoSegments: CumulativePathStats[] = []; + + /** + * TODO: These directly represent the subsegments comprising their + * corresponding path segments (in `_protoSegments`) produced by the + * prototype algorithm. Just... the stats analysis of each subsegment. + * + * In follow-up work, these should be used internally to build & update + * the internals of TBD cleaner, public-facing segment objects for the + * touch path. + */ private _protoSegmentSets: CumulativePathStats[][] = []; /** @@ -392,13 +528,17 @@ namespace com.keyman.osk { * that lie on already-fully-segmented parts of it. */ private choppedStats: CumulativePathStats = null; - private lastIntervalDuration = 0; constructor() { this.steppedCumulativeStats = []; this.lingeringSubsegmentations = []; } + /** + * Appends a new coordinate to the touch path and also kick-starts a timer for replicating it + * should neither further inputs be received nor termination of the touchpoint be indicated. + * @param sample + */ public add(sample: InputSample) { const repeater = (timeDelta: number) => { this.observe(sample, timeDelta); @@ -418,6 +558,10 @@ namespace com.keyman.osk { repeater(0); } + /** + * Used to finalize all segmentation for the touchpath whenever termination of the corresponding + * touchpoint has been terminated (mouse-up, touch-end). + */ public close() { // The Node clearTimeout & DOM clearTimeout appear to TS as overloads of each other, // and their type definitions will conflict. A simple @ts-ignore will bypass this issue. @@ -433,12 +577,23 @@ namespace com.keyman.osk { this.filterSubsegmentation(finalization, true); // forces out the final segment. // Hacky, but "enough" for now. - // FIXME: temporary statement to facilitate exploration, experimentation, & debugging + // TODO: these are temporary statements to facilitate exploration, experimentation, & debugging. + // We should be providing output to the touchpath object (`.path.segments`). + // But... that'll be left for a follow-up PR. console.log(this._protoSegments); console.log(this._protoSegmentSets); console.log(this._protoSegments.map((val) => (val.toJSON()))); } + /** + * Adds a statistic 'observation' of the state of the touchpath. + * + * This is either to be called directly upon reception of a new input-coordinate or + * upon replication of a prior input should the touchpath still be active without any + * indication of motion. + * @param sample + * @param timeDelta + */ private observe(sample: InputSample, timeDelta: number) { let cumulativeStats: CumulativePathStats; if(this.steppedCumulativeStats.length) { @@ -452,7 +607,7 @@ namespace com.keyman.osk { const extendedStats = cumulativeStats.extend(sample); this.steppedCumulativeStats.push(extendedStats); - this.attemptSegmentation(); + this.performSubsegmentation(); } private _debugLogSegmentationReport(candidateSplit: PotentialSegmentation) { @@ -484,7 +639,17 @@ namespace com.keyman.osk { // END: DO NOT RELEASE. } - private attemptSegmentation() { + // The "reported via event" aspect mentioned below is necessary because this may be + // called via setTimeout callback on a held touchpoint. We need that `setTimeout` + // (which replicates the most recently-seen input coordinate) in order to have good + // sample-data for detecting the boundary between motion and lack thereof during + // segmentation. + /** + * This is the "main method" for touchpath segmentation. Should a new segment result + * from its analysis, it will be reported via event. + * @returns + */ + private performSubsegmentation() { const cumulativeStats = this.steppedCumulativeStats[this.steppedCumulativeStats.length - 1]; const unsegmentedDuration = cumulativeStats.lastTimestamp - this.steppedCumulativeStats[0].lastTimestamp; @@ -511,9 +676,6 @@ namespace com.keyman.osk { // of both of the resulting intervals. let candidateSplit = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint); - const lastIntervalDuration = this.lastIntervalDuration; - this.lastIntervalDuration = unsegmentedDuration; - const xF = candidateSplit.segReg('x', 't'); const yF = candidateSplit.segReg('y', 't'); if(!candidateSplit.segmentationMerited) { @@ -535,7 +697,6 @@ namespace com.keyman.osk { // We either have the conditions to trigger segmentation or just became long enough to consider it. // If we're only just long enough to consider it, there may be a better segmentation point to start with. // Now... is there a better segmentation point? - class SplitSearchState { readonly xTest: typeof Segmentation.segmentationComparison.prototype; readonly yTest: typeof Segmentation.segmentationComparison.prototype; @@ -552,6 +713,8 @@ namespace com.keyman.osk { this.yTest = candidate.segReg('y', 't'); } + // The value for an 'objective function' to optimize in our search for a better candidate. + // The higher this is, the more we like its potential as the segmentation point. get segRating() { if(this.candidate) { return Math.max(this.xTest.coefficientOfDetermination, this.yTest.coefficientOfDetermination); @@ -560,6 +723,8 @@ namespace com.keyman.osk { } } + // But, if it turns out this potential point wouldn't actually result in segmentation, well... + // "abandon ship". get segmentationMerited() { return this.candidate?.segmentationMerited ?? false; } @@ -577,7 +742,6 @@ namespace com.keyman.osk { const criteria = [leftSplit.segRating, currentSplit.segRating, rightSplit.segRating]; let sortedCriteria = [...criteria].sort(); - // TODO: if we're better on both axes on one side, let's start shifting. const delta = criteria.indexOf(sortedCriteria[2])-1; // -1 if 'left' is best, 1 if 'right' is best. if(delta != 0) { @@ -624,9 +788,6 @@ namespace com.keyman.osk { // First phase of segmentation: complete! - // But... there are some cases where we want to prevent segmentation from fully happening. - // TODO: That. - // FIXME: DO NOT RELEASE. // This is exploratory / diagnostic code assisting development of the path segmentation // algorithm. @@ -635,24 +796,45 @@ namespace com.keyman.osk { this.steppedCumulativeStats = this.steppedCumulativeStats.slice(splitPoint); this.choppedStats = candidateSplit.chopPoint; - this.lastIntervalDuration = candidateSplit.post.duration * 1000; + + // There are some cases where we want to prevent segmentation from fully happening. + // The next method will both handle that and the signal of any completed segments that + // may result. this.filterSubsegmentation(candidateSplit); } - // NOTE: This function, as well as code called by it, are still very much still in prototyping. - private filterSubsegmentation(subsegmentation: PotentialSegmentation, force?: boolean) { - force = !!force; + /** + * Given the results of the subsegmentation process, this method analyzes the results + * in order to produce and update the detected segments. + * + * TODO: Should produce actual `Segment` objects and trigger their emission + * them via EventEmitter mechanics. + * @param subsegmentation + * @param finalize Set to `true` for a terminating touchpath, indicating that + * no further subsegmentation will occur for this touchpath. + */ + private filterSubsegmentation(subsegmentation: PotentialSegmentation, finalize?: boolean) { + finalize = !!finalize; + + // This function makes a LOT of assumptions particular to this method and class. + // It is not safe to make this class-level, and it's especially unsafe to make it `public`. + const mergeSubsegmentationAccumulations = function(array: PotentialSegmentation[]) { + if(array.length == 1) { + return array[0].pre; // It's pre-calculated, so just use it. + } else { + const finalSubsegmentation = array[array.length-1]; + return finalSubsegmentation.endOfPre.deaccumulate(array[0].baseChop); + } + } let predecessor = subsegmentation.pre; let firstChopPoint = subsegmentation.baseChop; if(this.lingeringSubsegmentations.length) { // First: check if the newly-finished subsegment should be merged with the lingering ones. - // const lastSubsegment = this.lingeringSubsegmentations[this.lingeringSubsegmentations.length-1]; - const mergedPrecursors = mergeSubsegmentations(this.lingeringSubsegmentations); + const mergedPrecursors = mergeSubsegmentationAccumulations(this.lingeringSubsegmentations); - // const tailUnion = mergeSubsegmentations([lastSubsegment, subsegmentation]); - const tailUnion = mergeSubsegmentations([...this.lingeringSubsegmentations, subsegmentation]); + const tailUnion = mergeSubsegmentationAccumulations([...this.lingeringSubsegmentations, subsegmentation]); console.log("Verifying linkage to pending merges: "); console.log(this.lingeringSubsegmentations); console.log("verification check:"); @@ -662,7 +844,7 @@ namespace com.keyman.osk { // becomes something VERY different. Validate that we should still merge the left-hand with its predecessors. if(!PathSegmenter.shouldMergeSubsegments(precursorMergeSegmentation)) { // Emit as separate subsegment. - const finishedSegment = mergeSubsegmentations(this.lingeringSubsegmentations); + const finishedSegment = mergeSubsegmentationAccumulations(this.lingeringSubsegmentations); this._protoSegments.push(finishedSegment); this._protoSegmentSets.push(this.lingeringSubsegmentations.map((val) => val.pre)); this.lingeringSubsegmentations = []; @@ -683,11 +865,10 @@ namespace com.keyman.osk { console.log("segmentation-prevention check:") console.log(fullMergeSegmentation); // if(!force && PathSegmenter.shouldMergeSubsegments(subsegmentation.pre, subsegmentation.post, subsegmentation.union)) { - if(!force && PathSegmenter.shouldMergeSubsegments(fullMergeSegmentation)) { + if(!finalize && PathSegmenter.shouldMergeSubsegments(fullMergeSegmentation)) { this.lingeringSubsegmentations.push(subsegmentation); } else { // Merge all as a completed segment! - // const finishedSegment = mergeSubsegmentations([...this.lingeringSubsegmentations, subsegmentation]); this._protoSegments.push(predecessor); this._protoSegmentSets.push([...this.lingeringSubsegmentations.map((val) => val.pre), subsegmentation.pre]); this.lingeringSubsegmentations = []; @@ -696,6 +877,14 @@ namespace com.keyman.osk { } } + /** + * Used by the 'filter' step (phase 2 of segmentation) to determine whether or not + * two subsegments should be 'linked' as members of the same segment. (That is, if + * a user would likely consider the two subsegments as belonging to the "same arc + * of motion".) + * @param segmentation + * @returns + */ private static shouldMergeSubsegments(segmentation: Segmentation): boolean { // // Known case #1: // // Near-identical direction, but heavy speed difference. @@ -705,11 +894,16 @@ namespace com.keyman.osk { // // Low-speed pivot; angle change caught on very low velocity for both subsegments. // // ... or should this be merged? Multiple mini-segments from 'wiggling' could just go ignored instead... - if(segmentation.pre.speedMean < 80 && segmentation.post.speedMean > 80) { + // This is kinda plain and arbitrary, but it seems to work "well enough" for now, + // at least at this stage of development. (Most testing was done with Chrome emulation of + // an iPhone SE.) + + // .mean('v') < 80: the mean speed (taken timestamp to timestamp) does not exceed 80px/sec. + if(segmentation.pre.mean('v') < 80 && segmentation.post.mean('v') > 80) { console.log("desegmentation exception"); - return; // SUPER TEMP: needs further work; avoids the "low-speed pivot" case. + return; } - if(segmentation.pre.speedMean > 80 && segmentation.post.speedMean < 80) { + if(segmentation.pre.mean('v') > 80 && segmentation.post.mean('v') < 80) { console.log("desegmentation exception"); return; }