erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-08-09 18:35:32 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

change(web): lots of docs, some cleanup

Browse source
This commit is contained in:
Joshua A. Horton 2022-08-19 13:52:56 +07:00
parent 58c15c3eec
commit cde550a3eb
2 changed files with 471 additions and 173 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

320
common/web/gesture-recognizer/src/cumulativePathStats.ts
View file

@ -1,13 +1,16 @@
namespace com.keyman.osk {
/**
* As the name suggests, this class exists to track cumulative mathematical values, etc
* necessary to provide statistical information. This information is used to facilitate
* path segmentation.
* As the name suggests, this class facilitates tracking of cumulative mathematical values, etc
* necessary to perform the statistical operations necessary for path segmentation.
*
* Instances of this class are immutable.
*/
export class CumulativePathStats {
// So... class-level "inner classes" are possible in TS... if defined via assignment to a field.
/**
* Provides linear-regression statistics & fitting values based on the underlying `CumulativePathStats`
* object used to generate it. All operations are O(1).
*/
static readonly regression = class RegressionFromSums {
readonly independent: 'x' | 'y' | 't';
readonly dependent: 'x' | 'y' | 't';
@ -15,6 +18,13 @@ namespace com.keyman.osk {
readonly accumulator: CumulativePathStats;
/**
*
* @param mainStats The `CumulativePathStats` instance to base all regression data on.
* @param dependentAxis The 'output' axis / dimension; the axis whose behavior should be predicted based on
* existing data of its relationship with the independent axis.
* @param independentAxis The 'input' axis/dimension.
*/
constructor(mainStats: CumulativePathStats, dependentAxis: 'x' | 'y' | 't', independentAxis: 'x' | 'y' | 't') {
if(dependentAxis == independentAxis) {
throw "Two different axes must be specified for the regression object.";
@ -32,6 +42,10 @@ namespace com.keyman.osk {
}
}
/**
* The 'slope' of the 'slope-intercept' form of the line that best fits the relationship between
* this regression's selected axes.
*/
get slope(): number {
// The technical definition is the commented-out line, but the denominator component of both
// cancels out - it's 'more efficient' to use the following line as a result.
@ -41,6 +55,10 @@ namespace com.keyman.osk {
return val;
}
/**
* The 'intercept' of the 'slope-intercept' form of the line that best fits the relationship between
* this regression's selected axes.
*/
get intercept(): number {
// Performing a regression based on these pre-summed values means that our obtained intercept is in
// the mapped coordinate system.
@ -52,10 +70,20 @@ namespace com.keyman.osk {
return val;
}
/**
* The total summed squared-distances of the best fitting line from actually-observed values;
* in other words, the "sum of the squared errors".
*
* Statistically, this is the portion of the dependent variable's variance (un-normalized)
* that is unexplained by this regression.
*/
get sumOfSquaredError(): number {
return this.accumulator.squaredSum(this.dependent) - this.sumOfSquaredModeled;
}
/**
* The portion of the dependent variable's variance that is successfully explained by this regression.
*/
get sumOfSquaredModeled(): number {
// If we have a perfectly straight vertical line, from the perspective of our independent axis,
// we get infinite slope. That's... not great for the math.
@ -68,6 +96,10 @@ namespace com.keyman.osk {
}
}
/**
* A statistical term that signals how successful the regression is. Always has values on
* the interval [0, 1], with 1 being a perfect fit.
*/
get coefficientOfDetermination(): number {
if(this.accumulator.squaredSum(this.dependent) == 0 || this.accumulator.squaredSum(this.independent) == 0) {
return 1;
@ -80,33 +112,41 @@ namespace com.keyman.osk {
return num / denom;
}
/**
* Gets the value of the dependent axis that lies on the regression's fitted line
* for a specified independent axis value.
*
* @param value The input value to use for the independent axis's variable.
* @returns The predicted dependent axis value.
*/
predictFromValue(value: number) {
return this.slope * value + this.intercept;
}
}
private rawLinearSums: {'x': number, 'y': number, 't': number} = {'x': 0, 'y': 0, 't': 0};
private xCentroidSum: number = 0;
private yCentroidSum: number = 0;
private rawSquaredSums: {'x': number, 'y': number, 't': number} = {'x': 0, 'y': 0, 't': 0};
private rawLinearSums: {'x': number, 'y': number, 't': number, 'v': number} = {'x': 0, 'y': 0, 't': 0, 'v': 0};
private rawSquaredSums: {'x': number, 'y': number, 't': number, 'v': number} = {'x': 0, 'y': 0, 't': 0, 'v': 0};
// Would 'tv' (time vs velocity) be worth it to track? And possibly even do a regression for?
// If so, maybe throw that in.
private rawCrossSums: {'tx': number, 'ty': number, 'xy': number} = {'tx': 0, 'ty': 0, 'xy': 0};
private coordArcSum: number = 0;
private arcSampleCount: number = 0;
private speedLinearSum: number = 0;
private speedQuadSum: number = 0;
// These two are kept separate because of their extreme interconnectedness - after all,
// they actually represent a SINGLE (polar) axis - the angle.
//
// Sadly, there's no straightforward, well-founded way to use these to give a proper
// statistical sense of 'fit' or 'regression' here, _especially_ in regard to segmentation.
// Proper angle-[other] cross-sums are pretty much impossible, at least as efficiently
// as the others are handled [O(1)].
private cosLinearSum: number = 0;
private sinLinearSum: number = 0;
private arcSampleCount: number = 0;
/**
* The base sample used to transpose all other received samples. Use of this helps
* avoid potential "catastrophic cancellation" effects that can occur when diffing two
* numbers far from the sample-space's mathematical origin.
* avoid potential "catastrophic cancellation" effects that can occur when diffing
* two numbers far from the sample-space's mathematical origin.
*
* Refer to https://en.wikipedia.org/wiki/Catastrophic_cancellation.
*/
@ -118,9 +158,9 @@ namespace com.keyman.osk {
*/
private initialSample?: InputSample;
/*private*/ lastSample?: InputSample;
private _lastSample?: InputSample;
private followingSample?: InputSample;
private sampleCount = 0;
private _sampleCount = 0;
constructor();
constructor(sample: InputSample);
@ -189,11 +229,6 @@ namespace com.keyman.osk {
result.coordArcSum += Math.sqrt(coordArcDeltaSq);
// Approximates weighting the time spent at each coord by splitting the time since
// last event evenly for both coordinates. Note: does NOT shift based upon .baseSample!
result.xCentroidSum += 0.5 * tDeltaInSec * (sample.targetX + this.lastSample.targetX);
result.yCentroidSum += 0.5 * tDeltaInSec * (sample.targetY + this.lastSample.targetY);
if(xDelta || yDelta) {
// We wish to measure angle clockwise from <0, -1> in the DOM. So, cos values should
// align with that axis, while sin values should align with the positive x-axis.
@ -206,12 +241,12 @@ namespace com.keyman.osk {
}
if(tDeltaInSec) {
result.speedLinearSum += Math.sqrt(coordArcDeltaSq) / tDeltaInSec;
result.speedQuadSum += coordArcDeltaSq / (tDeltaInSec * tDeltaInSec);
result.rawLinearSums['v'] += Math.sqrt(coordArcDeltaSq) / tDeltaInSec;
result.rawSquaredSums['v'] += coordArcDeltaSq / (tDeltaInSec * tDeltaInSec);
}
}
result.lastSample = sample;
result._lastSample = sample;
result.sampleCount = this.sampleCount + 1;
return result;
@ -225,18 +260,20 @@ namespace com.keyman.osk {
* @returns
*/
public deaccumulate(subsetStats?: CumulativePathStats): CumulativePathStats {
// Possible TODO: Because of the properties of statistical variance & mean...
// we could further prevent catastrophic cancellation by re-centering
// all the linear, cross, and quad sums.
// - mostly noteworthy for _long_ duration touches that wander long distances.
// - basically, for cases that'd cause the floating-point error to exceed our
// test thresholds. Re-centering would keep that error consistently below
// our thresholds.
// Possible addition: use `this.buildRenormalized` on the returned version
// if catastrophic cancellation effects (random, small floating point errors)
// are not sufficiently mitigated & handled by the measures currently in place.
//
// We could then take the new mean coordinates as a 'base sample'.
// Kinda has to be the new mean b/c of the stats identities we'd be abusing,
// but that's also the best catastrophic-cancellation prevention move we
// could take. So, this limitation's not really a negative.
// Even then, we'd need to apply such generated objects carefully - we can't
// re-merge the accumulated values or remap them to their old coordinate system
// afterward.`buildRenormalize`'s remapping maneuver is a one-way stats-abuse trick.
//
// Hint: we'd need to pay attention to the "lingering segments" aspects in which
// detected sub-segments might be "re-merged".
// - Whenever they're merged & cleared, we should be clear to recentralize
// the cumulative stats that follow. If any are still active, we can't
// recentralize.
const result = new CumulativePathStats(this);
// We actually WILL accept a `null` argument; makes some of the segmentation
@ -276,23 +313,13 @@ namespace com.keyman.osk {
result.coordArcSum -= Math.sqrt(coordArcSq);
result.coordArcSum -= subsetStats.coordArcSum;
// Centroid sum management!
// Same reasoning pattern as for the 'arc length stuff'.
result.xCentroidSum -= 0.5 * tDeltaInSec * (subsetStats.followingSample.targetX + subsetStats.lastSample.targetX)
result.xCentroidSum -= subsetStats.xCentroidSum;
result.yCentroidSum -= 0.5 * tDeltaInSec * (subsetStats.followingSample.targetY + subsetStats.lastSample.targetY)
result.yCentroidSum -= subsetStats.yCentroidSum;
result.cosLinearSum -= subsetStats.cosLinearSum;
result.sinLinearSum -= subsetStats.sinLinearSum;
result.arcSampleCount -= subsetStats.arcSampleCount;
result.speedLinearSum -= subsetStats.speedLinearSum;
result.speedQuadSum -= subsetStats.speedQuadSum;
if(tDeltaInSec) {
result.speedLinearSum -= Math.sqrt(coordArcSq) / tDeltaInSec;
result.speedQuadSum -= coordArcSq / (tDeltaInSec * tDeltaInSec);
result.rawLinearSums['v'] -= Math.sqrt(coordArcSq) / tDeltaInSec;
result.rawSquaredSums['v'] -= coordArcSq / (tDeltaInSec * tDeltaInSec);
}
}
@ -310,15 +337,30 @@ namespace com.keyman.osk {
return result;
}
public get lastSample() {
return this._lastSample;
}
public get lastTimestamp(): number {
return this.lastSample?.t;
}
public get count() {
return this.sampleCount;
public get sampleCount() {
return this._sampleCount;
}
private mappingConstant(dim: 'x' | 'y' | 't') {
private set sampleCount(value: number) {
this._sampleCount = value;
}
/**
* In order to mitigate the accumulation of small floating-point errors during the
* various accumulations performed by this class, the domain of incoming values
* is remapped near to the origin via axis-specific mapping constants.
* @param dim
* @returns
*/
private mappingConstant(dim: 'x' | 'y' | 't' | 'v') {
if(!this.baseSample) {
return undefined;
}
@ -327,33 +369,42 @@ namespace com.keyman.osk {
return this.baseSample.t;
} else if(dim == 'x') {
return this.baseSample.targetX;
} else {
} else if(dim == 'y') {
return this.baseSample.targetY;
} else {
return 0;
}
}
private mappedMean(dim: 'x' | 'y' | 't') {
/**
* Gets the statistical mean, utilizing the internal 'mapped' coordinate space.
* This is the version compatible with cross-sums and squared-sums.
* @param dim
* @returns
*/
private mappedMean(dim: 'x' | 'y' | 't' |'v') {
return this.rawLinearSums[dim] / this.sampleCount;
}
public get centroid(): {x: number, y: number} {
if(this.sampleCount == 0) {
return undefined;
} else if(this.sampleCount == 1) {
return {
x: this.lastSample.targetX,
y: this.lastSample.targetY
};
} else {
const coeff = 1 / (this.duration); // * (this.sampleCount-1));
return {
x: this.xCentroidSum * coeff,
y: this.yCentroidSum * coeff
};
}
/**
* Gets the statistical mean value of the samples observed during the represented
* interval on the specified axis.
* @param dim
* @returns
*/
public mean(dim: 'x' | 'y' | 't' | 'v') {
// This external-facing version needs to provide values in 'external'-friendly
// coordinate space.
return this.mappedMean(dim) + this.mappingConstant(dim);
}
public squaredSum(dim: 'x' | 'y' | 't') {
/**
* Gets the sum of the squared distance from the mean seen in samples observed
* during the represented interval on the specified axis.
* @param dim
* @returns
*/
public squaredSum(dim: 'x' | 'y' | 't' | 'v') {
const x2 = this.rawSquaredSums[dim];
const x1 = this.rawLinearSums[dim];
@ -362,6 +413,12 @@ namespace com.keyman.osk {
return val > 1e-8 ? val : 0;
}
/**
* Gets the sum of the statistical 'cross' term "distance" away from the mean
* observed during the represented interval on the specified axis.
* @param dimPair
* @returns
*/
public crossSum(dimPair: 'tx' | 'ty' | 'xy') {
const dim1 = dimPair.charAt(0);
const dim2 = dimPair.charAt(1);
@ -381,22 +438,39 @@ namespace com.keyman.osk {
return Math.abs(val) > 1e-8 ? val : 0;
}
/**
* Gets the unbiased covariance between the specified pair of axes for samples
* observed during the represented interval.
* @param dimPair
* @returns
*/
public covariance(dimPair: 'tx' | 'ty' | 'xy') {
return this.crossSum(dimPair) / (this.sampleCount - 1);
}
public variance(dim: 'x' | 'y' | 't') {
/**
* Gets the unbiased variance on the specified axis for samples observed
* during the represented interval.
*/
public variance(dim: 'x' | 'y' | 't' | 'v') {
return this.squaredSum(dim) / (this.sampleCount - 1);
}
/**
* Utilizing (and possibly abusing) statistical identities, this function produces
* an equivalent, but-recentered copy of this instance's statistical accumulations
* that will be less prone to catastrophic cancellation.
*
* In non-stats speak, the new instance will suffer smaller floating-point
* errors than the old instance whenever they do occur.
* @returns
*/
public buildRenormalized(): CumulativePathStats {
// Other (internal) notes: the internal mapping of the new instance will not
// match that of the old instance. This should not affect the practical
// results of any mapping to and from the external coordinate space, however.
let result = new CumulativePathStats(this);
// By abusing the statistical identities for calculating various expressions
// related to regression, we can re-center our mapped coordinate system on
// our current mean. We shouldn't do so too frequently, but this should help
// moderate effects from catastrophic cancellation.
let newBase: InputSample = {
targetX: this.mappedMean['x'] + this.baseSample.targetX,
targetY: this.mappedMean['y'] + this.baseSample.targetY,
@ -416,10 +490,23 @@ namespace com.keyman.osk {
return result;
}
/**
* Provides a linear-regression perspective on two specified axes over the represented
* interval.
* @param dependent
* @param independent
* @returns
*/
public fitRegression(dependent: 'x' | 'y' | 't', independent: 'x' | 'y' | 't') {
return new CumulativePathStats.regression(this, dependent, independent);
}
/**
* Provides the direct Euclidean distance between the start and end points of the segment
* (or curve) of the interval represented by this instance.
*
* This will likely not match the actual pixel distance traveled.
*/
public get netDistance() {
// No issue with a net distance of 0 due to a single point.
if(!this.lastSample || !this.initialSample) {
@ -432,6 +519,11 @@ namespace com.keyman.osk {
return Math.sqrt(xDelta * xDelta + yDelta * yDelta);
}
/**
* Gets the duration of the represented interval, in seconds.
*
* Note: input samples provide their timestamps in milliseconds.
*/
public get duration() {
// no issue with a duration of zero from just one sample.
if(!this.lastSample || !this.initialSample) {
@ -459,10 +551,21 @@ namespace com.keyman.osk {
return xDelta < 0 ? (2 * Math.PI - yAngleDiff) : yAngleDiff;
}
/**
* Returns the angle (in degrees) traveled by the corresponding segment clockwise
* from the unit vector <0, -1> in the DOM (the unit "upward" direction).
*/
public get angleInDegrees() {
return this.angle * 180 / Math.PI;
}
/**
* Returns the cardinal or intercardinal direction on the screen that most
* closely matches the direction of movement represented by the represented
* segment.
*
* @return A string one or two letters in length. (e.g: 'n', 'sw')
*/
public get cardinalDirection() {
if(this.sampleCount == 1 || !this.lastSample || !this.initialSample) {
return undefined;
@ -480,26 +583,22 @@ namespace com.keyman.osk {
return 'n';
}
// px per s.
/**
* Measured in pixels per second.
*/
public get speed() {
// this.duration is already in seconds, not milliseconds.
return this.duration ? this.netDistance / this.duration : Number.NaN;
}
// ... may not be "right".
public get speedMean() {
return this.speedLinearSum / (this.sampleCount-1);
}
public get speedVariance() {
return this.speedQuadSum / (this.sampleCount-1) - (this.speedMean * this.speedMean);
}
/**
* Returns the represented interval's 'mean angle' clockwise from the DOM's
* <0, -1> (the unit vector toward the top of the screen) in radians.
*
* Uses the 'circular mean'. Refer to https://en.wikipedia.org/wiki/Circular_mean.
* Based upon the 'circular mean'. Refer to https://en.wikipedia.org/wiki/Circular_mean.
*
* Note that very slow-moving segments may be heavily affected by pixel aliasing
* effects; mouse and touch events usually do not provide sub-pixel resolution.
*/
public get angleMean() {
if(this.arcSampleCount == 0) {
@ -533,22 +632,17 @@ namespace com.keyman.osk {
return rSquaredBase / (this.arcSampleCount * this.arcSampleCount);
}
/**
* The **circular variance** of the represented interval's angle observations.
*
* Refer to https://en.wikipedia.org/wiki/Directional_statistics#Variance.
*/
public get angleVariance() {
if(this.arcSampleCount == 0) {
return Number.NaN;
}
return 1 - (this.angleRSquared);
}
/**
* The **circular standard deviation** of the represented interval's angle observations.
*
* Refer to https://en.wikipedia.org/wiki/Directional_statistics#Standard_deviation.
*
* Note that very slow-moving segments may be heavily affected by pixel aliasing
* effects; mouse and touch events usually do not provide sub-pixel resolution.
* This can result in very high deviation values.
*
* In less-technical terms - the "stair-stepping" effect seen on high zoom levels means we don't
* get perfectly straight lines, and that can cause this value to be unexpectedly high.
*/
public get angleDeviation() {
if(this.arcSampleCount == 0) {
@ -558,24 +652,34 @@ namespace com.keyman.osk {
return Math.sqrt(-Math.log(this.angleRSquared));
}
/**
* Provides the actual, pixel-based distance actually traveled by the represented segment.
* May not be an integer (because diagonals are a thing).
*/
public get rawDistance() {
return this.coordArcSum;
}
// TODO: is this actually ideal? This was certainly useful for experimentation via interactive
// debugging, but it may not be the best thing long-term.
public toJSON() {
// Convert to a `toJSON` method for use during investigative debugging.
private toDebuggingJSON() {
return {
angleMean: this.angleMean,
angleMeanDegrees: this.angleMean * 180 / Math.PI,
angleVariance: this.angleVariance,
speedMean: this.speedMean,
speedVariance: this.speedVariance,
angle: this.angle,
speedMean: this.mean('v'),
rawDistance: this.rawDistance,
duration: this.duration,
sampleCount: this.sampleCount
sampleCount: this.sampleCount,
angleMeanDegrees: this.angleMean * 180 / Math.PI,
angleDeviation: this.angleDeviation,
speedVariance: this.variance('v')
}
}
public toJSON() {
// We're not actually saving the JSON out to anything yet or loading/parsing it
// for any use beyond direct human interpretation, so... it's "okay" to
// leave like this for now.
return this.toDebuggingJSON();
}
}
}

324
common/web/gesture-recognizer/src/pathSegmenter.ts
View file

@ -2,11 +2,11 @@
namespace com.keyman.osk {
// https://en.wikipedia.org/wiki/F-distribution
// Mostly used here to compare the sum-squared error components of segmented regressions
// to the sum-squared "modeled" components of their overall variance. Those are the two
// "independent random variables" we're examining. Variances (which are sums of squared
// values themselves) tend to be chi-squared distributed, fulfilling the conditions.
// That's stats-speak for "what this module does with it has a solid theoretical basis."
//
// We do NOT want to be computing this thing at run-time. (Just take one look at the
// equations found in that Wikipedia article!) Fortunately, it's very common in stats
@ -19,6 +19,17 @@ namespace com.keyman.osk {
// stats term; they'd then say that if the p-value is sufficiently low, then we\
// "reject the null hypothesis" - the theory that it actually DID happen randomly - in
// favor of the high likelihood that we really are onto something.
/**
* Acts as a lookup table for null-hypothesis tests utilizing the F-distribution from the
* domain of statistics. Refer to https://en.wikipedia.org/wiki/F-distribution for details
* on its properties.
*
* This is only a partial implementation; only a couple of numerator degrees-of-freedom are
* worth our consideration, and there's only marginal gain to be found supporting more
* denominator degrees-of-freedom than are already supported here. Keeping it small also
* helps with maintenance and readability.
*/
class FDistribution {
/**
@ -37,7 +48,11 @@ namespace com.keyman.osk {
*
* The f-statistic value must match or exceed its corresponding entry in
* the table below, based on the detected DoF (degrees of freedom) for the
* 'numerator' and 'denominator' components.
* 'numerator' and 'denominator' components. Exceeding the threshold specified
* for the p = .100 section indicates less than a 10% probability of the statistic
* being tested having arisen by purely random chance. Failure to exceed even
* that implies an unacceptably high chance of being "convenient but meaningless" -
* the "null hypothesis", in stats speak.
*/
private static readonly table = [
// p = .100
@ -94,11 +109,9 @@ namespace com.keyman.osk {
* @param numDoF
* @param denomDoF
*
* Tier 0: don't segment
* Tier 1: segment if the other axis also says to segment
* - p-value < 0.100 on the tested axis
* Tier 2: segment regardless of what the other axis says
* - p-value < 0.050 on the tested axis
* Returns 0.050 if the statistic indicates a p-value of 0.050 or lower.
* Failing that, returns 0.100 if a p-value of 0.100 or lower is indicated.
* Otherwise, returns 1.0 - we have no basis to "reject the null hypothesis".
*/
static thresholdTier(statistic: number, numDoF: number, denomDoF: number) {
// Former case: we'd never segment anyway
@ -128,22 +141,61 @@ namespace com.keyman.osk {
}
}
/**
* Represents the result of a segmentation of the search path and the related
* statistical accumulations needed to properly test the segmentation's
* validity.
*/
class Segmentation {
public static readonly SPLIT_CRITERION_THRESHOLD = 1.5;
/**
* The properties of the "left-hand" / earlier half of the time interval
* being segmented.
*/
readonly pre: CumulativePathStats;
/**
* The properties of the "right-hand" / later half of the time interval
* being segmented.
*/
readonly post: CumulativePathStats;
/**
* The properties of the full interval being segmented, before the split.
*/
readonly union: CumulativePathStats;
/**
* The full running accumulation for the ongoing touch path. May include
* accumulations from before the interval under examination.
*/
readonly endpoint: CumulativePathStats;
/**
* Provides segmented-regression statistics & fitting values on the two specified axes /
* dimensions based on the subsegments and their corresponding linear-regression
* statistics. All operations are O(1).
*/
static readonly segmentationComparison = class SegmentedRegression {
host: Segmentation;
readonly independent: 'x' | 'y' | 't';
readonly dependent: 'x' | 'y' | 't';
readonly paired: 'tx' | 'ty' | 'xy';
/**
* The linear regression for the underlying `pre` (earlier) subsegment.
*/
pre: typeof CumulativePathStats.regression.prototype;
/**
* The linear regression for the underlying `post` (later) subsegment.
*/
post: typeof CumulativePathStats.regression.prototype;
/**
* The linear, unsegmented regression for the underlying `union` (combined) subsegment.
*/
union: typeof CumulativePathStats.regression.prototype;
constructor(host: Segmentation, dependentAxis: 'x' | 'y' | 't', independentAxis: 'x' | 'y' | 't') {
@ -167,10 +219,22 @@ namespace com.keyman.osk {
this.union = this.host.union.fitRegression(dependentAxis, independentAxis);
}
/**
* The total summed squared-distances of the best fitting line from actually-observed values
* for their corresponding subsegment; in other words, the "sum of the squared errors" when
* utilizing segmented regression.
*
* Statistically, this is the portion of the dependent variable's variance (un-normalized)
* that is unexplained even after dividing the interval into separate subsegments.
*/
get remainingSumSquaredError(): number {
return this.pre.sumOfSquaredError + this.post.sumOfSquaredError;
}
/**
* Provides the coordinate that belongs to BOTH subsegments, as it ends one and begins
* the other.
*/
private get splitPoint() {
let splitPoint = this.host.pre.lastSample;
@ -194,6 +258,11 @@ namespace com.keyman.osk {
};
}
/**
* The total summed squared-distances of the best fitting line from actually-observed values
* when **not** utilizing segmented regression. Double-counts the subsegment's common point, as
* it is counted within both subsegments' sum-of-squared error term.
*/
get unsegmentedSumSquaredError(): number {
// What was our remaining error for the unsegmented regression?
let baseSSE = this.union.sumOfSquaredError;
@ -207,10 +276,19 @@ namespace com.keyman.osk {
return baseSSE + splitPointError * splitPointError;
}
/**
* Gets the amount of (unnormalized) variance not explained by a standard regression on
* the full interval but succesfully explained by splitting the interval in twain via
* segmented regression.
*/
get sumSquaredGainFromSegmentation(): number {
return this.unsegmentedSumSquaredError - this.remainingSumSquaredError;
}
/**
* A statistical term that signals how successful the segmented regression is. Always
* has values on the interval [0, 1], with 1 being a perfect fit.
*/
get coefficientOfDetermination(): number {
if(this.host.union.squaredSum(this.dependent) == 0 || this.host.union.squaredSum(this.independent) == 0) {
return 1;
@ -219,6 +297,11 @@ namespace com.keyman.osk {
return 1 - this.remainingSumSquaredError / this.host.union.squaredSum(this.dependent);
}
/**
* Gets the raw statistic value needed to test for validity of segmentation based on the axes
* under examination. The higher the proportion of newly-explained variance to still-unexplained
* variance is, the more confident we can _formally_ be in our segmentation.
*/
get fStat(): number {
const val = this.sumSquaredGainFromSegmentation / this.remainingSumSquaredError;
@ -226,6 +309,10 @@ namespace com.keyman.osk {
return isNaN(val) ? 0 : val;
}
/**
* Gets the degrees-of-freedom to be used for the numerator DoF parameter of
* the F-distribution needed for validity testing.
*/
get fDoF1(): number {
let numDoF = 3;
// Cases where there clearly may as well be 'no slope' at all.
@ -241,10 +328,21 @@ namespace com.keyman.osk {
return numDoF;
}
/**
* Gets the degrees-of-freedom to be used for the denominator DoF parameter of
* the F-distribution needed for validity testing.
*/
get fDoF2(): number {
return this.host.union.count - 2 - this.fDoF1;
return this.host.union.sampleCount - 2 - this.fDoF1;
}
/**
* States a statistically-founded level of confidence we may place in upholding
* the represented segmentation, as based on the regressions and axes under examination.
*
* If we cannot hold any confidence whatsoever (due to being unable to "reject the
* null hypothesis" from this specific regression), will return 0.
*/
get certaintyThreshold() {
return 1 - FDistribution.thresholdTier(this.fStat, this.fDoF1, this.fDoF2);
}
@ -260,10 +358,23 @@ namespace com.keyman.osk {
this.endpoint = cumulativeEndpoint;
}
/**
* Provides a segmented-regression perspective for the represented interval and proposed
* split point based upon the two specified axes.
* @param dependent
* @param independent
* @returns
*/
public segReg(dependentAxis: 'x' | 'y' | 't', independentAxis: 'x' | 'y' | 't') {
return new Segmentation.segmentationComparison(this, dependentAxis, independentAxis);
}
/**
* Defines our test for upholding a minimum of sub-segmentation based upon changes in the
* interval's coordinates over time. This may occur even with no change in spacial direction
* if there is significant enough change in _speed_, as that "curves the line" when one axis
* is time.
*/
get segmentationMerited(): boolean {
let totalThreshold = 0;
const xTest = new Segmentation.segmentationComparison(this, 'x', 't');
@ -278,13 +389,19 @@ namespace com.keyman.osk {
return totalThreshold >= 2;
}
/**
* Defines our test for considering two (or more) sub-segments as part of the same overall
* segment. This seeks to 'link' geometrically-related subsegments - especially those that
* maintain the same direction but differ only in observed speed.
*/
get mergeMerited(): boolean {
// Because of caret-like motions, we need to text for regression on both axes.
// I think? Or does it really make any sort of difference?
// Because of caret-like motions (as in, in the '^' shape), we need to text for
// regression on both axes. One may have notably higher variance than the other.
//
// These tests ignore time, and therefore speed. Only the raw geometry of the motion
// matters for this test.
const xTest = new Segmentation.segmentationComparison(this, 'x', 'y');
const yTest = new Segmentation.segmentationComparison(this, 'y', 'x');
// const xTestConfig = this.xyFTestConfiguration;
// const yTestConfig = this.yxFTestConfiguration;
// If we don't get a p-value less than .100, then as far as x & y are concerned - and thus the user
// is concerned - it's the same segment. Speed may be different, but not the direction.
@ -296,9 +413,32 @@ namespace com.keyman.osk {
}
}
/**
* A specialized form of the `Segmentation` class that assists in its
* construction from the interval directly under examination by the
* segmentation algorithm.
*/
class PotentialSegmentation extends Segmentation {
/**
* The full running accumulation for the ongoing touch path until the point
* before `post` / the later subsegment. May include
* accumulations from before the interval under examination.
*/
readonly chopPoint: CumulativePathStats;
/**
* The full running accumulation for the ongoing touch path until the point
* at the end of `pre` / the earlier subsegment. May include
* accumulations from before the interval under examination, and will
* include accumulations from the first point represented by `post`.
*/
readonly endOfPre: CumulativePathStats;
/**
* The full running accumulation for the ongoing touch path until the point
* before `pre` / the earlier subsegment. Will only include
* accumulations from before the interval under examination.
*/
readonly baseChop: CumulativePathStats;
constructor(steppedStats: CumulativePathStats[],
@ -318,30 +458,9 @@ namespace com.keyman.osk {
}
}
/* FIXME: Note that this function is a temporary development stopgap and will likely shift
* as development continues for a few reasons:
* 1. In its current form, it'd be better to return a completed `Segment`; this is being used
* to finalize `Segment`s, after all.
* 2. Except... we'll actually want it for uncompleted `Segment`s too, for the tail member of
* the public `path.segments` array, which'll need UPDATING, not replacement.
* 3. In some cases, subsegmentation provides an advantage for constructing / updating `Segment`s.
* E.g: Flicks threshold based on top speed, and the faster subsegment's stats are far better
* for this than the combined interval's stats.
*
* Also worth note: makes a LOT of assumptions about how it will be used; namely, that
* the members of the array originally were contiguous and are in their original
* segmentation order. This holds for current use cases, but this is why the func
* will not be exported.
/**
* The core logic, algorithm, and manager for touchpath segmentation.
*/
const mergeSubsegmentations = function(array: PotentialSegmentation[]) {
if(array.length == 1) {
return array[0].pre; // It's pre-calculated, so just use it.
} else {
const finalSubsegmentation = array[array.length-1];
return finalSubsegmentation.endOfPre.deaccumulate(array[0].baseChop);
}
}
export class PathSegmenter {
/**
* The minimum amount of time (in ms) to wait between sample repetitions
@ -363,13 +482,30 @@ namespace com.keyman.osk {
*/
private steppedCumulativeStats: CumulativePathStats[];
/**
* Tracks all subsegments awaiting completion of their overall segment.
*/
private lingeringSubsegmentations: PotentialSegmentation[];
// Currently used as an in-development diagnostic assist... but these
// directly represent actual path segments as produced by the prototype
// algorithm. Just... the stats analysis of the path segment, without
// obvious / public members to relevant coordinates.
/**
* TODO: These directly represent path segments produced by the prototype
* algorithm. Just... the stats analysis of the path segment, without
* obvious / public members to relevant coordinates.
*
* In follow-up work, this should be used to build & to update
* cleaner, public-facing segment objects for the touch path.
*/
private _protoSegments: CumulativePathStats[] = [];
/**
* TODO: These directly represent the subsegments comprising their
* corresponding path segments (in `_protoSegments`) produced by the
* prototype algorithm. Just... the stats analysis of each subsegment.
*
* In follow-up work, these should be used internally to build & update
* the internals of TBD cleaner, public-facing segment objects for the
* touch path.
*/
private _protoSegmentSets: CumulativePathStats[][] = [];
/**
@ -392,13 +528,17 @@ namespace com.keyman.osk {
* that lie on already-fully-segmented parts of it.
*/
private choppedStats: CumulativePathStats = null;
private lastIntervalDuration = 0;
constructor() {
this.steppedCumulativeStats = [];
this.lingeringSubsegmentations = [];
}
/**
* Appends a new coordinate to the touch path and also kick-starts a timer for replicating it
* should neither further inputs be received nor termination of the touchpoint be indicated.
* @param sample
*/
public add(sample: InputSample) {
const repeater = (timeDelta: number) => {
this.observe(sample, timeDelta);
@ -418,6 +558,10 @@ namespace com.keyman.osk {
repeater(0);
}
/**
* Used to finalize all segmentation for the touchpath whenever termination of the corresponding
* touchpoint has been terminated (mouse-up, touch-end).
*/
public close() {
// The Node clearTimeout & DOM clearTimeout appear to TS as overloads of each other,
// and their type definitions will conflict. A simple @ts-ignore will bypass this issue.
@ -433,12 +577,23 @@ namespace com.keyman.osk {
this.filterSubsegmentation(finalization, true); // forces out the final segment.
// Hacky, but "enough" for now.
// FIXME: temporary statement to facilitate exploration, experimentation, & debugging
// TODO: these are temporary statements to facilitate exploration, experimentation, & debugging.
// We should be providing output to the touchpath object (`.path.segments`).
// But... that'll be left for a follow-up PR.
console.log(this._protoSegments);
console.log(this._protoSegmentSets);
console.log(this._protoSegments.map((val) => (val.toJSON())));
}
/**
* Adds a statistic 'observation' of the state of the touchpath.
*
* This is either to be called directly upon reception of a new input-coordinate or
* upon replication of a prior input should the touchpath still be active without any
* indication of motion.
* @param sample
* @param timeDelta
*/
private observe(sample: InputSample, timeDelta: number) {
let cumulativeStats: CumulativePathStats;
if(this.steppedCumulativeStats.length) {
@ -452,7 +607,7 @@ namespace com.keyman.osk {
const extendedStats = cumulativeStats.extend(sample);
this.steppedCumulativeStats.push(extendedStats);
this.attemptSegmentation();
this.performSubsegmentation();
}
private _debugLogSegmentationReport(candidateSplit: PotentialSegmentation) {
@ -484,7 +639,17 @@ namespace com.keyman.osk {
// END: DO NOT RELEASE.
}
private attemptSegmentation() {
// The "reported via event" aspect mentioned below is necessary because this may be
// called via setTimeout callback on a held touchpoint. We need that `setTimeout`
// (which replicates the most recently-seen input coordinate) in order to have good
// sample-data for detecting the boundary between motion and lack thereof during
// segmentation.
/**
* This is the "main method" for touchpath segmentation. Should a new segment result
* from its analysis, it will be reported via event.
* @returns
*/
private performSubsegmentation() {
const cumulativeStats = this.steppedCumulativeStats[this.steppedCumulativeStats.length - 1];
const unsegmentedDuration = cumulativeStats.lastTimestamp - this.steppedCumulativeStats[0].lastTimestamp;
@ -511,9 +676,6 @@ namespace com.keyman.osk {
// of both of the resulting intervals.
let candidateSplit = new PotentialSegmentation(this.steppedCumulativeStats, this.choppedStats, splitPoint);
const lastIntervalDuration = this.lastIntervalDuration;
this.lastIntervalDuration = unsegmentedDuration;
const xF = candidateSplit.segReg('x', 't');
const yF = candidateSplit.segReg('y', 't');
if(!candidateSplit.segmentationMerited) {
@ -535,7 +697,6 @@ namespace com.keyman.osk {
// We either have the conditions to trigger segmentation or just became long enough to consider it.
// If we're only just long enough to consider it, there may be a better segmentation point to start with.
// Now... is there a better segmentation point?
class SplitSearchState {
readonly xTest: typeof Segmentation.segmentationComparison.prototype;
readonly yTest: typeof Segmentation.segmentationComparison.prototype;
@ -552,6 +713,8 @@ namespace com.keyman.osk {
this.yTest = candidate.segReg('y', 't');
}
// The value for an 'objective function' to optimize in our search for a better candidate.
// The higher this is, the more we like its potential as the segmentation point.
get segRating() {
if(this.candidate) {
return Math.max(this.xTest.coefficientOfDetermination, this.yTest.coefficientOfDetermination);
@ -560,6 +723,8 @@ namespace com.keyman.osk {
}
}
// But, if it turns out this potential point wouldn't actually result in segmentation, well...
// "abandon ship".
get segmentationMerited() {
return this.candidate?.segmentationMerited ?? false;
}
@ -577,7 +742,6 @@ namespace com.keyman.osk {
const criteria = [leftSplit.segRating, currentSplit.segRating, rightSplit.segRating];
let sortedCriteria = [...criteria].sort();
// TODO: if we're better on both axes on one side, let's start shifting.
const delta = criteria.indexOf(sortedCriteria[2])-1; // -1 if 'left' is best, 1 if 'right' is best.
if(delta != 0) {
@ -624,9 +788,6 @@ namespace com.keyman.osk {
// First phase of segmentation: complete!
// But... there are some cases where we want to prevent segmentation from fully happening.
// TODO: That.
// FIXME: DO NOT RELEASE.
// This is exploratory / diagnostic code assisting development of the path segmentation
// algorithm.
@ -635,24 +796,45 @@ namespace com.keyman.osk {
this.steppedCumulativeStats = this.steppedCumulativeStats.slice(splitPoint);
this.choppedStats = candidateSplit.chopPoint;
this.lastIntervalDuration = candidateSplit.post.duration * 1000;
// There are some cases where we want to prevent segmentation from fully happening.
// The next method will both handle that and the signal of any completed segments that
// may result.
this.filterSubsegmentation(candidateSplit);
}
// NOTE: This function, as well as code called by it, are still very much still in prototyping.
private filterSubsegmentation(subsegmentation: PotentialSegmentation, force?: boolean) {
force = !!force;
/**
* Given the results of the subsegmentation process, this method analyzes the results
* in order to produce and update the detected segments.
*
* TODO: Should produce actual `Segment` objects and trigger their emission
* them via EventEmitter mechanics.
* @param subsegmentation
* @param finalize Set to `true` for a terminating touchpath, indicating that
* no further subsegmentation will occur for this touchpath.
*/
private filterSubsegmentation(subsegmentation: PotentialSegmentation, finalize?: boolean) {
finalize = !!finalize;
// This function makes a LOT of assumptions particular to this method and class.
// It is not safe to make this class-level, and it's especially unsafe to make it `public`.
const mergeSubsegmentationAccumulations = function(array: PotentialSegmentation[]) {
if(array.length == 1) {
return array[0].pre; // It's pre-calculated, so just use it.
} else {
const finalSubsegmentation = array[array.length-1];
return finalSubsegmentation.endOfPre.deaccumulate(array[0].baseChop);
}
}
let predecessor = subsegmentation.pre;
let firstChopPoint = subsegmentation.baseChop;
if(this.lingeringSubsegmentations.length) {
// First: check if the newly-finished subsegment should be merged with the lingering ones.
// const lastSubsegment = this.lingeringSubsegmentations[this.lingeringSubsegmentations.length-1];
const mergedPrecursors = mergeSubsegmentations(this.lingeringSubsegmentations);
const mergedPrecursors = mergeSubsegmentationAccumulations(this.lingeringSubsegmentations);
// const tailUnion = mergeSubsegmentations([lastSubsegment, subsegmentation]);
const tailUnion = mergeSubsegmentations([...this.lingeringSubsegmentations, subsegmentation]);
const tailUnion = mergeSubsegmentationAccumulations([...this.lingeringSubsegmentations, subsegmentation]);
console.log("Verifying linkage to pending merges: ");
console.log(this.lingeringSubsegmentations);
console.log("verification check:");
@ -662,7 +844,7 @@ namespace com.keyman.osk {
// becomes something VERY different. Validate that we should still merge the left-hand with its predecessors.
if(!PathSegmenter.shouldMergeSubsegments(precursorMergeSegmentation)) {
// Emit as separate subsegment.
const finishedSegment = mergeSubsegmentations(this.lingeringSubsegmentations);
const finishedSegment = mergeSubsegmentationAccumulations(this.lingeringSubsegmentations);
this._protoSegments.push(finishedSegment);
this._protoSegmentSets.push(this.lingeringSubsegmentations.map((val) => val.pre));
this.lingeringSubsegmentations = [];
@ -683,11 +865,10 @@ namespace com.keyman.osk {
console.log("segmentation-prevention check:")
console.log(fullMergeSegmentation);
// if(!force && PathSegmenter.shouldMergeSubsegments(subsegmentation.pre, subsegmentation.post, subsegmentation.union)) {
if(!force && PathSegmenter.shouldMergeSubsegments(fullMergeSegmentation)) {
if(!finalize && PathSegmenter.shouldMergeSubsegments(fullMergeSegmentation)) {
this.lingeringSubsegmentations.push(subsegmentation);
} else {
// Merge all as a completed segment!
// const finishedSegment = mergeSubsegmentations([...this.lingeringSubsegmentations, subsegmentation]);
this._protoSegments.push(predecessor);
this._protoSegmentSets.push([...this.lingeringSubsegmentations.map((val) => val.pre), subsegmentation.pre]);
this.lingeringSubsegmentations = [];
@ -696,6 +877,14 @@ namespace com.keyman.osk {
}
}
/**
* Used by the 'filter' step (phase 2 of segmentation) to determine whether or not
* two subsegments should be 'linked' as members of the same segment. (That is, if
* a user would likely consider the two subsegments as belonging to the "same arc
* of motion".)
* @param segmentation
* @returns
*/
private static shouldMergeSubsegments(segmentation: Segmentation): boolean {
// // Known case #1:
// // Near-identical direction, but heavy speed difference.
@ -705,11 +894,16 @@ namespace com.keyman.osk {
// // Low-speed pivot; angle change caught on very low velocity for both subsegments.
// // ... or should this be merged? Multiple mini-segments from 'wiggling' could just go ignored instead...
if(segmentation.pre.speedMean < 80 && segmentation.post.speedMean > 80) {
// This is kinda plain and arbitrary, but it seems to work "well enough" for now,
// at least at this stage of development. (Most testing was done with Chrome emulation of
// an iPhone SE.)
// .mean('v') < 80: the mean speed (taken timestamp to timestamp) does not exceed 80px/sec.
if(segmentation.pre.mean('v') < 80 && segmentation.post.mean('v') > 80) {
console.log("desegmentation exception");
return; // SUPER TEMP: needs further work; avoids the "low-speed pivot" case.
return;
}
if(segmentation.pre.speedMean > 80 && segmentation.post.speedMean < 80) {
if(segmentation.pre.mean('v') > 80 && segmentation.post.mean('v') < 80) {
console.log("desegmentation exception");
return;
}