From 4cd7cd3363f00e7dec1ddf298bdd72145f994ab2 Mon Sep 17 00:00:00 2001 From: "Joshua A. Horton" Date: Mon, 12 Dec 2022 08:35:15 +0700 Subject: [PATCH] feat(common): TS-based srcmap remapper tool --- .../tools/sourcemap-path-remapper/.gitignore | 1 + .../sourcemap-path-remapper/package.json | 33 ++++ .../sourcemap-path-remapper/src/index.ts | 139 ++++++++++++++++ .../sourcemap-path-remapper/tsconfig.json | 25 +++ package-lock.json | 150 ++++++++++++++++++ package.json | 1 + 6 files changed, 349 insertions(+) create mode 100644 common/tools/sourcemap-path-remapper/.gitignore create mode 100644 common/tools/sourcemap-path-remapper/package.json create mode 100644 common/tools/sourcemap-path-remapper/src/index.ts create mode 100644 common/tools/sourcemap-path-remapper/tsconfig.json diff --git a/common/tools/sourcemap-path-remapper/.gitignore b/common/tools/sourcemap-path-remapper/.gitignore new file mode 100644 index 0000000000..07ed7069a2 --- /dev/null +++ b/common/tools/sourcemap-path-remapper/.gitignore @@ -0,0 +1 @@ +build/* \ No newline at end of file diff --git a/common/tools/sourcemap-path-remapper/package.json b/common/tools/sourcemap-path-remapper/package.json new file mode 100644 index 0000000000..9823fb22fe --- /dev/null +++ b/common/tools/sourcemap-path-remapper/package.json @@ -0,0 +1,33 @@ +{ + "name": "@keymanapp/sourcemap-path-remapper", + "description": "Sourcemap-path remapper utility", + "main": "build/index.js", + "type": "module", + "scripts": { + "tsc": "tsc" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/keymanapp/keyman.git" + }, + "author": "Joshua A. Horton ", + "license": "MIT", + "bugs": { + "url": "https://github.com/keymanapp/keyman/issues" + }, + "homepage": "https://github.com/keymanapp/keyman#readme", + "devDependencies": { + "@keymanapp/resources-gosh": "*", + "@types/node": "^10.17.21", + "chai": "^4.3.4", + "mocha": "^10.0.0", + "mocha-teamcity-reporter": "^4.0.0", + "sinon": "^7.1.1", + "ts-node": "^9.1.1", + "typescript": "^4.5.4" + }, + "dependencies": { + "combine-source-map": "^0.8.0", + "convert-source-map": "^2.0.0" + } +} diff --git a/common/tools/sourcemap-path-remapper/src/index.ts b/common/tools/sourcemap-path-remapper/src/index.ts new file mode 100644 index 0000000000..bfae8a5f19 --- /dev/null +++ b/common/tools/sourcemap-path-remapper/src/index.ts @@ -0,0 +1,139 @@ +// Sourcemap types! + +import * as fs from 'fs'; +import convertSourcemap from 'convert-source-map'; // Transforms sourcemaps among various common formats. + // Base64, stringified-JSON, end-of-file comment... + +// Because we're compiling the main project based on its TS build outputs, and our cross-module references +// also link to build outputs, we need to clean up the sourcemap-paths. Also, the individual module sourcemaps. + +export interface Mapping { + readonly from: RegExp | string; + readonly to: string; +} + +/** + * An intermediate state for sourcemap source path remapping operations + functions to + * mutate it. + */ +class RemappingState { + constructor(srcmap: Object) { + this.sourcemap = srcmap; + this.resetChangedSourcepathTracking(); + } + + public readonly sourcemap: any; + private _unchangedSourcepaths: string[] = []; + + public get unchangedSourcepaths() { + return this._unchangedSourcepaths; + } + + public resetChangedSourcepathTracking() { + this._unchangedSourcepaths = [...this.sourcemap.sources]; + } + + /** + * The sourcemap's `sourceRoot` property. + */ + public get sourceRoot() { + return this.sourcemap.sourceRoot; + } + + /** + * The sourcemap's `sourceRoot` property. + */ + public set sourceRoot(path: string) { + this.sourcemap.sourceRoot = path; + } + + /** + * Searches for source-file paths that match specified path-remapping rules and remaps them + * accordingly. + * @param transforms Path-remapping rules to apply. For each remappable path, only the first + * matching rule in the specified set will be applied. + * @returns `this` - the original instance - in order to facilitate function-call chaining. + */ + remapPaths(transforms: Mapping | Mapping[], logger?: (from: string, to: string) => void) { + if(!(transforms instanceof Array)) { + transforms = [transforms]; + } + + const unusedTransforms = [...transforms]; + + const sourcePaths = this.sourcemap.sources; + for(let i = 0; i < sourcePaths.length; i++) { + for(const transform of transforms) { + if(sourcePaths[i].match(transform.from)) { + // Note that this source path has been remapped (if it wasn't previously) + if(this.unchangedSourcepaths.includes(sourcePaths[i])) { + this.unchangedSourcepaths.splice(this.unchangedSourcepaths.indexOf(sourcePaths[i]), 1); + } + + // Note that this transform has been used. + if(unusedTransforms.includes(transform)) { + unusedTransforms.splice(unusedTransforms.indexOf(transform), 1); + } + + let startPath = sourcePaths[i]; + sourcePaths[i] = sourcePaths[i].replace(transform.from, transform.to); + if(logger) { + logger(startPath, sourcePaths[i]); + } + break; + } + } + } + + if(unusedTransforms.length > 0) { + console.warn(); + console.warn("Unused sourcemath mappings:"); + for(const transform of unusedTransforms) { + let isRegex = transform.from instanceof RegExp; + let delim = isRegex ? '' : '"'; + console.warn(`* ${delim}${transform.from.toString()}${delim} => "${transform.to}"`); + } + } + + return this; + } + + /** + * Writes the current state of the remapped sourcemap out to the specified file. + * @param file + */ + public toFile(file: fs.PathLike) { + fs.writeFileSync(file, convertSourcemap.fromObject(this.sourcemap).toJSON()); + } +} + +export default class SourcemapRemapper { + private constructor() {} + + /** + * Starts sourcemap source path remapping from a pre-loaded JSON sourcemap instance. + * @param obj + * @returns + */ + public static fromObject(obj: Object) { + return new RemappingState(obj); + } + + /** + * Starts sourcemap source path remapping from the specified .js.map file. + * @param file + * @returns + */ + public static fromFile(file: fs.PathLike) { + return this.fromBuffer(fs.readFileSync(file)); + } + + /** + * Starts sourcemap source path remapping from a loaded but unparsed file. + * @param buffer + * @returns + */ + public static fromBuffer(buffer: Buffer) { + return this.fromObject(convertSourcemap.fromJSON(buffer).toObject()); + } +} \ No newline at end of file diff --git a/common/tools/sourcemap-path-remapper/tsconfig.json b/common/tools/sourcemap-path-remapper/tsconfig.json new file mode 100644 index 0000000000..f9b531b311 --- /dev/null +++ b/common/tools/sourcemap-path-remapper/tsconfig.json @@ -0,0 +1,25 @@ +{ + "extends": "../../../tsconfig-base.json", + + "compilerOptions": { + "allowJs": false, + "allowSyntheticDefaultImports": true, + "declaration": true, + "module": "es6", + "moduleResolution": "node", + "inlineSourceMap": true, + "inlineSources": true, + "sourceRoot": "/common/tools/sourcemap-path-remapper/src", + "lib": ["dom", "es6"], + "target": "es5", + "types": ["node"], + "downlevelIteration": true, + "baseUrl": "./", + "outDir": "build/", + "tsBuildInfoFile": "build/tsconfig.tsbuildinfo", + "rootDir": "./src/" + }, + "include": [ + "src/**/*.ts" + ] +} diff --git a/package-lock.json b/package-lock.json index c3c9237622..cc9582664e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "developer/src/kmlmc", "developer/src/server", "common/models/*", + "common/tools/*", "common/web/*", "common/predictive-text", "web" @@ -197,6 +198,30 @@ "integrity": "sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==", "dev": true }, + "common/tools/sourcemap-path-remapper": { + "name": "@keymanapp/sourcemap-path-remapper", + "license": "MIT", + "dependencies": { + "combine-source-map": "^0.8.0", + "convert-source-map": "^2.0.0" + }, + "devDependencies": { + "@keymanapp/resources-gosh": "*", + "@types/node": "^10.17.21", + "chai": "^4.3.4", + "mocha": "^10.0.0", + "mocha-teamcity-reporter": "^4.0.0", + "sinon": "^7.1.1", + "ts-node": "^9.1.1", + "typescript": "^4.5.4" + } + }, + "common/tools/sourcemap-path-remapper/node_modules/@types/node": { + "version": "10.17.60", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz", + "integrity": "sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==", + "dev": true + }, "common/web/es6-shim": { "name": "@keymanapp/es6-shim", "extraneous": true, @@ -805,6 +830,10 @@ "resolved": "resources/gosh", "link": true }, + "node_modules/@keymanapp/sourcemap-path-remapper": { + "resolved": "common/tools/sourcemap-path-remapper", + "link": true + }, "node_modules/@keymanapp/web-sentry-manager": { "resolved": "common/web/sentry-manager", "link": true @@ -2178,6 +2207,30 @@ "color-support": "bin.js" } }, + "node_modules/combine-source-map": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/combine-source-map/-/combine-source-map-0.8.0.tgz", + "integrity": "sha512-UlxQ9Vw0b/Bt/KYwCFqdEwsQ1eL8d1gibiFb7lxQJFdvTgc2hIZi6ugsg+kyhzhPV+QEpUiEIwInIAIrgoEkrg==", + "dependencies": { + "convert-source-map": "~1.1.0", + "inline-source-map": "~0.6.0", + "lodash.memoize": "~3.0.3", + "source-map": "~0.5.3" + } + }, + "node_modules/combine-source-map/node_modules/convert-source-map": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.1.3.tgz", + "integrity": "sha512-Y8L5rp6jo+g9VEPgvqNfEopjTR4OTYct8lXlS8iVQdmnjDvbdbzYe9rjtFCB9egC86JoNCU61WRY+ScjkZpnIg==" + }, + "node_modules/combine-source-map/node_modules/source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -2288,6 +2341,11 @@ "node": ">= 0.6" } }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==" + }, "node_modules/cookie": { "version": "0.4.2", "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz", @@ -3448,6 +3506,22 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "node_modules/inline-source-map": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/inline-source-map/-/inline-source-map-0.6.2.tgz", + "integrity": "sha512-0mVWSSbNDvedDWIN4wxLsdPM4a7cIPcpyMxj3QZ406QRwQ6ePGB1YIHxVPjqpcUGbWQ5C+nHTwGNWAGvt7ggVA==", + "dependencies": { + "source-map": "~0.5.3" + } + }, + "node_modules/inline-source-map/node_modules/source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -4048,6 +4122,11 @@ "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==" }, + "node_modules/lodash.memoize": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-3.0.4.tgz", + "integrity": "sha512-eDn9kqrAmVUC1wmZvlQ6Uhde44n+tXpqPrN8olQJbttgh0oKclk+SF54P47VEGE9CEiMeRwAP8BaM7UHvBkz2A==" + }, "node_modules/lodash.set": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz", @@ -7533,6 +7612,29 @@ "@keymanapp/resources-gosh": { "version": "file:resources/gosh" }, + "@keymanapp/sourcemap-path-remapper": { + "version": "file:common/tools/sourcemap-path-remapper", + "requires": { + "@keymanapp/resources-gosh": "*", + "@types/node": "^10.17.21", + "chai": "^4.3.4", + "combine-source-map": "^0.8.0", + "convert-source-map": "^2.0.0", + "mocha": "^10.0.0", + "mocha-teamcity-reporter": "^4.0.0", + "sinon": "^7.1.1", + "ts-node": "^9.1.1", + "typescript": "^4.5.4" + }, + "dependencies": { + "@types/node": { + "version": "10.17.60", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz", + "integrity": "sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==", + "dev": true + } + } + }, "@keymanapp/web-sentry-manager": { "version": "file:common/web/sentry-manager", "requires": { @@ -8737,6 +8839,29 @@ "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==", "dev": true }, + "combine-source-map": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/combine-source-map/-/combine-source-map-0.8.0.tgz", + "integrity": "sha512-UlxQ9Vw0b/Bt/KYwCFqdEwsQ1eL8d1gibiFb7lxQJFdvTgc2hIZi6ugsg+kyhzhPV+QEpUiEIwInIAIrgoEkrg==", + "requires": { + "convert-source-map": "~1.1.0", + "inline-source-map": "~0.6.0", + "lodash.memoize": "~3.0.3", + "source-map": "~0.5.3" + }, + "dependencies": { + "convert-source-map": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.1.3.tgz", + "integrity": "sha512-Y8L5rp6jo+g9VEPgvqNfEopjTR4OTYct8lXlS8iVQdmnjDvbdbzYe9rjtFCB9egC86JoNCU61WRY+ScjkZpnIg==" + }, + "source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==" + } + } + }, "combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -8831,6 +8956,11 @@ "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" }, + "convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==" + }, "cookie": { "version": "0.4.2", "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.2.tgz", @@ -9733,6 +9863,21 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "inline-source-map": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/inline-source-map/-/inline-source-map-0.6.2.tgz", + "integrity": "sha512-0mVWSSbNDvedDWIN4wxLsdPM4a7cIPcpyMxj3QZ406QRwQ6ePGB1YIHxVPjqpcUGbWQ5C+nHTwGNWAGvt7ggVA==", + "requires": { + "source-map": "~0.5.3" + }, + "dependencies": { + "source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==" + } + } + }, "ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -10250,6 +10395,11 @@ "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==" }, + "lodash.memoize": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-3.0.4.tgz", + "integrity": "sha512-eDn9kqrAmVUC1wmZvlQ6Uhde44n+tXpqPrN8olQJbttgh0oKclk+SF54P47VEGE9CEiMeRwAP8BaM7UHvBkz2A==" + }, "lodash.set": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz", diff --git a/package.json b/package.json index df60de59f8..3740150c62 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "developer/src/kmlmc", "developer/src/server", "common/models/*", + "common/tools/*", "common/web/*", "common/predictive-text", "web"