From 35814bc29f05068f656d22ee33bd28ee125d39dc Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 9 Oct 2023 07:45:34 +0700 Subject: [PATCH] chore(developer): add --for-publishing to kmc The new --for-publishing flag allows us to enforce additional requirements for keyboards and lexical models that are to be published to the keymanapp repositories. This flag will always be switched on in the repository builds. This flag overrides the 'skipMetadata' option which is a project-level option. There is a bit of a delicate balance of requirements here: 1. We want to be able to automatically verify keyboards and package licenses if they are in the repository. 2. We do not want to force license checks on privately built keyboards and models. 3. We want to provide pathways for users to check locally before submitting to the repository. 4. If possible, we want to be able to build the .keyboard_info and .model_info files locally, but this should be up to the user for local builds. For now, the only additional check that this flag provides is to verify that the license is MIT, which was an unchecked requirement for the repositories in the past. Future checks can be added for file layout, additional required metadata files. Anticipate adding this as a tool to Keyman Developer IDE -- a 'pre-publish' check -- in the future. --- developer/src/kmc-keyboard-info/src/index.ts | 25 +++++++++++++------ .../test/test-keyboard-info-compiler.ts | 1 + .../kmc-model-info/src/model-info-compiler.ts | 3 +++ .../test/test-model-info-compiler.ts | 1 + developer/src/kmc/src/commands/build.ts | 8 +++--- .../buildClasses/BuildKeyboardInfo.ts | 8 +++--- .../commands/buildClasses/BuildModelInfo.ts | 8 +++--- .../src/commands/buildClasses/BuildProject.ts | 11 ++++---- .../kmc/src/util/extendedCompilerOptions.ts | 10 ++++++++ 9 files changed, 54 insertions(+), 21 deletions(-) create mode 100644 developer/src/kmc/src/util/extendedCompilerOptions.ts diff --git a/developer/src/kmc-keyboard-info/src/index.ts b/developer/src/kmc-keyboard-info/src/index.ts index 052c12dd63..611e7ea9a9 100644 --- a/developer/src/kmc-keyboard-info/src/index.ts +++ b/developer/src/kmc-keyboard-info/src/index.ts @@ -56,6 +56,9 @@ export interface KeyboardInfoSources { /** Last modification date for files in the project folder 'YYYY-MM-DDThh:mm:ssZ' */ lastCommitDate?: string; + + /** Return an error if project does not meet requirements of keyboards repository */ + forPublishing: boolean; }; export class KeyboardInfoCompiler { @@ -121,15 +124,23 @@ export class KeyboardInfoCompiler { // License - if(!kmpJsonData.options?.licenseFile) { - this.callbacks.reportMessage(KeyboardInfoCompilerMessages.Error_NoLicenseFound()); - return null; - } - - if(!this.isLicenseMIT(this.callbacks.resolveFilename(sources.kpsFilename, kmpJsonData.options.licenseFile))) { - return null; + if(sources.forPublishing) { + // We will only verify the license if asked to do so, so that all keyboard + // projects can be built even if license is not present. Keyboards + // repository will always verify license + if(!kmpJsonData.options?.licenseFile) { + this.callbacks.reportMessage(KeyboardInfoCompilerMessages.Error_NoLicenseFound()); + return null; + } + + if(!this.isLicenseMIT(this.callbacks.resolveFilename(sources.kpsFilename, kmpJsonData.options.licenseFile))) { + return null; + } } + // Even if license is not verified, we set the .keyboard_info license to + // 'mit' to meet the schema requirements. The .keyboard_info file is only + // used by the keyboards repository, so this is a fair assumption to make. keyboard_info.license = 'mit'; // isRTL diff --git a/developer/src/kmc-keyboard-info/test/test-keyboard-info-compiler.ts b/developer/src/kmc-keyboard-info/test/test-keyboard-info-compiler.ts index 0e71899184..a0fdb14ca2 100644 --- a/developer/src/kmc-keyboard-info/test/test-keyboard-info-compiler.ts +++ b/developer/src/kmc-keyboard-info/test/test-keyboard-info-compiler.ts @@ -26,6 +26,7 @@ describe('keyboard-info-compiler', function () { sourcePath: 'release/k/khmer_angkor', kpsFilename, jsFilename: jsFilename, + forPublishing: true, }); } catch(e) { callbacks.printMessages(); diff --git a/developer/src/kmc-model-info/src/model-info-compiler.ts b/developer/src/kmc-model-info/src/model-info-compiler.ts index 63644b9ef0..8a7101e86f 100644 --- a/developer/src/kmc-model-info/src/model-info-compiler.ts +++ b/developer/src/kmc-model-info/src/model-info-compiler.ts @@ -33,6 +33,9 @@ export class ModelInfoSources { /** Last modification date for files in the project folder 'YYYY-MM-DDThh:mm:ssZ' */ lastCommitDate?: string; + + /** Return an error if project does not meet requirements of lexical-models repository */ + forPublishing: boolean; }; /* c8 ignore stop */ diff --git a/developer/src/kmc-model-info/test/test-model-info-compiler.ts b/developer/src/kmc-model-info/test/test-model-info-compiler.ts index aa0c2ed65b..d74a2f5c60 100644 --- a/developer/src/kmc-model-info/test/test-model-info-compiler.ts +++ b/developer/src/kmc-model-info/test/test-model-info-compiler.ts @@ -29,6 +29,7 @@ describe('model-info-compiler', function () { modelFileName, sourcePath: 'release/sil/sil.cmo.bw', kpsFilename, + forPublishing: true, }); if(data == null) { callbacks.printMessages(); diff --git a/developer/src/kmc/src/commands/build.ts b/developer/src/kmc/src/commands/build.ts index 9cea460ec8..0d091680b7 100644 --- a/developer/src/kmc/src/commands/build.ts +++ b/developer/src/kmc/src/commands/build.ts @@ -11,10 +11,9 @@ import { expandFileLists } from '../util/fileLists.js'; import { isProject } from '../util/projectLoader.js'; import { buildTestData } from './buildTestData/index.js'; import { buildWindowsPackageInstaller } from './buildWindowsPackageInstaller/index.js'; -//import { buildWindowsPackageInstaller } from './buildWindowsPackageInstaller/index.js'; +import { ExtendedCompilerOptions } from 'src/util/extendedCompilerOptions.js'; - -function commandOptionsToCompilerOptions(options: any): CompilerOptions { +function commandOptionsToCompilerOptions(options: any): ExtendedCompilerOptions { // We don't want to rename command line options to match the precise // properties that we have in CompilerOptions, but nor do we want to rename // CompilerOptions properties... @@ -29,6 +28,8 @@ function commandOptionsToCompilerOptions(options: any): CompilerOptions { saveDebug: options.debug, compilerWarningsAsErrors: options.compilerWarningsAsErrors, warnDeprecatedCode: options.warnDeprecatedCode, + // ExtendedOptions + forPublishing: options.forPublishing, } } @@ -50,6 +51,7 @@ export function declareBuild(program: Command) { buildCommand.command('file [infile...]', {isDefault: true}) .description(`Compile one or more source files or projects ('file' subcommand is default).`) + .option('--for-publishing', 'Verify that project meets @keymanapp repository requirements') .addHelpText('after', ` Supported file types: * folder: Keyman project in folder diff --git a/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts b/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts index 0af8343b2d..8d8d66938f 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildKeyboardInfo.ts @@ -1,18 +1,19 @@ import * as fs from 'fs'; import { BuildActivity } from './BuildActivity.js'; -import { CompilerCallbacks, CompilerOptions, KeymanDeveloperProject, KeymanFileTypes } from '@keymanapp/common-types'; +import { CompilerCallbacks, KeymanDeveloperProject, KeymanFileTypes } from '@keymanapp/common-types'; import { KeyboardInfoCompiler } from '@keymanapp/kmc-keyboard-info'; import { loadProject } from '../../util/projectLoader.js'; import { InfrastructureMessages } from '../../messages/infrastructureMessages.js'; import { calculateSourcePath } from '../../util/calculateSourcePath.js'; import { getLastGitCommitDate } from '../../util/getLastGitCommitDate.js'; +import { ExtendedCompilerOptions } from 'src/util/extendedCompilerOptions.js'; export class BuildKeyboardInfo extends BuildActivity { public get name(): string { return 'Keyboard metadata'; } public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.Project; } public get compiledExtension(): KeymanFileTypes.Binary { return KeymanFileTypes.Binary.KeyboardInfo; } public get description(): string { return 'Build a keyboard metadata file'; } - public async build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { + public async build(infile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise { if(!KeymanFileTypes.filenameIs(infile, KeymanFileTypes.Source.Project)) { // Even if the project file does not exist, we use its name as our reference // in order to avoid ambiguity @@ -41,7 +42,8 @@ export class BuildKeyboardInfo extends BuildActivity { kpsFilename: project.resolveInputFilePath(kps), jsFilename: fs.existsSync(jsFilename) ? jsFilename : undefined, sourcePath: calculateSourcePath(infile), - lastCommitDate + lastCommitDate, + forPublishing: !!options.forPublishing, }); if(data == null) { diff --git a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts index 490543c0e6..bddfcc38db 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildModelInfo.ts @@ -1,12 +1,13 @@ import * as fs from 'fs'; import { BuildActivity } from './BuildActivity.js'; -import { CompilerCallbacks, CompilerOptions, KeymanFileTypes } from '@keymanapp/common-types'; +import { CompilerCallbacks, KeymanFileTypes } from '@keymanapp/common-types'; import { ModelInfoCompiler } from '@keymanapp/kmc-model-info'; import { KmpCompiler } from '@keymanapp/kmc-package'; import { loadProject } from '../../util/projectLoader.js'; import { InfrastructureMessages } from '../../messages/infrastructureMessages.js'; import { calculateSourcePath } from '../../util/calculateSourcePath.js'; import { getLastGitCommitDate } from '../../util/getLastGitCommitDate.js'; +import { ExtendedCompilerOptions } from 'src/util/extendedCompilerOptions.js'; export class BuildModelInfo extends BuildActivity { public get name(): string { return 'Lexical model metadata'; } @@ -24,7 +25,7 @@ export class BuildModelInfo extends BuildActivity { * @param options * @returns */ - public async build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { + public async build(infile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise { if(!KeymanFileTypes.filenameIs(infile, KeymanFileTypes.Source.Project)) { // Even if the project file does not exist, we use its name as our reference // in order to avoid ambiguity @@ -65,7 +66,8 @@ export class BuildModelInfo extends BuildActivity { modelFileName: project.resolveOutputFilePath(model, KeymanFileTypes.Source.Model, KeymanFileTypes.Binary.Model), kmpFileName: project.resolveOutputFilePath(kps, KeymanFileTypes.Source.Package, KeymanFileTypes.Binary.Package), kpsFilename: project.resolveInputFilePath(kps), - lastCommitDate + lastCommitDate, + forPublishing: !!options.forPublishing, }); if(data == null) { diff --git a/developer/src/kmc/src/commands/buildClasses/BuildProject.ts b/developer/src/kmc/src/commands/buildClasses/BuildProject.ts index db74ccec06..de0e55dfc1 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildProject.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildProject.ts @@ -1,17 +1,18 @@ import * as path from 'path'; import * as fs from 'fs'; -import { CompilerCallbacks, CompilerFileCallbacks, CompilerOptions, KeymanDeveloperProject, KeymanDeveloperProjectFile, KeymanFileTypes } from '@keymanapp/common-types'; +import { CompilerCallbacks, CompilerFileCallbacks, KeymanDeveloperProject, KeymanDeveloperProjectFile, KeymanFileTypes } from '@keymanapp/common-types'; import { BuildActivity } from './BuildActivity.js'; import { buildActivities, buildKeyboardInfoActivity, buildModelInfoActivity } from './buildActivities.js'; import { InfrastructureMessages } from '../../messages/infrastructureMessages.js'; import { loadProject } from '../../util/projectLoader.js'; +import { ExtendedCompilerOptions } from 'src/util/extendedCompilerOptions.js'; export class BuildProject extends BuildActivity { public get name(): string { return 'Project'; } public get sourceExtension(): KeymanFileTypes.Source { return KeymanFileTypes.Source.Project; } public get compiledExtension(): KeymanFileTypes.Binary { return null; } public get description(): string { return 'Build a keyboard or lexical model project'; } - public async build(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions): Promise { + public async build(infile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions): Promise { let builder = new ProjectBuilder(infile, callbacks, options); return builder.run(); } @@ -20,10 +21,10 @@ export class BuildProject extends BuildActivity { class ProjectBuilder { callbacks: CompilerCallbacks; infile: string; - options: CompilerOptions; + options: ExtendedCompilerOptions; project: KeymanDeveloperProject; - constructor(infile: string, callbacks: CompilerCallbacks, options: CompilerOptions) { + constructor(infile: string, callbacks: CompilerCallbacks, options: ExtendedCompilerOptions) { this.infile = path.resolve(infile); this.callbacks = new CompilerFileCallbacks(infile, options, callbacks); this.options = options; @@ -53,7 +54,7 @@ class ProjectBuilder { } // Build project metadata - if(!this.project.options.skipMetadataFiles) { + if(this.options.forPublishing || !this.project.options.skipMetadataFiles) { if(!await (this.buildProjectTargets( this.project.isKeyboardProject() ? buildKeyboardInfoActivity diff --git a/developer/src/kmc/src/util/extendedCompilerOptions.ts b/developer/src/kmc/src/util/extendedCompilerOptions.ts new file mode 100644 index 0000000000..8d2903e53c --- /dev/null +++ b/developer/src/kmc/src/util/extendedCompilerOptions.ts @@ -0,0 +1,10 @@ +import { CompilerOptions } from '@keymanapp/common-types'; + +export interface ExtendedCompilerOptions extends CompilerOptions { + /** + * Verify that the project meets the requirements of the keymanapp/keyboards + * or keymanapp/lexical-models repository, e.g. verify that project license is + * MIT + */ + forPublishing?: boolean; +};