From 33affc660f032f4a1d8c0ad61ec8e74447088708 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 3 Jul 2023 08:59:45 +0700 Subject: [PATCH] fix(common): legacy .kpj schema Fixes #9140. Fixes #9148. Keyman Developer 9.0 .kpj files included a lot of additional state metadata. We need a schema which validates these files, as they are otherwise valid to load (we'll never save them any more). Rather than add all the extra metadata to what is otherwise a fairly clean schema, we'll provide a legacy .schema.json. In the future, we may be able to merge these schemas, as we move towards the .kpj 2.0 format which doesn't list files. Ideally, all three formats (Keyman Developer 9.0 schema, call it legacy, 1.0 schema for Keyman Developer 10.0+ which has Options and Files listed, 2.0 schema for capturing just project settings for a folder) will be supported by a single schema file. --- common/schemas/kpj-9.0/README.md | 11 ++ common/schemas/kpj-9.0/kpj-9.0.schema.json | 134 ++++++++++++++++++ common/schemas/kpj/README.md | 3 + common/web/types/build.sh | 1 + common/web/types/src/kpj/kpj-file-reader.ts | 12 +- .../web/types/src/util/compiler-interfaces.ts | 1 + .../test/helpers/TestCompilerCallbacks.ts | 8 +- .../types/test/kpj/test-kpj-file-reader.ts | 2 +- developer/src/kmc-ldml/build.sh | 4 + developer/src/kmc/build.sh | 1 + .../src/commands/buildClasses/BuildProject.ts | 3 +- 11 files changed, 171 insertions(+), 9 deletions(-) create mode 100644 common/schemas/kpj-9.0/README.md create mode 100644 common/schemas/kpj-9.0/kpj-9.0.schema.json diff --git a/common/schemas/kpj-9.0/README.md b/common/schemas/kpj-9.0/README.md new file mode 100644 index 0000000000..9f28fcf2a1 --- /dev/null +++ b/common/schemas/kpj-9.0/README.md @@ -0,0 +1,11 @@ +# kpj-9.0.schema.json + +**Note:** `KeymanDeveloperProject.Options.Version` is currently implicitly +always '1.0'. It will be required for version 2.0 and later of the format. + +This is a schema for for supporting legacy versions of .kpj, from Keyman +Developer 9.0 and earlier, is now available. Note that this schema is not fully +validating on the legacy fields, but just sufficient to pass a valid +file. + +You should be using kpj.schema.json for most purposes diff --git a/common/schemas/kpj-9.0/kpj-9.0.schema.json b/common/schemas/kpj-9.0/kpj-9.0.schema.json new file mode 100644 index 0000000000..bfc8a3c498 --- /dev/null +++ b/common/schemas/kpj-9.0/kpj-9.0.schema.json @@ -0,0 +1,134 @@ +{ + "title": "kpj-9.0.xsd", + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "KeymanDeveloperProject": { + "properties": { + "Options": { + "$ref": "#/definitions/Options" + }, + "Files": { + "$ref": "#/definitions/Files" + }, + "templatepath": { "type":"string", "$comment": "Keyman Developer 9, no longer used in modern .kpj files" }, + "stringspath": { "type":"string", "$comment": "Keyman Developer 9, no longer used in modern .kpj files" }, + "state": { "type": "string", "$comment": "Keyman Developer 9, no longer used in modern .kpj files" }, + "Modules": { "oneOf": [{"type": "object"}, {"type": "string"}], "$comment": "Keyman Developer 9, no longer used in modern .kpj files" }, + "MRU": { "type": "object", "$comment": "Keyman Developer 9, no longer used in modern .kpj files" } + }, + "required": [ + ], + "additionalProperties": false, + "type": "object" + } + }, + "required": [ + "KeymanDeveloperProject" + ], + "additionalProperties": false, + "definitions": { + "Options": { + "type": "object", + "properties": { + "BuildPath": { + "type": "string" + }, + "SourcePath": { + "type": "string" + }, + "CompilerWarningsAsErrors": { + "type": "string", + "pattern": "^(True|False)$" + }, + "WarnDeprecatedCode": { + "type": "string", + "pattern": "^(True|False)$" + }, + "CheckFilenameConventions": { + "type": "string", + "pattern": "^(True|False)$" + }, + "ProjectType": { + "type": "string", + "pattern": "^(keyboard|lexicalmodel)$" + }, + "Version": { + "type": "string", + "pattern": "^(1\\.0|2\\.0)$" + } + }, + "required": [ + ], + "additionalProperties": false + }, + "Files": { + "type": "object", + "properties": { + "File": { + "type": "array", + "items": { + "$ref": "#/definitions/File" + } + } + }, + "additionalProperties": false, + "required": [ + "File" + ] + }, + "File": { + "type": "object", + "properties": { + "ID": { + "type": "string" + }, + "Filename": { + "type": "string" + }, + "Filepath": { + "type": "string" + }, + "FileVersion": { + "type": "string" + }, + "FileType": { + "type": "string" + }, + "Details": { + "$ref": "#/definitions/FileDetails" + }, + "ParentFileID": { + "type": "string" + }, + "FullPath": { "type":"string", "$comment": "Keyman Developer 9, no longer used in modern .kpj files" }, + "IDEState": { "type":"object", "$comment": "Keyman Developer 9, no longer used in modern .kpj files" } + }, + "required": [ + "Filename" + ], + "additionalProperties": false + }, + "FileDetails": { + "type": "object", + "properties": { + "Name": { + "type": "string" + }, + "Copyright": { + "type": "string" + }, + "Message": { + "type": "string" + }, + "Version": { + "type": "string" + }, + "Debug": { "type":"string", "$comment": "Keyman Developer 9, no longer used in modern .kpj files" } + }, + "required": [ + ], + "additionalProperties": false + } + } +} \ No newline at end of file diff --git a/common/schemas/kpj/README.md b/common/schemas/kpj/README.md index 533b672b65..0f8e55de60 100644 --- a/common/schemas/kpj/README.md +++ b/common/schemas/kpj/README.md @@ -3,6 +3,9 @@ **Note:** `KeymanDeveloperProject.Options.Version` is currently implicitly always '1.0'. It will be required for version 2.0 and later of the format. +**Note:** An additional schema file, kpj-9.0.schema.json, for supporting legacy + versions of .kpj, from Keyman Developer 9.0 and earlier, is now available. + ## 2023-02-27 2.0 * Version 2.0 makes 'Files' optional (internally, Files/File will be ignored, deleted on load and populated from folder structure). Adds Options/SourcePath, diff --git a/common/web/types/build.sh b/common/web/types/build.sh index 38c2eb1687..56cbf391f9 100755 --- a/common/web/types/build.sh +++ b/common/web/types/build.sh @@ -45,6 +45,7 @@ function copy_schemas() { cp "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards/techpreview/ldml-keyboardtest.schema.json" "$THIS_SCRIPT_PATH/build/src/" cp "$KEYMAN_ROOT/common/schemas/kvks/kvks.schema.json" "$THIS_SCRIPT_PATH/build/src/" cp "$KEYMAN_ROOT/common/schemas/kpj/kpj.schema.json" "$THIS_SCRIPT_PATH/build/src/" + cp "$KEYMAN_ROOT/common/schemas/kpj-9.0/kpj-9.0.schema.json" "$THIS_SCRIPT_PATH/build/src/" cp "$KEYMAN_ROOT/common/schemas/keyman-touch-layout/keyman-touch-layout.clean.spec.json" "$THIS_SCRIPT_PATH/build/src/" cp "$KEYMAN_ROOT/common/schemas/displaymap/displaymap.schema.json" "$THIS_SCRIPT_PATH/build/src/" # Store CLDR imports diff --git a/common/web/types/src/kpj/kpj-file-reader.ts b/common/web/types/src/kpj/kpj-file-reader.ts index 9ac1bc8dbc..1193bdaa2b 100644 --- a/common/web/types/src/kpj/kpj-file-reader.ts +++ b/common/web/types/src/kpj/kpj-file-reader.ts @@ -23,21 +23,27 @@ export class KPJFileReader { }); parser.parseString(file, (e: unknown, r: unknown) => { data = r as KPJFile }); + data = this.boxArrays(data); for(let file of data.KeymanDeveloperProject?.Files?.File) { // xml2js imports
as '' so we will just delete the empty string if(typeof file.Details == 'string') { delete file.Details; } } - data = this.boxArrays(data); return data as KPJFile; } - public validate(source: KPJFile, schemaBuffer: Uint8Array): void { + public validate(source: KPJFile, schemaBuffer: Uint8Array, legacySchemaBuffer: Uint8Array): void { const schema = JSON.parse(new TextDecoder().decode(schemaBuffer)); const ajv = new Ajv(); if(!ajv.validate(schema, source)) { - throw new Error(ajv.errorsText()); + const ajvLegacy = new Ajv(); + const legacySchema = JSON.parse(new TextDecoder().decode(legacySchemaBuffer)); + if(!ajvLegacy.validate(legacySchema, source)) { + // If the legacy schema also does not validate, then we will only report + // the errors against the modern schema + throw new Error(ajv.errorsText()); + } } } diff --git a/common/web/types/src/util/compiler-interfaces.ts b/common/web/types/src/util/compiler-interfaces.ts index e3d286a390..d4eaa672b1 100644 --- a/common/web/types/src/util/compiler-interfaces.ts +++ b/common/web/types/src/util/compiler-interfaces.ts @@ -116,6 +116,7 @@ export type CompilerSchema = 'ldml-keyboardtest' | 'kvks' | 'kpj' | + 'kpj-9.0' | 'displaymap'; // | 'keyman-touch-layout.clean'; TODO this has the wrong name pattern, .spec.json instead of .schema.json diff --git a/common/web/types/test/helpers/TestCompilerCallbacks.ts b/common/web/types/test/helpers/TestCompilerCallbacks.ts index d61e19596c..31cbb1c5cf 100644 --- a/common/web/types/test/helpers/TestCompilerCallbacks.ts +++ b/common/web/types/test/helpers/TestCompilerCallbacks.ts @@ -12,15 +12,15 @@ export class TestCompilerCallbacks implements CompilerCallbacks { loadSchema(schema: CompilerSchema): Buffer { switch (schema) { case 'kpj': - throw new Error('loadKpjJsonSchema not implemented.'); // not needed for this test + case 'kpj-9.0': case 'kvks': - throw new Error('loadKvksJsonSchema not implemented.'); + case 'displaymap': + // listing all unimplemented schemas rather than using 'default' + throw new Error(`loadSchema(${schema}) not implemented.`); // not needed for this test case 'ldml-keyboard': return loadSchema(schema); case 'ldml-keyboardtest': return loadSchema(schema); - case 'displaymap': - throw new Error('loadDisplayMapSchema not implemented.'); // not needed for this test } } clear() { diff --git a/common/web/types/test/kpj/test-kpj-file-reader.ts b/common/web/types/test/kpj/test-kpj-file-reader.ts index 95b5f1d01a..d9c863a1bc 100644 --- a/common/web/types/test/kpj/test-kpj-file-reader.ts +++ b/common/web/types/test/kpj/test-kpj-file-reader.ts @@ -16,7 +16,7 @@ describe('kpj-file-reader', function () { const reader = new KPJFileReader(callbacks); const kpj = reader.read(input); assert.doesNotThrow(() => { - reader.validate(kpj, loadSchema('kpj')); + reader.validate(kpj, loadSchema('kpj'), loadSchema('kpj-9.0')); }); assert.equal(kpj.KeymanDeveloperProject.Options.BuildPath, '$PROJECTPATH\\build'); assert.equal(kpj.KeymanDeveloperProject.Options.CheckFilenameConventions, 'False'); diff --git a/developer/src/kmc-ldml/build.sh b/developer/src/kmc-ldml/build.sh index 507b26274b..77fd1ec73b 100755 --- a/developer/src/kmc-ldml/build.sh +++ b/developer/src/kmc-ldml/build.sh @@ -45,6 +45,9 @@ fi SCHEMAS_COPIED=false copy_schemas() { + # TODO: why do we need a copy of the schemas here? kmc already has a copy of + # them; are they used only by tests, in which case, can't we use them in + # their source location? if $SCHEMAS_COPIED; then return 0 fi @@ -55,6 +58,7 @@ copy_schemas() { cp "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards/techpreview/ldml-keyboardtest.schema.json" "$THIS_SCRIPT_PATH/build/src/" cp "$KEYMAN_ROOT/common/schemas/kvks/kvks.schema.json" "$THIS_SCRIPT_PATH/build/src/" cp "$KEYMAN_ROOT/common/schemas/kpj/kpj.schema.json" "$THIS_SCRIPT_PATH/build/src/" + cp "$KEYMAN_ROOT/common/schemas/kpj-9.0/kpj-9.0.schema.json" "$THIS_SCRIPT_PATH/build/src/" } #------------------------------------------------------------------------------------------------------------------- diff --git a/developer/src/kmc/build.sh b/developer/src/kmc/build.sh index 6f4728b28f..2ec2c57377 100755 --- a/developer/src/kmc/build.sh +++ b/developer/src/kmc/build.sh @@ -61,6 +61,7 @@ function copy_schemas() { cp "$KEYMAN_ROOT/resources/standards-data/ldml-keyboards/techpreview/ldml-keyboardtest.schema.json" "$THIS_SCRIPT_PATH/build/src/util/" cp "$KEYMAN_ROOT/common/schemas/kvks/kvks.schema.json" "$THIS_SCRIPT_PATH/build/src/util/" cp "$KEYMAN_ROOT/common/schemas/kpj/kpj.schema.json" "$THIS_SCRIPT_PATH/build/src/util/" + cp "$KEYMAN_ROOT/common/schemas/kpj-9.0/kpj-9.0.schema.json" "$THIS_SCRIPT_PATH/build/src/util/" cp "$KEYMAN_ROOT/common/schemas/displaymap/displaymap.schema.json" "$THIS_SCRIPT_PATH/build/src/util/" } diff --git a/developer/src/kmc/src/commands/buildClasses/BuildProject.ts b/developer/src/kmc/src/commands/buildClasses/BuildProject.ts index 33db0c05fb..a24bb3a888 100644 --- a/developer/src/kmc/src/commands/buildClasses/BuildProject.ts +++ b/developer/src/kmc/src/commands/buildClasses/BuildProject.ts @@ -89,8 +89,9 @@ class ProjectBuilder { const reader = new KPJFileReader(this.callbacks); const kpj = reader.read(kpjData); const schema = this.callbacks.loadSchema('kpj'); + const legacySchema = this.callbacks.loadSchema('kpj-9.0'); try { - reader.validate(kpj, schema); + reader.validate(kpj, schema, legacySchema); } catch(e) { this.callbacks.reportMessage(InfrastructureMessages.Error_InvalidProjectFile({message: (e??'').toString()})); return null;