Merge pull request #13375 from keymanapp/feat/developer/10622-xml-parse-line-numbers-epic-ldml

feat(developer): line numbers for XML parsing and kmc-ldml 🙀
This commit is contained in:
Steven R. Loomis 2025-05-12 18:37:23 -05:00 committed by GitHub
commit c2b6a65f1b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 547 additions and 102 deletions

View file

@ -11,7 +11,7 @@
"dependencies": {
"@keymanapp/common-types": "*",
"eventemitter3": "^5.0.0",
"fast-xml-parser": "^5.0.9",
"fast-xml-parser": "^5.2.2",
"path-browserify": "^1.0.1",
"restructure": "^3.0.1",
"sax": ">=0.6.0",

View file

@ -7,6 +7,21 @@
import { CompilerEvent, CompilerCallbackOptions, CompilerErrorSeverity, CompilerError, CompilerMessageOverrideMap, CompilerErrorMask } from "./compiler-interfaces.js";
/**
* The EventResolver implementation is able to expand error messages,
* particularly to change a byte offset into a line number.
*/
export interface EventResolver {
resolve(event: CompilerEvent): void;
}
/** A do-nothing EventResolver. Implementations can use this as their default EventResolver. */
export class NullEventResolver implements EventResolver {
resolve(event: CompilerEvent): void {
// do nothing, this is the null event resolver.
}
}
/**
* Abstract interface for callbacks, to abstract out file i/o
*/
@ -40,6 +55,10 @@ export interface CompilerCallbacks {
*/
resolveFilename(baseFilename: string, filename: string): string;
/**
* Report a message from the compiler back to the caller.
* @param event
*/
reportMessage(event: CompilerEvent): void;
debug(msg: string): void;
@ -106,14 +125,64 @@ export interface CompilerFileSystemAsyncCallbacks {
resolveFilename(baseFilename: string, filename: string): string;
}
/** a CompilerCallbacks implementation that delegates to a parent */
export class DelegatingCompilerCallbacks implements CompilerCallbacks {
constructor(protected options: CompilerCallbackOptions, protected parent: CompilerCallbacks) {
}
loadFile(filename: string): Uint8Array {
return this.parent.loadFile(filename);
}
fileSize(filename: string): number {
return this.parent.fileSize(filename);
}
isDirectory(filename: string): boolean {
return this.parent.isDirectory(filename);
}
get path(): CompilerPathCallbacks {
return this.parent.path;
}
get fs(): CompilerFileSystemCallbacks {
return this.parent.fs;
}
get net(): CompilerNetAsyncCallbacks {
return this.parent.net;
}
get fsAsync(): CompilerFileSystemAsyncCallbacks {
return this.parent.fsAsync;
}
resolveFilename(baseFilename: string, filename: string): string {
return this.parent.resolveFilename(baseFilename, filename);
}
debug(msg: string): void {
return this.parent.debug(msg);
}
fileURLToPath(url: string | URL): string {
return this.parent.fileURLToPath(url);
}
reportMessage(event: CompilerEvent) {
this.parent.reportMessage(event);
}
}
/**
* Wrapper class for CompilerCallbacks for a given input file
*/
export class CompilerFileCallbacks implements CompilerCallbacks {
export class CompilerFileCallbacks extends DelegatingCompilerCallbacks {
messages: CompilerEvent[] = [];
constructor(private filename: string, private options: CompilerCallbackOptions, private parent: CompilerCallbacks) {
constructor(private filename: string, options: CompilerCallbackOptions, parent: CompilerCallbacks) {
super(options, parent);
}
/**
@ -170,38 +239,6 @@ export class CompilerFileCallbacks implements CompilerCallbacks {
this.messages = [];
}
loadFile(filename: string): Uint8Array {
return this.parent.loadFile(filename);
}
fileSize(filename: string): number {
return this.parent.fileSize(filename);
}
isDirectory(filename: string): boolean {
return this.parent.isDirectory(filename);
}
get path(): CompilerPathCallbacks {
return this.parent.path;
}
get fs(): CompilerFileSystemCallbacks {
return this.parent.fs;
}
get net(): CompilerNetAsyncCallbacks {
return this.parent.net;
}
get fsAsync(): CompilerFileSystemAsyncCallbacks {
return this.parent.fsAsync;
}
resolveFilename(baseFilename: string, filename: string): string {
return this.parent.resolveFilename(baseFilename, filename);
}
reportMessage(event: CompilerEvent): void {
const disable = CompilerFileCallbacks.applyMessageOverridesToEvent(event, this.options.messageOverrides);
this.messages.push(event);
@ -209,14 +246,6 @@ export class CompilerFileCallbacks implements CompilerCallbacks {
this.parent.reportMessage({ filename: this.filename, ...event });
}
}
debug(msg: string): void {
return this.parent.debug(msg);
}
fileURLToPath(url: string | URL): string {
return this.parent.fileURLToPath(url);
}
}
export class DefaultCompilerFileSystemAsyncCallbacks implements CompilerFileSystemAsyncCallbacks {
@ -242,3 +271,15 @@ export class DefaultCompilerFileSystemAsyncCallbacks implements CompilerFileSyst
return this.owner.resolveFilename(baseFilename, filename);
}
}
/** a CompilerCallbacks that applies the EventResolver to any reported message */
export class ResolvingCompilerCallbacks extends DelegatingCompilerCallbacks {
constructor(private eventResolver: EventResolver, options: CompilerCallbackOptions, parent: CompilerCallbacks) {
super(options, parent);
}
reportMessage(event: CompilerEvent) {
this.eventResolver.resolve(event);
this.parent.reportMessage(event);
}
}

View file

@ -5,7 +5,17 @@ import { CompilerCallbacks } from "./compiler-callbacks.js";
*/
export interface CompilerEvent {
filename?: string;
/** line where a message applies */
line?: number;
/**
* column where a message applies.
*/
column?: number;
/**
* offset where a message applies.
* If set, encompasses line and column.
*/
offset?: number;
code: number;
message: string;
/**

View file

@ -40,6 +40,10 @@ export {
CompilerPathCallbacks,
CompilerFileSystemCallbacksFolderEntry as FileSystemFolderEntry,
DefaultCompilerFileSystemAsyncCallbacks,
EventResolver,
NullEventResolver,
DelegatingCompilerCallbacks,
ResolvingCompilerCallbacks,
} from './compiler-callbacks.js';
export { defaultCompilerOptions, CompilerBaseOptions, CompilerOptions, CompilerEvent, CompilerErrorNamespace,
@ -63,8 +67,9 @@ export { UrlSubpathCompilerCallback } from './utils/UrlSubpathCompilerCallback.j
export { DeveloperUtilsMessages } from './developer-utils-messages.js';
export * as SourceFilenamePatterns from './source-filename-patterns.js';
export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader } from './xml-utils.js';
export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader, KeymanXMLMetadata, XML_FILENAME_SYMBOL } from './xml-utils.js';
export { SymbolUtils } from './symbol-utils.js';
export * as LineUtils from './line-utils.js';
export * as GitHubUrls from './github-urls.js';
export * as CloudUrls from './cloud-urls.js';
@ -72,4 +77,4 @@ export { getFontFamily, getFontFamilySync } from './font-family.js';
export * as ValidIds from './valid-ids.js';
export * as ProjectLoader from './project-loader.js';
export * as ProjectLoader from './project-loader.js';

View file

@ -0,0 +1,124 @@
/*
* Keyman is copyright (C) SIL Global. MIT License.
*
* Created by srl on 2025-04-03
*
* Abstraction for line number processing
*/
import { CompilerEvent, EventResolver } from "./index.js";
/** line number with an optional column number */
export interface LineColumn {
line: number;
column?: number;
}
export type LineList = number[];
/** a LineFinder is able to convert from an offset to a line/column */
export class LineFinder {
private list: LineList;
constructor(public text: string) {
}
public getLineList(): LineList {
if (!this.list) {
this.list = LineFinder.textToLines(this.text);
}
return this.list;
}
/**
* Convert an offset into line:column
* @param offset input offset into the text
* @returns line:column information
*/
public findOffset(offset: number): LineColumn {
return LineFinder.offsetToLineColumn(offset, this.getLineList());
}
/**
* Preprocess text to turn it into arrays of line lengths.
* This is effectively a 1-based line length, since line 0 has length of
* 0.
* Note that the fast-xml-parser converts \r\n to \n before processing,
* and this function does as well. So, any offsets passed in will need to
* have that taken into consideration.
*/
public static textToLines(text: string): LineList {
return [
0, // "line 0" is empty
...text.replaceAll("\r\n", "\n").split("\n")
.map(l => l.length + 1) // line length (counting the trailing newline)
];
}
/**
* convert a line number array to a line/column.
* Note that the fast-xml-parser converts \r\n to \n before processing,
* and this function does as well. So, any offsets passed in will need to
* have that taken into consideration.
*/
public static offsetToLineColumn(offset: number, list: LineList): LineColumn {
for (let line = 1; line < list.length; line++) { // 1-based (assume the first row is 0)
if (list[line] > offset) {
return { line, column: offset };
}
offset = offset - (list[line]); // count newline at end
}
// default: line 0, error
return { line: 0 }
}
}
/**
* Interface for a class which can receive file contents,
* keyed by the filename
*/
export interface FileConsumer {
/**
* @param filename name of the file
* @param contents string contents of the file
*/
addFile(filename: string, contents: string): void;
}
/** Cache of LineFinder elements, organized by filename */
export class LineFinderCache implements FileConsumer {
contentsCache: Map<string, LineFinder> = new Map();
/** add or update a source file */
addFile(filename: string, contents: string): void {
this.contentsCache.set(filename, new LineFinder(contents));
}
getByFilename(filename: string): LineFinder | undefined {
const lf = this.contentsCache.get(filename);
return lf;
}
}
/**
* This EventResolver expands events that have a filename and offset set,
* by looking up the event.offset in the LineFinder list.
* The expanded event has the event.line and event.column fields set.
*/
export class LineFinderEventResolver implements EventResolver, FileConsumer {
private lfcache = new LineFinderCache();
addFile(filename: string, contents: string): void {
this.lfcache.addFile(filename, contents);
}
/** resolve a CompilerEvent by expanding the line numbers */
resolve(event: CompilerEvent) {
if (event.offset && !event.line && event.filename) {
const lf = this.lfcache.getByFilename(event.filename);
const loc = lf.findOffset(event.offset);
event.line = loc.line;
event.column = loc.column;
}
return event;
}
}

View file

@ -0,0 +1,33 @@
/*
* Keyman is copyright (C) SIL Global. MIT License.
*
* Created by srl on 2025-03-27
*
* Utilities for manipulating Symbol properties
*/
export class SymbolUtils {
/**
* Copy symbols shallowly from 'from' onto 'onto'
* @param onto object to copy onto
* @param from source for symbols
* @returns the onto object
*/
public static copySymbols<T>(onto: T, from: any): T {
const o = onto as any;
for (const sym of Object.getOwnPropertySymbols(from)) {
o[sym] = from[sym];
}
return onto;
}
/** use Object.entries to remove all symbols, recursively. */
public static removeSymbols<T>(from: T): T {
if (Array.isArray(from)) {
return from.map(o => SymbolUtils.removeSymbols(o)) as T;
}
if (typeof from !== "object") return from;
return Object.fromEntries(Object.entries(from).map(([k, v]) => ([k, SymbolUtils.removeSymbols(v)]))) as T;
}
}

View file

@ -5,12 +5,14 @@
*/
import { SchemaValidators, util } from '@keymanapp/common-types';
import { DeveloperUtilsMessages } from '../../developer-utils-messages.js';
import { CompilerCallbacks } from "../../compiler-callbacks.js";
import { CompilerCallbacks, EventResolver } from "../../compiler-callbacks.js";
import { CompilerEvent } from "../../compiler-interfaces.js";
import { LDMLKeyboardXMLSourceFile, LKImport, ImportStatus } from './ldml-keyboard-xml.js';
import { constants } from '@keymanapp/ldml-keyboard-constants';
import { LDMLKeyboardTestDataXMLSourceFile, LKTTest, LKTTests } from './ldml-keyboard-testdata-xml.js';
import { KeymanXMLReader } from '@keymanapp/developer-utils';
import boxXmlArray = util.boxXmlArray;
import { LineFinderEventResolver } from '../../line-utils.js';
import { XML_FILENAME_SYMBOL, KeymanXMLReader } from '../../xml-utils.js';
interface NameAndProps {
'$'?: any; // content
@ -25,28 +27,44 @@ export class LDMLKeyboardXMLSourceFileReaderOptions {
localImportsPaths: string[];
};
export class LDMLKeyboardXMLSourceFileReader {
export class LDMLKeyboardXMLSourceFileReader implements EventResolver {
/** for resolving messages involving line numbers */
private eventResolver: LineFinderEventResolver = new LineFinderEventResolver();
constructor(private options: LDMLKeyboardXMLSourceFileReaderOptions, private callbacks : CompilerCallbacks) {
}
resolve(event: CompilerEvent): void {
this.eventResolver.resolve(event);
}
static get defaultImportsURL(): [string,string] {
return ['../import/', import.meta.url];
}
readImportFile(version: string, subpath: string): Uint8Array {
const importPath = this.callbacks.resolveFilename(this.options.cldrImportsPath, `${version}/${subpath}`);
return this.callbacks.loadFile(importPath);
/** bottleneck for reading keyboard XML files */
readFile(path: string): Uint8Array {
const data = this.callbacks.loadFile(path);
if (data) {
this.eventResolver.addFile(path, new TextDecoder().decode(data));
}
return data;
}
readLocalImportFile(path: string): Uint8Array {
/** @returns [data, filename] */
readImportFile(version: string, subpath: string): [Uint8Array, string] {
const importPath = this.callbacks.resolveFilename(this.options.cldrImportsPath, `${version}/${subpath}`);
return [this.readFile(importPath), importPath];
}
readLocalImportFile(path: string): [Uint8Array, string] {
// try each of the local imports paths
for (const localPath of this.options.localImportsPaths) {
const importPath = this.callbacks.path.join(localPath, path);
if(this.callbacks.fs.existsSync(importPath)) {
return this.callbacks.loadFile(importPath);
return [this.readFile(importPath), importPath];
}
}
return null; // was not able to load from any of the paths
return [null, null]; // was not able to load from any of the paths
}
/**
@ -223,6 +241,7 @@ export class LDMLKeyboardXMLSourceFileReader {
return false;
}
let importData: Uint8Array;
let importPath: string;
if (base === constants.cldr_import_base) {
// CLDR import
@ -235,10 +254,10 @@ export class LDMLKeyboardXMLSourceFileReader {
/** There's no data or DTD change in 45, 46, 46.1, 47 so map them all to 46 at present. */
paths[0] = constants.cldr_version_latest;
}
importData = this.readImportFile(paths[0], paths[1]);
[importData, importPath] = this.readImportFile(paths[0], paths[1]);
} else {
// local import
importData = this.readLocalImportFile(path);
[importData, importPath] = this.readLocalImportFile(path);
}
if (!importData || !importData.length) {
this.callbacks.reportMessage(DeveloperUtilsMessages.Error_ImportReadFail({base, path, subtag}));
@ -246,6 +265,7 @@ export class LDMLKeyboardXMLSourceFileReader {
}
const importXml: any = this.loadUnboxed(importData); // TODO-LDML: have to load as any because it is an arbitrary part
const importRootNode = importXml[subtag]; // e.g. <keys/>
this.eventResolver.addFile(importPath, new TextDecoder().decode(importData)); // TODO: double decode
// importXml will have one property: the root element.
if (!importRootNode) {
@ -263,7 +283,11 @@ export class LDMLKeyboardXMLSourceFileReader {
return false;
}
// Mark all children as an import
subsubval.forEach(o => o[ImportStatus.import] = basePath);
subsubval.forEach(o => {
o[ImportStatus.import] = basePath;
KeymanXMLReader.setMetaData(o, {[XML_FILENAME_SYMBOL as any]: importPath}); // mark overriding import path
});
if (implied) {
// mark all children as an implied import
subsubval.forEach(o => o[ImportStatus.impliedImport] = basePath);

View file

@ -6,7 +6,13 @@
* Abstraction for XML reading and writing
*/
import { XMLParser, XMLBuilder, XmlBuilderOptions, X2jOptions } from 'fast-xml-parser';
import { XMLParser, XMLBuilder, XMLMetaData, X2jOptions, XmlBuilderOptions } from 'fast-xml-parser';
import { SymbolUtils } from "./symbol-utils.js";
/** Symbol giving the start offset, in chars, of the node */
const XML_META_DATA_SYMBOL = XMLParser.getMetaDataSymbol();
/** Symbol giving an override which file a node came from */
export const XML_FILENAME_SYMBOL = Symbol("XML Filename");
export type KeymanXMLType =
'keyboard3' // LDML <keyboard3>
@ -48,6 +54,7 @@ const PARSER_OPTIONS: KeymanXMLParserOptionsBag = {
// if we do need elements in the future, we'd check the preserve-space attribute here.
return tagValue?.trim();
},
captureMetaData: true,
trimValues: false, // preserve spaces, but see tagValueProcessor
},
'keyboardTest3': {
@ -108,11 +115,54 @@ const GENERATOR_OPTIONS: KeymanXMLGeneratorOptionsBag = {
},
};
export interface KeymanXMLMetadata extends XMLMetaData {
/** override of name of XML file */
[XML_FILENAME_SYMBOL]?: string;
}
/** wrapper for XML parsing support */
export class KeymanXMLReader {
public constructor(public type: KeymanXMLType) {
}
/** Get metadata on a node if not already set */
static getMetaData(o: any) : KeymanXMLMetadata {
if(!o) return o;
const metadata : KeymanXMLMetadata = o[XML_META_DATA_SYMBOL as any];
return metadata;
}
/** Set metadata if not already set */
public static setMetaData(o: any, metadata: KeymanXMLMetadata) : KeymanXMLMetadata {
let m : KeymanXMLMetadata = KeymanXMLReader.getMetaData(o);
if (!m) {
m = {};
}
// copy non-symbols
m = {...metadata, ...m};
// copy symbols
SymbolUtils.copySymbols(m, metadata);
o[XML_META_DATA_SYMBOL as any] = m;
return m;
}
/** set metadata on this and children with the default filename - if not already set */
public static setDefaultFilename(data: any, filename: string) {
if (!data || !filename) return;
if (typeof data === 'object') {
const m = KeymanXMLReader.getMetaData(data) || {};
if (!m[XML_FILENAME_SYMBOL]) {
(m as any)[XML_FILENAME_SYMBOL] = filename;
KeymanXMLReader.setMetaData(data, m);
}
if (Array.isArray(data)) {
data.forEach(e => KeymanXMLReader.setDefaultFilename(e, filename));
} else for(const k of Object.keys(data)) {
KeymanXMLReader.setDefaultFilename(data[k], filename);
}
}
}
/** move `{ $abc: 4 }` into `{ $: { abc: 4 } }` */
private static fixupDollarAttributes(data: any) : any {
if (typeof data === 'object') {
@ -131,9 +181,9 @@ export class KeymanXMLReader {
}
});
if (attrs.length) {
e.push(['$', Object.fromEntries(attrs)]);
e.push(['$', SymbolUtils.copySymbols(Object.fromEntries(attrs), data)]);
}
return Object.fromEntries(e);
return SymbolUtils.copySymbols(Object.fromEntries(e), data);
} else {
return data;
}
@ -169,7 +219,7 @@ export class KeymanXMLReader {
}
}
});
return Object.fromEntries(e);
return SymbolUtils.copySymbols(Object.fromEntries(e), data);
} else {
return data;
}
@ -249,12 +299,11 @@ export class KeymanXMLReader {
}
public parser() {
let options = PARSER_OPTIONS[this.type];
const options = PARSER_OPTIONS[this.type];
if (!options) {
/* c8 ignore next 1 */
throw Error(`Internal error: unhandled XML type ${this.type}`);
}
options = Object.assign({}, options); // TODO: xml2js likes to mutate the options here. Shallow clone the object.
return new XMLParser(options);
}
}

View file

@ -6,6 +6,7 @@ import { KPJFileReader } from "../../src/types/kpj/kpj-file-reader.js";
import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers';
import { KPJFileWriter } from '../../src/types/kpj/kpj-file-writer.js';
import { KeymanDeveloperProjectOptions } from '../../src/types/kpj/keyman-developer-project.js';
import { SymbolUtils } from '../../src/symbol-utils.js';
const callbacks = new TestCompilerCallbacks();
@ -21,12 +22,12 @@ describe('kpj-file-writer', function () {
const writer = new KPJFileWriter();
const output = writer.write(project);
const outputKpj = reader.read(new TextEncoder().encode(output));
// Remove XML metadata symbols to reduce clutter for testing purposes
const outputKpj = SymbolUtils.removeSymbols(reader.read(new TextEncoder().encode(output)));
// The outputKpj may not contain all the fields from the inputKpj, only the
// essential fields. Many of the fields in .kpj are deprecated, when they
// relate to file content (e.g. parented files, file details)
assert.deepEqual(outputKpj.KeymanDeveloperProject.Options, {
"BuildPath": "$PROJECTPATH\\build",
"CompilerWarningsAsErrors": "True",
@ -61,7 +62,8 @@ describe('kpj-file-writer', function () {
const writer = new KPJFileWriter();
const output = writer.write(project);
const outputKpj = reader.read(new TextEncoder().encode(output));
// Remove XML metadata symbols to reduce clutter for testing purposes
const outputKpj = SymbolUtils.removeSymbols(reader.read(new TextEncoder().encode(output)));
// The outputKpj may not contain all the fields from the inputKpj, only the
// essential fields. Many of the fields in .kpj are deprecated, when they

View file

@ -14,6 +14,7 @@ import { makePathToFixture } from '../helpers/index.js';
import { KpsFileReader } from "../../src/types/kps/kps-file-reader.js";
import { KpsFileWriter } from '../../src/types/kps/kps-file-writer.js';
import { SymbolUtils } from '../../src/symbol-utils.js';
import { DeveloperUtilsMessages } from '../../src/developer-utils-messages.js';
const callbacks = new TestCompilerCallbacks();
@ -59,13 +60,15 @@ describe('kps-file-reader', function () {
it('kps-file-reader should round-trip with kps-file-writer', function() {
const input = fs.readFileSync(makePathToFixture('kps', 'khmer_angkor.kps'));
const reader = new KpsFileReader(callbacks);
const kps = reader.read(input);
// Remove XML metadata symbols to reduce clutter for testing purposes
const kps = SymbolUtils.removeSymbols(reader.read(input));
const writer = new KpsFileWriter();
const output = writer.write(kps);
// Round Trip
const kps2 = reader.read(new TextEncoder().encode(output));
// Remove XML metadata symbols to reduce clutter for testing purposes
const kps2 = SymbolUtils.removeSymbols(reader.read(new TextEncoder().encode(output)));
assert.deepEqual(kps2, kps);
});

View file

@ -5,6 +5,7 @@ import KvksFileReader from "../../src/types/kvks/kvks-file-reader.js";
import KvksFileWriter from "../../src/types/kvks/kvks-file-writer.js";
import { verify_khmer_angkor, verify_balochi_inpage } from './kvk-utils.tests.js';
import { assert } from 'chai';
import { SymbolUtils } from '../../src/symbol-utils.js';
describe('kvks-file-reader', function() {
it('should read a valid file', function() {
@ -63,7 +64,8 @@ describe('kvks-file-writer', function() {
const input = fs.readFileSync(path);
const reader = new KvksFileReader();
const kvksExpected = reader.read(input);
// Remove XML metadata symbols to reduce clutter for testing purposes
const kvksExpected = SymbolUtils.removeSymbols(reader.read(input));
const invalidVkeys: string[] = [];
const vk = reader.transform(kvksExpected, invalidVkeys);
assert.isEmpty(invalidVkeys);
@ -74,6 +76,7 @@ describe('kvks-file-writer', function() {
// We compare the (re)loaded data, because there may be
// minor, irrelevant formatting differences in the emitted xml
const kvks = reader.read(Buffer.from(output, 'utf8'));
assert.deepEqual(kvks, kvksExpected);
// Remove XML metadata symbols to reduce clutter for testing purposes
assert.deepEqual(SymbolUtils.removeSymbols(kvks), kvksExpected);
});
});

View file

@ -8,6 +8,7 @@ import { testReaderCases } from '../helpers/reader-callback-test.js';
import CLDRScanToVkey = Constants.CLDRScanToVkey;
import CLDRScanToKeyMap = Constants.CLDRScanToKeyMap;
import USVirtualKeyCodes = Constants.USVirtualKeyCodes;
import { KeymanXMLReader, XML_FILENAME_SYMBOL } from '../../src/xml-utils.js';
function pluckKeysFromKeybag(keys: LKKey[], ids: string[]) {
return keys.filter(({id}) => ids.indexOf(id) !== -1);
@ -140,10 +141,18 @@ describe('ldml keyboard xml reader tests', function () {
{ id: 'interrobang', output: '‽' },
{ id: 'snail', output: '@' },
]);
const snailKey = source?.keyboard3?.keys.key.find(({ id }) => id === 'snail');
// all of the keys are implied imports here
assert.isFalse(ImportStatus.isImpliedImport(source?.keyboard3?.keys.key.find(({id}) => id === 'snail')));
assert.isTrue(ImportStatus.isImport(source?.keyboard3?.keys.key.find(({id}) => id === 'snail')));
assert.isTrue(ImportStatus.isLocalImport(source?.keyboard3?.keys.key.find(({id}) => id === 'snail')));
assert.isFalse(ImportStatus.isImpliedImport(snailKey));
assert.isTrue(ImportStatus.isImport(snailKey));
assert.isTrue(ImportStatus.isLocalImport(snailKey));
// get the actual filename of where the import was located
const metadata = KeymanXMLReader.getMetaData(snailKey);
assert.ok(metadata);
const snailFilename = (metadata)[XML_FILENAME_SYMBOL];
assert.ok(snailFilename);
assert.ok(/keys-Zyyy-morepunctuation.xml$/.test(snailFilename)
, `snail key filename is ${snailFilename}`);
},
},
{

View file

@ -0,0 +1,28 @@
/*
* Keyman is copyright (C) SIL Global. MIT License.
*
* Created by srl on 2025-04-03
*
* Test for LineFinder
*/
import { assert } from 'chai';
import { readFileSync } from 'node:fs';
import 'mocha';
import { LineFinder } from '../src/line-utils.js';
import { makePathToFixture } from './helpers/index.js';
describe(`LineFinder test`, () => {
const path = 'tran_fail-empty.xml';
it(`Should be able to process ${path}`, () => {
const xmlPath = makePathToFixture('xml', `${path}`);
const data = readFileSync(xmlPath, 'utf-8');
assert.ok(data);
const lf = new LineFinder(data);
assert.ok(lf);
assert.deepEqual(lf.findOffset(40), { line: 3, column: 0 });
assert.deepEqual(lf.findOffset(136), { line: 4, column: 2 });
assert.deepEqual(lf.findOffset(186), { line: 8, column: 2 });
});
});

View file

@ -13,7 +13,9 @@ import { readFileSync, writeFileSync } from 'node:fs';
import { KeymanXMLType, KeymanXMLReader, KeymanXMLWriter } from '../src/xml-utils.js';
import { LineFinder } from '../src/line-utils.js';
import { makePathToFixture } from './helpers/index.js';
import { SymbolUtils } from '../src/symbol-utils.js';
// if true, attempt to WRITE the fixtures
const { GEN_XML_FIXTURES } = env;
@ -119,7 +121,7 @@ describe(`XML Reader Test ${GEN_XML_FIXTURES && '(update mode!)' || ''}`, () =>
writeJson(jsonPath, actual);
} else {
assert.ok(expect, `Could not read ${jsonPath} - run with env GEN_XML_FIXTURES=1 to update.`);
assert.deepEqual(actual, expect, `Mismatch of ${xmlPath} vs ${jsonPath}`);
assert.deepEqual(SymbolUtils.removeSymbols(actual), expect, `Mismatch of ${xmlPath} vs ${jsonPath}`);
}
});
}
@ -160,3 +162,41 @@ describe(`XML Writer Test ${GEN_XML_FIXTURES && '(update mode!)' || ''}`, () =>
});
}
});
describe(`XML Reader line number test`, () => {
const path = 'tran_fail-empty.xml';
const xmlPath = makePathToFixture('xml', `${path}`);
const type: KeymanXMLType = 'keyboard3';
it(`Should report line numbers on parse of ${type} ${path}`, () => {
let xml = readData(xmlPath);
assert.ok(xml, `Could not read ${xmlPath}`);
xml = xml.replace(/\r\n/g, '\n');
const reader = new KeymanXMLReader(type);
assert.ok(reader);
// now, parse. subsitute endings for Win
const actual = reader.parse(xml);
const lines = LineFinder.textToLines(xml);
assert.ok(actual, `Parser failed on ${xmlPath}`);
// now, assert char offset
const getMetaData = KeymanXMLReader.getMetaData;
assert.ok(getMetaData(actual.keyboard3));
assert.equal(
getMetaData(actual.keyboard3)?.startIndex, 40); // index of <keyboard3> element
assert.equal(
getMetaData(actual.keyboard3.info)?.startIndex, 136); // index of <info> etc
assert.equal(
getMetaData(actual.keyboard3.transforms)?.startIndex, 186);
assert.deepEqual(
LineFinder.offsetToLineColumn(
getMetaData(actual.keyboard3).startIndex, lines), { line: 3, column: 0 });
assert.deepEqual(
LineFinder.offsetToLineColumn(
getMetaData(actual.keyboard3.info).startIndex, lines), { line: 4, column: 2 });
assert.deepEqual(
LineFinder.offsetToLineColumn(
getMetaData(actual.keyboard3.transforms).startIndex, lines), { line: 8, column: 2 });
});
});

View file

@ -8,7 +8,9 @@ import {
CompilerCallbacks, KeymanCompiler, KeymanCompilerResult, KeymanCompilerArtifacts,
defaultCompilerOptions, LDMLKeyboardXMLSourceFileReader, LDMLKeyboard,
LDMLKeyboardTestDataXMLSourceFile, KMXBuilder,
KeymanCompilerArtifactOptional
KeymanCompilerArtifactOptional,
ResolvingCompilerCallbacks,
KeymanXMLReader,
} from "@keymanapp/developer-utils";
import { LdmlCompilerOptions } from './ldml-compiler-options.js';
import { LdmlCompilerMessages } from './ldml-compiler-messages.js';
@ -94,6 +96,7 @@ export class LdmlKeyboardCompiler implements KeymanCompiler {
// uset parser
private usetparser?: LdmlKeyboardTypes.UnicodeSetParser = undefined;
private reader?: LDMLKeyboardXMLSourceFileReader;
/**
* Initialize the compiler, including loading the WASM host for uset parsing.
@ -105,7 +108,9 @@ export class LdmlKeyboardCompiler implements KeymanCompiler {
*/
async init(callbacks: CompilerCallbacks, options: LdmlCompilerOptions): Promise<boolean> {
this.options = { ...options };
this.callbacks = callbacks;
this.reader = new LDMLKeyboardXMLSourceFileReader(this.options.readerOptions, callbacks);
// wrap the callbacks so that the eventresolver is called
this.callbacks = new ResolvingCompilerCallbacks(this.reader, this.options, callbacks);
return true;
}
@ -238,9 +243,9 @@ export class LdmlKeyboardCompiler implements KeymanCompiler {
* @returns the source file, or null if invalid
*/
public load(filename: string): LDMLKeyboardXMLSourceFile | null {
const reader = new LDMLKeyboardXMLSourceFileReader(this.options.readerOptions, this.callbacks);
const reader = this.reader;
// load the file from disk into a string
const data = this.callbacks.loadFile(filename);
const data = reader.readFile(filename);
if (!data) {
this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidFile({ errorText: 'Unable to read XML file' }));
return null;
@ -261,6 +266,9 @@ export class LdmlKeyboardCompiler implements KeymanCompiler {
return null;
}
// record the default filename - for error reporting.
KeymanXMLReader.setDefaultFilename(source, filename);
return source;
}
@ -272,7 +280,7 @@ export class LdmlKeyboardCompiler implements KeymanCompiler {
* @returns the source file, or null if invalid
*/
public loadTestData(filename: string): LDMLKeyboardTestDataXMLSourceFile | null {
const reader = new LDMLKeyboardXMLSourceFileReader(this.options.readerOptions, this.callbacks);
const reader = this.reader;
const data = this.callbacks.loadFile(filename);
if (!data) {
this.callbacks.reportMessage(LdmlCompilerMessages.Error_InvalidFile({ errorText: 'Unable to read XML file' }));

View file

@ -446,7 +446,7 @@ export class KeysCompiler extends SectionCompiler {
if (layer.row.length > keymap.length) {
this.callbacks.reportMessage(
LdmlCompilerMessages.Error_HardwareLayerHasTooManyRows()
LdmlCompilerMessages.Error_HardwareLayerHasTooManyRows(layer)
);
valid = false;
}
@ -485,7 +485,10 @@ export class KeysCompiler extends SectionCompiler {
}
if (!keydef.output && !keydef.gap && !keydef.layerId) {
this.callbacks.reportMessage(
LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch({ keyId: key })
LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch(
{ keyId: key },
keydef,
)
);
valid = false;
continue;

View file

@ -1,5 +1,5 @@
import { util } from "@keymanapp/common-types";
import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def } from '@keymanapp/developer-utils';
import { CompilerErrorNamespace, CompilerErrorSeverity, CompilerMessageSpec as m, CompilerMessageDef as def, XML_FILENAME_SYMBOL, CompilerEvent, KeymanXMLReader } from '@keymanapp/developer-utils';
// const SevInfo = CompilerErrorSeverity.Info | CompilerErrorNamespace.LdmlKeyboardCompiler;
const SevHint = CompilerErrorSeverity.Hint | CompilerErrorNamespace.LdmlKeyboardCompiler;
const SevWarn = CompilerErrorSeverity.Warn | CompilerErrorNamespace.LdmlKeyboardCompiler;
@ -9,6 +9,32 @@ const SevError = CompilerErrorSeverity.Error | CompilerErrorNamespace.LdmlKeyboa
// sub-numberspace for transform errors
const SevErrorTransform = SevError | 0xF00;
/**
* Any object with metadata, for line number errs.
* Could be for example an LKKeys or KMXPlus.KeysKeys object.
* Defined as 'any' here to reduce noise on the client side.
* @see {@link KeymanXMLReader.getMetaData()}
*/
type ObjectWithMetadata = any;
/**
* Convenience function for constructing CompilerEvents with line numbers
* @param code Unique numeric value of the event
* @param message A short description of the error presented to the user
* @param x Object to be used as a source for line number information
* @param detail Detailed Markdown-formatted description of the error
* including references to documentation, remediation options.
* @see CompilerMessageSpec
* @returns
*/
function CompilerMessageObjectSpec(code: number, message: string, x: ObjectWithMetadata, detail?: string): CompilerEvent {
let evt = m(code, message, detail); // raw message
evt = LdmlCompilerMessages.offset(evt, x); // with offset
return evt;
};
const mx = CompilerMessageObjectSpec;
/**
* @internal
*/
@ -20,7 +46,11 @@ export class LdmlCompilerMessages {
static Error_InvalidLocale = (o:{tag: string}) => m(this.ERROR_InvalidLocale, `Invalid BCP 47 locale form '${def(o.tag)}'`);
static ERROR_HardwareLayerHasTooManyRows = SevError | 0x0003;
static Error_HardwareLayerHasTooManyRows = () => m(this.ERROR_HardwareLayerHasTooManyRows, `'hardware' layer has too many rows`);
static Error_HardwareLayerHasTooManyRows = (x: any) => mx(
this.ERROR_HardwareLayerHasTooManyRows,
`'hardware' layer has too many rows`,
x,
);
static ERROR_RowOnHardwareLayerHasTooManyKeys = SevError | 0x0004;
static Error_RowOnHardwareLayerHasTooManyKeys = (o:{row: number, hardware: string, modifiers: string}) => m(this.ERROR_RowOnHardwareLayerHasTooManyKeys, `Row #${def(o.row)} on 'hardware' ${def(o.hardware)} layer for modifier ${o.modifiers || 'none'} has too many keys`);
@ -97,8 +127,11 @@ export class LdmlCompilerMessages {
m(this.ERROR_DisplayIsRepeated, `display ${LdmlCompilerMessages.outputOrKeyId(o)} has more than one display entry.`);
static ERROR_KeyMissingToGapOrSwitch = SevError | 0x0011;
static Error_KeyMissingToGapOrSwitch = (o:{keyId: string}) =>
m(this.ERROR_KeyMissingToGapOrSwitch, `key id='${def(o.keyId)}' must have either output=, gap=, or layerId=.`);
static Error_KeyMissingToGapOrSwitch = (o:{keyId: string}, x: ObjectWithMetadata) => mx(
this.ERROR_KeyMissingToGapOrSwitch,
`key id='${def(o.keyId)}' must have either output=, gap=, or layerId=.`,
x,
);
static ERROR_ExcessHardware = SevError | 0x0012;
static Error_ExcessHardware = (o:{formId: string}) => m(this.ERROR_ExcessHardware,
@ -274,5 +307,18 @@ export class LdmlCompilerMessages {
`Invalid transform to="${def(o.to)}": "${def(o.message)}"`,
);
/**
* Get an offset from o and set e's offset field
* @param event a compiler event, such as from functions in this class
* @param x any object parsed from XML or with the XML_META_DATA_SYMBOL symbol copied over
* @returns modified event object
*/
static offset(event: CompilerEvent, x?: any): CompilerEvent {
if(x) {
const metadata = KeymanXMLReader.getMetaData(x) || {};
event.offset = metadata?.startIndex;
event.filename = event.filename || metadata[XML_FILENAME_SYMBOL];
}
return event;
}
}

View file

@ -1,6 +1,6 @@
import {assert} from 'chai';
import {readFileSync} from 'node:fs';
import { KeymanXMLReader } from "@keymanapp/developer-utils";
import { KeymanXMLReader, SymbolUtils } from "@keymanapp/developer-utils";
/**
*
@ -21,5 +21,5 @@ export function compareXml(actual : string, expect: string, mutator?: (input: an
const actualParsed = mutator(reader.parse(actualStr));
const expectParsed = mutator(reader.parse(expectStr));
assert.deepEqual(actualParsed, expectParsed);
assert.deepEqual(SymbolUtils.removeSymbols(actualParsed), SymbolUtils.removeSymbols(expectParsed));
}

View file

@ -8,7 +8,7 @@ import * as path from 'path';
import { fileURLToPath } from 'url';
import { SectionCompiler, SectionCompilerNew } from '../../src/compiler/section-compiler.js';
import { util, KMXPlus, LdmlKeyboardTypes } from '@keymanapp/common-types';
import { CompilerEvent, compilerEventFormat, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboard, } from "@keymanapp/developer-utils";
import { CompilerEvent, compilerEventFormat, CompilerCallbacks, LDMLKeyboardXMLSourceFileReader, LDMLKeyboardTestDataXMLSourceFile, LDMLKeyboard, KeymanXMLMetadata, KeymanXMLReader } from "@keymanapp/developer-utils";
import { LdmlKeyboardCompiler } from '../../src/main.js'; // make sure main.js compiles
import { assert } from 'chai';
import { KMXPlusMetadataCompiler } from '../../src/compiler/metadata-compiler.js';
@ -290,7 +290,7 @@ export function testCompilationCases(compiler: SectionCompilerNew, cases : Compi
// no warnings, so expect zero messages
assert.sameDeepMembers(callbacks.messages, [], 'expected zero messages but got ' + callbacks.messages);
}
if (expectFailure) {
assert.isNull(section, 'expected compilation result failure (null)');
} else {
@ -327,3 +327,13 @@ const dontEscape = /[a-zA-Z0-9\.${}\[\]-]/;
export function hex_str(s?: string) : string {
return [...s].map(ch => dontEscape.test(ch) ? ch : util.escapeRegexChar(ch)).join('');
}
/** return an object simulating an XML object with a column number */
export function withColumn(c: number) : KeymanXMLMetadata {
// set metadata on an empty object
const o = {};
KeymanXMLReader.setMetaData(o, {
startIndex: c
});
return o;
}

View file

@ -1,7 +1,7 @@
import 'mocha';
import { assert } from 'chai';
import { KeysCompiler } from '../src/compiler/keys.js';
import { assertCodePoints, compilerTestCallbacks, loadSectionFixture, testCompilationCases } from './helpers/index.js';
import { assertCodePoints, compilerTestCallbacks, loadSectionFixture, testCompilationCases, withColumn } from './helpers/index.js';
import { KMXPlus, Constants, LdmlKeyboardTypes } from '@keymanapp/common-types';
import { LdmlCompilerMessages } from '../src/compiler/ldml-compiler-messages.js';
import { constants } from '@keymanapp/ldml-keyboard-constants';
@ -442,7 +442,7 @@ describe('keys.kmap', function () {
assert.isNull(keys);
assert.equal(compilerTestCallbacks.messages.length, 1);
assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_HardwareLayerHasTooManyRows());
assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_HardwareLayerHasTooManyRows(withColumn(276)));
});
it('should reject layouts with too many hardware keys', async function() {
@ -464,7 +464,10 @@ describe('keys.kmap', function () {
const keys = await loadSectionFixture(KeysCompiler, 'sections/keys/invalid-key-missing-attrs.xml', compilerTestCallbacks, keysDependencies) as Keys;
assert.isNull(keys);
assert.equal(compilerTestCallbacks.messages.length, 1);
assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch({keyId: 'Q'}));
assert.deepEqual(compilerTestCallbacks.messages[0], LdmlCompilerMessages.Error_KeyMissingToGapOrSwitch(
{keyId: 'Q'},
withColumn(188)
));
});
it('should accept layouts with gap/switch keys', async function() {
const keys = await loadSectionFixture(KeysCompiler, 'sections/keys/gap-switch.xml', compilerTestCallbacks, keysDependencies) as Keys;

View file

@ -45,6 +45,8 @@ export class NodeCompilerCallbacks implements CompilerCallbacks {
messages: CompilerEvent[] = [];
messageCount = 0;
messageFilename: string = '';
/** cache of the contentes of the text */
messageFiletext: string = '';
maxLogMessages = MaxMessagesDefault;
constructor(private options: CompilerCallbackOptions) {
@ -55,6 +57,7 @@ export class NodeCompilerCallbacks implements CompilerCallbacks {
this.messages = [];
this.messageCount = 0;
this.messageFilename = '';
this.messageFiletext = '';
}
/**
@ -154,6 +157,7 @@ export class NodeCompilerCallbacks implements CompilerCallbacks {
// Reset max message limit when a new file is being processed
this.messageFilename = event.filename;
this.messageCount = 0;
this.messageFiletext = '';
}
const disable = CompilerFileCallbacks.applyMessageOverridesToEvent(event, this.options.messageOverrides);

16
package-lock.json generated
View file

@ -341,7 +341,7 @@
"dependencies": {
"@keymanapp/common-types": "*",
"eventemitter3": "^5.0.0",
"fast-xml-parser": "^5.0.9",
"fast-xml-parser": "^5.2.2",
"path-browserify": "^1.0.1",
"restructure": "^3.0.1",
"sax": ">=0.6.0",
@ -9124,9 +9124,9 @@
"dev": true
},
"node_modules/fast-xml-parser": {
"version": "5.0.9",
"resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.0.9.tgz",
"integrity": "sha512-2mBwCiuW3ycKQQ6SOesSB8WeF+fIGb6I/GG5vU5/XEptwFFhp9PE8b9O7fbs2dpq9fXn4ULR3UsfydNUCntf5A==",
"version": "5.2.2",
"resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.2.2.tgz",
"integrity": "sha512-ZaCmslH75Jkfowo/x44Uq8KT5SutC5BFxHmY61nmTXPccw11PVuIXKUqC2hembMkJ3nPwTkQESXiUlsKutCbMg==",
"funding": [
{
"type": "github",
@ -9135,7 +9135,7 @@
],
"license": "MIT",
"dependencies": {
"strnum": "^2.0.5"
"strnum": "^2.1.0"
},
"bin": {
"fxparser": "src/cli/cli.js"
@ -13993,9 +13993,9 @@
"link": true
},
"node_modules/strnum": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/strnum/-/strnum-2.0.5.tgz",
"integrity": "sha512-YAT3K/sgpCUxhxNMrrdhtod3jckkpYwH6JAuwmUdXZsmzH1wUyzTMrrK2wYCEEqlKwrWDd35NeuUkbBy/1iK+Q==",
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/strnum/-/strnum-2.1.0.tgz",
"integrity": "sha512-w0S//9BqZZGw0L0Y8uLSelFGnDJgTyyNQLmSlPnVz43zPAiqu3w4t8J8sDqqANOGeZIZ/9jWuPguYcEnsoHv4A==",
"funding": [
{
"type": "github",