erabo/spiegel-keyman
1
0
Fork 0
mirror of https://github.com/keymanapp/keyman.git synced 2026-08-28 11:17:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

refactor(developer): complete fs move out of kmcmplib

Browse source 
* Moves filesystem access out of kmcmplib into kmc-kmn
* Adds filesystem access callback to kmcmplib unit tests
* Cleans up callback interface through wasm
* Adds unit tests for various file load scenarios
* Removes nodefs dependency from kmcmplib wasm build, and removes
  corresponding path mappings which were previously required for wasm
  builds; note that these are still present for the unit tests for
  kmcmplib.
This commit is contained in:
Marc Durdin 2023-05-31 07:44:35 +07:00
parent 980b893644
commit 47c8a95fb0
39 changed files with 406 additions and 308 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
developer/src/kmc-kmn/src/compiler/compiler.ts
View file

@ -42,13 +42,16 @@ const baseOptions: CompilerOptions = {
*/
let callbackProcIdentifier = 0;
const
callbackPrefix = 'kmnCompilerCallbacks_';
export class KmnCompiler {
private Module: any;
callbackName: string;
callbackID: string; // a unique numeric id added to globals with prefixed names
callbacks: CompilerCallbacks;
constructor() {
this.callbackName = 'kmnCompilerCallback' + callbackProcIdentifier;
this.callbackID = callbackPrefix + callbackProcIdentifier.toString();
callbackProcIdentifier++;
}
@ -58,6 +61,7 @@ export class KmnCompiler {
try {
this.Module = await loadWasmHost();
} catch(e: any) {
/* c8 ignore next 3 */
this.callbacks.reportMessage(CompilerMessages.Fatal_MissingWasmModule({e}));
return false;
}
@ -74,7 +78,9 @@ export class KmnCompiler {
// Can't report a message here.
throw Error('Must call Compiler.init(callbacks) before proceeding');
}
if(!this.Module) { // fail if wasm not loaded or function not found
if(!this.Module) {
/* c8 ignore next 4 */
// fail if wasm not loaded or function not found
this.callbacks.reportMessage(CompilerMessages.Fatal_MissingWasmModule({}));
return false;
}
@ -83,15 +89,17 @@ export class KmnCompiler {
public run(infile: string, outfile: string, options?: CompilerOptions): boolean {
if(!this.verifyInitialized()) {
/* c8 ignore next 2 */
return false;
}
options = {...baseOptions, ...options};
(globalThis as any)[this.callbackName] = this.compilerMessageCallback;
// TODO: use callbacks for file access -- so kmc-kmn is entirely fs agnostic
(globalThis as any)[this.callbackID] = {
message: this.compilerMessageCallback,
loadFile: this.loadFileCallback
};
let result = this.runCompiler(infile, outfile, options);
delete (globalThis as any)[this.callbackName];
//TODO: write the file out!
delete (globalThis as any)[this.callbackID];
if(result) {
if(result.kmx) {
this.callbacks.fs.writeFileSync(result.kmx.filename, result.kmx.data);
@ -111,6 +119,30 @@ export class KmnCompiler {
return 1;
}
private loadFileCallback = (filename: string, baseFilename: string, buffer: number, bufferSize: number): number => {
// TODO: we can optimize this in future by avoiding loading the file twice
let resolvedFilename = this.callbacks.resolveFilename(baseFilename, filename);
let data = this.callbacks.loadFile(resolvedFilename);
if(!data) {
return 0;
}
if(buffer == 0) {
/* We need to return buffer size required */
return data.byteLength;
}
if(bufferSize != data.byteLength) {
// TODO: consider chucking a wobbly because this is a bug
/* c8 ignore next 2 */
return 0;
}
this.Module.HEAP8.set(data, buffer);
return 1;
}
private runCompiler(infile: string, outfile: string, options: CompilerOptions): CompilerResult {
let result: CompilerResult = {};
let wasm_interface = new this.Module.CompilerInterface();
@ -122,8 +154,7 @@ export class KmnCompiler {
wasm_options.warnDeprecatedCode = options.warnDeprecatedCode;
wasm_options.shouldAddCompilerVersion = options.shouldAddCompilerVersion;
wasm_options.target = 0; //CKF_KEYMAN; TODO, support KMW
wasm_interface.messageCallback = this.callbackName;
wasm_interface.loadFileCallback = this.callbackName; // TODO: this is wrong, needs to be a new callback; not yet used though
wasm_interface.callbacksKey = this.callbackID; // key of object on globalThis
wasm_result = this.Module.kmcmp_compile(infile, wasm_options, wasm_interface);
if(!wasm_result.result) {
return null;
@ -143,6 +174,7 @@ export class KmnCompiler {
return result;
} catch(e) {
/* c8 ignore next 3 */
this.callbacks.reportMessage(CompilerMessages.Fatal_UnexpectedException({e:e}));
return null;
} finally {
@ -163,6 +195,7 @@ export class KmnCompiler {
reader.validate(kvks, this.callbacks.loadSchema('kvks'));
} catch(e) {
console.log(e);
// TODO: also unit test
// TODO: this.callbacks.reportMessage(CompilerMessages.Error_InvalidKvksFile({e}));
return null;
}
@ -170,6 +203,7 @@ export class KmnCompiler {
let vk = reader.transform(kvks, errors);
if(!vk || errors.length) {
console.dir(errors);
// TODO: also unit test
// TODO: this.callbacks.reportMessage(CompilerMessages.Error_InvalidKvksFile({e}));
return null;
}
@ -188,10 +222,12 @@ export class KmnCompiler {
*/
public parseUnicodeSet(pattern: string, bufferSize: number) : UnicodeSet | null {
if(!this.verifyInitialized()) {
/* c8 ignore next 2 */
return null;
}
if (!bufferSize) {
/* c8 ignore next 2 */
bufferSize = 100; // TODO-LDML: Preflight mode? Reuse buffer?
}
const buf = this.Module.asm.malloc(bufferSize * 2 * this.Module.HEAPU32.BYTES_PER_ELEMENT);
@ -239,6 +275,7 @@ function getUnicodeSetError(rc: number) : CompilerEvent {
case KMCMP_FATAL_OUT_OF_RANGE:
return CompilerMessages.Fatal_UnicodeSetOutOfRange();
default:
/* c8 ignore next */
return CompilerMessages.Fatal_UnexpectedException({e: `Unexpected UnicodeSet error code ${rc}`});
}
}

47
developer/src/kmcmplib/src/CheckFilenameConsistency.cpp
View file

@ -7,52 +7,12 @@
#include <string>
#include "CheckFilenameConsistency.h"
#include "kmx_u16.h"
#include "filesystem.h"
#ifdef _MSC_VER
#include <io.h>
#endif
namespace kmcmp {
extern KMX_CHAR CompileDir[260]; // TODO: this should not be a fixed buffer
}
bool IsRelativePath(KMX_CHAR const * p) {
// Relative path (returns TRUE):
// ..\...\BITMAP.BMP
// PATH\BITMAP.BMP
// BITMAP.BMP
// Semi-absolute path (returns FALSE):
// \...\BITMAP.BMP
// Absolute path (returns FALSE):
// C:\...\BITMAP.BMP
// \\SERVER\SHARE\...\BITMAP.BMP
if ((*p == '\\') || (*p == '/')) return FALSE;
if (*p && *(p + 1) == ':') return FALSE;
return TRUE;
}
bool IsRelativePath(KMX_WCHAR const * p) {
// Relative path (returns TRUE):
// ..\...\BITMAP.BMP
// PATH\BITMAP.BMP
// BITMAP.BMP
// Semi-absolute path (returns FALSE):
// \...\BITMAP.BMP
// Absolute path (returns FALSE):
// C:\...\BITMAP.BMP
// \\SERVER\SHARE\...\BITMAP.BMP
if ((*p == u'\\') || (*p == u'/'))return FALSE;
if (*p && *(p + 1) == u':') return FALSE;
return TRUE;
}
KMX_DWORD CheckFilenameConsistency( KMX_CHAR const * Filename, bool ReportMissingFile) {
PKMX_WCHAR WFilename = strtowstr(( KMX_CHAR *)Filename);
@ -62,6 +22,12 @@ KMX_DWORD CheckFilenameConsistency( KMX_CHAR const * Filename, bool ReportMissin
}
KMX_DWORD CheckFilenameConsistency(KMX_WCHAR const * Filename, bool ReportMissingFile) {
// TODO: we no longer have filesystem access here. We could move this check to
// kmc itself, and make it consistent across all compilers that use the same
// loader callback
return CERR_None;
#if 0
// not ready yet: needs more attention-> common includes for non-Windows platforms
KMX_WCHAR Name[260]; // TODO: fixed buffer sizes bad
@ -115,6 +81,7 @@ KMX_DWORD CheckFilenameConsistency(KMX_WCHAR const * Filename, bool ReportMissin
#endif
return CERR_None;
#endif
}
KMX_DWORD CheckFilenameConsistencyForCalls(PFILE_KEYBOARD fk) {

2
developer/src/kmcmplib/src/CheckFilenameConsistency.h
View file

@ -7,5 +7,3 @@ KMX_DWORD CheckFilenameConsistencyForCalls(PFILE_KEYBOARD fk);
KMX_DWORD CheckFilenameConsistency(KMX_CHAR const * Filename, bool ReportMissingFile);
KMX_DWORD CheckFilenameConsistency(KMX_WCHAR const * Filename, bool ReportMissingFile);
bool IsRelativePath(KMX_CHAR const * p);
bool IsRelativePath(KMX_WCHAR const * p);

102
developer/src/kmcmplib/src/Compiler.cpp
View file

@ -101,7 +101,6 @@
#include "UnreachableRules.h"
#include "CheckForDuplicates.h"
#include "kmx_u16.h"
#include "filesystem.h"
#include <CompMsg.h>
/* These macros are adapted from winnt.h and legacy use only */
@ -125,7 +124,6 @@ namespace kmcmp{
KMX_BOOL FMnemonicLayout = FALSE;
KMX_BOOL FOldCharPosMatching = FALSE;
int CompileTarget;
KMX_CHAR CompileDir[260]; // TODO: this should not be a fixed buffer
int BeginLine[4];
KMX_BOOL IsValidCallStore(PFILE_STORE fs);
@ -867,7 +865,6 @@ KMX_DWORD ProcessSystemStore(PFILE_KEYBOARD fk, KMX_DWORD SystemID, PFILE_STORE
int i, j;
KMX_DWORD msg;
PKMX_WCHAR p, q;
KMX_CHAR *pp;
if (!pssBuf) pssBuf = new KMX_WCHAR[GLOBAL_BUFSIZE];
PKMX_WCHAR buf = pssBuf;
@ -917,13 +914,9 @@ KMX_DWORD ProcessSystemStore(PFILE_KEYBOARD fk, KMX_DWORD SystemID, PFILE_STORE
case TSS_INCLUDECODES:
VERIFY_KEYBOARD_VERSION(fk, VERSION_60, CERR_60FeatureOnly_NamedCodes);
pp = wstrtostr(sp->dpString);
if (!kmcmp::CodeConstants->LoadFile(pp))
{
delete[] pp;
if (!kmcmp::CodeConstants->LoadFile(fk, sp->dpString)) {
return CERR_CannotLoadIncludeFile;
}
delete[] pp;
kmcmp::CodeConstants->reindex(); // I4982
break;
@ -3234,58 +3227,32 @@ KMX_BOOL IsSameToken(PKMX_WCHAR *p, KMX_WCHAR const * token)
return FALSE;
}
static bool endsWith(const std::string& str, const std::string& suffix)
{
return str.size() >= suffix.size() && 0 == str.compare(str.size()-suffix.size(), suffix.size(), suffix);
}
KMX_DWORD ImportBitmapFile(PFILE_KEYBOARD fk, PKMX_WCHAR szName, PKMX_DWORD FileSize, PKMX_BYTE *Buf)
{
FILE *fp;
KMX_WCHAR szNewName[260];
auto szNameUtf8 = string_from_u16string(szName);
if (IsRelativePath(szName))
{
PKMX_WCHAR WCompileDir = strtowstr(kmcmp::CompileDir);
u16ncpy(szNewName, WCompileDir, _countof(szNewName)); // I3481
u16ncat(szNewName,szName, _countof(szNewName )); // I3481
}
else
u16ncpy(szNewName, szName, _countof(szNewName)); // I3481
fp=Open_File(szNewName, u"rb");
if ( fp == NULL)
{
// else if filename.bmp is not in the folder -> attempt to open filename.bmp.bmp !
if ( u16cmp(szNewName+u16len(szNewName)-4, u".bmp") )
u16ncat(szNewName, u".bmp", _countof(szNewName)); // I3481
fp= Open_File(szNewName, u"rb");
if ( fp == NULL)
if(!loadfileproc(szNameUtf8.c_str(), fk->extra->kmnFilename.c_str(), nullptr, (int*) FileSize, msgprocContext)) {
// Append .bmp and try again
if(endsWith(szNameUtf8, ".bmp")) {
return CERR_CannotReadBitmapFile;
}
szNameUtf8.append(".bmp");
if(!loadfileproc(szNameUtf8.c_str(), fk->extra->kmnFilename.c_str(), nullptr, (int*) FileSize, msgprocContext)) {
return CERR_CannotReadBitmapFile;
}
}
KMX_DWORD msg;
if ((msg = CheckFilenameConsistency(szNewName, FALSE)) != CERR_None) {
return msg;
}
fseek(fp, 0, SEEK_END);
*FileSize = (KMX_DWORD)ftell(fp);
fseek(fp ,0,SEEK_SET);
if (*FileSize < 0) {
fclose(fp);
return CERR_CannotReadBitmapFile;
}
if (*FileSize < 2) return CERR_CannotReadBitmapFile;
*Buf = new KMX_BYTE[*FileSize];
if (fread(*Buf, 1, *FileSize, fp) < (size_t) *FileSize) {
delete[] * Buf;
*Buf = NULL;
if(!loadfileproc(szNameUtf8.c_str(), fk->extra->kmnFilename.c_str(), *Buf, (int*) FileSize, msgprocContext)) {
delete[] *Buf;
return CERR_CannotReadBitmapFile;
}
fclose(fp);
/* Test for version 7.0 icon support */
if (*((PKMX_CHAR)*Buf) != 'B' && *(((PKMX_CHAR)*Buf) + 1) != 'M') {
VERIFY_KEYBOARD_VERSION(fk, VERSION_70, CERR_70FeatureOnly);
@ -3447,6 +3414,8 @@ bool hasPreamble(std::u16string result) {
return result.size() > 0 && result[0] == 0xFEFF;
}
#include "unicode/ucnv.h"
bool UTF16TempFromUTF8(KMX_BYTE* infile, int sz, KMX_BYTE** tempfile, int *sz16) {
if(sz == 0) {
return FALSE;
@ -3456,23 +3425,36 @@ bool UTF16TempFromUTF8(KMX_BYTE* infile, int sz, KMX_BYTE** tempfile, int *sz16)
try {
std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> converter;
result = converter.from_bytes((char*)infile, (char*)infile+sz-1);
result = converter.from_bytes((char*)infile, (char*)infile+sz);
} catch(std::range_error e) {
std::wstring_convert<std::codecvt_utf16<char16_t>, char16_t> converter;
result = converter.from_bytes((char*)infile, (char*)infile+sz-1);
UErrorCode status = U_ZERO_ERROR;
// TODO: we need ICU data files here @srl295 plz help!
UConverter* conv = ucnv_open("windows-1252", &status);
if(U_FAILURE(status)) {
return FALSE;
}
char16_t* dest = new char16_t[sz*2];
ucnv_toUChars(conv, dest, sz*2, (char*)infile, sz, &status);
if(U_FAILURE(status)) {
delete[] dest;
return FALSE;
}
result = dest;
delete[] dest;
}
if(hasPreamble(result)) {
*sz16 = result.size() * 2 - 1;
*sz16 = result.size() * 2 - 2;
*tempfile = new KMX_BYTE[*sz16];
memcpy(*tempfile, result.c_str() + 2, *sz16);
memcpy(*tempfile, result.c_str() + 1, *sz16);
} else {
*sz16 = result.size() * 2;
*tempfile = new KMX_BYTE[*sz16];
memcpy(*tempfile, result.c_str(), *sz16);
}
*sz16 = result.size() * 2;
*tempfile = new KMX_BYTE[*sz16];
memcpy(*tempfile, result.c_str(), *sz16);
return TRUE;
}

44
developer/src/kmcmplib/src/CompilerInterfaces.cpp
View file

@ -3,7 +3,6 @@
#include <kmcmplibapi.h>
#include <kmn_compiler_errors.h>
#include "kmcmplib.h"
#include "filesystem.h"
#include "CheckFilenameConsistency.h"
#include "CheckNCapsConsistency.h"
#include "DeprecationChecks.h"
@ -19,7 +18,7 @@ bool CompileKeyboardHandle(KMX_BYTE* infile, int sz, PFILE_KEYBOARD fk);
WASM interface for compiler message callback
*/
EM_JS(int, wasm_msgproc, (int line, int msgcode, const char* text, char* context), {
const proc = globalThis[UTF8ToString(context)];
const proc = globalThis[UTF8ToString(context)].message;
if(!proc || typeof proc != 'function') {
console.log(`[${line}: ${msgcode}: ${UTF8ToString(text)}]`);
return 0;
@ -28,18 +27,27 @@ EM_JS(int, wasm_msgproc, (int line, int msgcode, const char* text, char* context
}
});
EM_JS(bool, wasm_loadfileproc, (const char* filename, const char* baseFilename, void* buffer, int* bufferSize, char* context), {
const proc = globalThis[UTF8ToString(context)];
EM_JS(int, wasm_loadfileproc, (const char* filename, const char* baseFilename, void* buffer, int bufferSize, char* context), {
const proc = globalThis[UTF8ToString(context)].loadFile;
if(!proc || typeof proc != 'function') {
return 0;
} else {
return proc(UTF8ToString(filename), UTF8ToString(baseFilename), buffer, bufferSize);
if(buffer == 0) {
return proc(UTF8ToString(filename), UTF8ToString(baseFilename), 0, 0);
} else {
return proc(UTF8ToString(filename), UTF8ToString(baseFilename), buffer, bufferSize);
}
}
});
bool wasm_LoadFileProc(const char* filename, const char* baseFilename, void* buffer, int* bufferSize, void* context) {
char* msgProc = static_cast<char*>(context);
return wasm_loadfileproc(filename, baseFilename, buffer, bufferSize, msgProc);
if(buffer == nullptr) {
*bufferSize = wasm_loadfileproc(filename, baseFilename, 0, 0, msgProc);
return *bufferSize != 0;
} else {
return wasm_loadfileproc(filename, baseFilename, buffer, *bufferSize, msgProc) == 1;
}
}
int wasm_CompilerMessageProc(int line, uint32_t dwMsgCode, const char* szText, void* context) {
@ -48,8 +56,7 @@ int wasm_CompilerMessageProc(int line, uint32_t dwMsgCode, const char* szText, v
}
struct WASM_COMPILER_INTERFACE {
std::string messageCallback; // int line, uint32_t dwMsgCode, char* szText
std::string loadFileCallback; // TODO: char* filename, char* baseFilename --> buffer
std::string callbacksKey; // key of callbacks object on globalThis
};
struct WASM_COMPILER_RESULT {
@ -76,7 +83,7 @@ WASM_COMPILER_RESULT kmcmp_wasm_compile(std::string pszInfile, const KMCMP_COMPI
options,
wasm_CompilerMessageProc,
wasm_LoadFileProc,
intf.messageCallback.c_str(),
intf.callbacksKey.c_str(),
kr
);
@ -103,8 +110,7 @@ EMSCRIPTEN_BINDINGS(compiler_interface) {
emscripten::class_<WASM_COMPILER_INTERFACE>("CompilerInterface")
.constructor<>()
.property("messageCallback", &WASM_COMPILER_INTERFACE::messageCallback)
.property("loadFileCallback", &WASM_COMPILER_INTERFACE::loadFileCallback)
.property("callbacksKey", &WASM_COMPILER_INTERFACE::callbacksKey)
;
emscripten::class_<WASM_COMPILER_RESULT>("CompilerResult")
@ -131,6 +137,8 @@ EXTERN bool kmcmp_CompileKeyboard(
) {
FILE_KEYBOARD fk;
fk.extra = new FILE_KEYBOARD_EXTRA;
fk.extra->kmnFilename = pszInfile;
kmcmp::FSaveDebug = options.saveDebug; // I3681
kmcmp::FCompilerWarningsAsErrors = options.compilerWarningsAsErrors; // I4865
@ -143,16 +151,6 @@ EXTERN bool kmcmp_CompileKeyboard(
return FALSE;
}
PKMX_STR p;
if ((p = strrchr_slash((char*)pszInfile)) != nullptr) {
strncpy(kmcmp::CompileDir, pszInfile, (int)(p - pszInfile + 1)); // I3481
kmcmp::CompileDir[(int)(p - pszInfile + 1)] = 0;
}
else {
kmcmp::CompileDir[0] = 0;
}
msgproc = messageProc;
loadfileproc = loadFileProc;
msgprocContext = (void*)procContext;
@ -172,7 +170,7 @@ EXTERN bool kmcmp_CompileKeyboard(
return FALSE;
}
KMX_BYTE* infile = new KMX_BYTE[sz];
KMX_BYTE* infile = new KMX_BYTE[sz+1];
if(!infile) {
AddCompileError(CERR_CannotAllocateMemory);
return FALSE;
@ -182,6 +180,7 @@ EXTERN bool kmcmp_CompileKeyboard(
AddCompileError(CERR_CannotReadInfile);
return FALSE;
}
infile[sz] = 0; // zero-terminate for safety, not technically needed but helps avoid memory bugs
int offset = 0;
if(infile[0] == (KMX_BYTE) UTF16Sig[0] && infile[1] == (KMX_BYTE) UTF16Sig[1]) {
@ -266,7 +265,6 @@ bool CompileKeyboardHandle(KMX_BYTE* infile, int sz, PFILE_KEYBOARD fk)
fk->dpDeadKeyArray = NULL;
fk->cxVKDictionary = 0; // I3438
fk->dpVKDictionary = NULL; // I3438
fk->extra = new FILE_KEYBOARD_EXTRA;
fk->extra->kvksFilename = u"";
/* fk->szMessage[0] = 0;
fk->szLanguageName[0] = 0;*/

97
developer/src/kmcmplib/src/NamedCodeConstants.cpp
View file

@ -27,15 +27,12 @@
#include "CheckFilenameConsistency.h"
#include <kmcmplib.h>
#include "kmcompx.h"
#include "filesystem.h"
using namespace kmcmp;
int IsHangulSyllable(const KMX_WCHAR *codename, int *code);
namespace kmcmp {
extern KMX_CHAR CompileDir[];
int __cdecl sort_entries(const void *elem1, const void *elem2)
{
return u16icmp(
@ -117,85 +114,61 @@ char *kmc_strupr(char *s) {
return s;
}
KMX_BOOL NamedCodeConstants::IntLoadFile(const KMX_CHAR *filename)
{
KMX_BOOL NamedCodeConstants::LoadFile(PFILE_KEYBOARD fk, const KMX_WCHAR *filename) {
const int str_size = 256;
FILE *fp = NULL;
if (CheckFilenameConsistency(filename, FALSE) != 0) {
return FALSE;
}
fp = Open_File(filename, "rt");
if(fp == NULL) {
return FALSE; // I3481
auto szNameUtf8 = string_from_u16string(filename);
int FileSize;
KMX_BYTE* Buf;
if(!loadfileproc(szNameUtf8.c_str(), fk->extra->kmnFilename.c_str(), nullptr, &FileSize, msgprocContext)) {
return FALSE;
}
KMX_CHAR str[str_size], *p, *q, *context = NULL;
KMX_BOOL isEol , first = TRUE;
Buf = new KMX_BYTE[FileSize+1];
if(!loadfileproc(szNameUtf8.c_str(), fk->extra->kmnFilename.c_str(), Buf, &FileSize, msgprocContext)) {
delete[] Buf;
return FALSE;
}
Buf[FileSize] = 0; // zero-terminate for strtok
while(fgets(str, str_size, fp))
{
isEol = *(strchr(str, 0) - 1) == '\n';
p = strtok_r(str, ";", &context); // I3481
q = strtok_r(NULL, ";\n", &context);
if(p && q)
{
if(first && *p == (KMX_CHAR)0xEF && *(p+1) == (KMX_CHAR)0xBB && *(p+2) == (KMX_CHAR)0xBF) p += 3; // I3056 UTF-8 // I3512
first = FALSE;
char* filetok;
char* filecontext;
filetok = strtok_r((char*)Buf, "\n", &filecontext);
if(*filetok == (KMX_CHAR)0xEF && *(filetok+1) == (KMX_CHAR)0xBB && *(filetok+2) == (KMX_CHAR)0xBF) filetok += 3; // I3056 UTF-8 // I3512
while(filetok) {
KMX_CHAR str[str_size], *p, *q, *context = NULL;
if(strlen(filetok) >= str_size) {
delete[] Buf;
// TODO chuck a wobbly
return FALSE;
}
strcpy(str, filetok);
p = strtok_r(str, ";\r", &context); // I3481
q = strtok_r(nullptr, ";\r", &context);
if(p && q) {
kmc_strupr(q); // I3481 // I3641
long n = strtol(p, NULL, 16);
long n = strtol(p, nullptr, 16);
if (*q != '<') {
PKMX_WCHAR q0 = strtowstr(q);
AddCode_IncludedCodes((int)n, q0);
delete[] q0;
}
}
if(!isEol )
{
while(fgets(str, str_size, fp)) if(*(strchr(str, 0)-1) == '\n') break;
}
filetok = strtok_r(nullptr, "\n", &filecontext);
}
fclose(fp);
return TRUE;
}
KMX_BOOL NamedCodeConstants::LoadFile(const KMX_CHAR *filename)
{
const int buf_size = 260;
KMX_CHAR buf[buf_size];
// Look in current directory first -- REMOVED AS DANGEROUS
/* strncpy(buf, filename, (buf_size-1)); buf[buf_size-1] = 0; // I3481
if(kmcmp_FileExists(buf))
return IntLoadFile(buf);
*/
// Then look in keyboard file directory (CompileDir)
strncpy(buf, CompileDir, (buf_size-1)); buf[buf_size-1] = 0; // I3481
strncat(buf, filename, (buf_size-1)-strlen(CompileDir)); buf[buf_size-1] = 0;
if(kmcmp_FileExists(buf))
return IntLoadFile(buf);
//TODO: sort out how to find common includes in non-Windows platforms:
#ifdef _WINDOWS_
// Finally look in kmcmpdll.dll directory
GetModuleFileName(0, buf, buf_size);
KMX_CHAR *p = strrchr_slash(buf);
if(p)
p++;
else
p = buf;
*p = 0;
strncat_s(buf, _countof(buf), filename, (buf_size-1)-strlen(buf)); buf[buf_size-1] = 0; // I3481 // I3641
if(kmcmp_FileExists(buf))
return IntLoadFile(buf);
#endif
delete[] Buf;
reindex();
return FALSE;
return TRUE;
}
void NamedCodeConstants::reindex()

5
developer/src/kmcmplib/src/NamedCodeConstants.h
View file

@ -2,6 +2,8 @@
#ifndef NAMEDCODECONSTANTS_H
#define NAMEDCODECONSTANTS_H
#include "compfile.h"
#define MAX_ENAME 128
#define ALLOC_SIZE 256
@ -23,14 +25,13 @@ namespace kmcmp{
int GetCode_IncludedCodes(const KMX_WCHAR *codename);
void AddCode_IncludedCodes(int n, const KMX_WCHAR *p);
KMX_BOOL IntLoadFile(const KMX_CHAR *filename);
public:
NamedCodeConstants();
~NamedCodeConstants();
void reindex();
void AddCode(int n, const KMX_WCHAR *p, KMX_DWORD storeIndex);
KMX_BOOL LoadFile(const KMX_CHAR *filename);
KMX_BOOL LoadFile(PFILE_KEYBOARD fk, const KMX_WCHAR *filename);
int GetCode(const KMX_WCHAR *codename, KMX_DWORD *storeIndex);
};
}

3
developer/src/kmcmplib/src/compfile.h
View file

@ -123,7 +123,8 @@ typedef FILE_VKDICTIONARY *PFILE_VKDICTIONARY;
* Extra metadata for API consumers
*/
struct FILE_KEYBOARD_EXTRA {
std::u16string kvksFilename; // original TSS_VISUALKEYBOARD value
std::string kmnFilename; // utf-8
std::u16string kvksFilename; // utf-16, original TSS_VISUALKEYBOARD value
};
typedef struct FILE_KEYBOARD_EXTRA* PFILE_KEYBOARD_EXTRA;

1
developer/src/kmcmplib/src/kmcmplib.h
View file

@ -14,7 +14,6 @@ namespace kmcmp {
extern KMX_BOOL FMnemonicLayout;
extern KMX_BOOL FOldCharPosMatching;
extern int CompileTarget;
extern KMX_CHAR CompileDir[260]; // TODO: this should not be a fixed buffer
extern int BeginLine[4];
extern int currentLine;
extern NamedCodeConstants *CodeConstants;

9
developer/src/kmcmplib/src/meson.build
View file

@ -25,8 +25,10 @@ endif
name_suffix = []
if cpp_compiler.get_id() == 'emscripten'
lib_links = ['--whole-archive', '--bind', '-sMODULARIZE', '-sEXPORT_ES6']
links += ['-lnodefs.js', '--bind', '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\']']
# wasm-exceptions supported in Node 18+, Chrome 95+, Firefox 100+, Safari 15.2+
flags += ['-fwasm-exceptions']
lib_links = ['--whole-archive', '-sMODULARIZE', '-sEXPORT_ES6']
links += ['-fwasm-exceptions', '--bind', '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\']']
# tests are building as ES6 so we need to declare the file extension
# note that meson currently struggles with the sanitycheckc_cross.exe
# program, because it has a hard coded extension (.exe) which is not
@ -44,7 +46,6 @@ lib = library('kmcmplib',
'CompilerInterfaces.cpp',
'DeprecationChecks.cpp',
'Edition.cpp',
'filesystem.cpp',
'NamedCodeConstants.cpp',
'versioning.cpp',
'virtualcharkeys.cpp',
@ -79,7 +80,7 @@ if cpp_compiler.get_id() == 'emscripten'
host = executable('wasm-host', #'wasm-host.cpp',
cpp_args: defns,
include_directories: inc,
link_args: links,
link_args: links + lib_links,
objects: lib.extract_all_objects(),
dependencies: icuuc_dep)
endif

2
developer/src/kmcmplib/src/pch.h
View file

@ -10,6 +10,4 @@
#include "../../../../common/windows/cpp/include/crc32.h"
#include <ctype.h>
#include <stdio.h>
#include <string.h>

53
developer/src/kmcmplib/tests/api-test.cpp
View file

@ -18,54 +18,12 @@
#include <kmn_compiler_errors.h>
#include "../src/compfile.h"
#include <test_assert.h>
#include "../src/filesystem.h"
#include "util_filesystem.h"
#include "util_callbacks.h"
void setup();
void test_kmcmp_CompileKeyboard(char *kmn_file);
std::vector<int> error_vec;
int msgproc(int line, uint32_t dwMsgCode, const char* szText, void* context) {
error_vec.push_back(dwMsgCode);
const char*t = "unknown";
switch(dwMsgCode & 0xF000) {
case CERR_HINT: t=" hint"; break;
case CERR_WARNING: t="warning"; break;
case CERR_ERROR: t=" error"; break;
case CERR_FATAL: t=" fatal"; break;
}
printf("line %d %s %04.4x: %s\n", line, t, (unsigned int)dwMsgCode, szText);
return 1;
}
bool loadfileProc(const char* filename, const char* baseFilename, void* data, int* size, void* context) {
FILE* fp = Open_File(filename, "rb");
if(!fp) {
return false;
}
if(!data) {
// return size
if(fseek(fp, 0, SEEK_END) != 0) {
fclose(fp);
return false;
}
*size = ftell(fp);
if(*size == -1L) {
fclose(fp);
return false;
}
} else {
// return data
if(fread(data, 1, *size, fp) != *size) {
fclose(fp);
return false;
}
}
fclose(fp);
return true;
}
int main(int argc, char *argv[]) {
if(argc < 1) {
puts("Usage: api-test <full-path-to-blank_keyboard.kmn>");
@ -82,6 +40,13 @@ void setup() {
error_vec.clear();
}
/*
TODO: tests to run:
4. ANSI (no BOM of course)
8. file without blank last line (cannot compare with fixture due to bug in kmcmpdll...)
Hint to add: k004_ansi.kmn: Hint: 10A6 Keyman Developer has detected that the file has ANSI encoding. Consider converting this file to UTF-8
*/
void test_kmcmp_CompileKeyboard(char *kmn_file) {
// Create an empty file
FILE *fp = Open_File(kmn_file, "wb");

4
developer/src/kmcmplib/tests/fixtures/valid-keyboards/compile_legacy.bat vendored Normal file
View file

@ -0,0 +1,4 @@
@echo off
echo Compiles the keyboards using the legacy kmcomp.exe
echo to use as baseline comparisons for kmcmplib
for %%d in (*.kmn) do kmcomp -no-compiler-version -d %%d

BIN
developer/src/kmcmplib/tests/fixtures/valid-keyboards/k001_utf16.kmn vendored Normal file
View file

Binary file not shown.

BIN
developer/src/kmcmplib/tests/fixtures/valid-keyboards/k001_utf16.kmx vendored Normal file
View file

Binary file not shown.

11
developer/src/kmcmplib/tests/fixtures/valid-keyboards/k002_utf8_without_bom.kmn vendored Normal file
View file

@ -0,0 +1,11 @@
c Description: Verifies that kmcmplib can compile a UTF-8 file (without BOM)
store(&NAME) 'k002_utf8_without_bom'
store(&VERSION) '9.0'
begin unicode > use(main)
group(main) using keys
+ [K_A] > 'a'
'a' + [K_B] > 'ខ្មែរ'

BIN
developer/src/kmcmplib/tests/fixtures/valid-keyboards/k002_utf8_without_bom.kmx vendored Normal file
View file

Binary file not shown.

11
developer/src/kmcmplib/tests/fixtures/valid-keyboards/k003_utf8_with_bom.kmn vendored Normal file
View file

@ -0,0 +1,11 @@
c Description: Verifies that kmcmplib can compile a UTF-8 file (with BOM)
store(&NAME) 'k003_utf8_with_bom'
store(&VERSION) '9.0'
begin unicode > use(main)
group(main) using keys
+ [K_A] > 'a'
'a' + [K_B] > 'ខ្មែរ'

BIN
developer/src/kmcmplib/tests/fixtures/valid-keyboards/k003_utf8_with_bom.kmx vendored Normal file
View file

Binary file not shown.

13
developer/src/kmcmplib/tests/fixtures/valid-keyboards/k004_ansi.kmn vendored Normal file
View file

@ -0,0 +1,13 @@
c Description: Verifies that kmcmplib can compile an ANSI file
c This has some high-ascii letters in cp1252 to ensure that
c it fails to load as 'utf8 without bom'
store(&NAME) 'k004_ansi'
store(&VERSION) '9.0'
begin unicode > use(main)
group(main) using keys
+ [K_A] > 'a'
'a' + [K_B] > 'ÀÐ'

BIN
developer/src/kmcmplib/tests/fixtures/valid-keyboards/k004_ansi.kmx vendored Normal file
View file

Binary file not shown.

BIN
developer/src/kmcmplib/tests/fixtures/valid-keyboards/k005_bitmap.bmp vendored Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 246 B

12
developer/src/kmcmplib/tests/fixtures/valid-keyboards/k005_bitmap.kmn vendored Normal file
View file

@ -0,0 +1,12 @@
c Description: Verifies that kmcmplib can load a bitmap without file extension
store(&NAME) 'k005_bitmap'
store(&VERSION) '9.0'
store(&BITMAP) 'k005_bitmap'
begin unicode > use(main)
group(main) using keys
+ [K_A] > 'a'
'a' + [K_B] > 'ខ្មែរ'

BIN
developer/src/kmcmplib/tests/fixtures/valid-keyboards/k005_bitmap.kmx vendored Normal file
View file

Binary file not shown.

BIN
developer/src/kmcmplib/tests/fixtures/valid-keyboards/k006_icon.ico vendored Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 318 B

12
developer/src/kmcmplib/tests/fixtures/valid-keyboards/k006_icon.kmn vendored Normal file
View file

@ -0,0 +1,12 @@
c Description: Verifies that kmcmplib can load an icon with file extension
store(&NAME) 'k006_icon'
store(&VERSION) '9.0'
store(&BITMAP) 'k006_icon.ico'
begin unicode > use(main)
group(main) using keys
+ [K_A] > 'a'
'a' + [K_B] > 'ខ្មែរ'

BIN
developer/src/kmcmplib/tests/fixtures/valid-keyboards/k006_icon.kmx vendored Normal file
View file

Binary file not shown.

12
developer/src/kmcmplib/tests/fixtures/valid-keyboards/k007_includecodes_r_n.kmn vendored Normal file
View file

@ -0,0 +1,12 @@
c Description: Verifies that kmcmplib can load an includecodes file with \r\n line endings
store(&NAME) 'k007_includecodes_r_n'
store(&VERSION) '9.0'
store(&includecodes) 'k007_includecodes_r_n.txt'
begin unicode > use(main)
group(main) using keys
+ [K_A] > $LOWER_A
$LOWER_A + [K_B] > $LOWER_B

BIN
developer/src/kmcmplib/tests/fixtures/valid-keyboards/k007_includecodes_r_n.kmx vendored Normal file
View file

Binary file not shown.

2
developer/src/kmcmplib/tests/fixtures/valid-keyboards/k007_includecodes_r_n.txt vendored Normal file
View file

@ -0,0 +1,2 @@
0061;LOWER_A
0062;LOWER_B

12
developer/src/kmcmplib/tests/fixtures/valid-keyboards/k008_includecodes_n.kmn vendored Normal file
View file

@ -0,0 +1,12 @@
c Description: Verifies that kmcmplib can load an includecodes file with \n line endings
store(&NAME) 'k008_includecodes_n'
store(&VERSION) '9.0'
store(&includecodes) 'k008_includecodes_n.txt'
begin unicode > use(main)
group(main) using keys
+ [K_A] > $LOWER_A
$LOWER_A + [K_B] > $LOWER_B

BIN
developer/src/kmcmplib/tests/fixtures/valid-keyboards/k008_includecodes_n.kmx vendored Normal file
View file

Binary file not shown.

2
developer/src/kmcmplib/tests/fixtures/valid-keyboards/k008_includecodes_n.txt vendored Normal file
View file

@ -0,0 +1,2 @@
0061;LOWER_A
0062;LOWER_B

49
developer/src/kmcmplib/tests/kmcompxtest.cpp
View file

@ -13,7 +13,8 @@
#include <sstream>
#include <kmcmplibapi.h>
#include <kmx_file.h>
#include "../src/filesystem.h"
#include "util_filesystem.h"
#include "util_callbacks.h"
#ifdef _MSC_VER
#else
@ -22,57 +23,11 @@
using namespace std;
vector < int > error_vec;
#define CERR_FATAL 0x00008000
#define CERR_ERROR 0x00004000
#define CERR_WARNING 0x00002000
#define CERR_HINT 0x00001000
int msgproc(int line, uint32_t dwMsgCode, const char* szText, void* context)
{
error_vec.push_back(dwMsgCode);
const char*t = "unknown";
switch(dwMsgCode & 0xF000) {
case CERR_HINT: t=" hint"; break;
case CERR_WARNING: t="warning"; break;
case CERR_ERROR: t=" error"; break;
case CERR_FATAL: t=" fatal"; break;
}
printf("line %d %s %04.4x: %s\n", line, t, (unsigned int)dwMsgCode, szText);
return 1;
}
bool loadfileProc(const char* filename, const char* baseFilename, void* data, int* size, void* context) {
FILE* fp = Open_File(filename, "rb");
if(!fp) {
return false;
}
if(!data) {
// return size
if(fseek(fp, 0, SEEK_END) != 0) {
fclose(fp);
return false;
}
*size = ftell(fp);
if(*size == -1L) {
fclose(fp);
return false;
}
} else {
// return data
if(fread(data, 1, *size, fp) != *size) {
fclose(fp);
return false;
}
}
fclose(fp);
return true;
}
#include "../src/filesystem.h"
int main(int argc, char *argv[])
{
if(argc < 4) {

41
developer/src/kmcmplib/tests/meson.build
View file

@ -6,16 +6,20 @@
fs = import('fs')
tests_flags = []
tests_links = []
if cpp_compiler.get_id() == 'emscripten'
tests_links += ['-lnodefs.js']
endif
input_path = meson.current_source_dir() / '../../../../common/test/keyboards/baseline'
output_path = meson.current_build_dir()
kmcompxtest = executable('kmcompxtest', 'kmcompxtest.cpp',
cpp_args: defns,
kmcompxtest = executable('kmcompxtest', ['kmcompxtest.cpp','util_filesystem.cpp','util_callbacks.cpp'],
cpp_args: defns + flags,
include_directories: inc,
name_suffix: name_suffix,
link_args: links + tests_flags,
link_args: links + tests_links,
objects: lib.extract_all_objects(),
dependencies: icuuc_dep,
)
@ -76,6 +80,25 @@ foreach kbd : tests
test(kbd, kmcompxtest, args: [kbd_src, kbd_obj, join_paths(input_path, kbd) + '.kmx'])
endforeach
valid_keyboard_tests = [
'k001_utf16',
'k002_utf8_without_bom',
'k003_utf8_with_bom',
# 'k004_ansi', # TODO: enable ansi test when we have the icu datafiles
'k005_bitmap',
'k006_icon',
'k007_includecodes_r_n',
'k008_includecodes_n'
]
fixtures_path = meson.current_source_dir() / 'fixtures/valid-keyboards'
foreach kbd : valid_keyboard_tests
kbd_src = join_paths(fixtures_path, kbd) + '.kmn'
kbd_obj = join_paths(output_path, kbd) + '.kmx'
test(kbd, kmcompxtest, args: [kbd_src, kbd_obj, join_paths(fixtures_path, kbd) + '.kmx'])
endforeach
# Test fixtures that come from keyboards repo -- but only for a "full" test,
# which typically we run on CI no more than once a day, because it's expensive.
@ -107,11 +130,11 @@ endif
# Test the API endpoints
apitest = executable('api-test', 'api-test.cpp',
cpp_args: defns,
apitest = executable('api-test', ['api-test.cpp','util_filesystem.cpp','util_callbacks.cpp'],
cpp_args: defns + flags,
include_directories: inc,
name_suffix: name_suffix,
link_args: links + tests_flags,
link_args: links + tests_links,
objects: lib.extract_all_objects(),
dependencies: icuuc_dep
)
@ -119,10 +142,10 @@ apitest = executable('api-test', 'api-test.cpp',
test('api-test', apitest, args: [output_path / 'blank_keyboard.kmx'])
usetapitest = executable('uset-api-test', 'uset-api-test.cpp',
cpp_args: defns,
cpp_args: defns + flags,
include_directories: inc,
name_suffix: name_suffix,
link_args: links + tests_flags,
link_args: links + tests_links,
objects: lib.extract_all_objects(),
dependencies: icuuc_dep,
)

59
developer/src/kmcmplib/tests/util_callbacks.cpp Normal file
View file

@ -0,0 +1,59 @@
#include <stdio.h>
#include <string>
#include <kmcmplibapi.h>
#include "util_filesystem.h"
#include "../src/compfile.h"
#include <kmn_compiler_errors.h>
std::vector<int> error_vec;
int msgproc(int line, uint32_t dwMsgCode, const char* szText, void* context) {
error_vec.push_back(dwMsgCode);
const char*t = "unknown";
switch(dwMsgCode & 0xF000) {
case CERR_HINT: t=" hint"; break;
case CERR_WARNING: t="warning"; break;
case CERR_ERROR: t=" error"; break;
case CERR_FATAL: t=" fatal"; break;
}
printf("line %d %s %04.4x: %s\n", line, t, (unsigned int)dwMsgCode, szText);
return 1;
}
bool loadfileProc(const char* filename, const char* baseFilename, void* data, int* size, void* context) {
std::string resolvedFilename = filename;
if(baseFilename && *baseFilename && IsRelativePath(filename)) {
char* p;
if ((p = strrchr_slash((char*)baseFilename)) != nullptr) {
std::string basePath = std::string(baseFilename, (int)(p - baseFilename + 1));
resolvedFilename = basePath;
resolvedFilename.append(filename);
}
}
FILE* fp = Open_File(resolvedFilename.c_str(), "rb");
if(!fp) {
return false;
}
if(!data) {
// return size
if(fseek(fp, 0, SEEK_END) != 0) {
fclose(fp);
return false;
}
*size = ftell(fp);
if(*size == -1L) {
fclose(fp);
return false;
}
} else {
// return data
if(fread(data, 1, *size, fp) != *size) {
fclose(fp);
return false;
}
}
fclose(fp);
return true;
}

8
developer/src/kmcmplib/tests/util_callbacks.h Normal file
View file

@ -0,0 +1,8 @@
#pragma once
#include <vector>
int msgproc(int line, uint32_t dwMsgCode, const char* szText, void* context);
bool loadfileProc(const char* filename, const char* baseFilename, void* data, int* size, void* context);
extern std::vector<int> error_vec;

41
developer/src/kmcmplib/src/filesystem.cpp → developer/src/kmcmplib/tests/util_filesystem.cpp
View file

@ -7,7 +7,7 @@
#include <cassert>
#include <algorithm>
#include <iostream>
#include "filesystem.h"
#include "util_filesystem.h"
#ifdef _MSC_VER
#include <io.h>
@ -186,3 +186,42 @@ KMX_BOOL kmcmp_FileExists(const KMX_WCHAR* filename) {
return FALSE;
};
bool IsRelativePath(KMX_CHAR const * p) {
// Relative path (returns TRUE):
// ..\...\BITMAP.BMP
// PATH\BITMAP.BMP
// BITMAP.BMP
// Semi-absolute path (returns FALSE):
// \...\BITMAP.BMP
// Absolute path (returns FALSE):
// C:\...\BITMAP.BMP
// \\SERVER\SHARE\...\BITMAP.BMP
if ((*p == '\\') || (*p == '/')) return FALSE;
if (*p && *(p + 1) == ':') return FALSE;
return TRUE;
}
bool IsRelativePath(KMX_WCHAR const * p) {
// Relative path (returns TRUE):
// ..\...\BITMAP.BMP
// PATH\BITMAP.BMP
// BITMAP.BMP
// Semi-absolute path (returns FALSE):
// \...\BITMAP.BMP
// Absolute path (returns FALSE):
// C:\...\BITMAP.BMP
// \\SERVER\SHARE\...\BITMAP.BMP
if ((*p == u'\\') || (*p == u'/'))return FALSE;
if (*p && *(p + 1) == u':') return FALSE;
return TRUE;
}

5
developer/src/kmcmplib/src/filesystem.h → developer/src/kmcmplib/tests/util_filesystem.h
View file

@ -1,7 +1,7 @@
#pragma once
#include <stdio.h>
#include "kmx_u16.h"
#include "../src/kmx_u16.h"
// Opens files on windows and non-windows platforms. Datatypes for Filename and mode must be the same.
// returns FILE* if file could be opened; FILE needs to be closed in calling function
@ -10,3 +10,6 @@ FILE* Open_File(const KMX_WCHART* Filename, const KMX_WCHART* mode);
FILE* Open_File(const KMX_WCHAR* Filename, const KMX_WCHAR* mode);
KMX_BOOL kmcmp_FileExists(const KMX_CHAR *filename);
KMX_BOOL kmcmp_FileExists(const KMX_WCHAR *filename);
bool IsRelativePath(KMX_CHAR const * p);
bool IsRelativePath(KMX_WCHAR const * p);