From 980b893644536d322f4a5196b35425de0581d5f5 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 30 May 2023 10:52:05 +0700 Subject: [PATCH 1/6] refactor(developer): move fs for kmn load to caller --- developer/src/kmcmplib/include/kmcmplibapi.h | 4 +- developer/src/kmcmplib/src/Compiler.cpp | 132 +++++++----------- .../src/kmcmplib/src/CompilerInterfaces.cpp | 89 ++++++++---- developer/src/kmcmplib/src/compfile.h | 29 ---- developer/src/kmcmplib/src/kmcmplib.h | 5 +- developer/src/kmcmplib/src/meson.build | 8 +- developer/src/kmcmplib/tests/api-test.cpp | 50 +++++-- developer/src/kmcmplib/tests/kmcompxtest.cpp | 33 ++++- developer/src/kmcmplib/tests/meson.build | 4 +- 9 files changed, 194 insertions(+), 160 deletions(-) diff --git a/developer/src/kmcmplib/include/kmcmplibapi.h b/developer/src/kmcmplib/include/kmcmplibapi.h index 4054d81e81..5c2ae46fda 100644 --- a/developer/src/kmcmplib/include/kmcmplibapi.h +++ b/developer/src/kmcmplib/include/kmcmplibapi.h @@ -36,7 +36,7 @@ struct KMCMP_COMPILER_RESULT { }; // TODO: parameters in UTF-8 -typedef int (*kmcmp_CompilerMessageProc)(int line, uint32_t dwMsgCode, char* szText, void* context); +typedef int (*kmcmp_CompilerMessageProc)(int line, uint32_t dwMsgCode, const char* szText, void* context); // parameters in UTF-8 // TODO typical usage: @@ -48,7 +48,7 @@ typedef int (*kmcmp_CompilerMessageProc)(int line, uint32_t dwMsgCode, char* szT // delete[] buf; // return error; // } -typedef bool (*kmcmp_LoadFileProc)(char* loadFilename, char* baseFilename, void* buffer, int* bufferSize); +typedef bool (*kmcmp_LoadFileProc)(const char* loadFilename, const char* baseFilename, void* buffer, int* bufferSize, void* context); // Parameters in UTF-8 EXTERN bool kmcmp_CompileKeyboard( diff --git a/developer/src/kmcmplib/src/Compiler.cpp b/developer/src/kmcmplib/src/Compiler.cpp index b09e36cbc1..8e0f94cb29 100644 --- a/developer/src/kmcmplib/src/Compiler.cpp +++ b/developer/src/kmcmplib/src/Compiler.cpp @@ -94,6 +94,9 @@ #include "CasedKeys.h" #include #include +#include +#include + #include "CheckFilenameConsistency.h" #include "UnreachableRules.h" #include "CheckForDuplicates.h" @@ -230,6 +233,8 @@ enum LinePrefixType { lptNone, lptKeymanAndKeymanWeb, lptKeymanWebOnly, lptKeyma /* Compile target */ kmcmp_CompilerMessageProc msgproc = NULL; +kmcmp_LoadFileProc loadfileproc = NULL; + void* msgprocContext = NULL; int kmcmp::currentLine = 0; @@ -3068,7 +3073,7 @@ KMX_DWORD WriteCompiledKeyboard(PFILE_KEYBOARD fk, KMX_BYTE**data, size_t& dataS return CERR_None; } -KMX_DWORD ReadLine(FILE* fp_in , PKMX_WCHAR wstr, KMX_BOOL PreProcess) +KMX_DWORD ReadLine(KMX_BYTE* infile, int sz, int& offset, PKMX_WCHAR wstr, KMX_BOOL PreProcess) { KMX_DWORD len; PKMX_WCHAR p; @@ -3076,21 +3081,26 @@ KMX_DWORD ReadLine(FILE* fp_in , PKMX_WCHAR wstr, KMX_BOOL PreProcess) KMX_DWORD n; KMX_WCHAR currentQuotes = 0; KMX_WCHAR str[LINESIZE + 3]; - len = (KMX_DWORD)fread( str , 1 ,LINESIZE * 2,fp_in); - if (ferror(fp_in) ) return CERR_CannotReadInfile; + + if(offset >= sz) { + return CERR_EndOfFile; + } + + len = offset + LINESIZE*2 > sz ? sz-offset : LINESIZE*2; + memcpy(str, infile+offset, len); + offset += len; len /= 2; - str[len] = 0; auto cur = ftell(fp_in); - fseek(fp_in, 0, SEEK_END); - auto fsize = ftell(fp_in); - fseek(fp_in, cur, SEEK_SET); + str[len] = 0; - if (cur == fsize) + if(offset == sz) { + // \r\n is still added here even though Linux doesn`t use \r. + // This is to ensure to still have a working windows-only-version + u16ncat(str, u"\r\n", _countof(str)); // I3481 // Always a "\r\n" to the EOF, avoids funny bugs + } - // \r\n is still added here even though Linux doesn`t use \r. - // This is to ensure to still have a working windows-only-version - u16ncat(str, u"\r\n", _countof(str)); // I3481 // Always a "\r\n" to the EOF, avoids funny bugs - - if (len == 0) return CERR_EndOfFile; + if (len == 0) { + return CERR_EndOfFile; + } // neccessary to add this block for using on non-windows platforms (removes all \r for platforms that use \n instead of \r\n) for (p = str, n = 0; n < len; n++, p++) { @@ -3172,9 +3182,13 @@ KMX_DWORD ReadLine(FILE* fp_in , PKMX_WCHAR wstr, KMX_BOOL PreProcess) return (PreProcess ? CERR_None : CERR_LineTooLong); } - if (*p == L'\n') kmcmp::currentLine++; + kmcmp::currentLine++; - fseek(fp_in, -(int)(len * 2 - (int)(p - str) * 2 - 2), SEEK_CUR); + offset -= (int)(len * 2 - (int)(p - str) * 2 - 2); + if(offset >= sz) { + // If we've appended a \n, we can go past EOF + offset = sz; + } p--; while (p >= str && iswspace(*p)) p--; @@ -3427,81 +3441,39 @@ KMX_BOOL kmcmp::IsValidCallStore(PFILE_STORE fs) return i == 1; } -FILE* CreateTempFile() -{ - return tmpfile(); -} - /////////////////// -FILE* UTF16TempFromUTF8(FILE* fp_in , KMX_BOOL hasPreamble) -{ - FILE *fp_out = CreateTempFile(); - if(fp_out == NULL) // I3228 // I3510 - { - fclose(fp_in); - return NULL; //return INVALID_HANDLE_VALUE; _S2 can I exchange that? +bool hasPreamble(std::u16string result) { + return result.size() > 0 && result[0] == 0xFEFF; +} + +bool UTF16TempFromUTF8(KMX_BYTE* infile, int sz, KMX_BYTE** tempfile, int *sz16) { + if(sz == 0) { + return FALSE; } - PKMX_BYTE buf, p; - PKMX_WCHAR outbuf, poutbuf; - KMX_DWORD len; - KMX_DWORD len2; - KMX_WCHAR prolog = 0xFEFF; - fwrite(&prolog,2, 1, fp_out); + std::u16string result; - fseek(fp_in, 0, SEEK_END); - len = (KMX_DWORD)ftell(fp_in); - fseek(fp_in, 0, SEEK_SET); - if (hasPreamble) { - fseek( fp_in,3,SEEK_SET); // Cut off UTF-8 marker - len -= 3; + try { + std::wstring_convert, char16_t> converter; + result = converter.from_bytes((char*)infile, (char*)infile+sz-1); + } catch(std::range_error e) { + std::wstring_convert, char16_t> converter; + result = converter.from_bytes((char*)infile, (char*)infile+sz-1); } - buf = new KMX_BYTE[len + 1]; // null terminated - outbuf = new KMX_WCHAR[len + 1]; + if(hasPreamble(result)) { + *sz16 = result.size() * 2 - 1; + *tempfile = new KMX_BYTE[*sz16]; + memcpy(*tempfile, result.c_str() + 2, *sz16); - len2= (KMX_DWORD)fread(buf,1,len,fp_in); - if (len2) { - buf[len2] = 0; - p = buf; - poutbuf = outbuf; - if (hasPreamble) { - // We have a preamble, so we attempt to read as UTF-8 and allow conversion errors to be filtered. This is not great for a - // compiler but matches existing behaviour -- in future versions we may not do lenient conversion. - ConvertUTF8toUTF16(&p, &buf[len2], (UTF16 **)&poutbuf, (const UTF16 *)&outbuf[len], lenientConversion); - fwrite(outbuf, (KMX_DWORD)(poutbuf - outbuf) * 2 , 1, fp_out); - } - else { - // No preamble, so we attempt to read as strict UTF-8 and fall back to ANSI if that fails - ConversionResult cr = ConvertUTF8toUTF16(&p, &buf[len2], (UTF16 **)&poutbuf, (const UTF16 *)&outbuf[len], strictConversion); - if (cr == sourceIllegal) { - // Not a valid UTF-8 file, so fall back to ANSI - // AddCompileError(CHINT_NonUnicodeFile); - // note, while this message is defined, for now we will not emit it - // because we don't support HINT/INFO messages yet and we don't want - // this to cause a blocking compile at this stage - // do strtowstr only when no invalid characters are found - if( p==0){ - poutbuf = strtowstr((PKMX_STR)buf); - fwrite(poutbuf, (KMX_DWORD)u16len(poutbuf) * 2 , 1, fp_out); - delete[] poutbuf; - } - else - AddCompileError(CERR_InvalidCharacter); - } - - else { - fwrite(outbuf, (KMX_DWORD)(poutbuf - outbuf) * 2 , 1, fp_out); - } - } } - fclose( fp_in); - delete[] buf; - delete[] outbuf; - fseek( fp_out,2,SEEK_SET); - return fp_out; + *sz16 = result.size() * 2; + *tempfile = new KMX_BYTE[*sz16]; + memcpy(*tempfile, result.c_str(), *sz16); + + return TRUE; } PFILE_STORE FindSystemStore(PFILE_KEYBOARD fk, KMX_DWORD dwSystemID) diff --git a/developer/src/kmcmplib/src/CompilerInterfaces.cpp b/developer/src/kmcmplib/src/CompilerInterfaces.cpp index 71e1e895f2..fd297f4332 100644 --- a/developer/src/kmcmplib/src/CompilerInterfaces.cpp +++ b/developer/src/kmcmplib/src/CompilerInterfaces.cpp @@ -11,14 +11,14 @@ #include "../../../../common/windows/cpp/include/ConvertUTF.h" #include "../../../../common/windows/cpp/include/keymanversion.h" -bool CompileKeyboardHandle(FILE* fp_in, PFILE_KEYBOARD fk); +bool CompileKeyboardHandle(KMX_BYTE* infile, int sz, PFILE_KEYBOARD fk); #ifdef __EMSCRIPTEN__ /* WASM interface for compiler message callback */ -EM_JS(int, wasm_msgproc, (int line, int msgcode, char* text, char* context), { +EM_JS(int, wasm_msgproc, (int line, int msgcode, const char* text, char* context), { const proc = globalThis[UTF8ToString(context)]; if(!proc || typeof proc != 'function') { console.log(`[${line}: ${msgcode}: ${UTF8ToString(text)}]`); @@ -28,7 +28,21 @@ EM_JS(int, wasm_msgproc, (int line, int msgcode, char* text, char* context), { } }); -int wasm_CompilerMessageProc(int line, uint32_t dwMsgCode, char* szText, void* context) { +EM_JS(bool, wasm_loadfileproc, (const char* filename, const char* baseFilename, void* buffer, int* bufferSize, char* context), { + const proc = globalThis[UTF8ToString(context)]; + if(!proc || typeof proc != 'function') { + return 0; + } else { + return proc(UTF8ToString(filename), UTF8ToString(baseFilename), buffer, bufferSize); + } +}); + +bool wasm_LoadFileProc(const char* filename, const char* baseFilename, void* buffer, int* bufferSize, void* context) { + char* msgProc = static_cast(context); + return wasm_loadfileproc(filename, baseFilename, buffer, bufferSize, msgProc); +} + +int wasm_CompilerMessageProc(int line, uint32_t dwMsgCode, const char* szText, void* context) { char* msgProc = static_cast(context); return wasm_msgproc(line, dwMsgCode, szText, msgProc); } @@ -61,7 +75,7 @@ WASM_COMPILER_RESULT kmcmp_wasm_compile(std::string pszInfile, const KMCMP_COMPI pszInfile.c_str(), options, wasm_CompilerMessageProc, - nullptr, //wasm_LoadFileProc, + wasm_LoadFileProc, intf.messageCallback.c_str(), kr ); @@ -116,8 +130,6 @@ EXTERN bool kmcmp_CompileKeyboard( KMCMP_COMPILER_RESULT& result ) { - FILE* fp_in = NULL; - KMX_CHAR str[260]; FILE_KEYBOARD fk; kmcmp::FSaveDebug = options.saveDebug; // I3681 @@ -126,7 +138,7 @@ EXTERN bool kmcmp_CompileKeyboard( kmcmp::FShouldAddCompilerVersion = options.shouldAddCompilerVersion; kmcmp::CompileTarget = options.target; - if (!messageProc || !pszInfile) { // TODO: add loadFileProc + if (!messageProc || !loadFileProc || !pszInfile) { AddCompileError(CERR_BadCallParams); return FALSE; } @@ -142,43 +154,58 @@ EXTERN bool kmcmp_CompileKeyboard( } msgproc = messageProc; - //TODO: loadfileproc = loadFileProc; + loadfileproc = loadFileProc; msgprocContext = (void*)procContext; kmcmp::currentLine = 0; kmcmp::nErrors = 0; - fp_in = Open_File(pszInfile, "rb"); - - if (fp_in == NULL) { + int sz; + if(!loadFileProc(pszInfile, "", nullptr, &sz, msgprocContext)) { AddCompileError(CERR_InfileNotExist); return FALSE; } - // Transfer the file to a memory stream for processing UTF-8 or ANSI to UTF-16? - // What about really large files? Transfer to a temp file... - if (!fread(str, 1, 3, fp_in)) { - fclose(fp_in); + if(sz < 3) { + // Technically, a 3 byte file can never be a valid .kmn, so we can shortcut + // here and avoid testing outside memory bounds for looking at BOM AddCompileError(CERR_CannotReadInfile); return FALSE; } - fseek(fp_in, 0, SEEK_SET); - if (str[0] == UTF8Sig[0] && str[1] == UTF8Sig[1] && str[2] == UTF8Sig[2]) - fp_in = UTF16TempFromUTF8(fp_in, TRUE); - else if (str[0] == UTF16Sig[0] && str[1] == UTF16Sig[1]) - fseek(fp_in, 2, SEEK_SET); - else - fp_in = UTF16TempFromUTF8(fp_in, FALSE); - if (fp_in == NULL) { - AddCompileError(CERR_CannotCreateTempfile); + KMX_BYTE* infile = new KMX_BYTE[sz]; + if(!infile) { + AddCompileError(CERR_CannotAllocateMemory); + return FALSE; + } + if(!loadFileProc(pszInfile, "", infile, &sz, msgprocContext)) { + delete[] infile; + AddCompileError(CERR_CannotReadInfile); return FALSE; } + int offset = 0; + if(infile[0] == (KMX_BYTE) UTF16Sig[0] && infile[1] == (KMX_BYTE) UTF16Sig[1]) { + // UTF-16 source file + offset = 2; + } else { + // UTF-8 source file + KMX_BYTE* infile16; + int sz16; + if(!UTF16TempFromUTF8(infile, sz, &infile16, &sz16)) { + delete[] infile; + AddCompileError(CERR_CannotCreateTempfile); + return FALSE; + } + delete[] infile; + infile = infile16; + sz = sz16; + } + kmcmp::CodeConstants = new kmcmp::NamedCodeConstants; - bool success = CompileKeyboardHandle(fp_in, &fk); + bool success = CompileKeyboardHandle(infile+offset, sz-offset, &fk); delete kmcmp::CodeConstants; - fclose(fp_in); + delete[] infile; if (kmcmp::nErrors > 0 || !success) { return FALSE; @@ -204,7 +231,7 @@ EXTERN bool kmcmp_CompileKeyboard( return TRUE; } -bool CompileKeyboardHandle(FILE* fp_in, PFILE_KEYBOARD fk) +bool CompileKeyboardHandle(KMX_BYTE* infile, int sz, PFILE_KEYBOARD fk) { PKMX_WCHAR str, p; @@ -263,8 +290,10 @@ bool CompileKeyboardHandle(FILE* fp_in, PFILE_KEYBOARD fk) AddStore(fk, TSS_CUSTOMKEYMANEDITION, u"0"); AddStore(fk, TSS_CUSTOMKEYMANEDITIONNAME, u"Keyman"); + int offset = 0; + // must preprocess for group and store names -> this isn't really necessary, but never mind! - while ((msg = ReadLine(fp_in, str, TRUE)) == CERR_None) + while ((msg = ReadLine(infile, sz, offset, str, TRUE)) == CERR_None) { p = str; switch (LineTokenType(&p)) @@ -301,7 +330,7 @@ bool CompileKeyboardHandle(FILE* fp_in, PFILE_KEYBOARD fk) return FALSE; } - fseek( fp_in,2,SEEK_SET); + offset = 0; kmcmp::currentLine = 0; /* Reindex the list of codeconstants after stores added */ @@ -309,7 +338,7 @@ bool CompileKeyboardHandle(FILE* fp_in, PFILE_KEYBOARD fk) kmcmp::CodeConstants->reindex(); /* ReadLine will automatically skip over $Keyman lines, and parse wrapped lines */ - while ((msg = ReadLine(fp_in, str, FALSE)) == CERR_None) + while ((msg = ReadLine(infile, sz, offset, str, FALSE)) == CERR_None) { msg = ParseLine(fk, str); if (msg != CERR_None) { diff --git a/developer/src/kmcmplib/src/compfile.h b/developer/src/kmcmplib/src/compfile.h index 595d1fecec..7c262be090 100644 --- a/developer/src/kmcmplib/src/compfile.h +++ b/developer/src/kmcmplib/src/compfile.h @@ -197,33 +197,4 @@ struct COMPILEMESSAGES { typedef COMPILEMESSAGES *PCOMPILEMESSAGES; -/* -struct TVersion -{ - //int MinVersion; // 0x0500 usually - //int CompilerVersion[4]; - //int MinCompilerVersion[4]; - int KeyboardVersion; // 0x0500 usually -}; - -extern TVersion FVersionInfo; -*/ - -/* -#define bstrcpy(c,d) (LPBYTE)strcpy((LPSTR)(c),(LPSTR)(d)) -#define bstrlen(c) strlen((LPSTR)(c)) -#define bstrcmp(c,d) strcmp((LPSTR)(c),(LPSTR)(d)) -#define bstrncmp(c,d,n) strncmp((LPSTR)(c),(LPSTR)(d),(n)) -#define bstrnicmp(c,d,n) strnicmp((LPSTR)(c),(LPSTR)(d),(n)) -#define bstricmp(c,d) stricmp((LPSTR)(c),(LPSTR)(d)) -#define bstrchr(c,ch) (LPBYTE)strchr((LPSTR)(c),(char)ch) -#define bstrncpy(c,d,n) (LPBYTE)strncpy((LPSTR)(c),(LPSTR)(d),(n)) -#define bstrtok(c,d) (LPBYTE)strtok((LPSTR)(c),(LPSTR)(d)) -#define bstrcat(c,d) (LPBYTE)strcat((LPSTR)(c),(LPSTR)(d)) -#define bstrncat(c,d,n) (LPBYTE)strncat((LPSTR)(c),(LPSTR)(d),(n)) -#define bstrrev(c) (LPBYTE)strrev((LPSTR)(c)) -#define batoi(c) atoi((LPSTR)(c)) -#define bstrtol(c,d,n) strtol((LPSTR)(c),(LPSTR *)(d),(n)) -*/ - #endif // _COMPFILE_H diff --git a/developer/src/kmcmplib/src/kmcmplib.h b/developer/src/kmcmplib/src/kmcmplib.h index 78e945ee0d..70718f98bc 100644 --- a/developer/src/kmcmplib/src/kmcmplib.h +++ b/developer/src/kmcmplib/src/kmcmplib.h @@ -24,6 +24,7 @@ namespace kmcmp { } extern kmcmp_CompilerMessageProc msgproc; +extern kmcmp_LoadFileProc loadfileproc; extern void* msgprocContext; extern KMX_BOOL AWarnDeprecatedCode_GLOBAL_LIB; @@ -40,10 +41,10 @@ KMX_BOOL AddCompileError(KMX_DWORD msg); PKMX_WCHAR strtowstr(PKMX_STR in); PFILE_STORE FindSystemStore(PFILE_KEYBOARD fk, KMX_DWORD dwSystemID); -FILE* UTF16TempFromUTF8(FILE* fp_in , KMX_BOOL hasPreamble); +bool UTF16TempFromUTF8(KMX_BYTE* infile, int sz, KMX_BYTE** tempfile, int *sz16); KMX_DWORD WriteCompiledKeyboard(PFILE_KEYBOARD fk, KMX_BYTE**data, size_t& dataSize); KMX_DWORD AddStore(PFILE_KEYBOARD fk, KMX_DWORD SystemID, const KMX_WCHAR * str, KMX_DWORD *dwStoreID= NULL); -KMX_DWORD ReadLine(FILE* fp_in , PKMX_WCHAR wstr, KMX_BOOL PreProcess); +KMX_DWORD ReadLine(KMX_BYTE* infile, int sz, int& offset, PKMX_WCHAR wstr, KMX_BOOL PreProcess); KMX_DWORD ParseLine(PFILE_KEYBOARD fk, PKMX_WCHAR str); KMX_DWORD ProcessGroupLine(PFILE_KEYBOARD fk, PKMX_WCHAR p); KMX_DWORD ProcessGroupFinish(PFILE_KEYBOARD fk); diff --git a/developer/src/kmcmplib/src/meson.build b/developer/src/kmcmplib/src/meson.build index 8264c3d28e..727d6831e6 100644 --- a/developer/src/kmcmplib/src/meson.build +++ b/developer/src/kmcmplib/src/meson.build @@ -7,6 +7,7 @@ # TODO: is this required? It should be Keyman Core only defns += ['-DKMN_KBP_EXPORTING'] version_res = [] +lib_links = [] if cpp_compiler.get_id() == 'gcc' or cpp_compiler.get_id() == 'clang' warns += [ @@ -24,12 +25,13 @@ endif name_suffix = [] if cpp_compiler.get_id() == 'emscripten' - links += ['-lnodefs.js', '-sMODULARIZE', '-sEXPORT_ES6', '--whole-archive', '--bind', '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\']'] + lib_links = ['--whole-archive', '--bind', '-sMODULARIZE', '-sEXPORT_ES6'] + links += ['-lnodefs.js', '--bind', '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\']'] # tests are building as ES6 so we need to declare the file extension # note that meson currently struggles with the sanitycheckc_cross.exe # program, because it has a hard coded extension (.exe) which is not # valid for node programs in module mode. - name_suffix = '.mjs' + # name_suffix = '.mjs' endif icu = subproject('icu-for-uset', default_options: [ 'default_library=static', 'cpp_std=c++17', 'warning_level=0', 'werror=false']) @@ -63,7 +65,7 @@ lib = library('kmcmplib', version_res, cpp_args: defns + warns + flags, - link_args: links, + link_args: links + lib_links, version: meson.project_version(), include_directories: inc, install: true, diff --git a/developer/src/kmcmplib/tests/api-test.cpp b/developer/src/kmcmplib/tests/api-test.cpp index 4bb0276a79..b347773220 100644 --- a/developer/src/kmcmplib/tests/api-test.cpp +++ b/developer/src/kmcmplib/tests/api-test.cpp @@ -18,13 +18,14 @@ #include #include "../src/compfile.h" #include +#include "../src/filesystem.h" void setup(); -void test_kmcmp_CompileKeyboard(); +void test_kmcmp_CompileKeyboard(char *kmn_file); std::vector error_vec; -int msgproc(int line, uint32_t dwMsgCode, char* szText, void* context) { +int msgproc(int line, uint32_t dwMsgCode, const char* szText, void* context) { error_vec.push_back(dwMsgCode); const char*t = "unknown"; switch(dwMsgCode & 0xF000) { @@ -37,9 +38,42 @@ int msgproc(int line, uint32_t dwMsgCode, char* szText, void* context) { return 1; } +bool loadfileProc(const char* filename, const char* baseFilename, void* data, int* size, void* context) { + FILE* fp = Open_File(filename, "rb"); + if(!fp) { + return false; + } + + if(!data) { + // return size + if(fseek(fp, 0, SEEK_END) != 0) { + fclose(fp); + return false; + } + *size = ftell(fp); + if(*size == -1L) { + fclose(fp); + return false; + } + } else { + // return data + if(fread(data, 1, *size, fp) != *size) { + fclose(fp); + return false; + } + } + fclose(fp); + return true; +} + int main(int argc, char *argv[]) { + if(argc < 1) { + puts("Usage: api-test "); + puts("Warning: blank_keyboard will be overwritten"); + return 1; + } setup(); - test_kmcmp_CompileKeyboard(); + test_kmcmp_CompileKeyboard(argv[1]); return 0; } @@ -48,13 +82,9 @@ void setup() { error_vec.clear(); } -void test_kmcmp_CompileKeyboard() { - char kmn_file[L_tmpnam], kmx_file[L_tmpnam]; - tmpnam(kmn_file); - tmpnam(kmx_file); - +void test_kmcmp_CompileKeyboard(char *kmn_file) { // Create an empty file - FILE *fp = fopen(kmn_file, "w"); + FILE *fp = Open_File(kmn_file, "wb"); fclose(fp); // It should fail when a zero-byte file is passed in @@ -65,7 +95,7 @@ void test_kmcmp_CompileKeyboard() { options.warnDeprecatedCode = true; options.shouldAddCompilerVersion = false; options.target = CKF_KEYMAN; - assert(!kmcmp_CompileKeyboard(kmn_file, options, msgproc, nullptr, nullptr, result)); + assert(!kmcmp_CompileKeyboard(kmn_file, options, msgproc, loadfileProc, nullptr, result)); assert(error_vec.size() == 1); assert(error_vec[0] == CERR_CannotReadInfile); diff --git a/developer/src/kmcmplib/tests/kmcompxtest.cpp b/developer/src/kmcmplib/tests/kmcompxtest.cpp index 87d9e53cf7..c5d189798b 100644 --- a/developer/src/kmcmplib/tests/kmcompxtest.cpp +++ b/developer/src/kmcmplib/tests/kmcompxtest.cpp @@ -13,6 +13,7 @@ #include #include #include +#include "../src/filesystem.h" #ifdef _MSC_VER #else @@ -28,7 +29,7 @@ vector < int > error_vec; #define CERR_WARNING 0x00002000 #define CERR_HINT 0x00001000 -int msgproc(int line, uint32_t dwMsgCode, char* szText, void* context) +int msgproc(int line, uint32_t dwMsgCode, const char* szText, void* context) { error_vec.push_back(dwMsgCode); const char*t = "unknown"; @@ -42,6 +43,34 @@ int msgproc(int line, uint32_t dwMsgCode, char* szText, void* context) return 1; } +bool loadfileProc(const char* filename, const char* baseFilename, void* data, int* size, void* context) { + FILE* fp = Open_File(filename, "rb"); + if(!fp) { + return false; + } + + if(!data) { + // return size + if(fseek(fp, 0, SEEK_END) != 0) { + fclose(fp); + return false; + } + *size = ftell(fp); + if(*size == -1L) { + fclose(fp); + return false; + } + } else { + // return data + if(fread(data, 1, *size, fp) != *size) { + fclose(fp); + return false; + } + } + fclose(fp); + return true; +} + #include "../src/filesystem.h" int main(int argc, char *argv[]) @@ -79,7 +108,7 @@ int main(int argc, char *argv[]) options.shouldAddCompilerVersion = false; options.target = CKF_KEYMAN; - if(kmcmp_CompileKeyboard(kmn_file, options, msgproc, nullptr, nullptr, result)) { + if(kmcmp_CompileKeyboard(kmn_file, options, msgproc, loadfileProc, nullptr, result)) { char* testname = strrchr( (char*) kmn_file, '/') + 1; if(strncmp(testname, pfirst5, 5) == 0){ return __LINE__; // exit code: CERR_ in Name + no Error found diff --git a/developer/src/kmcmplib/tests/meson.build b/developer/src/kmcmplib/tests/meson.build index 49924f2d78..2468ac53b1 100644 --- a/developer/src/kmcmplib/tests/meson.build +++ b/developer/src/kmcmplib/tests/meson.build @@ -113,10 +113,10 @@ apitest = executable('api-test', 'api-test.cpp', name_suffix: name_suffix, link_args: links + tests_flags, objects: lib.extract_all_objects(), - dependencies: icuuc_dep, + dependencies: icuuc_dep ) -test('api-test', apitest) +test('api-test', apitest, args: [output_path / 'blank_keyboard.kmx']) usetapitest = executable('uset-api-test', 'uset-api-test.cpp', cpp_args: defns, From 47c8a95fb0743d471eee87b90ee2353469e99ad2 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 31 May 2023 07:44:35 +0700 Subject: [PATCH 2/6] refactor(developer): complete fs move out of kmcmplib * Moves filesystem access out of kmcmplib into kmc-kmn * Adds filesystem access callback to kmcmplib unit tests * Cleans up callback interface through wasm * Adds unit tests for various file load scenarios * Removes nodefs dependency from kmcmplib wasm build, and removes corresponding path mappings which were previously required for wasm builds; note that these are still present for the unit tests for kmcmplib. --- .../src/kmc-kmn/src/compiler/compiler.ts | 55 ++++++++-- .../kmcmplib/src/CheckFilenameConsistency.cpp | 47 ++------ .../kmcmplib/src/CheckFilenameConsistency.h | 2 - developer/src/kmcmplib/src/Compiler.cpp | 102 ++++++++---------- .../src/kmcmplib/src/CompilerInterfaces.cpp | 44 ++++---- .../src/kmcmplib/src/NamedCodeConstants.cpp | 97 ++++++----------- .../src/kmcmplib/src/NamedCodeConstants.h | 5 +- developer/src/kmcmplib/src/compfile.h | 3 +- developer/src/kmcmplib/src/kmcmplib.h | 1 - developer/src/kmcmplib/src/meson.build | 9 +- developer/src/kmcmplib/src/pch.h | 2 - developer/src/kmcmplib/tests/api-test.cpp | 53 ++------- .../valid-keyboards/compile_legacy.bat | 4 + .../fixtures/valid-keyboards/k001_utf16.kmn | Bin 0 -> 452 bytes .../fixtures/valid-keyboards/k001_utf16.kmx | Bin 0 -> 356 bytes .../valid-keyboards/k002_utf8_without_bom.kmn | 11 ++ .../valid-keyboards/k002_utf8_without_bom.kmx | Bin 0 -> 378 bytes .../valid-keyboards/k003_utf8_with_bom.kmn | 11 ++ .../valid-keyboards/k003_utf8_with_bom.kmx | Bin 0 -> 372 bytes .../fixtures/valid-keyboards/k004_ansi.kmn | 13 +++ .../fixtures/valid-keyboards/k004_ansi.kmx | Bin 0 -> 350 bytes .../fixtures/valid-keyboards/k005_bitmap.bmp | Bin 0 -> 246 bytes .../fixtures/valid-keyboards/k005_bitmap.kmn | 12 +++ .../fixtures/valid-keyboards/k005_bitmap.kmx | Bin 0 -> 656 bytes .../fixtures/valid-keyboards/k006_icon.ico | Bin 0 -> 318 bytes .../fixtures/valid-keyboards/k006_icon.kmn | 12 +++ .../fixtures/valid-keyboards/k006_icon.kmx | Bin 0 -> 728 bytes .../valid-keyboards/k007_includecodes_r_n.kmn | 12 +++ .../valid-keyboards/k007_includecodes_r_n.kmx | Bin 0 -> 462 bytes .../valid-keyboards/k007_includecodes_r_n.txt | 2 + .../valid-keyboards/k008_includecodes_n.kmn | 12 +++ .../valid-keyboards/k008_includecodes_n.kmx | Bin 0 -> 454 bytes .../valid-keyboards/k008_includecodes_n.txt | 2 + developer/src/kmcmplib/tests/kmcompxtest.cpp | 49 +-------- developer/src/kmcmplib/tests/meson.build | 41 +++++-- .../src/kmcmplib/tests/util_callbacks.cpp | 59 ++++++++++ developer/src/kmcmplib/tests/util_callbacks.h | 8 ++ .../util_filesystem.cpp} | 41 ++++++- .../filesystem.h => tests/util_filesystem.h} | 5 +- 39 files changed, 406 insertions(+), 308 deletions(-) create mode 100644 developer/src/kmcmplib/tests/fixtures/valid-keyboards/compile_legacy.bat create mode 100644 developer/src/kmcmplib/tests/fixtures/valid-keyboards/k001_utf16.kmn create mode 100644 developer/src/kmcmplib/tests/fixtures/valid-keyboards/k001_utf16.kmx create mode 100644 developer/src/kmcmplib/tests/fixtures/valid-keyboards/k002_utf8_without_bom.kmn create mode 100644 developer/src/kmcmplib/tests/fixtures/valid-keyboards/k002_utf8_without_bom.kmx create mode 100644 developer/src/kmcmplib/tests/fixtures/valid-keyboards/k003_utf8_with_bom.kmn create mode 100644 developer/src/kmcmplib/tests/fixtures/valid-keyboards/k003_utf8_with_bom.kmx create mode 100644 developer/src/kmcmplib/tests/fixtures/valid-keyboards/k004_ansi.kmn create mode 100644 developer/src/kmcmplib/tests/fixtures/valid-keyboards/k004_ansi.kmx create mode 100644 developer/src/kmcmplib/tests/fixtures/valid-keyboards/k005_bitmap.bmp create mode 100644 developer/src/kmcmplib/tests/fixtures/valid-keyboards/k005_bitmap.kmn create mode 100644 developer/src/kmcmplib/tests/fixtures/valid-keyboards/k005_bitmap.kmx create mode 100644 developer/src/kmcmplib/tests/fixtures/valid-keyboards/k006_icon.ico create mode 100644 developer/src/kmcmplib/tests/fixtures/valid-keyboards/k006_icon.kmn create mode 100644 developer/src/kmcmplib/tests/fixtures/valid-keyboards/k006_icon.kmx create mode 100644 developer/src/kmcmplib/tests/fixtures/valid-keyboards/k007_includecodes_r_n.kmn create mode 100644 developer/src/kmcmplib/tests/fixtures/valid-keyboards/k007_includecodes_r_n.kmx create mode 100644 developer/src/kmcmplib/tests/fixtures/valid-keyboards/k007_includecodes_r_n.txt create mode 100644 developer/src/kmcmplib/tests/fixtures/valid-keyboards/k008_includecodes_n.kmn create mode 100644 developer/src/kmcmplib/tests/fixtures/valid-keyboards/k008_includecodes_n.kmx create mode 100644 developer/src/kmcmplib/tests/fixtures/valid-keyboards/k008_includecodes_n.txt create mode 100644 developer/src/kmcmplib/tests/util_callbacks.cpp create mode 100644 developer/src/kmcmplib/tests/util_callbacks.h rename developer/src/kmcmplib/{src/filesystem.cpp => tests/util_filesystem.cpp} (84%) rename developer/src/kmcmplib/{src/filesystem.h => tests/util_filesystem.h} (82%) diff --git a/developer/src/kmc-kmn/src/compiler/compiler.ts b/developer/src/kmc-kmn/src/compiler/compiler.ts index 606bec638c..b27440fd06 100644 --- a/developer/src/kmc-kmn/src/compiler/compiler.ts +++ b/developer/src/kmc-kmn/src/compiler/compiler.ts @@ -42,13 +42,16 @@ const baseOptions: CompilerOptions = { */ let callbackProcIdentifier = 0; +const + callbackPrefix = 'kmnCompilerCallbacks_'; + export class KmnCompiler { private Module: any; - callbackName: string; + callbackID: string; // a unique numeric id added to globals with prefixed names callbacks: CompilerCallbacks; constructor() { - this.callbackName = 'kmnCompilerCallback' + callbackProcIdentifier; + this.callbackID = callbackPrefix + callbackProcIdentifier.toString(); callbackProcIdentifier++; } @@ -58,6 +61,7 @@ export class KmnCompiler { try { this.Module = await loadWasmHost(); } catch(e: any) { + /* c8 ignore next 3 */ this.callbacks.reportMessage(CompilerMessages.Fatal_MissingWasmModule({e})); return false; } @@ -74,7 +78,9 @@ export class KmnCompiler { // Can't report a message here. throw Error('Must call Compiler.init(callbacks) before proceeding'); } - if(!this.Module) { // fail if wasm not loaded or function not found + if(!this.Module) { + /* c8 ignore next 4 */ + // fail if wasm not loaded or function not found this.callbacks.reportMessage(CompilerMessages.Fatal_MissingWasmModule({})); return false; } @@ -83,15 +89,17 @@ export class KmnCompiler { public run(infile: string, outfile: string, options?: CompilerOptions): boolean { if(!this.verifyInitialized()) { + /* c8 ignore next 2 */ return false; } options = {...baseOptions, ...options}; - (globalThis as any)[this.callbackName] = this.compilerMessageCallback; - // TODO: use callbacks for file access -- so kmc-kmn is entirely fs agnostic + (globalThis as any)[this.callbackID] = { + message: this.compilerMessageCallback, + loadFile: this.loadFileCallback + }; let result = this.runCompiler(infile, outfile, options); - delete (globalThis as any)[this.callbackName]; - //TODO: write the file out! + delete (globalThis as any)[this.callbackID]; if(result) { if(result.kmx) { this.callbacks.fs.writeFileSync(result.kmx.filename, result.kmx.data); @@ -111,6 +119,30 @@ export class KmnCompiler { return 1; } + private loadFileCallback = (filename: string, baseFilename: string, buffer: number, bufferSize: number): number => { + // TODO: we can optimize this in future by avoiding loading the file twice + let resolvedFilename = this.callbacks.resolveFilename(baseFilename, filename); + let data = this.callbacks.loadFile(resolvedFilename); + if(!data) { + return 0; + } + + if(buffer == 0) { + /* We need to return buffer size required */ + return data.byteLength; + } + + if(bufferSize != data.byteLength) { + // TODO: consider chucking a wobbly because this is a bug + /* c8 ignore next 2 */ + return 0; + } + + this.Module.HEAP8.set(data, buffer); + + return 1; + } + private runCompiler(infile: string, outfile: string, options: CompilerOptions): CompilerResult { let result: CompilerResult = {}; let wasm_interface = new this.Module.CompilerInterface(); @@ -122,8 +154,7 @@ export class KmnCompiler { wasm_options.warnDeprecatedCode = options.warnDeprecatedCode; wasm_options.shouldAddCompilerVersion = options.shouldAddCompilerVersion; wasm_options.target = 0; //CKF_KEYMAN; TODO, support KMW - wasm_interface.messageCallback = this.callbackName; - wasm_interface.loadFileCallback = this.callbackName; // TODO: this is wrong, needs to be a new callback; not yet used though + wasm_interface.callbacksKey = this.callbackID; // key of object on globalThis wasm_result = this.Module.kmcmp_compile(infile, wasm_options, wasm_interface); if(!wasm_result.result) { return null; @@ -143,6 +174,7 @@ export class KmnCompiler { return result; } catch(e) { + /* c8 ignore next 3 */ this.callbacks.reportMessage(CompilerMessages.Fatal_UnexpectedException({e:e})); return null; } finally { @@ -163,6 +195,7 @@ export class KmnCompiler { reader.validate(kvks, this.callbacks.loadSchema('kvks')); } catch(e) { console.log(e); + // TODO: also unit test // TODO: this.callbacks.reportMessage(CompilerMessages.Error_InvalidKvksFile({e})); return null; } @@ -170,6 +203,7 @@ export class KmnCompiler { let vk = reader.transform(kvks, errors); if(!vk || errors.length) { console.dir(errors); + // TODO: also unit test // TODO: this.callbacks.reportMessage(CompilerMessages.Error_InvalidKvksFile({e})); return null; } @@ -188,10 +222,12 @@ export class KmnCompiler { */ public parseUnicodeSet(pattern: string, bufferSize: number) : UnicodeSet | null { if(!this.verifyInitialized()) { + /* c8 ignore next 2 */ return null; } if (!bufferSize) { + /* c8 ignore next 2 */ bufferSize = 100; // TODO-LDML: Preflight mode? Reuse buffer? } const buf = this.Module.asm.malloc(bufferSize * 2 * this.Module.HEAPU32.BYTES_PER_ELEMENT); @@ -239,6 +275,7 @@ function getUnicodeSetError(rc: number) : CompilerEvent { case KMCMP_FATAL_OUT_OF_RANGE: return CompilerMessages.Fatal_UnicodeSetOutOfRange(); default: + /* c8 ignore next */ return CompilerMessages.Fatal_UnexpectedException({e: `Unexpected UnicodeSet error code ${rc}`}); } } diff --git a/developer/src/kmcmplib/src/CheckFilenameConsistency.cpp b/developer/src/kmcmplib/src/CheckFilenameConsistency.cpp index 627298bf37..3a185bc403 100644 --- a/developer/src/kmcmplib/src/CheckFilenameConsistency.cpp +++ b/developer/src/kmcmplib/src/CheckFilenameConsistency.cpp @@ -7,52 +7,12 @@ #include #include "CheckFilenameConsistency.h" #include "kmx_u16.h" -#include "filesystem.h" #ifdef _MSC_VER #include #endif -namespace kmcmp { - extern KMX_CHAR CompileDir[260]; // TODO: this should not be a fixed buffer -} -bool IsRelativePath(KMX_CHAR const * p) { - // Relative path (returns TRUE): - // ..\...\BITMAP.BMP - // PATH\BITMAP.BMP - // BITMAP.BMP - // Semi-absolute path (returns FALSE): - // \...\BITMAP.BMP - - // Absolute path (returns FALSE): - // C:\...\BITMAP.BMP - // \\SERVER\SHARE\...\BITMAP.BMP - - if ((*p == '\\') || (*p == '/')) return FALSE; - if (*p && *(p + 1) == ':') return FALSE; - - return TRUE; -} - -bool IsRelativePath(KMX_WCHAR const * p) { - // Relative path (returns TRUE): - // ..\...\BITMAP.BMP - // PATH\BITMAP.BMP - // BITMAP.BMP - - // Semi-absolute path (returns FALSE): - // \...\BITMAP.BMP - - // Absolute path (returns FALSE): - // C:\...\BITMAP.BMP - // \\SERVER\SHARE\...\BITMAP.BMP - - if ((*p == u'\\') || (*p == u'/'))return FALSE; - if (*p && *(p + 1) == u':') return FALSE; - - return TRUE; -} KMX_DWORD CheckFilenameConsistency( KMX_CHAR const * Filename, bool ReportMissingFile) { PKMX_WCHAR WFilename = strtowstr(( KMX_CHAR *)Filename); @@ -62,6 +22,12 @@ KMX_DWORD CheckFilenameConsistency( KMX_CHAR const * Filename, bool ReportMissin } KMX_DWORD CheckFilenameConsistency(KMX_WCHAR const * Filename, bool ReportMissingFile) { + // TODO: we no longer have filesystem access here. We could move this check to + // kmc itself, and make it consistent across all compilers that use the same + // loader callback + return CERR_None; + +#if 0 // not ready yet: needs more attention-> common includes for non-Windows platforms KMX_WCHAR Name[260]; // TODO: fixed buffer sizes bad @@ -115,6 +81,7 @@ KMX_DWORD CheckFilenameConsistency(KMX_WCHAR const * Filename, bool ReportMissin #endif return CERR_None; +#endif } KMX_DWORD CheckFilenameConsistencyForCalls(PFILE_KEYBOARD fk) { diff --git a/developer/src/kmcmplib/src/CheckFilenameConsistency.h b/developer/src/kmcmplib/src/CheckFilenameConsistency.h index 39f86db391..eba8f2d8bf 100644 --- a/developer/src/kmcmplib/src/CheckFilenameConsistency.h +++ b/developer/src/kmcmplib/src/CheckFilenameConsistency.h @@ -7,5 +7,3 @@ KMX_DWORD CheckFilenameConsistencyForCalls(PFILE_KEYBOARD fk); KMX_DWORD CheckFilenameConsistency(KMX_CHAR const * Filename, bool ReportMissingFile); KMX_DWORD CheckFilenameConsistency(KMX_WCHAR const * Filename, bool ReportMissingFile); -bool IsRelativePath(KMX_CHAR const * p); -bool IsRelativePath(KMX_WCHAR const * p); diff --git a/developer/src/kmcmplib/src/Compiler.cpp b/developer/src/kmcmplib/src/Compiler.cpp index 8e0f94cb29..91e02e14e7 100644 --- a/developer/src/kmcmplib/src/Compiler.cpp +++ b/developer/src/kmcmplib/src/Compiler.cpp @@ -101,7 +101,6 @@ #include "UnreachableRules.h" #include "CheckForDuplicates.h" #include "kmx_u16.h" -#include "filesystem.h" #include /* These macros are adapted from winnt.h and legacy use only */ @@ -125,7 +124,6 @@ namespace kmcmp{ KMX_BOOL FMnemonicLayout = FALSE; KMX_BOOL FOldCharPosMatching = FALSE; int CompileTarget; - KMX_CHAR CompileDir[260]; // TODO: this should not be a fixed buffer int BeginLine[4]; KMX_BOOL IsValidCallStore(PFILE_STORE fs); @@ -867,7 +865,6 @@ KMX_DWORD ProcessSystemStore(PFILE_KEYBOARD fk, KMX_DWORD SystemID, PFILE_STORE int i, j; KMX_DWORD msg; PKMX_WCHAR p, q; - KMX_CHAR *pp; if (!pssBuf) pssBuf = new KMX_WCHAR[GLOBAL_BUFSIZE]; PKMX_WCHAR buf = pssBuf; @@ -917,13 +914,9 @@ KMX_DWORD ProcessSystemStore(PFILE_KEYBOARD fk, KMX_DWORD SystemID, PFILE_STORE case TSS_INCLUDECODES: VERIFY_KEYBOARD_VERSION(fk, VERSION_60, CERR_60FeatureOnly_NamedCodes); - pp = wstrtostr(sp->dpString); - if (!kmcmp::CodeConstants->LoadFile(pp)) - { - delete[] pp; + if (!kmcmp::CodeConstants->LoadFile(fk, sp->dpString)) { return CERR_CannotLoadIncludeFile; } - delete[] pp; kmcmp::CodeConstants->reindex(); // I4982 break; @@ -3234,58 +3227,32 @@ KMX_BOOL IsSameToken(PKMX_WCHAR *p, KMX_WCHAR const * token) return FALSE; } +static bool endsWith(const std::string& str, const std::string& suffix) +{ + return str.size() >= suffix.size() && 0 == str.compare(str.size()-suffix.size(), suffix.size(), suffix); +} + KMX_DWORD ImportBitmapFile(PFILE_KEYBOARD fk, PKMX_WCHAR szName, PKMX_DWORD FileSize, PKMX_BYTE *Buf) { - FILE *fp; - KMX_WCHAR szNewName[260]; + auto szNameUtf8 = string_from_u16string(szName); - if (IsRelativePath(szName)) - { - PKMX_WCHAR WCompileDir = strtowstr(kmcmp::CompileDir); - u16ncpy(szNewName, WCompileDir, _countof(szNewName)); // I3481 - u16ncat(szNewName,szName, _countof(szNewName )); // I3481 - } - else - u16ncpy(szNewName, szName, _countof(szNewName)); // I3481 - - fp=Open_File(szNewName, u"rb"); - - if ( fp == NULL) - { - // else if filename.bmp is not in the folder -> attempt to open filename.bmp.bmp ! - if ( u16cmp(szNewName+u16len(szNewName)-4, u".bmp") ) - u16ncat(szNewName, u".bmp", _countof(szNewName)); // I3481 - - fp= Open_File(szNewName, u"rb"); - - if ( fp == NULL) + if(!loadfileproc(szNameUtf8.c_str(), fk->extra->kmnFilename.c_str(), nullptr, (int*) FileSize, msgprocContext)) { + // Append .bmp and try again + if(endsWith(szNameUtf8, ".bmp")) { return CERR_CannotReadBitmapFile; + } + szNameUtf8.append(".bmp"); + if(!loadfileproc(szNameUtf8.c_str(), fk->extra->kmnFilename.c_str(), nullptr, (int*) FileSize, msgprocContext)) { + return CERR_CannotReadBitmapFile; + } } - KMX_DWORD msg; - if ((msg = CheckFilenameConsistency(szNewName, FALSE)) != CERR_None) { - return msg; - } - - fseek(fp, 0, SEEK_END); - *FileSize = (KMX_DWORD)ftell(fp); - fseek(fp ,0,SEEK_SET); - if (*FileSize < 0) { - fclose(fp); - return CERR_CannotReadBitmapFile; - } - - if (*FileSize < 2) return CERR_CannotReadBitmapFile; *Buf = new KMX_BYTE[*FileSize]; - - if (fread(*Buf, 1, *FileSize, fp) < (size_t) *FileSize) { - delete[] * Buf; - *Buf = NULL; + if(!loadfileproc(szNameUtf8.c_str(), fk->extra->kmnFilename.c_str(), *Buf, (int*) FileSize, msgprocContext)) { + delete[] *Buf; return CERR_CannotReadBitmapFile; } - fclose(fp); - /* Test for version 7.0 icon support */ if (*((PKMX_CHAR)*Buf) != 'B' && *(((PKMX_CHAR)*Buf) + 1) != 'M') { VERIFY_KEYBOARD_VERSION(fk, VERSION_70, CERR_70FeatureOnly); @@ -3447,6 +3414,8 @@ bool hasPreamble(std::u16string result) { return result.size() > 0 && result[0] == 0xFEFF; } +#include "unicode/ucnv.h" + bool UTF16TempFromUTF8(KMX_BYTE* infile, int sz, KMX_BYTE** tempfile, int *sz16) { if(sz == 0) { return FALSE; @@ -3456,23 +3425,36 @@ bool UTF16TempFromUTF8(KMX_BYTE* infile, int sz, KMX_BYTE** tempfile, int *sz16) try { std::wstring_convert, char16_t> converter; - result = converter.from_bytes((char*)infile, (char*)infile+sz-1); + result = converter.from_bytes((char*)infile, (char*)infile+sz); } catch(std::range_error e) { - std::wstring_convert, char16_t> converter; - result = converter.from_bytes((char*)infile, (char*)infile+sz-1); + UErrorCode status = U_ZERO_ERROR; + // TODO: we need ICU data files here @srl295 plz help! + UConverter* conv = ucnv_open("windows-1252", &status); + if(U_FAILURE(status)) { + return FALSE; + } + + char16_t* dest = new char16_t[sz*2]; + ucnv_toUChars(conv, dest, sz*2, (char*)infile, sz, &status); + if(U_FAILURE(status)) { + delete[] dest; + return FALSE; + } + + result = dest; + delete[] dest; } if(hasPreamble(result)) { - *sz16 = result.size() * 2 - 1; + *sz16 = result.size() * 2 - 2; *tempfile = new KMX_BYTE[*sz16]; - memcpy(*tempfile, result.c_str() + 2, *sz16); - + memcpy(*tempfile, result.c_str() + 1, *sz16); + } else { + *sz16 = result.size() * 2; + *tempfile = new KMX_BYTE[*sz16]; + memcpy(*tempfile, result.c_str(), *sz16); } - *sz16 = result.size() * 2; - *tempfile = new KMX_BYTE[*sz16]; - memcpy(*tempfile, result.c_str(), *sz16); - return TRUE; } diff --git a/developer/src/kmcmplib/src/CompilerInterfaces.cpp b/developer/src/kmcmplib/src/CompilerInterfaces.cpp index fd297f4332..1a4e77f8e8 100644 --- a/developer/src/kmcmplib/src/CompilerInterfaces.cpp +++ b/developer/src/kmcmplib/src/CompilerInterfaces.cpp @@ -3,7 +3,6 @@ #include #include #include "kmcmplib.h" -#include "filesystem.h" #include "CheckFilenameConsistency.h" #include "CheckNCapsConsistency.h" #include "DeprecationChecks.h" @@ -19,7 +18,7 @@ bool CompileKeyboardHandle(KMX_BYTE* infile, int sz, PFILE_KEYBOARD fk); WASM interface for compiler message callback */ EM_JS(int, wasm_msgproc, (int line, int msgcode, const char* text, char* context), { - const proc = globalThis[UTF8ToString(context)]; + const proc = globalThis[UTF8ToString(context)].message; if(!proc || typeof proc != 'function') { console.log(`[${line}: ${msgcode}: ${UTF8ToString(text)}]`); return 0; @@ -28,18 +27,27 @@ EM_JS(int, wasm_msgproc, (int line, int msgcode, const char* text, char* context } }); -EM_JS(bool, wasm_loadfileproc, (const char* filename, const char* baseFilename, void* buffer, int* bufferSize, char* context), { - const proc = globalThis[UTF8ToString(context)]; +EM_JS(int, wasm_loadfileproc, (const char* filename, const char* baseFilename, void* buffer, int bufferSize, char* context), { + const proc = globalThis[UTF8ToString(context)].loadFile; if(!proc || typeof proc != 'function') { return 0; } else { - return proc(UTF8ToString(filename), UTF8ToString(baseFilename), buffer, bufferSize); + if(buffer == 0) { + return proc(UTF8ToString(filename), UTF8ToString(baseFilename), 0, 0); + } else { + return proc(UTF8ToString(filename), UTF8ToString(baseFilename), buffer, bufferSize); + } } }); bool wasm_LoadFileProc(const char* filename, const char* baseFilename, void* buffer, int* bufferSize, void* context) { char* msgProc = static_cast(context); - return wasm_loadfileproc(filename, baseFilename, buffer, bufferSize, msgProc); + if(buffer == nullptr) { + *bufferSize = wasm_loadfileproc(filename, baseFilename, 0, 0, msgProc); + return *bufferSize != 0; + } else { + return wasm_loadfileproc(filename, baseFilename, buffer, *bufferSize, msgProc) == 1; + } } int wasm_CompilerMessageProc(int line, uint32_t dwMsgCode, const char* szText, void* context) { @@ -48,8 +56,7 @@ int wasm_CompilerMessageProc(int line, uint32_t dwMsgCode, const char* szText, v } struct WASM_COMPILER_INTERFACE { - std::string messageCallback; // int line, uint32_t dwMsgCode, char* szText - std::string loadFileCallback; // TODO: char* filename, char* baseFilename --> buffer + std::string callbacksKey; // key of callbacks object on globalThis }; struct WASM_COMPILER_RESULT { @@ -76,7 +83,7 @@ WASM_COMPILER_RESULT kmcmp_wasm_compile(std::string pszInfile, const KMCMP_COMPI options, wasm_CompilerMessageProc, wasm_LoadFileProc, - intf.messageCallback.c_str(), + intf.callbacksKey.c_str(), kr ); @@ -103,8 +110,7 @@ EMSCRIPTEN_BINDINGS(compiler_interface) { emscripten::class_("CompilerInterface") .constructor<>() - .property("messageCallback", &WASM_COMPILER_INTERFACE::messageCallback) - .property("loadFileCallback", &WASM_COMPILER_INTERFACE::loadFileCallback) + .property("callbacksKey", &WASM_COMPILER_INTERFACE::callbacksKey) ; emscripten::class_("CompilerResult") @@ -131,6 +137,8 @@ EXTERN bool kmcmp_CompileKeyboard( ) { FILE_KEYBOARD fk; + fk.extra = new FILE_KEYBOARD_EXTRA; + fk.extra->kmnFilename = pszInfile; kmcmp::FSaveDebug = options.saveDebug; // I3681 kmcmp::FCompilerWarningsAsErrors = options.compilerWarningsAsErrors; // I4865 @@ -143,16 +151,6 @@ EXTERN bool kmcmp_CompileKeyboard( return FALSE; } - PKMX_STR p; - - if ((p = strrchr_slash((char*)pszInfile)) != nullptr) { - strncpy(kmcmp::CompileDir, pszInfile, (int)(p - pszInfile + 1)); // I3481 - kmcmp::CompileDir[(int)(p - pszInfile + 1)] = 0; - } - else { - kmcmp::CompileDir[0] = 0; - } - msgproc = messageProc; loadfileproc = loadFileProc; msgprocContext = (void*)procContext; @@ -172,7 +170,7 @@ EXTERN bool kmcmp_CompileKeyboard( return FALSE; } - KMX_BYTE* infile = new KMX_BYTE[sz]; + KMX_BYTE* infile = new KMX_BYTE[sz+1]; if(!infile) { AddCompileError(CERR_CannotAllocateMemory); return FALSE; @@ -182,6 +180,7 @@ EXTERN bool kmcmp_CompileKeyboard( AddCompileError(CERR_CannotReadInfile); return FALSE; } + infile[sz] = 0; // zero-terminate for safety, not technically needed but helps avoid memory bugs int offset = 0; if(infile[0] == (KMX_BYTE) UTF16Sig[0] && infile[1] == (KMX_BYTE) UTF16Sig[1]) { @@ -266,7 +265,6 @@ bool CompileKeyboardHandle(KMX_BYTE* infile, int sz, PFILE_KEYBOARD fk) fk->dpDeadKeyArray = NULL; fk->cxVKDictionary = 0; // I3438 fk->dpVKDictionary = NULL; // I3438 - fk->extra = new FILE_KEYBOARD_EXTRA; fk->extra->kvksFilename = u""; /* fk->szMessage[0] = 0; fk->szLanguageName[0] = 0;*/ diff --git a/developer/src/kmcmplib/src/NamedCodeConstants.cpp b/developer/src/kmcmplib/src/NamedCodeConstants.cpp index e6b1ae6d70..30091b7cb8 100644 --- a/developer/src/kmcmplib/src/NamedCodeConstants.cpp +++ b/developer/src/kmcmplib/src/NamedCodeConstants.cpp @@ -27,15 +27,12 @@ #include "CheckFilenameConsistency.h" #include #include "kmcompx.h" -#include "filesystem.h" using namespace kmcmp; int IsHangulSyllable(const KMX_WCHAR *codename, int *code); namespace kmcmp { - extern KMX_CHAR CompileDir[]; - int __cdecl sort_entries(const void *elem1, const void *elem2) { return u16icmp( @@ -117,85 +114,61 @@ char *kmc_strupr(char *s) { return s; } -KMX_BOOL NamedCodeConstants::IntLoadFile(const KMX_CHAR *filename) -{ +KMX_BOOL NamedCodeConstants::LoadFile(PFILE_KEYBOARD fk, const KMX_WCHAR *filename) { const int str_size = 256; - FILE *fp = NULL; if (CheckFilenameConsistency(filename, FALSE) != 0) { return FALSE; } - fp = Open_File(filename, "rt"); - if(fp == NULL) { - return FALSE; // I3481 + auto szNameUtf8 = string_from_u16string(filename); + + int FileSize; + KMX_BYTE* Buf; + if(!loadfileproc(szNameUtf8.c_str(), fk->extra->kmnFilename.c_str(), nullptr, &FileSize, msgprocContext)) { + return FALSE; } - KMX_CHAR str[str_size], *p, *q, *context = NULL; - KMX_BOOL isEol , first = TRUE; + Buf = new KMX_BYTE[FileSize+1]; + if(!loadfileproc(szNameUtf8.c_str(), fk->extra->kmnFilename.c_str(), Buf, &FileSize, msgprocContext)) { + delete[] Buf; + return FALSE; + } + Buf[FileSize] = 0; // zero-terminate for strtok - while(fgets(str, str_size, fp)) - { - isEol = *(strchr(str, 0) - 1) == '\n'; - p = strtok_r(str, ";", &context); // I3481 - q = strtok_r(NULL, ";\n", &context); - if(p && q) - { - if(first && *p == (KMX_CHAR)0xEF && *(p+1) == (KMX_CHAR)0xBB && *(p+2) == (KMX_CHAR)0xBF) p += 3; // I3056 UTF-8 // I3512 - first = FALSE; + char* filetok; + char* filecontext; + filetok = strtok_r((char*)Buf, "\n", &filecontext); + + if(*filetok == (KMX_CHAR)0xEF && *(filetok+1) == (KMX_CHAR)0xBB && *(filetok+2) == (KMX_CHAR)0xBF) filetok += 3; // I3056 UTF-8 // I3512 + + while(filetok) { + KMX_CHAR str[str_size], *p, *q, *context = NULL; + + if(strlen(filetok) >= str_size) { + delete[] Buf; + // TODO chuck a wobbly + return FALSE; + } + strcpy(str, filetok); + p = strtok_r(str, ";\r", &context); // I3481 + q = strtok_r(nullptr, ";\r", &context); + if(p && q) { kmc_strupr(q); // I3481 // I3641 - long n = strtol(p, NULL, 16); + long n = strtol(p, nullptr, 16); if (*q != '<') { PKMX_WCHAR q0 = strtowstr(q); AddCode_IncludedCodes((int)n, q0); delete[] q0; } } - if(!isEol ) - { - while(fgets(str, str_size, fp)) if(*(strchr(str, 0)-1) == '\n') break; - } + filetok = strtok_r(nullptr, "\n", &filecontext); } - fclose(fp); - - return TRUE; -} - -KMX_BOOL NamedCodeConstants::LoadFile(const KMX_CHAR *filename) -{ - const int buf_size = 260; - KMX_CHAR buf[buf_size]; - // Look in current directory first -- REMOVED AS DANGEROUS - /* strncpy(buf, filename, (buf_size-1)); buf[buf_size-1] = 0; // I3481 - if(kmcmp_FileExists(buf)) - return IntLoadFile(buf); - */ - // Then look in keyboard file directory (CompileDir) - strncpy(buf, CompileDir, (buf_size-1)); buf[buf_size-1] = 0; // I3481 - strncat(buf, filename, (buf_size-1)-strlen(CompileDir)); buf[buf_size-1] = 0; - if(kmcmp_FileExists(buf)) - return IntLoadFile(buf); - - //TODO: sort out how to find common includes in non-Windows platforms: - #ifdef _WINDOWS_ - // Finally look in kmcmpdll.dll directory - GetModuleFileName(0, buf, buf_size); - - KMX_CHAR *p = strrchr_slash(buf); - if(p) - p++; - else - p = buf; - *p = 0; - strncat_s(buf, _countof(buf), filename, (buf_size-1)-strlen(buf)); buf[buf_size-1] = 0; // I3481 // I3641 - if(kmcmp_FileExists(buf)) - return IntLoadFile(buf); - #endif + delete[] Buf; reindex(); - - return FALSE; + return TRUE; } void NamedCodeConstants::reindex() diff --git a/developer/src/kmcmplib/src/NamedCodeConstants.h b/developer/src/kmcmplib/src/NamedCodeConstants.h index 83d707d092..c537a73361 100644 --- a/developer/src/kmcmplib/src/NamedCodeConstants.h +++ b/developer/src/kmcmplib/src/NamedCodeConstants.h @@ -2,6 +2,8 @@ #ifndef NAMEDCODECONSTANTS_H #define NAMEDCODECONSTANTS_H +#include "compfile.h" + #define MAX_ENAME 128 #define ALLOC_SIZE 256 @@ -23,14 +25,13 @@ namespace kmcmp{ int GetCode_IncludedCodes(const KMX_WCHAR *codename); void AddCode_IncludedCodes(int n, const KMX_WCHAR *p); - KMX_BOOL IntLoadFile(const KMX_CHAR *filename); public: NamedCodeConstants(); ~NamedCodeConstants(); void reindex(); void AddCode(int n, const KMX_WCHAR *p, KMX_DWORD storeIndex); - KMX_BOOL LoadFile(const KMX_CHAR *filename); + KMX_BOOL LoadFile(PFILE_KEYBOARD fk, const KMX_WCHAR *filename); int GetCode(const KMX_WCHAR *codename, KMX_DWORD *storeIndex); }; } diff --git a/developer/src/kmcmplib/src/compfile.h b/developer/src/kmcmplib/src/compfile.h index 7c262be090..c54663aba7 100644 --- a/developer/src/kmcmplib/src/compfile.h +++ b/developer/src/kmcmplib/src/compfile.h @@ -123,7 +123,8 @@ typedef FILE_VKDICTIONARY *PFILE_VKDICTIONARY; * Extra metadata for API consumers */ struct FILE_KEYBOARD_EXTRA { - std::u16string kvksFilename; // original TSS_VISUALKEYBOARD value + std::string kmnFilename; // utf-8 + std::u16string kvksFilename; // utf-16, original TSS_VISUALKEYBOARD value }; typedef struct FILE_KEYBOARD_EXTRA* PFILE_KEYBOARD_EXTRA; diff --git a/developer/src/kmcmplib/src/kmcmplib.h b/developer/src/kmcmplib/src/kmcmplib.h index 70718f98bc..f22c836bae 100644 --- a/developer/src/kmcmplib/src/kmcmplib.h +++ b/developer/src/kmcmplib/src/kmcmplib.h @@ -14,7 +14,6 @@ namespace kmcmp { extern KMX_BOOL FMnemonicLayout; extern KMX_BOOL FOldCharPosMatching; extern int CompileTarget; - extern KMX_CHAR CompileDir[260]; // TODO: this should not be a fixed buffer extern int BeginLine[4]; extern int currentLine; extern NamedCodeConstants *CodeConstants; diff --git a/developer/src/kmcmplib/src/meson.build b/developer/src/kmcmplib/src/meson.build index 727d6831e6..a81efac562 100644 --- a/developer/src/kmcmplib/src/meson.build +++ b/developer/src/kmcmplib/src/meson.build @@ -25,8 +25,10 @@ endif name_suffix = [] if cpp_compiler.get_id() == 'emscripten' - lib_links = ['--whole-archive', '--bind', '-sMODULARIZE', '-sEXPORT_ES6'] - links += ['-lnodefs.js', '--bind', '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\']'] + # wasm-exceptions supported in Node 18+, Chrome 95+, Firefox 100+, Safari 15.2+ + flags += ['-fwasm-exceptions'] + lib_links = ['--whole-archive', '-sMODULARIZE', '-sEXPORT_ES6'] + links += ['-fwasm-exceptions', '--bind', '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\']'] # tests are building as ES6 so we need to declare the file extension # note that meson currently struggles with the sanitycheckc_cross.exe # program, because it has a hard coded extension (.exe) which is not @@ -44,7 +46,6 @@ lib = library('kmcmplib', 'CompilerInterfaces.cpp', 'DeprecationChecks.cpp', 'Edition.cpp', - 'filesystem.cpp', 'NamedCodeConstants.cpp', 'versioning.cpp', 'virtualcharkeys.cpp', @@ -79,7 +80,7 @@ if cpp_compiler.get_id() == 'emscripten' host = executable('wasm-host', #'wasm-host.cpp', cpp_args: defns, include_directories: inc, - link_args: links, + link_args: links + lib_links, objects: lib.extract_all_objects(), dependencies: icuuc_dep) endif diff --git a/developer/src/kmcmplib/src/pch.h b/developer/src/kmcmplib/src/pch.h index f878ad11be..358e28998a 100644 --- a/developer/src/kmcmplib/src/pch.h +++ b/developer/src/kmcmplib/src/pch.h @@ -10,6 +10,4 @@ #include "../../../../common/windows/cpp/include/crc32.h" #include -#include - #include diff --git a/developer/src/kmcmplib/tests/api-test.cpp b/developer/src/kmcmplib/tests/api-test.cpp index b347773220..cdbabea4d5 100644 --- a/developer/src/kmcmplib/tests/api-test.cpp +++ b/developer/src/kmcmplib/tests/api-test.cpp @@ -18,54 +18,12 @@ #include #include "../src/compfile.h" #include -#include "../src/filesystem.h" +#include "util_filesystem.h" +#include "util_callbacks.h" void setup(); void test_kmcmp_CompileKeyboard(char *kmn_file); -std::vector error_vec; - -int msgproc(int line, uint32_t dwMsgCode, const char* szText, void* context) { - error_vec.push_back(dwMsgCode); - const char*t = "unknown"; - switch(dwMsgCode & 0xF000) { - case CERR_HINT: t=" hint"; break; - case CERR_WARNING: t="warning"; break; - case CERR_ERROR: t=" error"; break; - case CERR_FATAL: t=" fatal"; break; - } - printf("line %d %s %04.4x: %s\n", line, t, (unsigned int)dwMsgCode, szText); - return 1; -} - -bool loadfileProc(const char* filename, const char* baseFilename, void* data, int* size, void* context) { - FILE* fp = Open_File(filename, "rb"); - if(!fp) { - return false; - } - - if(!data) { - // return size - if(fseek(fp, 0, SEEK_END) != 0) { - fclose(fp); - return false; - } - *size = ftell(fp); - if(*size == -1L) { - fclose(fp); - return false; - } - } else { - // return data - if(fread(data, 1, *size, fp) != *size) { - fclose(fp); - return false; - } - } - fclose(fp); - return true; -} - int main(int argc, char *argv[]) { if(argc < 1) { puts("Usage: api-test "); @@ -82,6 +40,13 @@ void setup() { error_vec.clear(); } +/* + TODO: tests to run: + 4. ANSI (no BOM of course) + 8. file without blank last line (cannot compare with fixture due to bug in kmcmpdll...) + Hint to add: k004_ansi.kmn: Hint: 10A6 Keyman Developer has detected that the file has ANSI encoding. Consider converting this file to UTF-8 +*/ + void test_kmcmp_CompileKeyboard(char *kmn_file) { // Create an empty file FILE *fp = Open_File(kmn_file, "wb"); diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/compile_legacy.bat b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/compile_legacy.bat new file mode 100644 index 0000000000..e346418eb4 --- /dev/null +++ b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/compile_legacy.bat @@ -0,0 +1,4 @@ +@echo off +echo Compiles the keyboards using the legacy kmcomp.exe +echo to use as baseline comparisons for kmcmplib +for %%d in (*.kmn) do kmcomp -no-compiler-version -d %%d \ No newline at end of file diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k001_utf16.kmn b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k001_utf16.kmn new file mode 100644 index 0000000000000000000000000000000000000000..2b3180c36ab4b473e74b1b3487fc8f0a61d623f6 GIT binary patch literal 452 zcmZvX%}T>S6otRF;Ci;RG8IwmMz9N^r4>Z2P-{0*q)pQpYC6SO#HDW`K8}yz1F2_b zEO8;>PVStad(V7-rC7RTRHQsou;NLlV@XTQNQ_74DzC9(@0sYdEGXqE<#-S~6_Scs zhQAAVAtuv(qPk(oDf=`z;)0$KKQreYOjH}hfM HO;rB?W(q}E literal 0 HcmV?d00001 diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k001_utf16.kmx b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k001_utf16.kmx new file mode 100644 index 0000000000000000000000000000000000000000..13ea2a686e09c16db68ff7bb355e114b9125d76d GIT binary patch literal 356 zcmX|-y-EW?6h@CC{(@LJjfK^g5*smH5)hGKgd~cM=wcK?h&EbS+F6LTi1-vfKuBe6 zY2hPS_yT(Fx_IF%-}l|QGdt5f>~@%g>e1pi)hKINm-ap;#+1e%<2!;qYd!Vk use(main) + +group(main) using keys + ++ [K_A] > 'a' +'a' + [K_B] > 'ខ្មែរ' diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k002_utf8_without_bom.kmx b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k002_utf8_without_bom.kmx new file mode 100644 index 0000000000000000000000000000000000000000..0b9c9294182d9387165c98e1ffc84c8fe45452e3 GIT binary patch literal 378 zcmX|-zb}JP6owxn6pNv86O*ZdP*T+eoFYo)Fd(S=j8hhJK#^I(|T)`TOA8S;iuF8z!+fx45AU`;5awoSE5eup-GIf`;l(pDb^*mi- zgBt5Bs~4F|6HDysw`j7)rhS|MtM+EoD_MGH-mK1!I^)1;9k+mrTP?|74J)FJ+x_0F i=_X>k3LRTrscR?m>{x#(?dZ7 use(main) + +group(main) using keys + ++ [K_A] > 'a' +'a' + [K_B] > 'ខ្មែរ' diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k003_utf8_with_bom.kmx b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k003_utf8_with_bom.kmx new file mode 100644 index 0000000000000000000000000000000000000000..ee1cf4225502e3c7944b58f301e5be649af4ec00 GIT binary patch literal 372 zcmX|-y-EW?6h@B`{B116X{}O;1cXJqBp?!k5sZqJk{HD$Xi`MP(k_Kq2MIojmG}T+ zXuY}scO@6Efi@` zVV7<77VFupB1igtn$*~{k9%Rq-fA{K%}z+nsvN3Q&Yafqd|2~NN&bIY3KDI+*JNKs gw;=I(PHeTMs$I;pj=nFg=(!IcVH94&lMfV=KM7PUb^rhX literal 0 HcmV?d00001 diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k004_ansi.kmn b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k004_ansi.kmn new file mode 100644 index 0000000000..50a1b31ea1 --- /dev/null +++ b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k004_ansi.kmn @@ -0,0 +1,13 @@ +c Description: Verifies that kmcmplib can compile an ANSI file +c This has some high-ascii letters in cp1252 to ensure that +c it fails to load as 'utf8 without bom' + +store(&NAME) 'k004_ansi' +store(&VERSION) '9.0' + +begin unicode > use(main) + +group(main) using keys + ++ [K_A] > 'a' +'a' + [K_B] > '' diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k004_ansi.kmx b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k004_ansi.kmx new file mode 100644 index 0000000000000000000000000000000000000000..6c41da979cb2a3ec20aadf6de0ca778fb8ce5ec1 GIT binary patch literal 350 zcmX|-y-EW?6h@CCYQTV1Y%Ef3DG6A_&MX88!36RnVxa_+Vi8S|DsPZN@&q9dkp~bf zYfGyKTe*wxmCUXD) literal 0 HcmV?d00001 diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k005_bitmap.bmp b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k005_bitmap.bmp new file mode 100644 index 0000000000000000000000000000000000000000..509b89ae6df7f09a442ae60f215a6c7b3211b34f GIT binary patch literal 246 zcmZvUF%Ez*2t_e5>FO~&hr4&_cK5)zF7R>OXdqB#E&96yg&T8zj G@{Su^DQ%(v literal 0 HcmV?d00001 diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k005_bitmap.kmn b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k005_bitmap.kmn new file mode 100644 index 0000000000..926ccdeeaf --- /dev/null +++ b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k005_bitmap.kmn @@ -0,0 +1,12 @@ +c Description: Verifies that kmcmplib can load a bitmap without file extension + +store(&NAME) 'k005_bitmap' +store(&VERSION) '9.0' +store(&BITMAP) 'k005_bitmap' + +begin unicode > use(main) + +group(main) using keys + ++ [K_A] > 'a' +'a' + [K_B] > 'ខ្មែរ' diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k005_bitmap.kmx b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k005_bitmap.kmx new file mode 100644 index 0000000000000000000000000000000000000000..bc9bc9b671724050e169e59fce976a18d54a030e GIT binary patch literal 656 zcmZ{hyGjF55QhJV5JkZh;`acRmwz7&=*hh@R?A}o^(Fw$>p)BB*B zffhV~2T#xjoA1IijKLecfV&^}PM(8l@|;2L6X&2nE|qnpYn`g0GgqI`FK9t~+E=Ix zr$i`ut*fMpR>+q$@22zGrmm`{HEp0v|4#>;sWV^Frn^B}<&If*f9SrwNI%dPd8h-O z>yVd8RbILaLmMHn(q5d*m8d36E=qSuO$aC_)80MxBQ&hmt@rA+ysp>gH(DpC*#9-V z0w3%j85xb@rP%qu9Eyo?oCG{6#Oic92&~e!$x2D|BF%`g<4g%dtuD`P6eIRAp=916x*GXhr`_H use(main) + +group(main) using keys + ++ [K_A] > 'a' +'a' + [K_B] > 'ខ្មែរ' diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k006_icon.kmx b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k006_icon.kmx new file mode 100644 index 0000000000000000000000000000000000000000..f994a1dcac9e5ae797eb88eed9cee4b0e6f85b00 GIT binary patch literal 728 zcmbu6JxhX76vzKYU{S*}M9=CF;v`N*4WU9rNH3^GLlE{dNK~#(-JqeGBgC;O`Vb+F zjedqcfJ@|NzjNS>L1T*-NfuF<9K>=T?=vWszRYPY!J|SPwy!O@9q0W6a zs|}U3t`#~KmG_fb?GRU0)0#G^OMfTx$qp&2GJSP+l(glyNUQA0_>-Idlm+r-ZR3ZN zkGPps<)(Wuk}(_$?Z#oQL^NgPqI6B^2FirfN$!sL0U8$TD(=PY_%-hEj3JX^Ulc{? zasM{<);X}Z%7EofoYz5F%?RQ`(K+8^8W)_p-7YPBsr@Qn>kZm|*}mHT`SFvrMGq4+QP0yrw1lmGw# literal 0 HcmV?d00001 diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k007_includecodes_r_n.kmn b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k007_includecodes_r_n.kmn new file mode 100644 index 0000000000..71d72d5120 --- /dev/null +++ b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k007_includecodes_r_n.kmn @@ -0,0 +1,12 @@ +c Description: Verifies that kmcmplib can load an includecodes file with \r\n line endings + +store(&NAME) 'k007_includecodes_r_n' +store(&VERSION) '9.0' +store(&includecodes) 'k007_includecodes_r_n.txt' + +begin unicode > use(main) + +group(main) using keys + ++ [K_A] > $LOWER_A +$LOWER_A + [K_B] > $LOWER_B diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k007_includecodes_r_n.kmx b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k007_includecodes_r_n.kmx new file mode 100644 index 0000000000000000000000000000000000000000..f8b4eac1b0d9103c7767e487b5109a06aed8e1ec GIT binary patch literal 462 zcma)%J4%C55JrzSDq4tzwy9Ev-%9L~fQZBwiHe=Z#3J|zmm`-ru-1)wlxv$dkL4`TUG*`bcreZN>By?W7ilL#L#M^jq*~&ofWJMn2 zUSj{dJj$$m%7-jT;@)rbqU@`;C3M|ihyS0LqfJlZ use(main) + +group(main) using keys + ++ [K_A] > $LOWER_A +$LOWER_A + [K_B] > $LOWER_B diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k008_includecodes_n.kmx b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k008_includecodes_n.kmx new file mode 100644 index 0000000000000000000000000000000000000000..aa158cdd546cdde915e4a55c1db521e57d4abb04 GIT binary patch literal 454 zcma)%zb=De6o;Q$N*Khz%F>vcZrIv}AYW9XNQWlUmZ3onNVox$!C+{51>y<}(!t;Y zT!2O58f-l0Q{9?4$@8Ay@0|1gOKQ7SCSc(<`M}7;Vq_(BQ96omLphAs!QiZwuH4I< zJV;Ms|C>C@xO~dH%*dj9zs%FJpf~T9ja`yZ6E%3TAmB8Bu*W5Yup0s-ZP)RHs)YS%IvAz p!xzGhzq^ya7*m9eU;geZ=_aFHgeP0iQr1olYJL5SG{V!AP(Sg;I(+~D literal 0 HcmV?d00001 diff --git a/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k008_includecodes_n.txt b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k008_includecodes_n.txt new file mode 100644 index 0000000000..4cc3fe4351 --- /dev/null +++ b/developer/src/kmcmplib/tests/fixtures/valid-keyboards/k008_includecodes_n.txt @@ -0,0 +1,2 @@ +0061;LOWER_A +0062;LOWER_B diff --git a/developer/src/kmcmplib/tests/kmcompxtest.cpp b/developer/src/kmcmplib/tests/kmcompxtest.cpp index c5d189798b..5dbaee88bc 100644 --- a/developer/src/kmcmplib/tests/kmcompxtest.cpp +++ b/developer/src/kmcmplib/tests/kmcompxtest.cpp @@ -13,7 +13,8 @@ #include #include #include -#include "../src/filesystem.h" +#include "util_filesystem.h" +#include "util_callbacks.h" #ifdef _MSC_VER #else @@ -22,57 +23,11 @@ using namespace std; -vector < int > error_vec; - #define CERR_FATAL 0x00008000 #define CERR_ERROR 0x00004000 #define CERR_WARNING 0x00002000 #define CERR_HINT 0x00001000 -int msgproc(int line, uint32_t dwMsgCode, const char* szText, void* context) -{ - error_vec.push_back(dwMsgCode); - const char*t = "unknown"; - switch(dwMsgCode & 0xF000) { - case CERR_HINT: t=" hint"; break; - case CERR_WARNING: t="warning"; break; - case CERR_ERROR: t=" error"; break; - case CERR_FATAL: t=" fatal"; break; - } - printf("line %d %s %04.4x: %s\n", line, t, (unsigned int)dwMsgCode, szText); - return 1; -} - -bool loadfileProc(const char* filename, const char* baseFilename, void* data, int* size, void* context) { - FILE* fp = Open_File(filename, "rb"); - if(!fp) { - return false; - } - - if(!data) { - // return size - if(fseek(fp, 0, SEEK_END) != 0) { - fclose(fp); - return false; - } - *size = ftell(fp); - if(*size == -1L) { - fclose(fp); - return false; - } - } else { - // return data - if(fread(data, 1, *size, fp) != *size) { - fclose(fp); - return false; - } - } - fclose(fp); - return true; -} - -#include "../src/filesystem.h" - int main(int argc, char *argv[]) { if(argc < 4) { diff --git a/developer/src/kmcmplib/tests/meson.build b/developer/src/kmcmplib/tests/meson.build index 2468ac53b1..cc51189a63 100644 --- a/developer/src/kmcmplib/tests/meson.build +++ b/developer/src/kmcmplib/tests/meson.build @@ -6,16 +6,20 @@ fs = import('fs') -tests_flags = [] +tests_links = [] + +if cpp_compiler.get_id() == 'emscripten' + tests_links += ['-lnodefs.js'] +endif input_path = meson.current_source_dir() / '../../../../common/test/keyboards/baseline' output_path = meson.current_build_dir() -kmcompxtest = executable('kmcompxtest', 'kmcompxtest.cpp', - cpp_args: defns, +kmcompxtest = executable('kmcompxtest', ['kmcompxtest.cpp','util_filesystem.cpp','util_callbacks.cpp'], + cpp_args: defns + flags, include_directories: inc, name_suffix: name_suffix, - link_args: links + tests_flags, + link_args: links + tests_links, objects: lib.extract_all_objects(), dependencies: icuuc_dep, ) @@ -76,6 +80,25 @@ foreach kbd : tests test(kbd, kmcompxtest, args: [kbd_src, kbd_obj, join_paths(input_path, kbd) + '.kmx']) endforeach +valid_keyboard_tests = [ + 'k001_utf16', + 'k002_utf8_without_bom', + 'k003_utf8_with_bom', + # 'k004_ansi', # TODO: enable ansi test when we have the icu datafiles + 'k005_bitmap', + 'k006_icon', + 'k007_includecodes_r_n', + 'k008_includecodes_n' +] + +fixtures_path = meson.current_source_dir() / 'fixtures/valid-keyboards' + +foreach kbd : valid_keyboard_tests + kbd_src = join_paths(fixtures_path, kbd) + '.kmn' + kbd_obj = join_paths(output_path, kbd) + '.kmx' + test(kbd, kmcompxtest, args: [kbd_src, kbd_obj, join_paths(fixtures_path, kbd) + '.kmx']) +endforeach + # Test fixtures that come from keyboards repo -- but only for a "full" test, # which typically we run on CI no more than once a day, because it's expensive. @@ -107,11 +130,11 @@ endif # Test the API endpoints -apitest = executable('api-test', 'api-test.cpp', - cpp_args: defns, +apitest = executable('api-test', ['api-test.cpp','util_filesystem.cpp','util_callbacks.cpp'], + cpp_args: defns + flags, include_directories: inc, name_suffix: name_suffix, - link_args: links + tests_flags, + link_args: links + tests_links, objects: lib.extract_all_objects(), dependencies: icuuc_dep ) @@ -119,10 +142,10 @@ apitest = executable('api-test', 'api-test.cpp', test('api-test', apitest, args: [output_path / 'blank_keyboard.kmx']) usetapitest = executable('uset-api-test', 'uset-api-test.cpp', - cpp_args: defns, + cpp_args: defns + flags, include_directories: inc, name_suffix: name_suffix, - link_args: links + tests_flags, + link_args: links + tests_links, objects: lib.extract_all_objects(), dependencies: icuuc_dep, ) diff --git a/developer/src/kmcmplib/tests/util_callbacks.cpp b/developer/src/kmcmplib/tests/util_callbacks.cpp new file mode 100644 index 0000000000..e610a54c09 --- /dev/null +++ b/developer/src/kmcmplib/tests/util_callbacks.cpp @@ -0,0 +1,59 @@ +#include +#include +#include +#include "util_filesystem.h" +#include "../src/compfile.h" +#include + +std::vector error_vec; + +int msgproc(int line, uint32_t dwMsgCode, const char* szText, void* context) { + error_vec.push_back(dwMsgCode); + const char*t = "unknown"; + switch(dwMsgCode & 0xF000) { + case CERR_HINT: t=" hint"; break; + case CERR_WARNING: t="warning"; break; + case CERR_ERROR: t=" error"; break; + case CERR_FATAL: t=" fatal"; break; + } + printf("line %d %s %04.4x: %s\n", line, t, (unsigned int)dwMsgCode, szText); + return 1; +} + +bool loadfileProc(const char* filename, const char* baseFilename, void* data, int* size, void* context) { + std::string resolvedFilename = filename; + if(baseFilename && *baseFilename && IsRelativePath(filename)) { + char* p; + if ((p = strrchr_slash((char*)baseFilename)) != nullptr) { + std::string basePath = std::string(baseFilename, (int)(p - baseFilename + 1)); + resolvedFilename = basePath; + resolvedFilename.append(filename); + } + } + + FILE* fp = Open_File(resolvedFilename.c_str(), "rb"); + if(!fp) { + return false; + } + + if(!data) { + // return size + if(fseek(fp, 0, SEEK_END) != 0) { + fclose(fp); + return false; + } + *size = ftell(fp); + if(*size == -1L) { + fclose(fp); + return false; + } + } else { + // return data + if(fread(data, 1, *size, fp) != *size) { + fclose(fp); + return false; + } + } + fclose(fp); + return true; +} \ No newline at end of file diff --git a/developer/src/kmcmplib/tests/util_callbacks.h b/developer/src/kmcmplib/tests/util_callbacks.h new file mode 100644 index 0000000000..345de64b9f --- /dev/null +++ b/developer/src/kmcmplib/tests/util_callbacks.h @@ -0,0 +1,8 @@ +#pragma once + +#include + +int msgproc(int line, uint32_t dwMsgCode, const char* szText, void* context); +bool loadfileProc(const char* filename, const char* baseFilename, void* data, int* size, void* context); + +extern std::vector error_vec; \ No newline at end of file diff --git a/developer/src/kmcmplib/src/filesystem.cpp b/developer/src/kmcmplib/tests/util_filesystem.cpp similarity index 84% rename from developer/src/kmcmplib/src/filesystem.cpp rename to developer/src/kmcmplib/tests/util_filesystem.cpp index a343641acc..7f9edd6b36 100644 --- a/developer/src/kmcmplib/src/filesystem.cpp +++ b/developer/src/kmcmplib/tests/util_filesystem.cpp @@ -7,7 +7,7 @@ #include #include #include -#include "filesystem.h" +#include "util_filesystem.h" #ifdef _MSC_VER #include @@ -186,3 +186,42 @@ KMX_BOOL kmcmp_FileExists(const KMX_WCHAR* filename) { return FALSE; }; + + +bool IsRelativePath(KMX_CHAR const * p) { + // Relative path (returns TRUE): + // ..\...\BITMAP.BMP + // PATH\BITMAP.BMP + // BITMAP.BMP + + // Semi-absolute path (returns FALSE): + // \...\BITMAP.BMP + + // Absolute path (returns FALSE): + // C:\...\BITMAP.BMP + // \\SERVER\SHARE\...\BITMAP.BMP + + if ((*p == '\\') || (*p == '/')) return FALSE; + if (*p && *(p + 1) == ':') return FALSE; + + return TRUE; +} + +bool IsRelativePath(KMX_WCHAR const * p) { + // Relative path (returns TRUE): + // ..\...\BITMAP.BMP + // PATH\BITMAP.BMP + // BITMAP.BMP + + // Semi-absolute path (returns FALSE): + // \...\BITMAP.BMP + + // Absolute path (returns FALSE): + // C:\...\BITMAP.BMP + // \\SERVER\SHARE\...\BITMAP.BMP + + if ((*p == u'\\') || (*p == u'/'))return FALSE; + if (*p && *(p + 1) == u':') return FALSE; + + return TRUE; +} \ No newline at end of file diff --git a/developer/src/kmcmplib/src/filesystem.h b/developer/src/kmcmplib/tests/util_filesystem.h similarity index 82% rename from developer/src/kmcmplib/src/filesystem.h rename to developer/src/kmcmplib/tests/util_filesystem.h index 786b7268ac..4c94dae6bf 100644 --- a/developer/src/kmcmplib/src/filesystem.h +++ b/developer/src/kmcmplib/tests/util_filesystem.h @@ -1,7 +1,7 @@ #pragma once #include -#include "kmx_u16.h" +#include "../src/kmx_u16.h" // Opens files on windows and non-windows platforms. Datatypes for Filename and mode must be the same. // returns FILE* if file could be opened; FILE needs to be closed in calling function @@ -10,3 +10,6 @@ FILE* Open_File(const KMX_WCHART* Filename, const KMX_WCHART* mode); FILE* Open_File(const KMX_WCHAR* Filename, const KMX_WCHAR* mode); KMX_BOOL kmcmp_FileExists(const KMX_CHAR *filename); KMX_BOOL kmcmp_FileExists(const KMX_WCHAR *filename); + +bool IsRelativePath(KMX_CHAR const * p); +bool IsRelativePath(KMX_WCHAR const * p); From 6f4f20acb8e90d9c25f6b5da56299b3148580f07 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 31 May 2023 12:13:59 +1000 Subject: [PATCH 3/6] chore(developer): Add TODO issue numbers to PR --- developer/src/kmc-kmn/src/compiler/compiler.ts | 8 ++++---- developer/src/kmcmplib/include/kmcmplibapi.h | 2 +- developer/src/kmcmplib/src/CheckFilenameConsistency.cpp | 2 +- developer/src/kmcmplib/src/Compiler.cpp | 2 +- developer/src/kmcmplib/src/CompilerInterfaces.cpp | 3 +++ developer/src/kmcmplib/src/meson.build | 5 ----- developer/src/kmcmplib/tests/api-test.cpp | 2 +- developer/src/kmcmplib/tests/meson.build | 2 +- 8 files changed, 12 insertions(+), 14 deletions(-) diff --git a/developer/src/kmc-kmn/src/compiler/compiler.ts b/developer/src/kmc-kmn/src/compiler/compiler.ts index b27440fd06..3c6df80e7e 100644 --- a/developer/src/kmc-kmn/src/compiler/compiler.ts +++ b/developer/src/kmc-kmn/src/compiler/compiler.ts @@ -120,7 +120,7 @@ export class KmnCompiler { } private loadFileCallback = (filename: string, baseFilename: string, buffer: number, bufferSize: number): number => { - // TODO: we can optimize this in future by avoiding loading the file twice + // TODO: we can optimize this in future by avoiding loading the file twice #8885 let resolvedFilename = this.callbacks.resolveFilename(baseFilename, filename); let data = this.callbacks.loadFile(resolvedFilename); if(!data) { @@ -133,7 +133,7 @@ export class KmnCompiler { } if(bufferSize != data.byteLength) { - // TODO: consider chucking a wobbly because this is a bug + // TODO: consider chucking a wobbly because this is a bug #8885 /* c8 ignore next 2 */ return 0; } @@ -195,7 +195,7 @@ export class KmnCompiler { reader.validate(kvks, this.callbacks.loadSchema('kvks')); } catch(e) { console.log(e); - // TODO: also unit test + // TODO: also unit test #8886 // TODO: this.callbacks.reportMessage(CompilerMessages.Error_InvalidKvksFile({e})); return null; } @@ -203,7 +203,7 @@ export class KmnCompiler { let vk = reader.transform(kvks, errors); if(!vk || errors.length) { console.dir(errors); - // TODO: also unit test + // TODO: also unit test #8886 // TODO: this.callbacks.reportMessage(CompilerMessages.Error_InvalidKvksFile({e})); return null; } diff --git a/developer/src/kmcmplib/include/kmcmplibapi.h b/developer/src/kmcmplib/include/kmcmplibapi.h index 5c2ae46fda..0b41af6990 100644 --- a/developer/src/kmcmplib/include/kmcmplibapi.h +++ b/developer/src/kmcmplib/include/kmcmplibapi.h @@ -35,7 +35,7 @@ struct KMCMP_COMPILER_RESULT { std::string kvksFilename; }; -// TODO: parameters in UTF-8 +// TODO: parameters in UTF-8 #8887 typedef int (*kmcmp_CompilerMessageProc)(int line, uint32_t dwMsgCode, const char* szText, void* context); // parameters in UTF-8 diff --git a/developer/src/kmcmplib/src/CheckFilenameConsistency.cpp b/developer/src/kmcmplib/src/CheckFilenameConsistency.cpp index 3a185bc403..6d113d116b 100644 --- a/developer/src/kmcmplib/src/CheckFilenameConsistency.cpp +++ b/developer/src/kmcmplib/src/CheckFilenameConsistency.cpp @@ -24,7 +24,7 @@ KMX_DWORD CheckFilenameConsistency( KMX_CHAR const * Filename, bool ReportMissin KMX_DWORD CheckFilenameConsistency(KMX_WCHAR const * Filename, bool ReportMissingFile) { // TODO: we no longer have filesystem access here. We could move this check to // kmc itself, and make it consistent across all compilers that use the same - // loader callback + // loader callback -- see #8883 return CERR_None; #if 0 diff --git a/developer/src/kmcmplib/src/Compiler.cpp b/developer/src/kmcmplib/src/Compiler.cpp index 91e02e14e7..49484b1acc 100644 --- a/developer/src/kmcmplib/src/Compiler.cpp +++ b/developer/src/kmcmplib/src/Compiler.cpp @@ -3428,7 +3428,7 @@ bool UTF16TempFromUTF8(KMX_BYTE* infile, int sz, KMX_BYTE** tempfile, int *sz16) result = converter.from_bytes((char*)infile, (char*)infile+sz); } catch(std::range_error e) { UErrorCode status = U_ZERO_ERROR; - // TODO: we need ICU data files here @srl295 plz help! + // TODO: we need ICU data files here #8884 UConverter* conv = ucnv_open("windows-1252", &status); if(U_FAILURE(status)) { return FALSE; diff --git a/developer/src/kmcmplib/src/CompilerInterfaces.cpp b/developer/src/kmcmplib/src/CompilerInterfaces.cpp index 1a4e77f8e8..16b5b98654 100644 --- a/developer/src/kmcmplib/src/CompilerInterfaces.cpp +++ b/developer/src/kmcmplib/src/CompilerInterfaces.cpp @@ -13,6 +13,9 @@ bool CompileKeyboardHandle(KMX_BYTE* infile, int sz, PFILE_KEYBOARD fk); #ifdef __EMSCRIPTEN__ +// TODO: move emscripten wrappers into their own .cpp. Also move CompileKeyboardHandle +// into its own .cpp, so CompilerInterfaces.cpp has only C public API functions listed. +// #8889 /* WASM interface for compiler message callback diff --git a/developer/src/kmcmplib/src/meson.build b/developer/src/kmcmplib/src/meson.build index a81efac562..e697a0dced 100644 --- a/developer/src/kmcmplib/src/meson.build +++ b/developer/src/kmcmplib/src/meson.build @@ -29,11 +29,6 @@ if cpp_compiler.get_id() == 'emscripten' flags += ['-fwasm-exceptions'] lib_links = ['--whole-archive', '-sMODULARIZE', '-sEXPORT_ES6'] links += ['-fwasm-exceptions', '--bind', '-sEXPORTED_RUNTIME_METHODS=[\'UTF8ToString\']'] - # tests are building as ES6 so we need to declare the file extension - # note that meson currently struggles with the sanitycheckc_cross.exe - # program, because it has a hard coded extension (.exe) which is not - # valid for node programs in module mode. - # name_suffix = '.mjs' endif icu = subproject('icu-for-uset', default_options: [ 'default_library=static', 'cpp_std=c++17', 'warning_level=0', 'werror=false']) diff --git a/developer/src/kmcmplib/tests/api-test.cpp b/developer/src/kmcmplib/tests/api-test.cpp index cdbabea4d5..cb0337caef 100644 --- a/developer/src/kmcmplib/tests/api-test.cpp +++ b/developer/src/kmcmplib/tests/api-test.cpp @@ -42,7 +42,7 @@ void setup() { /* TODO: tests to run: - 4. ANSI (no BOM of course) + 4. ANSI (no BOM of course) #8884 8. file without blank last line (cannot compare with fixture due to bug in kmcmpdll...) Hint to add: k004_ansi.kmn: Hint: 10A6 Keyman Developer has detected that the file has ANSI encoding. Consider converting this file to UTF-8 */ diff --git a/developer/src/kmcmplib/tests/meson.build b/developer/src/kmcmplib/tests/meson.build index cc51189a63..0d4c078ed3 100644 --- a/developer/src/kmcmplib/tests/meson.build +++ b/developer/src/kmcmplib/tests/meson.build @@ -84,7 +84,7 @@ valid_keyboard_tests = [ 'k001_utf16', 'k002_utf8_without_bom', 'k003_utf8_with_bom', - # 'k004_ansi', # TODO: enable ansi test when we have the icu datafiles + # 'k004_ansi', # TODO: enable ansi test when we have the icu datafiles #8884 'k005_bitmap', 'k006_icon', 'k007_includecodes_r_n', From e2930fa08058d17cbb4be9da462bea520e1d4505 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 31 May 2023 10:23:42 +0700 Subject: [PATCH 4/6] chore(developer): fix npm pack for kmc-ldml --- package-lock.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package-lock.json b/package-lock.json index 20d7ce98c4..75d9e1abef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1030,6 +1030,7 @@ } }, "developer/src/kmc-ldml": { + "name": "@keymanapp/kmc-ldml", "license": "MIT", "dependencies": { "@keymanapp/keyman-version": "*", From 9d351f4798d80ef71044ae80812a7cc7b2e08322 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 31 May 2023 15:22:54 +0700 Subject: [PATCH 5/6] chore(developer): verify kvks files and report errors Fixes #8886. .kvks compiler now returns helpful errors for xml parse failures and schema validation errors, and for invalid virtual key codes. Introduces extra infrastructure for reporting and unit testing messages to kmc-kmn. This also fixes unhandled xml load exceptions and simplifies the error reporting coming out of the kvks loader, on the basis that only one type of error was being reported anyway. --- common/web/types/src/kvk/kvks-file-reader.ts | 25 ++++------ .../web/types/test/kvk/test-kvk-round-trip.ts | 14 +++--- common/web/types/test/kvk/test-kvks-file.ts | 14 +++--- .../src/kmc-kmn/src/compiler/compiler.ts | 48 +++++++++---------- .../src/kmc-kmn/src/compiler/messages.ts | 8 ++++ .../error_invalid_kvks_file.kmn | 12 +++++ .../error_invalid_kvks_file.kvks | 9 ++++ .../warn_invalid_vkey_in_kvks_file.kmn | 12 +++++ .../warn_invalid_vkey_in_kvks_file.kvks | 15 ++++++ developer/src/kmc-kmn/test/helpers/index.ts | 17 +++++++ developer/src/kmc-kmn/test/test-messages.ts | 47 +++++++++++++++++- developer/src/kmc-kmn/test/tsconfig.json | 3 +- 12 files changed, 167 insertions(+), 57 deletions(-) create mode 100644 developer/src/kmc-kmn/test/fixtures/invalid-keyboards/error_invalid_kvks_file.kmn create mode 100644 developer/src/kmc-kmn/test/fixtures/invalid-keyboards/error_invalid_kvks_file.kvks create mode 100644 developer/src/kmc-kmn/test/fixtures/invalid-keyboards/warn_invalid_vkey_in_kvks_file.kmn create mode 100644 developer/src/kmc-kmn/test/fixtures/invalid-keyboards/warn_invalid_vkey_in_kvks_file.kvks create mode 100644 developer/src/kmc-kmn/test/helpers/index.ts diff --git a/common/web/types/src/kvk/kvks-file-reader.ts b/common/web/types/src/kvk/kvks-file-reader.ts index c12b7036e4..184c162210 100644 --- a/common/web/types/src/kvk/kvks-file-reader.ts +++ b/common/web/types/src/kvk/kvks-file-reader.ts @@ -7,12 +7,6 @@ import { VisualKeyboard, VisualKeyboardHeaderFlags, VisualKeyboardKey, VisualKey import { USVirtualKeyCodes } from '../consts/virtual-key-constants.js'; import { BUILDER_KVK_HEADER_VERSION } from './kvk-file.js'; -export enum KVKSParseErrorType { invalidVkey }; -export class KVKSParseError extends Error { - public type: KVKSParseErrorType; - public vkey: string; -}; - export default class KVKSFileReader { public read(file: Uint8Array): KVKSourceFile { let source: KVKSourceFile; @@ -33,10 +27,12 @@ export default class KVKSFileReader { // rather than using the version tagged on npmjs.com. }); - parser.parseString(file, (e: unknown, r: unknown) => { source = r as KVKSourceFile }); - source = this.boxArrays(source); - this.cleanupFlags(source); - this.cleanupUnderscore('visualkeyboard', source.visualkeyboard); + parser.parseString(file, (e: unknown, r: unknown) => { if(e) { throw e }; source = r as KVKSourceFile }); + if(source) { + source = this.boxArrays(source); + this.cleanupFlags(source); + this.cleanupUnderscore('visualkeyboard', source.visualkeyboard); + } return source; } @@ -80,7 +76,7 @@ export default class KVKSFileReader { } } - public transform(source: KVKSourceFile, errors?: KVKSParseError[]): VisualKeyboard { + public transform(source: KVKSourceFile, invalidVkeys?: string[]): VisualKeyboard { // NOTE: at this point, the xml should have been validated // and matched the schema result so we can assume properties exist let result: VisualKeyboard = { @@ -118,11 +114,8 @@ export default class KVKSFileReader { for(let sourceKey of layer.key) { let vkey = (USVirtualKeyCodes as any)[sourceKey.$?.vkey]; if(!vkey) { - if(errors) { - let e = new KVKSParseError(); - e.type = KVKSParseErrorType.invalidVkey; - e.vkey = sourceKey.$?.vkey; - errors.push(e); + if(typeof invalidVkeys !== 'undefined') { + invalidVkeys.push(sourceKey.$?.vkey); } continue; } diff --git a/common/web/types/test/kvk/test-kvk-round-trip.ts b/common/web/types/test/kvk/test-kvk-round-trip.ts index 41dbe2dbc0..031ad60b9e 100644 --- a/common/web/types/test/kvk/test-kvk-round-trip.ts +++ b/common/web/types/test/kvk/test-kvk-round-trip.ts @@ -5,7 +5,7 @@ import Hexy from 'hexy'; import gitDiff from 'git-diff'; const { hexy } = Hexy; import { loadSchema, makePathToFixture } from '../helpers/index.js'; -import KvksFileReader, { KVKSParseError } from "../../src/kvk/kvks-file-reader.js"; +import KvksFileReader from "../../src/kvk/kvks-file-reader.js"; import KvkFileReader from "../../src/kvk/kvk-file-reader.js"; import KvkFileWriter from "../../src/kvk/kvk-file-writer.js"; import KvksFileWriter from "../../src/kvk/kvks-file-writer.js"; @@ -53,9 +53,9 @@ describe('kvks-file-reader', function () { assert.doesNotThrow(() => { reader.validate(kvks, loadSchema('kvks')); }); - const errors: KVKSParseError[] = []; - const vk = reader.transform(kvks, errors); - assert.isEmpty(errors); + const invalidVkeys: string[] = []; + const vk = reader.transform(kvks, invalidVkeys); + assert.isEmpty(invalidVkeys); const writer = new KvkFileWriter(); const output = writer.write(vk); assertBufferMatch(Buffer.from(output), compiled); @@ -76,9 +76,9 @@ describe('kvks-file-reader', function () { assert.doesNotThrow(() => { kvksReader.validate(kvks, loadSchema('kvks')); }); - const errors: KVKSParseError[] = []; - const vk2 = kvksReader.transform(kvks, errors); - assert.isEmpty(errors); + const invalidVkeys: string[] = []; + const vk2 = kvksReader.transform(kvks, invalidVkeys); + assert.isEmpty(invalidVkeys); // make sure the binary is the same assert.deepEqual(vk2, vk); diff --git a/common/web/types/test/kvk/test-kvks-file.ts b/common/web/types/test/kvk/test-kvks-file.ts index 523bfaf54d..4cbb299a00 100644 --- a/common/web/types/test/kvk/test-kvks-file.ts +++ b/common/web/types/test/kvk/test-kvks-file.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import 'mocha'; import { loadSchema, makePathToFixture } from '../helpers/index.js'; -import KvksFileReader, { KVKSParseError } from "../../src/kvk/kvks-file-reader.js"; +import KvksFileReader from "../../src/kvk/kvks-file-reader.js"; import KvksFileWriter from "../../src/kvk/kvks-file-writer.js"; import { verify_khmer_angkor } from './test-kvk-utils.js'; import { assert } from 'chai'; @@ -16,9 +16,9 @@ describe('kvks-file-reader', function() { assert.doesNotThrow(() => { reader.validate(kvks, loadSchema('kvks')); }); - const errors: KVKSParseError[] = []; - const vk = reader.transform(kvks, errors); - assert.isEmpty(errors); + const invalidVkeys: string[] = []; + const vk = reader.transform(kvks, invalidVkeys); + assert.isEmpty(invalidVkeys); verify_khmer_angkor(vk); }); }); @@ -30,9 +30,9 @@ describe('kvks-file-writer', function() { const reader = new KvksFileReader(); const kvksExpected = reader.read(input); - const errors: KVKSParseError[] = []; - const vk = reader.transform(kvksExpected, errors); - assert.isEmpty(errors); + const invalidVkeys: string[] = []; + const vk = reader.transform(kvksExpected, invalidVkeys); + assert.isEmpty(invalidVkeys); const writer = new KvksFileWriter(); const output = writer.write(vk); diff --git a/developer/src/kmc-kmn/src/compiler/compiler.ts b/developer/src/kmc-kmn/src/compiler/compiler.ts index 3c6df80e7e..4761fc5c3b 100644 --- a/developer/src/kmc-kmn/src/compiler/compiler.ts +++ b/developer/src/kmc-kmn/src/compiler/compiler.ts @@ -88,18 +88,7 @@ export class KmnCompiler { } public run(infile: string, outfile: string, options?: CompilerOptions): boolean { - if(!this.verifyInitialized()) { - /* c8 ignore next 2 */ - return false; - } - - options = {...baseOptions, ...options}; - (globalThis as any)[this.callbackID] = { - message: this.compilerMessageCallback, - loadFile: this.loadFileCallback - }; let result = this.runCompiler(infile, outfile, options); - delete (globalThis as any)[this.callbackID]; if(result) { if(result.kmx) { this.callbacks.fs.writeFileSync(result.kmx.filename, result.kmx.data); @@ -133,9 +122,8 @@ export class KmnCompiler { } if(bufferSize != data.byteLength) { - // TODO: consider chucking a wobbly because this is a bug #8885 /* c8 ignore next 2 */ - return 0; + throw new Error(`Second call, expected file size ${bufferSize} == ${data.byteLength}`); } this.Module.HEAP8.set(data, buffer); @@ -143,7 +131,19 @@ export class KmnCompiler { return 1; } - private runCompiler(infile: string, outfile: string, options: CompilerOptions): CompilerResult { + public runCompiler(infile: string, outfile: string, options: CompilerOptions): CompilerResult { + if(!this.verifyInitialized()) { + /* c8 ignore next 2 */ + return null; + } + + options = {...baseOptions, ...options}; + + (globalThis as any)[this.callbackID] = { + message: this.compilerMessageCallback, + loadFile: this.loadFileCallback + }; + let result: CompilerResult = {}; let wasm_interface = new this.Module.CompilerInterface(); let wasm_options = new this.Module.CompilerOptions(); @@ -183,6 +183,7 @@ export class KmnCompiler { } wasm_interface.delete(); wasm_options.delete(); + delete (globalThis as any)[this.callbackID]; } } @@ -190,22 +191,19 @@ export class KmnCompiler { // The compiler detected a .kvks file, which needs to be captured let reader = new KvksFileReader(); kvksFilename = this.callbacks.resolveFilename(kmnFilename, kvksFilename); - let kvks = reader.read(this.callbacks.loadFile(kvksFilename)); + let filename = this.callbacks.path.basename(kvksFilename); + let kvks = null; try { + kvks = reader.read(this.callbacks.loadFile(kvksFilename)); reader.validate(kvks, this.callbacks.loadSchema('kvks')); } catch(e) { - console.log(e); - // TODO: also unit test #8886 - // TODO: this.callbacks.reportMessage(CompilerMessages.Error_InvalidKvksFile({e})); + this.callbacks.reportMessage(CompilerMessages.Error_InvalidKvksFile({filename, e})); return null; } - let errors: any = []; //TODO: KVKSParseError[]; - let vk = reader.transform(kvks, errors); - if(!vk || errors.length) { - console.dir(errors); - // TODO: also unit test #8886 - // TODO: this.callbacks.reportMessage(CompilerMessages.Error_InvalidKvksFile({e})); - return null; + let invalidVkeys: string[] = []; + let vk = reader.transform(kvks, invalidVkeys); + for(let invalidVkey of invalidVkeys) { + this.callbacks.reportMessage(CompilerMessages.Warn_InvalidVkeyInKvksFile({filename, invalidVkey})); } let writer = new KvkFileWriter(); return { diff --git a/developer/src/kmc-kmn/src/compiler/messages.ts b/developer/src/kmc-kmn/src/compiler/messages.ts index 10578870d7..0769096e07 100644 --- a/developer/src/kmc-kmn/src/compiler/messages.ts +++ b/developer/src/kmc-kmn/src/compiler/messages.ts @@ -67,6 +67,14 @@ export class CompilerMessages { static Error_UnicodeSetSyntaxError = () => m(this.ERROR_UnicodeSetSyntaxError, `UnicodeSet had a Syntax Error while parsing`); static ERROR_UnicodeSetSyntaxError = SevError | 0x1007; + + static Error_InvalidKvksFile = (o:{filename: string, e: any}) => m(this.ERROR_InvalidKvksFile, + `Error encountered parsing ${o.filename}: ${o.e}`); + static ERROR_InvalidKvksFile = SevError | 0x1008; + + static Warn_InvalidVkeyInKvksFile = (o:{filename: string, invalidVkey: string}) => m(this.WARN_InvalidVkeyInKvksFile, + `Invalid virtual key ${o.invalidVkey} found in ${o.filename}`); + static WARN_InvalidVkeyInKvksFile = SevWarn | 0x1009; } export function mapErrorFromKmcmplib(line: number, code: number, msg: string): CompilerEvent { diff --git a/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/error_invalid_kvks_file.kmn b/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/error_invalid_kvks_file.kmn new file mode 100644 index 0000000000..f37d6397c5 --- /dev/null +++ b/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/error_invalid_kvks_file.kmn @@ -0,0 +1,12 @@ +c Description: Verifies that kmc throws an error with an invalid .kvks file + +store(&NAME) 'error_invalid_kvks_file' +store(&VERSION) '9.0' +store(&VISUALKEYBOARD) 'error_invalid_kvks_file.kvks' + +begin unicode > use(main) + +group(main) using keys + ++ [K_A] > 'a' +'a' + [K_B] > 'ខ្មែរ' diff --git a/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/error_invalid_kvks_file.kvks b/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/error_invalid_kvks_file.kvks new file mode 100644 index 0000000000..c681064f5b --- /dev/null +++ b/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/error_invalid_kvks_file.kvks @@ -0,0 +1,9 @@ + + +
+ 10.0 + caps_lock_layer_3620 + + +
+ diff --git a/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/warn_invalid_vkey_in_kvks_file.kmn b/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/warn_invalid_vkey_in_kvks_file.kmn new file mode 100644 index 0000000000..18a88973e9 --- /dev/null +++ b/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/warn_invalid_vkey_in_kvks_file.kmn @@ -0,0 +1,12 @@ +c Description: Verifies that kmc reports a warning with an invalid vkey in the kvks + +store(&NAME) 'warn_invalid_vkey_in_kvks_file' +store(&VERSION) '9.0' +store(&VISUALKEYBOARD) 'warn_invalid_vkey_in_kvks_file.kvks' + +begin unicode > use(main) + +group(main) using keys + ++ [K_A] > 'a' +'a' + [K_B] > 'ខ្មែរ' diff --git a/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/warn_invalid_vkey_in_kvks_file.kvks b/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/warn_invalid_vkey_in_kvks_file.kvks new file mode 100644 index 0000000000..ff441fd7c5 --- /dev/null +++ b/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/warn_invalid_vkey_in_kvks_file.kvks @@ -0,0 +1,15 @@ + + +
+ 10.0 + caps_lock_layer_3620 + + +
+ + + + ឞ + + + diff --git a/developer/src/kmc-kmn/test/helpers/index.ts b/developer/src/kmc-kmn/test/helpers/index.ts new file mode 100644 index 0000000000..8250dfd101 --- /dev/null +++ b/developer/src/kmc-kmn/test/helpers/index.ts @@ -0,0 +1,17 @@ +/** + * Helpers and utilities for the Mocha tests. + */ +import * as path from 'path'; +import { fileURLToPath } from 'url'; + +/** + * Builds a path to the fixture with the given path components. + * + * e.g., makePathToFixture('example.qaa.trivial') + * e.g., makePathToFixture('example.qaa.trivial', 'model.ts') + * + * @param components One or more path components. + */ + export function makePathToFixture(...components: string[]): string { + return fileURLToPath(new URL(path.join('..', '..', '..', 'test', 'fixtures', ...components), import.meta.url)); +} diff --git a/developer/src/kmc-kmn/test/test-messages.ts b/developer/src/kmc-kmn/test/test-messages.ts index 0dee53f121..dce0c2477d 100644 --- a/developer/src/kmc-kmn/test/test-messages.ts +++ b/developer/src/kmc-kmn/test/test-messages.ts @@ -1,9 +1,54 @@ import 'mocha'; +import { assert } from 'chai'; import { CompilerMessages } from '../src/compiler/messages.js'; -import { verifyCompilerMessagesObject } from '@keymanapp/developer-test-helpers'; +import { TestCompilerCallbacks, verifyCompilerMessagesObject } from '@keymanapp/developer-test-helpers'; +import { makePathToFixture } from './helpers/index.js'; +import { KmnCompiler } from '../src/main.js'; describe('CompilerMessages', function () { + const callbacks = new TestCompilerCallbacks(); + it('should have a valid CompilerMessages object', function() { return verifyCompilerMessagesObject(CompilerMessages); }); + + // + // Message tests + // + + async function testForMessage(context: Mocha.Context, fixture: string[], messageId?: number) { + context.timeout(10000); + + callbacks.clear(); + + const compiler = new KmnCompiler(); + assert(await compiler.init(callbacks)); + assert(compiler.verifyInitialized()); + + const kmnPath = makePathToFixture(...fixture); + const outfile = callbacks.path.basename(kmnPath, '.kmn') + '.kmx'; + + // Note: throwing away compile results (just to memory) + compiler.runCompiler(kmnPath, outfile, {saveDebug: true, shouldAddCompilerVersion: false}); + + if(messageId) { + assert.isTrue(callbacks.hasMessage(messageId), `messageId ${messageId.toString(16)} not generated, instead got: `+JSON.stringify(callbacks.messages,null,2)); + assert.lengthOf(callbacks.messages, 1); + } else { + assert.lengthOf(callbacks.messages, 0, `messages should be empty, but instead got: `+JSON.stringify(callbacks.messages,null,2)); + } + } + + // ERROR_InvalidKvksFile + + it('should generate ERROR_InvalidKvksFile if the kvks is not valid XML', async function() { + await testForMessage(this, ['invalid-keyboards', 'error_invalid_kvks_file.kmn'], CompilerMessages.ERROR_InvalidKvksFile); + }); + + // WARN_InvalidVkeyInKvksFile + + it('should generate WARN_InvalidVkeyInKvksFile if the kvks contains an invalid virtual key', async function() { + await testForMessage(this, ['invalid-keyboards', 'warn_invalid_vkey_in_kvks_file.kmn'], CompilerMessages.WARN_InvalidVkeyInKvksFile); + }); + }); diff --git a/developer/src/kmc-kmn/test/tsconfig.json b/developer/src/kmc-kmn/test/tsconfig.json index 115412f57f..f61df15d4a 100644 --- a/developer/src/kmc-kmn/test/tsconfig.json +++ b/developer/src/kmc-kmn/test/tsconfig.json @@ -13,7 +13,8 @@ }, }, "include": [ - "**/test-*.ts" + "**/test-*.ts", + "./helpers/index.ts" ], "references": [ { "path": "../../../../common/web/keyman-version/tsconfig.esm.json" }, From 717b1eea9889db65c12c27e33ed871a7c0e85f6f Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 1 Jun 2023 08:04:34 +1000 Subject: [PATCH 6/6] chore: Apply code review suggestion Co-authored-by: Eberhard Beilharz --- .../invalid-keyboards/warn_invalid_vkey_in_kvks_file.kvks | 1 - 1 file changed, 1 deletion(-) diff --git a/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/warn_invalid_vkey_in_kvks_file.kvks b/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/warn_invalid_vkey_in_kvks_file.kvks index ff441fd7c5..f7b247db34 100644 --- a/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/warn_invalid_vkey_in_kvks_file.kvks +++ b/developer/src/kmc-kmn/test/fixtures/invalid-keyboards/warn_invalid_vkey_in_kvks_file.kvks @@ -3,7 +3,6 @@
10.0 caps_lock_layer_3620 -