From 035c447a315b1612435b21653dcc5c3ac5adc722 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 10 Apr 2023 15:21:45 +0700 Subject: [PATCH 1/5] chore(core): move test_assert.h and test_color.h to common --- {core/tests/unit => common/include}/test_assert.h | 0 {core/tests/unit => common/include}/test_color.h | 0 core/tests/unit/kmnkbd/action_items.hpp | 2 +- core/tests/unit/kmnkbd/debug_api.cpp | 2 +- core/tests/unit/kmnkbd/debug_items.hpp | 2 +- core/tests/unit/kmnkbd/state_api.cpp | 2 +- core/tests/unit/kmnkbd/test_kmx_context.cpp | 2 +- core/tests/unit/kmnkbd/test_kmx_xstring.cpp | 2 +- core/tests/unit/kmx/kmx.cpp | 4 ++-- core/tests/unit/kmx/kmx_imx.cpp | 4 ++-- core/tests/unit/kmx/kmx_key_list.cpp | 4 ++-- core/tests/unit/ldml/ldml.cpp | 4 ++-- core/tests/unit/ldml/ldml_test_source.cpp | 2 +- core/tests/unit/ldml/test_kmx_plus.cpp | 2 +- 14 files changed, 16 insertions(+), 16 deletions(-) rename {core/tests/unit => common/include}/test_assert.h (100%) rename {core/tests/unit => common/include}/test_color.h (100%) diff --git a/core/tests/unit/test_assert.h b/common/include/test_assert.h similarity index 100% rename from core/tests/unit/test_assert.h rename to common/include/test_assert.h diff --git a/core/tests/unit/test_color.h b/common/include/test_color.h similarity index 100% rename from core/tests/unit/test_color.h rename to common/include/test_color.h diff --git a/core/tests/unit/kmnkbd/action_items.hpp b/core/tests/unit/kmnkbd/action_items.hpp index 676e904eed..253b8c740a 100644 --- a/core/tests/unit/kmnkbd/action_items.hpp +++ b/core/tests/unit/kmnkbd/action_items.hpp @@ -2,7 +2,7 @@ #include #include -#include "../test_assert.h" +#include namespace { diff --git a/core/tests/unit/kmnkbd/debug_api.cpp b/core/tests/unit/kmnkbd/debug_api.cpp index 18e1e3c108..b4cfae6ab3 100644 --- a/core/tests/unit/kmnkbd/debug_api.cpp +++ b/core/tests/unit/kmnkbd/debug_api.cpp @@ -18,7 +18,7 @@ #include "action_items.hpp" #include "debug_items.hpp" -#include "../test_assert.h" +#include #include "../emscripten_filesystem.h" using namespace km::kbp::kmx; diff --git a/core/tests/unit/kmnkbd/debug_items.hpp b/core/tests/unit/kmnkbd/debug_items.hpp index 137325d731..95703005c4 100644 --- a/core/tests/unit/kmnkbd/debug_items.hpp +++ b/core/tests/unit/kmnkbd/debug_items.hpp @@ -4,7 +4,7 @@ #include "kmx/kmx_base.h" #include "kmx/kmx_xstring.h" -#include "../test_assert.h" +#include using namespace km::kbp::kmx; diff --git a/core/tests/unit/kmnkbd/state_api.cpp b/core/tests/unit/kmnkbd/state_api.cpp index 37964489b7..3ae446541c 100644 --- a/core/tests/unit/kmnkbd/state_api.cpp +++ b/core/tests/unit/kmnkbd/state_api.cpp @@ -15,7 +15,7 @@ #include "state.hpp" #include "action_items.hpp" -#include "../test_assert.h" +#include #if defined(__GNUC__) || defined(__clang__) #define PRAGMA(X) _Pragma(#X) diff --git a/core/tests/unit/kmnkbd/test_kmx_context.cpp b/core/tests/unit/kmnkbd/test_kmx_context.cpp index a05dfed543..1023d1611e 100644 --- a/core/tests/unit/kmnkbd/test_kmx_context.cpp +++ b/core/tests/unit/kmnkbd/test_kmx_context.cpp @@ -13,7 +13,7 @@ #include "../../../src/kmx/kmx_context.h" #include "../../../src/kmx/kmx_processevent.h" // for Debug_UnicodeString #include "../../../src/kmx/kmx_xstring.h" -#include "../test_assert.h" +#include using namespace km::kbp::kmx; using namespace std; diff --git a/core/tests/unit/kmnkbd/test_kmx_xstring.cpp b/core/tests/unit/kmnkbd/test_kmx_xstring.cpp index c4b94c52b2..d578fa7e38 100644 --- a/core/tests/unit/kmnkbd/test_kmx_xstring.cpp +++ b/core/tests/unit/kmnkbd/test_kmx_xstring.cpp @@ -19,7 +19,7 @@ #include #include "../../../src/kmx/kmx_xstring.h" #include -#include "../test_assert.h" +#include using namespace km::kbp::kmx; using namespace std; diff --git a/core/tests/unit/kmx/kmx.cpp b/core/tests/unit/kmx/kmx.cpp index 251651e882..a0c8eaa1b2 100644 --- a/core/tests/unit/kmx/kmx.cpp +++ b/core/tests/unit/kmx/kmx.cpp @@ -24,8 +24,8 @@ #include "state.hpp" #include "utfcodec.hpp" -#include "../test_assert.h" -#include "../test_color.h" +#include +#include #include "../emscripten_filesystem.h" #include "kmx_test_source.hpp" diff --git a/core/tests/unit/kmx/kmx_imx.cpp b/core/tests/unit/kmx/kmx_imx.cpp index eed4f18ad3..b935257246 100644 --- a/core/tests/unit/kmx/kmx_imx.cpp +++ b/core/tests/unit/kmx/kmx_imx.cpp @@ -11,8 +11,8 @@ #include "path.hpp" #include "state.hpp" #include "../kmnkbd/action_items.hpp" -#include "../test_assert.h" -#include "../test_color.h" +#include +#include #include "../emscripten_filesystem.h" #include diff --git a/core/tests/unit/kmx/kmx_key_list.cpp b/core/tests/unit/kmx/kmx_key_list.cpp index dc40082173..d4750c44d3 100644 --- a/core/tests/unit/kmx/kmx_key_list.cpp +++ b/core/tests/unit/kmx/kmx_key_list.cpp @@ -12,8 +12,8 @@ #include "state.hpp" #include -#include "../test_assert.h" -#include "../test_color.h" +#include +#include #include "../emscripten_filesystem.h" using namespace km::kbp::kmx; diff --git a/core/tests/unit/ldml/ldml.cpp b/core/tests/unit/ldml/ldml.cpp index 26210ab2f3..6308625c0c 100644 --- a/core/tests/unit/ldml/ldml.cpp +++ b/core/tests/unit/ldml/ldml.cpp @@ -21,8 +21,8 @@ #include "state.hpp" #include "utfcodec.hpp" -#include "../test_assert.h" -#include "../test_color.h" +#include +#include #include // for surrogate pair macros diff --git a/core/tests/unit/ldml/ldml_test_source.cpp b/core/tests/unit/ldml/ldml_test_source.cpp index fa7afa15eb..817116bf97 100644 --- a/core/tests/unit/ldml/ldml_test_source.cpp +++ b/core/tests/unit/ldml/ldml_test_source.cpp @@ -45,7 +45,7 @@ namespace km { namespace tests { -#include "../test_color.h" +#include LdmlTestSource::LdmlTestSource() { diff --git a/core/tests/unit/ldml/test_kmx_plus.cpp b/core/tests/unit/ldml/test_kmx_plus.cpp index 9518c0bd78..d0ed53b683 100644 --- a/core/tests/unit/ldml/test_kmx_plus.cpp +++ b/core/tests/unit/ldml/test_kmx_plus.cpp @@ -1,4 +1,4 @@ -#include "../test_assert.h" +#include #include "kmx/kmx_plus.h" #include "kmx/kmx_xstring.h" #include "../../../src/ldml/ldml_vkeys.hpp" From 59b981880ff2ecd5cf621020a79f66999829ee87 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 10 Apr 2023 15:23:17 +0700 Subject: [PATCH 2/5] chore(developer): cleanup SetError and return types for public apis in kmcmplib --- .../src/kmc-kmn/src/compiler/compiler.ts | 2 +- developer/src/kmcmpdll/Compiler.cpp | 12 -- developer/src/kmcmplib/include/kmcmplibapi.h | 9 +- developer/src/kmcmplib/src/Compiler.cpp | 103 ++++++++++-------- 4 files changed, 65 insertions(+), 61 deletions(-) diff --git a/developer/src/kmc-kmn/src/compiler/compiler.ts b/developer/src/kmc-kmn/src/compiler/compiler.ts index 5a5e08df6c..a380c21e94 100644 --- a/developer/src/kmc-kmn/src/compiler/compiler.ts +++ b/developer/src/kmc-kmn/src/compiler/compiler.ts @@ -1,7 +1,7 @@ /* TODO: implement additional interfaces: - extern "C" uint32_t kmcmp_CompileKeyboardFileToBuffer( + extern "C" bool kmcmp_CompileKeyboardFileToBuffer( char* pszInfile, void* pfkBuffer, bool ACompilerWarningsAsErrors, diff --git a/developer/src/kmcmpdll/Compiler.cpp b/developer/src/kmcmpdll/Compiler.cpp index ae522b987a..d7994d98b2 100644 --- a/developer/src/kmcmpdll/Compiler.cpp +++ b/developer/src/kmcmpdll/Compiler.cpp @@ -90,19 +90,7 @@ #include "UnreachableRules.h" #include "CheckForDuplicates.h" -/* - Following kmcmplib definitions are translated to Windows types - because we can't import the kmcmplib headers. The structure - sizes and alignments have been manually verified. -*/ - - #include "../kmcmplib/include/kmcmplibapi.h" -/*typedef int (*kmcmp_CompilerMessageProc)(int line, KMX_DWORD dwMsgCode, PKMX_STR szText, void* context); -extern "C" BOOL kmcmp_CompileKeyboardFile(PSTR pszInfile, PSTR pszOutfile, BOOL FSaveDebug, BOOL ACompilerWarningsAsErrors, BOOL AWarnDeprecatedCode, kmcmp_CompilerMessageProc pMsgProc, void* context); // I4865 // I4866 -extern "C" BOOL kmcmp_CompileKeyboardFileToBuffer(PSTR pszInfile, void* pfkBuffer, BOOL ACompilerWarningsAsErrors, BOOL AWarnDeprecatedCode, kmcmp_CompilerMessageProc pMsgProc, void* context, int Target); // I4865 // I4866 -extern "C" void kmcmp_Keyman_Diagnostic(int mode) ; -extern "C" BOOL kmcmp_SetCompilerOptions(PCOMPILER_OPTIONS options);*/ int xatoi(PWSTR *p); int atoiW(PWSTR p); diff --git a/developer/src/kmcmplib/include/kmcmplibapi.h b/developer/src/kmcmplib/include/kmcmplibapi.h index 7ea39ea53e..9cf28613fd 100644 --- a/developer/src/kmcmplib/include/kmcmplibapi.h +++ b/developer/src/kmcmplib/include/kmcmplibapi.h @@ -25,7 +25,7 @@ EXTERN bool kmcmp_SetCompilerOptions( typedef int (*kmcmp_CompilerMessageProc)(int line, uint32_t dwMsgCode, char* szText, void* context); -EXTERN uint32_t kmcmp_CompileKeyboardFile( +EXTERN bool kmcmp_CompileKeyboardFile( char* pszInfile, char* pszOutfile, bool ASaveDebug, @@ -35,7 +35,12 @@ EXTERN uint32_t kmcmp_CompileKeyboardFile( void* AmsgprocContext ); -EXTERN uint32_t kmcmp_CompileKeyboardFileToBuffer( +/* Compile target */ + +#define CKF_KEYMAN 0 +#define CKF_KEYMANWEB 1 + +EXTERN bool kmcmp_CompileKeyboardFileToBuffer( char* pszInfile, void* pfkBuffer, bool ACompilerWarningsAsErrors, diff --git a/developer/src/kmcmplib/src/Compiler.cpp b/developer/src/kmcmplib/src/Compiler.cpp index 73f230c022..83025c6823 100644 --- a/developer/src/kmcmplib/src/Compiler.cpp +++ b/developer/src/kmcmplib/src/Compiler.cpp @@ -163,7 +163,7 @@ KMX_DWORD ExpandKp(PFILE_KEYBOARD fk, PFILE_KEY kpp, KMX_DWORD storeIndex); KMX_DWORD ReadLine(FILE* fp_in , PKMX_WCHAR wstr, KMX_BOOL PreProcess); KMX_DWORD WriteCompiledKeyboard(PFILE_KEYBOARD fk, FILE* fp_out); -KMX_BOOL CompileKeyboardHandle(FILE* fp_in, PFILE_KEYBOARD fk); +bool CompileKeyboardHandle(FILE* fp_in, PFILE_KEYBOARD fk); int GetVKCode(PFILE_KEYBOARD fk, PKMX_WCHAR p); // I3438 // TODO: Consolidate GetDeadKey and GetVKCode? KMX_DWORD BuildVKDictionary(PFILE_KEYBOARD fk); // I3438 @@ -253,11 +253,6 @@ int kmcmp::currentLine = 0; kmcmp::NamedCodeConstants *CodeConstants = NULL; -/* Compile target */ - -#define CKF_KEYMAN 0 -#define CKF_KEYMANWEB 1 - PKMX_WCHAR strtowstr(PKMX_STR in) { PKMX_WCHAR result; @@ -351,14 +346,14 @@ int wasm_CompilerMessageProc(int line, uint32_t dwMsgCode, char* szText, void* c return wasm_msgproc(line, dwMsgCode, szText, msgProc); } -EXTERN uint32_t kmcmp_Wasm_SetCompilerOptions(int ShouldAddCompilerVersion) { +EXTERN bool kmcmp_Wasm_SetCompilerOptions(int ShouldAddCompilerVersion) { KMCMP_COMPILER_OPTIONS options; options.dwSize = sizeof(KMCMP_COMPILER_OPTIONS); options.ShouldAddCompilerVersion = ShouldAddCompilerVersion; return kmcmp_SetCompilerOptions(&options); } -EXTERN uint32_t kmcmp_Wasm_CompileKeyboardFile(char* pszInfile, +EXTERN bool kmcmp_Wasm_CompileKeyboardFile(char* pszInfile, char* pszOutfile, int ASaveDebug, int ACompilerWarningsAsErrors, int AWarnDeprecatedCode, char* msgProc ) { @@ -374,13 +369,12 @@ EXTERN uint32_t kmcmp_Wasm_CompileKeyboardFile(char* pszInfile, } #endif -EXTERN uint32_t kmcmp_CompileKeyboardFile(char* pszInfile, +EXTERN bool kmcmp_CompileKeyboardFile(char* pszInfile, char* pszOutfile, bool ASaveDebug, bool ACompilerWarningsAsErrors, bool AWarnDeprecatedCode, kmcmp_CompilerMessageProc pMsgproc, void* AmsgprocContext ) { FILE* fp_in = NULL; FILE* fp_out = NULL; - KMX_BOOL err; KMX_CHAR str[260]; //printf("°°-> changed to CompileKeyboardFile() of kmcmplib \n"); @@ -410,15 +404,15 @@ EXTERN uint32_t kmcmp_CompileKeyboardFile(char* pszInfile, fp_in = Open_File(pszInfile, "rb"); - if (fp_in == NULL) SetError(CERR_InfileNotExist); - + if (fp_in == NULL) { + SetError(CERR_InfileNotExist); + } // Transfer the file to a memory stream for processing UTF-8 or ANSI to UTF-16? // What about really large files? Transfer to a temp file... - if (!fread(str, 1, 3, fp_in)) - { + if (!fread(str, 1, 3, fp_in)) { fclose(fp_in); - return CERR_CannotReadInfile; + SetError(CERR_CannotReadInfile); } fseek(fp_in, 0, SEEK_SET); @@ -428,27 +422,28 @@ EXTERN uint32_t kmcmp_CompileKeyboardFile(char* pszInfile, fseek(fp_in, 2, SEEK_SET); else fp_in = UTF16TempFromUTF8(fp_in, FALSE); - if (fp_in == NULL) - { - return CERR_CannotCreateTempfile; + if (fp_in == NULL) { + SetError(CERR_CannotCreateTempfile); } fp_out = Open_File(pszOutfile, "wb"); - if (fp_out == NULL) SetError(CERR_CannotCreateOutfile); + if (fp_out == NULL) { + SetError(CERR_CannotCreateOutfile); + } - - KMX_DWORD msg; FILE_KEYBOARD fk; CodeConstants = new kmcmp::NamedCodeConstants; - err = CompileKeyboardHandle(fp_in, &fk); - if (err) - { - if ((msg = WriteCompiledKeyboard(&fk, fp_out)) != CERR_None) + bool result = CompileKeyboardHandle(fp_in, &fk); + if(result) { + KMX_DWORD msg; + if ((msg = WriteCompiledKeyboard(&fk, fp_out)) != CERR_None) { + result = FALSE; AddCompileError(msg); - } - else + } + } else { AddCompileError(CERR_InvalidValue); + } fclose(fp_in); fclose(fp_out); @@ -461,17 +456,16 @@ EXTERN uint32_t kmcmp_CompileKeyboardFile(char* pszInfile, return FALSE; } - return err; + return result; } -EXTERN uint32_t kmcmp_CompileKeyboardFileToBuffer(char* pszInfile, void* pfkBuffer, bool ACompilerWarningsAsErrors, bool AWarnDeprecatedCode, +EXTERN bool kmcmp_CompileKeyboardFileToBuffer(char* pszInfile, void* pfkBuffer, bool ACompilerWarningsAsErrors, bool AWarnDeprecatedCode, kmcmp_CompilerMessageProc pMsgproc, void* AmsgprocContext, int Target) // I4865 // I4866 { //printf("°°-> changed to CompileKeyboardFileToBuffer() of kmcmplib \n"); FILE* fp_in = NULL; - KMX_BOOL err; KMX_CHAR str[260]; kmcmp::FSaveDebug = TRUE; // I3681 @@ -479,17 +473,19 @@ EXTERN uint32_t kmcmp_CompileKeyboardFileToBuffer(char* pszInfile, void* pfkBuff AWarnDeprecatedCode_GLOBAL_LIB = AWarnDeprecatedCode; kmcmp::CompileTarget = Target; - if (!pMsgproc || !pszInfile || !pfkBuffer) SetError(CERR_BadCallParams); + if (!pMsgproc || !pszInfile || !pfkBuffer) { + SetError(CERR_BadCallParams); + } PKMX_STR p; - if ((p = strrchr_slash(pszInfile)) != nullptr) - { + if ((p = strrchr_slash(pszInfile)) != nullptr) { strncpy(kmcmp::CompileDir, pszInfile, (int)(p - pszInfile + 1)); // I3481 kmcmp::CompileDir[(int)(p - pszInfile + 1)] = 0; } - else + else { kmcmp::CompileDir[0] = 0; + } msgproc = pMsgproc; msgprocContext = AmsgprocContext; @@ -498,7 +494,9 @@ EXTERN uint32_t kmcmp_CompileKeyboardFileToBuffer(char* pszInfile, void* pfkBuff fp_in = Open_File(pszInfile,"rb"); - if (fp_in == NULL) SetError(CERR_InfileNotExist); + if (fp_in == NULL) { + SetError(CERR_InfileNotExist); + } // Transfer the file to a memory stream for processing UTF-8 or ANSI to UTF-16? // What about really large files? Transfer to a temp file... @@ -506,7 +504,7 @@ EXTERN uint32_t kmcmp_CompileKeyboardFileToBuffer(char* pszInfile, void* pfkBuff if( !fread(str,1,3,fp_in)) { fclose(fp_in); - return CERR_CannotReadInfile; + SetError(CERR_CannotReadInfile); } fseek( fp_in,0,SEEK_SET); @@ -519,16 +517,17 @@ EXTERN uint32_t kmcmp_CompileKeyboardFileToBuffer(char* pszInfile, void* pfkBuff CodeConstants = new kmcmp::NamedCodeConstants; - err = CompileKeyboardHandle(fp_in, static_cast(pfkBuffer)); + bool result = CompileKeyboardHandle(fp_in, static_cast(pfkBuffer)); delete CodeConstants; fclose(fp_in); - if (kmcmp::nErrors > 0) + if (kmcmp::nErrors > 0) { return FALSE; - return err; + } + return result; } -KMX_BOOL CompileKeyboardHandle(FILE* fp_in, PFILE_KEYBOARD fk) +bool CompileKeyboardHandle(FILE* fp_in, PFILE_KEYBOARD fk) { PKMX_WCHAR str, p; @@ -612,7 +611,9 @@ KMX_BOOL CompileKeyboardHandle(FILE* fp_in, PFILE_KEYBOARD fk) } } - if (msg != CERR_EndOfFile) SetError(msg); + if (msg != CERR_EndOfFile) { + SetError(msg); + } fseek( fp_in,2,SEEK_SET); kmcmp::currentLine = 0; @@ -625,21 +626,31 @@ KMX_BOOL CompileKeyboardHandle(FILE* fp_in, PFILE_KEYBOARD fk) while ((msg = ReadLine(fp_in, str, FALSE)) == CERR_None) { msg = ParseLine(fk, str); - if (msg != CERR_None) SetError(msg); + if (msg != CERR_None) { + SetError(msg); + } } - if (msg != CERR_EndOfFile) SetError(msg); + if (msg != CERR_EndOfFile) { + SetError(msg); + } ProcessGroupFinish(fk); if (kmcmp::FSaveDebug) kmcmp::RecordDeadkeyNames(fk); /* Add the compiler version as a system store */ - if ((msg = kmcmp::AddCompilerVersionStore(fk)) != CERR_None) SetError(msg); + if ((msg = kmcmp::AddCompilerVersionStore(fk)) != CERR_None) { + SetError(msg); + } - if ((msg = BuildVKDictionary(fk)) != CERR_None) SetError(msg); // I3438 + if ((msg = BuildVKDictionary(fk)) != CERR_None) { + SetError(msg); // I3438 + } - if ((msg = CheckFilenameConsistencyForCalls(fk)) != CERR_None) SetError(msg); + if ((msg = CheckFilenameConsistencyForCalls(fk)) != CERR_None) { + SetError(msg); + } delete str; From 732c0e47860811e383a09a457222d517decf5629 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 10 Apr 2023 15:23:43 +0700 Subject: [PATCH 3/5] chore(developer): -d flag should not be overridden in build.sh --- developer/src/kmcmplib/build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/developer/src/kmcmplib/build.sh b/developer/src/kmcmplib/build.sh index 6607e4946e..68ed92b869 100755 --- a/developer/src/kmcmplib/build.sh +++ b/developer/src/kmcmplib/build.sh @@ -52,7 +52,6 @@ Libraries will be built in 'build///src'. "build" \ "test" \ "${archtargets[@]}" \ - "--debug,-d configuration is 'debug', not 'release'" \ "--test=opt_tests,-t test[s] to run (space separated)" builder_parse "$@" From 5e761f56ebe5d89a4989d5bbc22164926bedbd77 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 10 Apr 2023 15:24:03 +0700 Subject: [PATCH 4/5] chore(developer): add basic unit test for kmcmplib apis --- developer/src/kmcmplib/tests/api-test.cpp | 89 +++++++++++++++++++++++ developer/src/kmcmplib/tests/meson.build | 9 +++ 2 files changed, 98 insertions(+) create mode 100644 developer/src/kmcmplib/tests/api-test.cpp diff --git a/developer/src/kmcmplib/tests/api-test.cpp b/developer/src/kmcmplib/tests/api-test.cpp new file mode 100644 index 0000000000..83b83d896b --- /dev/null +++ b/developer/src/kmcmplib/tests/api-test.cpp @@ -0,0 +1,89 @@ +/* + * Keyman is copyright (C) SIL International. MIT License. + * + * API endpoint unit tests for kmcmplib + */ + +#include + +#ifdef _MSC_VER +#include +#else +#include +#endif + +#include +#include +#include +#include "../src/comperr.h" +#include "../src/compfile.h" +#include + +void setup(); +void test_kmcmp_CompileKeyboardFile(); +void test_kmcmp_CompileKeyboardFileToBuffer(); + +std::vector error_vec; + +int msgproc(int line, uint32_t dwMsgCode, char* szText, void* context) { + error_vec.push_back(dwMsgCode); + const char*t = "unknown"; + switch(dwMsgCode & 0xF000) { + case CERR_HINT: t=" hint"; break; + case CERR_WARNING: t="warning"; break; + case CERR_ERROR: t=" error"; break; + case CERR_FATAL: t=" fatal"; break; + } + printf("line %d %s %04.4x: %s\n", line, t, (unsigned int)dwMsgCode, szText); + return 1; +} + +int main(int argc, char *argv[]) { + setup(); + test_kmcmp_CompileKeyboardFile(); + + setup(); + test_kmcmp_CompileKeyboardFileToBuffer(); + + return 0; +} + +void setup() { + error_vec.clear(); +} + +void test_kmcmp_CompileKeyboardFile() { + char kmn_file[L_tmpnam], kmx_file[L_tmpnam]; + tmpnam(kmn_file); + tmpnam(kmx_file); + + // Create an empty file + FILE *fp = fopen(kmn_file, "w"); + fclose(fp); + + // It should fail when a zero-byte file is passed in + assert(!kmcmp_CompileKeyboardFile(kmn_file, kmx_file, true, false, true, msgproc, nullptr)); + assert(error_vec.size() == 1); + assert(error_vec[0] == CERR_CannotReadInfile); + + unlink(kmn_file); +} + +void test_kmcmp_CompileKeyboardFileToBuffer() { + char kmn_file[L_tmpnam], kmx_file[L_tmpnam]; + tmpnam(kmn_file); + tmpnam(kmx_file); + + // Create an empty file + FILE *fp = fopen(kmn_file, "w"); + fclose(fp); + + FILE_KEYBOARD fk; + + // It should fail when a zero-byte file is passed in + assert(!kmcmp_CompileKeyboardFileToBuffer(kmn_file, &fk, true, false, msgproc, nullptr, CKF_KEYMAN)); + assert(error_vec.size() == 1); + assert(error_vec[0] == CERR_CannotReadInfile); + + unlink(kmn_file); +} \ No newline at end of file diff --git a/developer/src/kmcmplib/tests/meson.build b/developer/src/kmcmplib/tests/meson.build index 0acfa7bac7..02f1a5e612 100644 --- a/developer/src/kmcmplib/tests/meson.build +++ b/developer/src/kmcmplib/tests/meson.build @@ -98,3 +98,12 @@ foreach kbd : tests endif endforeach + +apitest = executable('api-test', 'api-test.cpp', + cpp_args: defns, + include_directories: inc, + name_suffix: name_suffix, + link_args: links + tests_flags, + objects: lib.extract_all_objects()) + +test('api-test', apitest) \ No newline at end of file From e31ad60d3c9e882cd4defe0a14c67cdfc7dfdcc3 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 10 Apr 2023 16:04:42 +0700 Subject: [PATCH 5/5] chore(developer): Further AddWarning and SetError cleanup * Splits compiler interface functions that return bool into CompilerInterfaces.cpp, and moves `SetError` define to that unit so it can't be used accidentally elsewhere * Audit of `AddWarning` calls and only those that are used in bool functions now `return FALSE` (with new `AddWarningBool`) whereas all others `return CERR_Break` --- .../kmcmplib/src/CheckNCapsConsistency.cpp | 2 +- developer/src/kmcmplib/src/Compiler.cpp | 405 +----------------- .../src/kmcmplib/src/CompilerInterfaces.cpp | 368 ++++++++++++++++ .../src/kmcmplib/src/DeprecationChecks.cpp | 4 +- developer/src/kmcmplib/src/kmcmplib.h | 37 +- developer/src/kmcmplib/src/meson.build | 1 + 6 files changed, 423 insertions(+), 394 deletions(-) create mode 100644 developer/src/kmcmplib/src/CompilerInterfaces.cpp diff --git a/developer/src/kmcmplib/src/CheckNCapsConsistency.cpp b/developer/src/kmcmplib/src/CheckNCapsConsistency.cpp index 25e8c85b40..8b1a708d22 100644 --- a/developer/src/kmcmplib/src/CheckNCapsConsistency.cpp +++ b/developer/src/kmcmplib/src/CheckNCapsConsistency.cpp @@ -87,7 +87,7 @@ bool CheckNCapsConsistency(PFILE_KEYBOARD fk) { if (caps_ncaps_usage[i].neither_line && (caps_ncaps_usage[i].caps_line || caps_ncaps_usage[i].ncaps_line)) { // We set the current line to one needing work: the developer should add the NCAPS flag kmcmp::currentLine = caps_ncaps_usage[i].neither_line; - AddWarning(CWARN_KeyShouldIncludeNCaps); + AddWarningBool(CWARN_KeyShouldIncludeNCaps); } } diff --git a/developer/src/kmcmplib/src/Compiler.cpp b/developer/src/kmcmplib/src/Compiler.cpp index 83025c6823..3d9600bd30 100644 --- a/developer/src/kmcmplib/src/Compiler.cpp +++ b/developer/src/kmcmplib/src/Compiler.cpp @@ -94,7 +94,6 @@ #include "CasedKeys.h" #include #include -#include "CheckNCapsConsistency.h" #include "CheckFilenameConsistency.h" #include "UnreachableRules.h" #include "CheckForDuplicates.h" @@ -126,11 +125,9 @@ namespace kmcmp{ int BeginLine[4]; KMX_BOOL IsValidCallStore(PFILE_STORE fs); - void RecordDeadkeyNames(PFILE_KEYBOARD fk); - KMX_DWORD AddCompilerVersionStore(PFILE_KEYBOARD fk); KMX_BOOL CheckStoreUsage(PFILE_KEYBOARD fk, int storeIndex, KMX_BOOL fIsStore, KMX_BOOL fIsOption, KMX_BOOL fIsCall); - int UTF32ToUTF16(int n, int *n1, int *n2); - int CheckUTF16(int n); + KMX_DWORD UTF32ToUTF16(int n, int *n1, int *n2); + KMX_DWORD CheckUTF16(int n); int cmpkeys(const void *key, const void *elem); } @@ -145,13 +142,7 @@ PKMX_WCHAR GetDelimitedString(PKMX_WCHAR *p, KMX_WCHAR const * Delimiters, KMX_W KMX_DWORD GetXString(PFILE_KEYBOARD fk, PKMX_WCHAR str, KMX_WCHAR const * token, PKMX_WCHAR output, int max, int offset, PKMX_WCHAR *newp, int isVKey, int isUnicode); int GetGroupNum(PFILE_KEYBOARD fk, PKMX_WCHAR p); -int LineTokenType(PKMX_WCHAR *str); -KMX_DWORD ParseLine(PFILE_KEYBOARD fk, PKMX_WCHAR str); - -KMX_DWORD ProcessGroupFinish(PFILE_KEYBOARD fk); -KMX_DWORD ProcessGroupLine(PFILE_KEYBOARD fk, PKMX_WCHAR p); -KMX_DWORD ProcessStoreLine(PFILE_KEYBOARD fk, PKMX_WCHAR p); KMX_DWORD AddDebugStore(PFILE_KEYBOARD fk, KMX_WCHAR const * str); KMX_DWORD ProcessKeyLine(PFILE_KEYBOARD fk, PKMX_WCHAR str, KMX_BOOL IsUnicode); KMX_DWORD ProcessEthnologueStore(PKMX_WCHAR p); // I2646 @@ -160,14 +151,7 @@ KMX_DWORD ImportBitmapFile(PFILE_KEYBOARD fk, PKMX_WCHAR szName, PKMX_DWORD File KMX_DWORD ExpandKp(PFILE_KEYBOARD fk, PFILE_KEY kpp, KMX_DWORD storeIndex); -KMX_DWORD ReadLine(FILE* fp_in , PKMX_WCHAR wstr, KMX_BOOL PreProcess); - -KMX_DWORD WriteCompiledKeyboard(PFILE_KEYBOARD fk, FILE* fp_out); -bool CompileKeyboardHandle(FILE* fp_in, PFILE_KEYBOARD fk); - int GetVKCode(PFILE_KEYBOARD fk, PKMX_WCHAR p); // I3438 // TODO: Consolidate GetDeadKey and GetVKCode? -KMX_DWORD BuildVKDictionary(PFILE_KEYBOARD fk); // I3438 -KMX_DWORD AddStore(PFILE_KEYBOARD fk, KMX_DWORD SystemID, KMX_WCHAR const * str, KMX_DWORD *dwStoreID= NULL); KMX_DWORD ProcessSystemStore(PFILE_KEYBOARD fk, KMX_DWORD SystemID, PFILE_STORE sp); KMX_DWORD process_if(PFILE_KEYBOARD fk, PKMX_WCHAR q, PKMX_WCHAR tstr, int *mx); @@ -181,8 +165,6 @@ KMX_DWORD process_expansion(PFILE_KEYBOARD fk, PKMX_WCHAR q, PKMX_WCHAR tstr, in KMX_BOOL IsValidKeyboardVersion(KMX_WCHAR *dpString); -FILE* UTF16TempFromUTF8(FILE* fp_in , KMX_BOOL hasPreamble); - const KMX_WCHAR * LineTokens[] = { u"SVNBHBGMNSCCLLCMLB", u"store", u"VERSION ", u"NAME ", u"BITMAP ", u"HOTKEY ", u"begin", u"group", u"match", u"nomatch", @@ -251,7 +233,7 @@ void* msgprocContext = NULL; int kmcmp::currentLine = 0; -kmcmp::NamedCodeConstants *CodeConstants = NULL; +kmcmp::NamedCodeConstants *kmcmp::CodeConstants = NULL; PKMX_WCHAR strtowstr(PKMX_STR in) { @@ -317,358 +299,6 @@ KMX_BOOL AddCompileError(KMX_DWORD msg) return FALSE; } -EXTERN bool kmcmp_SetCompilerOptions(KMCMP_COMPILER_OPTIONS* options) { - //printf("°°-> changed to SetCompilerOptions() of kmcmplib \n"); - if(!options || options->dwSize < sizeof(KMCMP_COMPILER_OPTIONS)) { - return FALSE; - } - kmcmp::FShouldAddCompilerVersion = options->ShouldAddCompilerVersion; - return TRUE; -} - -#ifdef __EMSCRIPTEN__ - -/* - WASM interface for compiler message callback -*/ -EM_JS(int, wasm_msgproc, (int line, int msgcode, char* text, char* context), { - const proc = globalThis[context]; - if(!proc || typeof proc != 'function') { - console.log(`[${line}: ${msgcode}: ${UTF8ToString(text)}]`); - return 0; - } else { - return proc(line, msgcode, UTF8ToString(text)); - } -}); - -int wasm_CompilerMessageProc(int line, uint32_t dwMsgCode, char* szText, void* context) { - char* msgProc = static_cast(context); - return wasm_msgproc(line, dwMsgCode, szText, msgProc); -} - -EXTERN bool kmcmp_Wasm_SetCompilerOptions(int ShouldAddCompilerVersion) { - KMCMP_COMPILER_OPTIONS options; - options.dwSize = sizeof(KMCMP_COMPILER_OPTIONS); - options.ShouldAddCompilerVersion = ShouldAddCompilerVersion; - return kmcmp_SetCompilerOptions(&options); -} - -EXTERN bool kmcmp_Wasm_CompileKeyboardFile(char* pszInfile, - char* pszOutfile, int ASaveDebug, int ACompilerWarningsAsErrors, - int AWarnDeprecatedCode, char* msgProc -) { - return kmcmp_CompileKeyboardFile( - pszInfile, - pszOutfile, - ASaveDebug, - ACompilerWarningsAsErrors, - AWarnDeprecatedCode, - wasm_CompilerMessageProc, - msgProc - ); -} -#endif - -EXTERN bool kmcmp_CompileKeyboardFile(char* pszInfile, - char* pszOutfile, bool ASaveDebug, bool ACompilerWarningsAsErrors, - bool AWarnDeprecatedCode, kmcmp_CompilerMessageProc pMsgproc, void* AmsgprocContext -) { - FILE* fp_in = NULL; - FILE* fp_out = NULL; - KMX_CHAR str[260]; - - //printf("°°-> changed to CompileKeyboardFile() of kmcmplib \n"); - - kmcmp::FSaveDebug = ASaveDebug; - kmcmp::FCompilerWarningsAsErrors = ACompilerWarningsAsErrors; // I4865 - AWarnDeprecatedCode_GLOBAL_LIB = AWarnDeprecatedCode; - - kmcmp::CompileTarget = CKF_KEYMAN; - - if (!pMsgproc || !pszInfile || !pszOutfile) SetError(CERR_BadCallParams); - - PKMX_STR p; - - if ((p = strrchr_slash(pszInfile)) != nullptr) - { - strncpy(kmcmp::CompileDir, pszInfile, (int)(p - pszInfile + 1)); // I3481 - kmcmp::CompileDir[(int)(p - pszInfile + 1)] = 0; - } - else - kmcmp::CompileDir[0] = 0; - - msgproc = pMsgproc; - msgprocContext = AmsgprocContext; - kmcmp::currentLine = 0; - kmcmp::nErrors = 0; - - fp_in = Open_File(pszInfile, "rb"); - - if (fp_in == NULL) { - SetError(CERR_InfileNotExist); - } - - // Transfer the file to a memory stream for processing UTF-8 or ANSI to UTF-16? - // What about really large files? Transfer to a temp file... - if (!fread(str, 1, 3, fp_in)) { - fclose(fp_in); - SetError(CERR_CannotReadInfile); - } - - fseek(fp_in, 0, SEEK_SET); - if (str[0] == UTF8Sig[0] && str[1] == UTF8Sig[1] && str[2] == UTF8Sig[2]) - fp_in = UTF16TempFromUTF8(fp_in, TRUE); - else if (str[0] == UTF16Sig[0] && str[1] == UTF16Sig[1]) - fseek(fp_in, 2, SEEK_SET); - else - fp_in = UTF16TempFromUTF8(fp_in, FALSE); - if (fp_in == NULL) { - SetError(CERR_CannotCreateTempfile); - } - - fp_out = Open_File(pszOutfile, "wb"); - - if (fp_out == NULL) { - SetError(CERR_CannotCreateOutfile); - } - - FILE_KEYBOARD fk; - CodeConstants = new kmcmp::NamedCodeConstants; - bool result = CompileKeyboardHandle(fp_in, &fk); - if(result) { - KMX_DWORD msg; - if ((msg = WriteCompiledKeyboard(&fk, fp_out)) != CERR_None) { - result = FALSE; - AddCompileError(msg); - } - } else { - AddCompileError(CERR_InvalidValue); - } - - fclose(fp_in); - fclose(fp_out); - - delete CodeConstants; - - if (kmcmp::nErrors > 0) - { - remove(pszOutfile); - return FALSE; - } - - return result; -} - - - -EXTERN bool kmcmp_CompileKeyboardFileToBuffer(char* pszInfile, void* pfkBuffer, bool ACompilerWarningsAsErrors, bool AWarnDeprecatedCode, - kmcmp_CompilerMessageProc pMsgproc, void* AmsgprocContext, int Target) // I4865 // I4866 -{ - //printf("°°-> changed to CompileKeyboardFileToBuffer() of kmcmplib \n"); - FILE* fp_in = NULL; - KMX_CHAR str[260]; - - kmcmp::FSaveDebug = TRUE; // I3681 - kmcmp::FCompilerWarningsAsErrors = ACompilerWarningsAsErrors; // I4865 - AWarnDeprecatedCode_GLOBAL_LIB = AWarnDeprecatedCode; - kmcmp::CompileTarget = Target; - - if (!pMsgproc || !pszInfile || !pfkBuffer) { - SetError(CERR_BadCallParams); - } - - PKMX_STR p; - - if ((p = strrchr_slash(pszInfile)) != nullptr) { - strncpy(kmcmp::CompileDir, pszInfile, (int)(p - pszInfile + 1)); // I3481 - kmcmp::CompileDir[(int)(p - pszInfile + 1)] = 0; - } - else { - kmcmp::CompileDir[0] = 0; - } - - msgproc = pMsgproc; - msgprocContext = AmsgprocContext; - kmcmp::currentLine = 0; - kmcmp::nErrors = 0; - - fp_in = Open_File(pszInfile,"rb"); - - if (fp_in == NULL) { - SetError(CERR_InfileNotExist); - } - - // Transfer the file to a memory stream for processing UTF-8 or ANSI to UTF-16? - // What about really large files? Transfer to a temp file... - - if( !fread(str,1,3,fp_in)) - { - fclose(fp_in); - SetError(CERR_CannotReadInfile); - } - - fseek( fp_in,0,SEEK_SET); - if (str[0] == UTF8Sig[0] && str[1] == UTF8Sig[1] && str[2] == UTF8Sig[2]) - fp_in = UTF16TempFromUTF8(fp_in, TRUE); - else if (str[0] == UTF16Sig[0] && str[1] == UTF16Sig[1]) - fseek( fp_in,2,SEEK_SET); - else - fp_in = UTF16TempFromUTF8(fp_in, FALSE); - - CodeConstants = new kmcmp::NamedCodeConstants; - - bool result = CompileKeyboardHandle(fp_in, static_cast(pfkBuffer)); - delete CodeConstants; - fclose(fp_in); - - if (kmcmp::nErrors > 0) { - return FALSE; - } - return result; -} - -bool CompileKeyboardHandle(FILE* fp_in, PFILE_KEYBOARD fk) -{ - PKMX_WCHAR str, p; - - KMX_DWORD msg; - - kmcmp::FMnemonicLayout = FALSE; - - if (!fk) { - SetError(CERR_SomewhereIGotItWrong); - } - - str = new KMX_WCHAR[LINESIZE]; - if (!str) { - SetError(CERR_CannotAllocateMemory); - } - - fk->KeyboardID = 0; - fk->version = 0; - fk->dpStoreArray = NULL; - fk->dpGroupArray = NULL; - fk->cxStoreArray = 0; - fk->cxGroupArray = 0; - fk->StartGroup[0] = fk->StartGroup[1] = -1; - fk->szName[0] = 0; - fk->szCopyright[0] = 0; - fk->dwFlags = KF_AUTOMATICVERSION; - fk->currentGroup = 0xFFFFFFFF; - fk->currentStore = 0; - fk->cxDeadKeyArray = 0; - fk->dpDeadKeyArray = NULL; - fk->cxVKDictionary = 0; // I3438 - fk->dpVKDictionary = NULL; // I3438 - -/* fk->szMessage[0] = 0; - fk->szLanguageName[0] = 0;*/ - fk->dwBitmapSize = 0; - fk->dwHotKey = 0; - - kmcmp::BeginLine[BEGIN_ANSI] = -1; - kmcmp::BeginLine[BEGIN_UNICODE] = -1; - kmcmp::BeginLine[BEGIN_NEWCONTEXT] = -1; - kmcmp::BeginLine[BEGIN_POSTKEYSTROKE] = -1; - - - /* Add a store for the Keyman 6.0 copyright information string */ - - if(kmcmp::FShouldAddCompilerVersion) { - u16sprintf(str,LINESIZE, L"Created with Keyman Developer version %d.%d.%d.%d", KEYMAN_VersionMajor, KEYMAN_VersionMinor, KEYMAN_VersionPatch, 0); - AddStore(fk, TSS_KEYMANCOPYRIGHT, str); - } - - /* Add a system store for the Keyman edition number */ - u16sprintf(str, LINESIZE, L"%d", 0); // I3481 - AddStore(fk, TSS_CUSTOMKEYMANEDITION, str); - - PKMX_WCHAR tbuf = strtowstr((KMX_CHAR*) "Keyman"); - AddStore(fk, TSS_CUSTOMKEYMANEDITIONNAME, tbuf); - delete tbuf; - - // must preprocess for group and store names -> this isn't really necessary, but never mind! - while ((msg = ReadLine(fp_in, str, TRUE)) == CERR_None) - { - p = str; - switch (LineTokenType(&p)) - { - case T_VERSION: - *(p + 4) = 0; - if ((msg = AddStore(fk, TSS_VERSION, p)) != CERR_None) SetError(msg); - break; - - case T_GROUP: - if ((msg = ProcessGroupLine(fk, p)) != CERR_None) SetError(msg); - break; - - case T_STORE: - if ((msg = ProcessStoreLine(fk, p)) != CERR_None) SetError(msg); - break; - - default: - break; - } - } - - if (msg != CERR_EndOfFile) { - SetError(msg); - } - - fseek( fp_in,2,SEEK_SET); - kmcmp::currentLine = 0; - - /* Reindex the list of codeconstants after stores added */ - - CodeConstants->reindex(); - - /* ReadLine will automatically skip over $Keyman lines, and parse wrapped lines */ - while ((msg = ReadLine(fp_in, str, FALSE)) == CERR_None) - { - msg = ParseLine(fk, str); - if (msg != CERR_None) { - SetError(msg); - } - } - - if (msg != CERR_EndOfFile) { - SetError(msg); - } - - ProcessGroupFinish(fk); - - if (kmcmp::FSaveDebug) kmcmp::RecordDeadkeyNames(fk); - - /* Add the compiler version as a system store */ - if ((msg = kmcmp::AddCompilerVersionStore(fk)) != CERR_None) { - SetError(msg); - } - - if ((msg = BuildVKDictionary(fk)) != CERR_None) { - SetError(msg); // I3438 - } - - if ((msg = CheckFilenameConsistencyForCalls(fk)) != CERR_None) { - SetError(msg); - } - - delete str; - - if (!kmcmp::CheckKeyboardFinalVersion(fk)) { - return FALSE; - } - - /* Warn on inconsistent use of NCAPS */ - if (!kmcmp::FMnemonicLayout) { - CheckNCapsConsistency(fk); - } - - /* Flag presence of deprecated features */ - kmcmp::CheckForDeprecatedFeatures(fk); - - return TRUE; -} - KMX_DWORD ProcessBeginLine(PFILE_KEYBOARD fk, PKMX_WCHAR p) { KMX_WCHAR tstr[128]; @@ -1137,10 +767,10 @@ KMX_DWORD ProcessStoreLine(PFILE_KEYBOARD fk, PKMX_WCHAR p) VERIFY_KEYBOARD_VERSION(fk, VERSION_60, CERR_60FeatureOnly_NamedCodes); // Add a single char store as a defined character constant if (Uni_IsSurrogate1(*sp->dpString)) - CodeConstants->AddCode(Uni_SurrogateToUTF32(sp->dpString[0], sp->dpString[1]), sp->szName, fk->cxStoreArray); + kmcmp::CodeConstants->AddCode(Uni_SurrogateToUTF32(sp->dpString[0], sp->dpString[1]), sp->szName, fk->cxStoreArray); else - CodeConstants->AddCode(sp->dpString[0], sp->szName, fk->cxStoreArray); - CodeConstants->reindex(); // has to be done after every character add due to possible use in another store. // I4982 + kmcmp::CodeConstants->AddCode(sp->dpString[0], sp->szName, fk->cxStoreArray); + kmcmp::CodeConstants->reindex(); // has to be done after every character add due to possible use in another store. // I4982 } fk->cxStoreArray++; // increment now, because GetXString refers to stores @@ -1280,13 +910,13 @@ KMX_DWORD ProcessSystemStore(PFILE_KEYBOARD fk, KMX_DWORD SystemID, PFILE_STORE case TSS_INCLUDECODES: VERIFY_KEYBOARD_VERSION(fk, VERSION_60, CERR_60FeatureOnly_NamedCodes); pp = wstrtostr(sp->dpString); - if (!CodeConstants->LoadFile(pp)) + if (!kmcmp::CodeConstants->LoadFile(pp)) { delete[] pp; return CERR_CannotLoadIncludeFile; } delete[] pp; - CodeConstants->reindex(); // I4982 + kmcmp::CodeConstants->reindex(); // I4982 break; case TSS_LANGUAGE: @@ -1617,13 +1247,13 @@ KMX_BOOL CheckContextStatementPositions(PKMX_WCHAR context) { switch (*(p + 1)) { case CODE_NUL: if (p > context) { - AddWarning(CWARN_NulNotFirstStatementInContext); + AddWarningBool(CWARN_NulNotFirstStatementInContext); } break; case CODE_IFOPT: case CODE_IFSYSTEMSTORE: if (hadContextChar) { - AddWarning(CWARN_IfShouldBeAtStartOfContext); + AddWarningBool(CWARN_IfShouldBeAtStartOfContext); } break; default: @@ -2659,7 +2289,7 @@ KMX_DWORD GetXStringImpl(PKMX_WCHAR tstr, PFILE_KEYBOARD fk, PKMX_WCHAR str, KMX q = p + 1; while (*q && !iswspace(*q)) q++; c = *q; *q = 0; - n = CodeConstants->GetCode(p + 1, &i); + n = kmcmp::CodeConstants->GetCode(p + 1, &i); *q = c; if (n == 0) return CERR_InvalidNamedCode; if (i < 0xFFFFFFFFL) kmcmp::CheckStoreUsage(fk, i, TRUE, FALSE, FALSE); // I2993 @@ -3236,19 +2866,19 @@ KMX_BOOL kmcmp::CheckStoreUsage(PFILE_KEYBOARD fk, int storeIndex, KMX_BOOL fIsS if (fIsStore && !sp->fIsStore) { if (sp->fIsDebug || sp->fIsOption || sp->fIsReserved || sp->fIsCall) - AddWarning(CWARN_StoreAlreadyUsedAsOptionOrCall); + AddWarningBool(CWARN_StoreAlreadyUsedAsOptionOrCall); sp->fIsStore = TRUE; } else if (fIsOption && !sp->fIsOption) { if (sp->fIsDebug || sp->fIsStore || sp->fIsReserved || sp->fIsCall) - AddWarning(CWARN_StoreAlreadyUsedAsStoreOrCall); + AddWarningBool(CWARN_StoreAlreadyUsedAsStoreOrCall); sp->fIsOption = TRUE; } else if (fIsCall && !sp->fIsCall) { if (sp->fIsDebug || sp->fIsStore || sp->fIsReserved || sp->fIsOption) - AddWarning(CWARN_StoreAlreadyUsedAsStoreOrOption); + AddWarningBool(CWARN_StoreAlreadyUsedAsStoreOrOption); sp->fIsCall = TRUE; } @@ -3656,7 +3286,7 @@ int atoiW(PKMX_WCHAR p) return i; } -int kmcmp::CheckUTF16(int n) +KMX_DWORD kmcmp::CheckUTF16(int n) { const int res[] = { 0xFDD0, 0xFDD1, 0xFDD2, 0xFDD3, 0xFDD4, 0xFDD5, 0xFDD6, 0xFDD7, @@ -3675,7 +3305,7 @@ int kmcmp::CheckUTF16(int n) return CERR_None; } -int kmcmp::UTF32ToUTF16(int n, int *n1, int *n2) +KMX_DWORD kmcmp::UTF32ToUTF16(int n, int *n1, int *n2) { *n2 = -1; if (n <= 0xFFFF) @@ -3690,7 +3320,8 @@ int kmcmp::UTF32ToUTF16(int n, int *n1, int *n2) n = n - 0x10000; *n1 = (n / 0x400) + 0xD800; *n2 = (n % 0x400) + 0xDC00; - if ((n = kmcmp::CheckUTF16(*n1)) != CERR_None) return n; + KMX_DWORD msg; + if ((msg = kmcmp::CheckUTF16(*n1)) != CERR_None) return msg; return kmcmp::CheckUTF16(*n2); } diff --git a/developer/src/kmcmplib/src/CompilerInterfaces.cpp b/developer/src/kmcmplib/src/CompilerInterfaces.cpp new file mode 100644 index 0000000000..09afa1d2cb --- /dev/null +++ b/developer/src/kmcmplib/src/CompilerInterfaces.cpp @@ -0,0 +1,368 @@ +#include "pch.h" + +#include +#include +#include "kmcmplib.h" +#include "filesystem.h" +#include "CheckFilenameConsistency.h" +#include "CheckNCapsConsistency.h" +#include "DeprecationChecks.h" +#include "versioning.h" +#include "../../../../common/windows/cpp/include/ConvertUTF.h" +#include "../../../../common/windows/cpp/include/keymanversion.h" + +#define SetError(err) { if(AddCompileError(err) || (err & CERR_FATAL)) return FALSE; } + +bool CompileKeyboardHandle(FILE* fp_in, PFILE_KEYBOARD fk); + +EXTERN bool kmcmp_SetCompilerOptions(KMCMP_COMPILER_OPTIONS* options) { + //printf("°°-> changed to SetCompilerOptions() of kmcmplib \n"); + if(!options || options->dwSize < sizeof(KMCMP_COMPILER_OPTIONS)) { + return FALSE; + } + kmcmp::FShouldAddCompilerVersion = options->ShouldAddCompilerVersion; + return TRUE; +} + +#ifdef __EMSCRIPTEN__ + +/* + WASM interface for compiler message callback +*/ +EM_JS(int, wasm_msgproc, (int line, int msgcode, char* text, char* context), { + const proc = globalThis[context]; + if(!proc || typeof proc != 'function') { + console.log(`[${line}: ${msgcode}: ${UTF8ToString(text)}]`); + return 0; + } else { + return proc(line, msgcode, UTF8ToString(text)); + } +}); + +int wasm_CompilerMessageProc(int line, uint32_t dwMsgCode, char* szText, void* context) { + char* msgProc = static_cast(context); + return wasm_msgproc(line, dwMsgCode, szText, msgProc); +} + +EXTERN bool kmcmp_Wasm_SetCompilerOptions(int ShouldAddCompilerVersion) { + KMCMP_COMPILER_OPTIONS options; + options.dwSize = sizeof(KMCMP_COMPILER_OPTIONS); + options.ShouldAddCompilerVersion = ShouldAddCompilerVersion; + return kmcmp_SetCompilerOptions(&options); +} + +EXTERN bool kmcmp_Wasm_CompileKeyboardFile(char* pszInfile, + char* pszOutfile, int ASaveDebug, int ACompilerWarningsAsErrors, + int AWarnDeprecatedCode, char* msgProc +) { + return kmcmp_CompileKeyboardFile( + pszInfile, + pszOutfile, + ASaveDebug, + ACompilerWarningsAsErrors, + AWarnDeprecatedCode, + wasm_CompilerMessageProc, + msgProc + ); +} +#endif + +EXTERN bool kmcmp_CompileKeyboardFile(char* pszInfile, + char* pszOutfile, bool ASaveDebug, bool ACompilerWarningsAsErrors, + bool AWarnDeprecatedCode, kmcmp_CompilerMessageProc pMsgproc, void* AmsgprocContext +) { + FILE* fp_in = NULL; + FILE* fp_out = NULL; + KMX_CHAR str[260]; + + //printf("°°-> changed to CompileKeyboardFile() of kmcmplib \n"); + + kmcmp::FSaveDebug = ASaveDebug; + kmcmp::FCompilerWarningsAsErrors = ACompilerWarningsAsErrors; // I4865 + AWarnDeprecatedCode_GLOBAL_LIB = AWarnDeprecatedCode; + + kmcmp::CompileTarget = CKF_KEYMAN; + + if (!pMsgproc || !pszInfile || !pszOutfile) SetError(CERR_BadCallParams); + + PKMX_STR p; + + if ((p = strrchr_slash(pszInfile)) != nullptr) + { + strncpy(kmcmp::CompileDir, pszInfile, (int)(p - pszInfile + 1)); // I3481 + kmcmp::CompileDir[(int)(p - pszInfile + 1)] = 0; + } + else + kmcmp::CompileDir[0] = 0; + + msgproc = pMsgproc; + msgprocContext = AmsgprocContext; + kmcmp::currentLine = 0; + kmcmp::nErrors = 0; + + fp_in = Open_File(pszInfile, "rb"); + + if (fp_in == NULL) { + SetError(CERR_InfileNotExist); + } + + // Transfer the file to a memory stream for processing UTF-8 or ANSI to UTF-16? + // What about really large files? Transfer to a temp file... + if (!fread(str, 1, 3, fp_in)) { + fclose(fp_in); + SetError(CERR_CannotReadInfile); + } + + fseek(fp_in, 0, SEEK_SET); + if (str[0] == UTF8Sig[0] && str[1] == UTF8Sig[1] && str[2] == UTF8Sig[2]) + fp_in = UTF16TempFromUTF8(fp_in, TRUE); + else if (str[0] == UTF16Sig[0] && str[1] == UTF16Sig[1]) + fseek(fp_in, 2, SEEK_SET); + else + fp_in = UTF16TempFromUTF8(fp_in, FALSE); + if (fp_in == NULL) { + SetError(CERR_CannotCreateTempfile); + } + + fp_out = Open_File(pszOutfile, "wb"); + + if (fp_out == NULL) { + SetError(CERR_CannotCreateOutfile); + } + + FILE_KEYBOARD fk; + kmcmp::CodeConstants = new kmcmp::NamedCodeConstants; + bool result = CompileKeyboardHandle(fp_in, &fk); + if(result) { + KMX_DWORD msg; + if ((msg = WriteCompiledKeyboard(&fk, fp_out)) != CERR_None) { + result = FALSE; + AddCompileError(msg); + } + } else { + AddCompileError(CERR_InvalidValue); + } + + fclose(fp_in); + fclose(fp_out); + + delete kmcmp::CodeConstants; + + if (kmcmp::nErrors > 0) + { + remove(pszOutfile); + return FALSE; + } + + return result; +} + + + +EXTERN bool kmcmp_CompileKeyboardFileToBuffer(char* pszInfile, void* pfkBuffer, bool ACompilerWarningsAsErrors, bool AWarnDeprecatedCode, + kmcmp_CompilerMessageProc pMsgproc, void* AmsgprocContext, int Target) // I4865 // I4866 +{ + //printf("°°-> changed to CompileKeyboardFileToBuffer() of kmcmplib \n"); + FILE* fp_in = NULL; + KMX_CHAR str[260]; + + kmcmp::FSaveDebug = TRUE; // I3681 + kmcmp::FCompilerWarningsAsErrors = ACompilerWarningsAsErrors; // I4865 + AWarnDeprecatedCode_GLOBAL_LIB = AWarnDeprecatedCode; + kmcmp::CompileTarget = Target; + + if (!pMsgproc || !pszInfile || !pfkBuffer) { + SetError(CERR_BadCallParams); + } + + PKMX_STR p; + + if ((p = strrchr_slash(pszInfile)) != nullptr) { + strncpy(kmcmp::CompileDir, pszInfile, (int)(p - pszInfile + 1)); // I3481 + kmcmp::CompileDir[(int)(p - pszInfile + 1)] = 0; + } + else { + kmcmp::CompileDir[0] = 0; + } + + msgproc = pMsgproc; + msgprocContext = AmsgprocContext; + kmcmp::currentLine = 0; + kmcmp::nErrors = 0; + + fp_in = Open_File(pszInfile,"rb"); + + if (fp_in == NULL) { + SetError(CERR_InfileNotExist); + } + + // Transfer the file to a memory stream for processing UTF-8 or ANSI to UTF-16? + // What about really large files? Transfer to a temp file... + + if( !fread(str,1,3,fp_in)) + { + fclose(fp_in); + SetError(CERR_CannotReadInfile); + } + + fseek( fp_in,0,SEEK_SET); + if (str[0] == UTF8Sig[0] && str[1] == UTF8Sig[1] && str[2] == UTF8Sig[2]) + fp_in = UTF16TempFromUTF8(fp_in, TRUE); + else if (str[0] == UTF16Sig[0] && str[1] == UTF16Sig[1]) + fseek( fp_in,2,SEEK_SET); + else + fp_in = UTF16TempFromUTF8(fp_in, FALSE); + + kmcmp::CodeConstants = new kmcmp::NamedCodeConstants; + + bool result = CompileKeyboardHandle(fp_in, static_cast(pfkBuffer)); + delete kmcmp::CodeConstants; + fclose(fp_in); + + if (kmcmp::nErrors > 0) { + return FALSE; + } + return result; +} + +bool CompileKeyboardHandle(FILE* fp_in, PFILE_KEYBOARD fk) +{ + PKMX_WCHAR str, p; + + KMX_DWORD msg; + + kmcmp::FMnemonicLayout = FALSE; + + if (!fk) { + SetError(CERR_SomewhereIGotItWrong); + } + + str = new KMX_WCHAR[LINESIZE]; + if (!str) { + SetError(CERR_CannotAllocateMemory); + } + + fk->KeyboardID = 0; + fk->version = 0; + fk->dpStoreArray = NULL; + fk->dpGroupArray = NULL; + fk->cxStoreArray = 0; + fk->cxGroupArray = 0; + fk->StartGroup[0] = fk->StartGroup[1] = -1; + fk->szName[0] = 0; + fk->szCopyright[0] = 0; + fk->dwFlags = KF_AUTOMATICVERSION; + fk->currentGroup = 0xFFFFFFFF; + fk->currentStore = 0; + fk->cxDeadKeyArray = 0; + fk->dpDeadKeyArray = NULL; + fk->cxVKDictionary = 0; // I3438 + fk->dpVKDictionary = NULL; // I3438 + +/* fk->szMessage[0] = 0; + fk->szLanguageName[0] = 0;*/ + fk->dwBitmapSize = 0; + fk->dwHotKey = 0; + + kmcmp::BeginLine[BEGIN_ANSI] = -1; + kmcmp::BeginLine[BEGIN_UNICODE] = -1; + kmcmp::BeginLine[BEGIN_NEWCONTEXT] = -1; + kmcmp::BeginLine[BEGIN_POSTKEYSTROKE] = -1; + + + /* Add a store for the Keyman 6.0 copyright information string */ + + if(kmcmp::FShouldAddCompilerVersion) { + u16sprintf(str,LINESIZE, L"Created with Keyman Developer version %d.%d.%d.%d", KEYMAN_VersionMajor, KEYMAN_VersionMinor, KEYMAN_VersionPatch, 0); + AddStore(fk, TSS_KEYMANCOPYRIGHT, str); + } + + /* Add a system store for the Keyman edition number */ + u16sprintf(str, LINESIZE, L"%d", 0); // I3481 + AddStore(fk, TSS_CUSTOMKEYMANEDITION, str); + + PKMX_WCHAR tbuf = strtowstr((KMX_CHAR*) "Keyman"); + AddStore(fk, TSS_CUSTOMKEYMANEDITIONNAME, tbuf); + delete tbuf; + + // must preprocess for group and store names -> this isn't really necessary, but never mind! + while ((msg = ReadLine(fp_in, str, TRUE)) == CERR_None) + { + p = str; + switch (LineTokenType(&p)) + { + case T_VERSION: + *(p + 4) = 0; + if ((msg = AddStore(fk, TSS_VERSION, p)) != CERR_None) SetError(msg); + break; + + case T_GROUP: + if ((msg = ProcessGroupLine(fk, p)) != CERR_None) SetError(msg); + break; + + case T_STORE: + if ((msg = ProcessStoreLine(fk, p)) != CERR_None) SetError(msg); + break; + + default: + break; + } + } + + if (msg != CERR_EndOfFile) { + SetError(msg); + } + + fseek( fp_in,2,SEEK_SET); + kmcmp::currentLine = 0; + + /* Reindex the list of codeconstants after stores added */ + + kmcmp::CodeConstants->reindex(); + + /* ReadLine will automatically skip over $Keyman lines, and parse wrapped lines */ + while ((msg = ReadLine(fp_in, str, FALSE)) == CERR_None) + { + msg = ParseLine(fk, str); + if (msg != CERR_None) { + SetError(msg); + } + } + + if (msg != CERR_EndOfFile) { + SetError(msg); + } + + ProcessGroupFinish(fk); + + if (kmcmp::FSaveDebug) kmcmp::RecordDeadkeyNames(fk); + + /* Add the compiler version as a system store */ + if ((msg = kmcmp::AddCompilerVersionStore(fk)) != CERR_None) { + SetError(msg); + } + + if ((msg = BuildVKDictionary(fk)) != CERR_None) { + SetError(msg); // I3438 + } + + if ((msg = CheckFilenameConsistencyForCalls(fk)) != CERR_None) { + SetError(msg); + } + + delete str; + + if (!kmcmp::CheckKeyboardFinalVersion(fk)) { + return FALSE; + } + + /* Warn on inconsistent use of NCAPS */ + if (!kmcmp::FMnemonicLayout) { + CheckNCapsConsistency(fk); + } + + /* Flag presence of deprecated features */ + kmcmp::CheckForDeprecatedFeatures(fk); + + return TRUE; +} \ No newline at end of file diff --git a/developer/src/kmcmplib/src/DeprecationChecks.cpp b/developer/src/kmcmplib/src/DeprecationChecks.cpp index 7153a6f723..ef160cc96b 100644 --- a/developer/src/kmcmplib/src/DeprecationChecks.cpp +++ b/developer/src/kmcmplib/src/DeprecationChecks.cpp @@ -8,7 +8,7 @@ KMX_BOOL kmcmp::WarnDeprecatedHeader() { // I4866 if( AWarnDeprecatedCode_GLOBAL_LIB){ - AddWarning(CWARN_HeaderStatementIsDeprecated); + AddWarningBool(CWARN_HeaderStatementIsDeprecated); } return TRUE; } @@ -42,7 +42,7 @@ KMX_BOOL kmcmp::CheckForDeprecatedFeatures(PFILE_KEYBOARD fk) { sp->dwSystemID == TSS_ETHNOLOGUECODE || sp->dwSystemID == TSS_WINDOWSLANGUAGES) { kmcmp::currentLine = sp->line; - AddWarning(CWARN_LanguageHeadersDeprecatedInKeyman10); + AddWarningBool(CWARN_LanguageHeadersDeprecatedInKeyman10); } } } diff --git a/developer/src/kmcmplib/src/kmcmplib.h b/developer/src/kmcmplib/src/kmcmplib.h index 896ed8e10a..13d871f0ec 100644 --- a/developer/src/kmcmplib/src/kmcmplib.h +++ b/developer/src/kmcmplib/src/kmcmplib.h @@ -1,21 +1,50 @@ #pragma once +#include #include "compfile.h" +#include "NamedCodeConstants.h" namespace kmcmp { KMX_BOOL AddCompileWarning(char* buf); extern int currentLine; + extern KMX_BOOL FShouldAddCompilerVersion; + extern KMX_BOOL FSaveDebug, FCompilerWarningsAsErrors; // I4865 // I4866 + extern int ErrChr; + extern int nErrors; + extern KMX_BOOL FMnemonicLayout; + extern KMX_BOOL FOldCharPosMatching; + extern int CompileTarget; + extern KMX_CHAR CompileDir[260]; // TODO: this should not be a fixed buffer + extern int BeginLine[4]; + extern int currentLine; + extern NamedCodeConstants *CodeConstants; + + void RecordDeadkeyNames(PFILE_KEYBOARD fk); + KMX_DWORD AddCompilerVersionStore(PFILE_KEYBOARD fk); } +extern kmcmp_CompilerMessageProc msgproc; +extern void* msgprocContext; + extern KMX_BOOL AWarnDeprecatedCode_GLOBAL_LIB; extern char ErrExtraLIB[256]; extern KMX_WCHAR ErrExtraW[256]; KMX_BOOL AddCompileError(KMX_DWORD msg); -// TODO: These macros can return FALSE in functions that expect a DWORD CERR_x -// return value type. This is just plain wrong! -#define SetError(err) { if(AddCompileError(err) || (err & CERR_FATAL)) return FALSE; } -#define AddWarning(warn) { if(AddCompileError(warn)) return FALSE; } +/// Use AddWarningBool for functions that return bool or KMX_BOOL +#define AddWarningBool(warn) { if(AddCompileError(warn)) return FALSE; } +/// Use AddWarning for functions that return KMX_DWORD +#define AddWarning(warn) { if(AddCompileError(warn)) return CERR_Break; } PKMX_WCHAR strtowstr(PKMX_STR in); PFILE_STORE FindSystemStore(PFILE_KEYBOARD fk, KMX_DWORD dwSystemID); +FILE* UTF16TempFromUTF8(FILE* fp_in , KMX_BOOL hasPreamble); +KMX_DWORD WriteCompiledKeyboard(PFILE_KEYBOARD fk, FILE* fp_out); +KMX_DWORD AddStore(PFILE_KEYBOARD fk, KMX_DWORD SystemID, KMX_WCHAR const * str, KMX_DWORD *dwStoreID= NULL); +KMX_DWORD ReadLine(FILE* fp_in , PKMX_WCHAR wstr, KMX_BOOL PreProcess); +KMX_DWORD ParseLine(PFILE_KEYBOARD fk, PKMX_WCHAR str); +KMX_DWORD ProcessGroupLine(PFILE_KEYBOARD fk, PKMX_WCHAR p); +KMX_DWORD ProcessGroupFinish(PFILE_KEYBOARD fk); +KMX_DWORD ProcessStoreLine(PFILE_KEYBOARD fk, PKMX_WCHAR p); +int LineTokenType(PKMX_WCHAR *str); +KMX_DWORD BuildVKDictionary(PFILE_KEYBOARD fk); // I3438 diff --git a/developer/src/kmcmplib/src/meson.build b/developer/src/kmcmplib/src/meson.build index 5188655ef5..6ac2b97cec 100644 --- a/developer/src/kmcmplib/src/meson.build +++ b/developer/src/kmcmplib/src/meson.build @@ -37,6 +37,7 @@ lib = library('kmcmplib', 'CasedKeys.cpp', 'CharToKeyConversion.cpp', 'Compiler.cpp', + 'CompilerInterfaces.cpp', 'DeprecationChecks.cpp', 'Edition.cpp', 'filesystem.cpp',