// kmanalyze.cpp : This file contains the 'main' function. Program execution begins and ends there. // #include "pch.h" #include "../../../common/windows/cpp/include/crc32.h" #include "../../../common/windows/cpp/include/keymanversion.h" #include #include #include #include BOOL LoadKeyboard(LPSTR fileName, LPKEYBOARD *lpKeyboard); BOOL VerifyChecksum(LPBYTE buf, LPDWORD CheckSum, DWORD sz); void Err(const char *p); int DoKeyboardAnalysis(LPKEYBOARD kbd, char *keyboardID, char *keyboardJSFilename, char *outputfilename); void MapVirtualKeys(void); struct TEST { UINT key; UINT shift; std::wstring context; }; typedef std::vector TESTS; enum GROUPREFTYPE { rule, match, nomatch }; struct GROUPTREE; struct GROUPREF { GROUPREFTYPE type; GROUPTREE* target; LPKEY rule; }; struct GROUPTREE { LPGROUP group; std::vector refs; }; TESTS *DoGroupAnalysis(LPKEYBOARD kbd, LPGROUP gp, std::vector & tree, TEST *test); int run(int argc, char *argv[]); wchar_t VKToChar[256][2]; UINT CharToVK[256]; UINT CharToShift[256]; #define KEYMAN_SENTRY_LOGGER_DEVELOPER_TOOLS_KMANALYZE KEYMAN_SENTRY_LOGGER_DEVELOPER_TOOLS ".kmanalyze" int main(int argc, char *argv[]) { return keyman_sentry_main(TRUE, KEYMAN_SENTRY_LOGGER_DEVELOPER_TOOLS_KMANALYZE, argc, argv, run); } int run(int argc, char *argv[]) { LPKEYBOARD kbd; char buf[_MAX_PATH], drive[_MAX_DRIVE], dir[_MAX_DIR], filename[_MAX_FNAME], ext[_MAX_EXT], jsfilename[_MAX_PATH]; if (argc < 2 || !strcmp(argv[1], "--help")) { printf( "KMANALYZE: Extract rules from a Keyman .kmx keyboard to use for building automated tests\n" "Version %s, %s\n" "Usage: KMANALYZE [outputfilename]\n\n" "Will create a keyboard.tests from keyboard.js; if outputfilename is not specified,\n" "then will put the output file in the same folder as filename.", KEYMAN_VersionWithTag, KEYMAN_Copyright ); return 1; } MapVirtualKeys(); if (!LoadKeyboard(argv[1], &kbd)) return 2; _splitpath_s(argv[1], drive, dir, filename, ext); if (argc >= 3) { strcpy_s(buf, argv[2]); } else { _makepath_s(buf, drive, dir, filename, ".tests"); } _makepath_s(jsfilename, drive, dir, filename, ".js"); int n = DoKeyboardAnalysis(kbd, filename, jsfilename, buf); delete kbd; return n; } void Err(const char *p) { printf("Fatal Error: %s\n", p); } BOOL LoadKeyboard(LPSTR fileName, LPKEYBOARD *lpKeyboard) { DWORD sz, i, j; LPBYTE buf; HANDLE hFile; PCOMP_KEYBOARD ckbp; PCOMP_GROUP cgp; PCOMP_STORE csp; PCOMP_KEY ckp; LPKEYBOARD kbp; LPGROUP gp; LPSTORE sp; LPKEY kp; if (!lpKeyboard) { Err("Internal error 001"); return FALSE; } if (!fileName) { Err("Bad Filename"); return FALSE; } hFile = CreateFile(fileName, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, 0, NULL); if (hFile == INVALID_HANDLE_VALUE) { Err("Could not open file"); return FALSE; } sz = GetFileSize(hFile, NULL); buf = new BYTE[sz]; if (!buf) { CloseHandle(hFile); Err("Could not allocate memory"); return FALSE; } ReadFile(hFile, buf, sz, &sz, NULL); CloseHandle(hFile); kbp = (LPKEYBOARD)buf; ckbp = (PCOMP_KEYBOARD)buf; if (kbp->dwIdentifier != FILEID_COMPILED) { delete buf; Err("errNotFileID"); return FALSE; } /* Check file version */ if (ckbp->dwFileVersion < VERSION_MIN || ckbp->dwFileVersion > VERSION_MAX) { /* Old or new version -- identify the desired program version */ if (VerifyChecksum(buf, &kbp->dwCheckSum, sz)) { kbp->dpStoreArray = (LPSTORE)(buf + ckbp->dpStoreArray); for (sp = kbp->dpStoreArray, i = 0; i < kbp->cxStoreArray; i++, sp++) if (sp->dwSystemID == TSS_COMPILEDVERSION) { char buf2[256]; wsprintf(buf2, "Wrong File Version: file version is %ls", ((PBYTE)kbp) + (INT_PTR)sp->dpString); delete buf; Err(buf2); return FALSE; } } delete buf; Err("Unknown File Version: try using the latest version of KMDECOMP"); return FALSE; } if (!VerifyChecksum(buf, &kbp->dwCheckSum, sz)) { delete buf; Err("Bad Checksum in file"); return FALSE; } kbp->dpStoreArray = (LPSTORE)(buf + ckbp->dpStoreArray); kbp->dpGroupArray = (LPGROUP)(buf + ckbp->dpGroupArray); //kbp->dpName = (PWSTR) (buf + ckbp->dpName); //kbp->dpCopyright = (PWSTR) (buf + ckbp->dpCopyright); //kbp->dpMessage = (PWSTR) (buf + ckbp->dpMessage); //kbp->dpLanguageName = (PWSTR) (buf + ckbp->dpLanguageName); for (sp = kbp->dpStoreArray, csp = (PCOMP_STORE)sp, i = 0; i < kbp->cxStoreArray; i++, sp++, csp++) { if (csp->dpName > 0) sp->dpName = (PWSTR)(buf + csp->dpName); else sp->dpName = NULL; sp->dpString = (PWSTR)(buf + csp->dpString); } for (gp = kbp->dpGroupArray, cgp = (PCOMP_GROUP)gp, i = 0; i < kbp->cxGroupArray; i++, gp++, cgp++) { if (cgp->dpName > 0) gp->dpName = (PWSTR)(buf + cgp->dpName); else gp->dpName = NULL; gp->dpKeyArray = (LPKEY)(buf + cgp->dpKeyArray); if (cgp->dpMatch != NULL) gp->dpMatch = (PWSTR)(buf + cgp->dpMatch); if (cgp->dpNoMatch != NULL) gp->dpNoMatch = (PWSTR)(buf + cgp->dpNoMatch); for (kp = gp->dpKeyArray, ckp = (PCOMP_KEY)kp, j = 0; j < gp->cxKeyArray; j++, kp++, ckp++) { kp->dpOutput = (PWSTR)(buf + ckp->dpOutput); kp->dpContext = (PWSTR)(buf + ckp->dpContext); } } *lpKeyboard = kbp; return TRUE; } BOOL VerifyChecksum(LPBYTE buf, LPDWORD CheckSum, DWORD sz) { DWORD tempcs; tempcs = *CheckSum; *CheckSum = 0; BuildCRCTable(); return tempcs == CalculateBufferCRC(sz, buf); } //void DoContextAnalysis(LPKEYBOARD kbd, GROUPTREE *gpref, std::vector tree, LPKEY kp, PWCHAR pc, GROUPREFTYPE type) { //} TESTS *DoGroupAnalysis(LPKEYBOARD kbd, LPGROUP gp, std::vector & tree, TEST *base_test) { // // Prevent recursion // std::vector::iterator it = std::find(tree.begin(), tree.end(), gp); if(it != tree.end()) { puts("Recursive groups are not yet supported"); return NULL; } tree.push_back(gp); //printf("%*.*sEntering group %d [depth %d]\n", tree.size(), tree.size(), " ", gpref->group - kbd->dpGroupArray + 1, tree.size()); LPKEY kp = gp->dpKeyArray; // Set of tests for this group. TESTS *tests = new TESTS; for (DWORD i = 0; i < gp->cxKeyArray; i++, kp++) { // Set of expanding tests for this rule TESTS *new_tests = NULL; TEST t0; if (gp->fUsingKeys && base_test && base_test->key != 0 && (base_test->key != kp->Key || base_test->shift != kp->ShiftFlags)) { // We are in a subgroup that uses keys, and this rule's key doesn't match the parent group's test's key. continue; } std::wstring context; for (PWCHAR pc = kp->dpContext; pc && *pc; pc = incxstr(pc)) { if (*pc == UC_SENTINEL) { switch (*(pc + 1)) { case CODE_ANY: { PWCHAR ps = kbd->dpStoreArray[*(pc + 2) - 1].dpString; // TODO: Support any character in store if (ps[0] == UC_SENTINEL) break; context += ps[0]; if (Uni_IsSurrogate1(ps[0])) context += ps[1]; } break; case CODE_NUL: // TODO: SUpport nul at start of context break; case CODE_DEADKEY: // TODO: Support deadkeys for testing break; case CODE_CONTEXTEX: break; case CODE_NOTANY: break; } } else { context += *pc; if (Uni_IsSurrogate1(*pc)) context += *(pc + 1); } } if (base_test) { std::wstring c1 = context.length() > base_test->context.length() ? context : base_test->context; //std::wstring c2 = context.length() > base_test->context.length() ? base_test->context : context; //if(c1.compare(c1.length() - c2.length(), c2.length(), c2) != 0) { // TODO: We are not comparing like with like ... so continue; but this naive test does not take into account // stores, which we need to, in order to make this work well. //} // TODO: For now we are assuming that the contexts are congruent, so we take the longer one for our test. // TODO: In reality, we could probably do both the longer and shorter one as separate tests... //TODO: merge_context(context); context = c1; } t0.context = context; if (!gp->fUsingKeys) { if (base_test) { t0.key = base_test->key; t0.shift = base_test->shift; } } else { t0.key = kp->Key; t0.shift = kp->ShiftFlags; } BOOL found = FALSE; for (PWCHAR pc = kp->dpOutput; pc && *pc; pc = incxstr(pc)) { if (*pc == UC_SENTINEL && *(pc + 1) == CODE_USE) { //printf("%*.*s: Matched rule %d of group %d\n", tree.size(), tree.size(), " ", kp ? kp - gpref->group->dpKeyArray + 1 : -1, gpref->group - kbd->dpGroupArray + 1); LPGROUP gpTarget = &kbd->dpGroupArray[*(pc + 2) - 1]; new_tests = DoGroupAnalysis(kbd, gpTarget, tree, &t0); if (!new_tests) return NULL; // TODO: Then multiplying new_tests for each subsequent group I guess!? found = TRUE; break; } } if (!found) { // TODO: Support use() in rule and use() in match // check for use in match or extra PWCHAR pc = gp->dpMatch; if (pc && *pc == UC_SENTINEL && *(pc + 1) == CODE_USE) { new_tests = DoGroupAnalysis(kbd, &kbd->dpGroupArray[*(pc + 2) - 1], tree, &t0); if (!new_tests) return NULL; // TODO: JUST A SINGLE GROUP FOR NOW } } tests->push_back(t0); if (new_tests) { for (size_t i = 0; i < new_tests->size(); i++) tests->push_back(new_tests->at(i)); delete new_tests; } //t0.context = ""; //PWC //t0.context = kp->dpContext; //for (int i = 0; i < new_tests.size(); i++) { //DoContextAnalysis(kbd, gp, tree, kp, kp->dpOutput, rule, gpref->group->dpMatch); } PWCHAR pc = gp->dpNoMatch; if (pc && *pc == UC_SENTINEL && *(pc + 1) == CODE_USE) { TESTS *new_tests = DoGroupAnalysis(kbd, &kbd->dpGroupArray[*(pc+2)-1], tree, base_test); if (!new_tests) return NULL; for (size_t i = 0; i < new_tests->size(); i++) tests->push_back(new_tests->at(i)); delete new_tests; // TODO: JUST A SINGLE GROUP FOR NOW } //printf("%*.*sExiting group %d [depth %d]\n", tree.size(), tree.size(), " ", gpref->group - kbd->dpGroupArray + 1, tree.size()); tree.pop_back(); return tests; } void PrintTree(LPKEYBOARD kbd, GROUPTREE *t, int depth); std::string unicode_escape(std::wstring s) { std::string r = "\""; for (auto ch = s.begin(); ch != s.end(); ch++) { char buf[16]; sprintf_s(buf, "\\u%04.4x", *ch); r += buf; } r += "\""; return r; } #include "../../../common/windows/cpp/include/vkeys.h" std::wstring GetKeyName(UINT key) { WCHAR buf[64]; // if (key < 256) { // wsprintfW(buf, L"keyCodes.%s /* 0x%x */", VKeyNames[key], key); // } // else { wsprintfW(buf, L"0x%x", key); // } std::wstring str(buf); return str; } struct ModifierNames { PCWSTR name; UINT modifier; }; // see kmwosk.ts ModifierNames modifierNames[] = { { L"LCTRL", 0x0001 }, { L"RCTRL", 0x0002 }, { L"LALT", 0x0004 }, { L"RALT", 0x0008 }, { L"SHIFT", 0x0010 }, { L"CTRL", 0x0020 }, { L"ALT", 0x0040 }, { L"CAPS", 0x0100 }, { L"NO_CAPS", 0x0200 }, { L"NUM_LOCK", 0x0400 }, { L"NO_NUM_LOCK", 0x0800 }, { L"SCROLL_LOCK", 0x1000 }, { L"NO_SCROLL_LOCK", 0x2000 }, { L"VIRTUAL_KEY", 0x4000 }, { NULL, 0 } }; std::wstring GetModifierName(UINT modifier) { modifier &= ~ISVIRTUALKEY; // we don't need to dump this in every rule std::wstring str; WCHAR buf[64]; UINT originalModifier = modifier; for (int i = 0; modifierNames[i].name; i++) { if (modifier & modifierNames[i].modifier) { wsprintfW(buf, L"modCodes.%s", modifierNames[i].name); if (str.size()) str += L" | "; str += buf; modifier &= ~modifierNames[i].modifier; } } if (modifier) { wsprintfW(buf, L"%x", modifier); if (str.size()) str += L" | "; str += buf; } wsprintfW(buf, L" /* 0x%x */", originalModifier); if (!str.size()) str += L"0"; str += buf; return str; } void PrintTests(TESTS *tests, char *keyboardID, wchar_t *keyboardName, char *keyboardJSFilename, char *outputfilename) { FILE *fp; fopen_s(&fp, outputfilename, "wt"); std::wstring_convert> utf8_conv; //std::wstring keyboardName_t = keyboardName; fprintf(fp, /*"(function(){\n" "\"use strict\";\n" "var modCodes = testRunner.modCodes;\n" "var keyCodes = testRunner.keyCodes;\n" "testRunner.register(*/"{\n" " \"keyboard\": {\n" " \"id\": \"%s\",\n" //" \"name\" : \"%s\",\n" //TODO: utf8 //" \"filename\" : \"%s\",\n" " \"languages\" : [{\n" " \"id\": \"en\",\n" " \"name\" : \"English\",\n" " \"region\" : \"Europe\"\n" " }]\n" " },\n" " \"inputTests\": {\n", keyboardID //utf8_conv.to_bytes(keyboardName).c_str(), //keyboardJSFilename ); for (size_t i = 0; i < tests->size(); i++) { UINT key = tests->at(i).key, shift = tests->at(i).shift; if (!(shift & ISVIRTUALKEY)) { shift = CharToShift[key] | ISVIRTUALKEY; key = CharToVK[key]; } std::wstring keyName = GetKeyName(key); std::wstring modifierName = GetModifierName(shift); fprintf(fp, " \"%d\": {\"key\": %d", i, key); // keyName.c_str()); if (shift != ISVIRTUALKEY && shift != 0) fprintf(fp, ", \"modifier\": %d", shift & ~ISVIRTUALKEY); // %ws", modifierName.c_str()); if (tests->at(i).context.size()) fprintf(fp, ", \"context\": %s", unicode_escape(tests->at(i).context).c_str()); fprintf(fp, "}%s\n", (i == tests->size() - 1 ? "" : ",")); } fprintf(fp, " }\n" "}"/*);\n" "})();\n"*/ ); fclose(fp); } void RemoveDuplicateTests(TESTS *tests) { // TODO: sort tests by key, modifier, context // Then remove duplicates } int DoKeyboardAnalysis(LPKEYBOARD kbd, char *keyboardID, char *keyboardJSFilename, char *outputfilename) { std::vector> groupPaths; //GROUPTREE tree; LPGROUP gp = &kbd->dpGroupArray[kbd->StartGroup[1]]; // Unicode start group std::vector tt; // each rule in the base group is a base rule. Then, each // rule can be expanded out by the use() tree from the rule. TESTS *tests = DoGroupAnalysis(kbd, gp, tt, NULL); if (!tests) { // Error exit return 3; } wchar_t *keyboardName = NULL, keyboardNameBuf[32]; for (UINT i = 0; i < kbd->cxStoreArray; i++) { if (kbd->dpStoreArray[i].dwSystemID == TSS_NAME) { keyboardName = kbd->dpStoreArray[i].dpString; } } if (!keyboardName) { wsprintfW(keyboardNameBuf, L"%s", keyboardID); keyboardName = keyboardNameBuf; } RemoveDuplicateTests(tests); PrintTests(tests, keyboardID, keyboardName, keyboardJSFilename, outputfilename); delete tests; // print the tree //PrintTree(kbd, &tree, 0); return 0; } DWORD NextUTF32(PWCHAR pc) { if (Uni_IsSurrogate1(*pc)) { return Uni_SurrogateToUTF32(*pc, *(pc + 1)); } return *pc; } /* void PrintRule(LPKEYBOARD kbd, LPKEY kp) { LPSTORE sp; std::vector context, output; // Always take first char in each referenced store for (PWCHAR pc = kp->dpContext; pc && *pc; pc = incxstr(pc)) { if (*pc == UC_SENTINEL) { switch (*(pc + 1)) { case CODE_ANY: sp = &kbd->dpStoreArray[*(pc + 2) - 1]; context.push_back(NextUTF32(sp->dpString)); break; case CODE_NOTANY: assert(FALSE); //TODO case CODE_INDEX: assert(FALSE); //TODO case CODE_DEADKEY: case CODE_EXTENDED: p += 2; while (*p != UC_SENTINEL_EXTENDEDEND) p++; return p + 1; case CODE_CLEARCONTEXT: return p + 1; case CODE_CALL: return p + 1; case CODE_CONTEXTEX: return p + 1; case CODE_IFOPT: return p + 3; case CODE_IFSYSTEMSTORE: return p + 3; case CODE_SETOPT: return p + 2; case CODE_SETSYSTEMSTORE: return p + 2; case CODE_RESETOPT: return p + 1; case CODE_SAVEOPT: return p + 1; } } else { // Character (either 1 or 2 word) } } } */ int groupindex(LPKEYBOARD kbd, LPGROUP gp) { return gp - kbd->dpGroupArray; } void PrintTree(LPKEYBOARD kbd, GROUPTREE *t, int depth) { if (t->group->dpName) printf("%*.*s%ws\n", depth * 2, depth * 2, " ", t->group->dpName); else printf("%*.*sgroup%d\n", depth * 2, depth * 2, " ", groupindex(kbd, t->group)); // LPKEY kp = t->group->dpKeyArray; // for (DWORD i = 0; i < t->group->cxKeyArray; i++) { // AnalyzeRule(kp); // } for (size_t i = 0; i < t->refs.size(); i++) { PrintTree(kbd, t->refs[i].target, (depth + 1)); } } // MapVirtualKeys is copied from syskbd.cpp (keyman32) #define VK_COLON 0xBA #define VK_EQUAL 0xBB #define VK_COMMA 0xBC #define VK_HYPHEN 0xBD #define VK_PERIOD 0xBE #define VK_SLASH 0xBF #define VK_ACCENT 0xC0 #define VK_LBRKT 0xDB #define VK_BKSLASH 0xDC #define VK_RBRKT 0xDD #define VK_QUOTE 0xDE #define VK_xDF 0xDF WCHAR MapVirtualKeys(WORD keyCode, UINT shiftFlags) { char shiftedDigit[] = ")!@#$%^&*("; int n, Shift; if (shiftFlags & (LCTRLFLAG | RCTRLFLAG | LALTFLAG | RALTFLAG)) return 0; if (keyCode >= '0' && keyCode <= '9') { n = keyCode - '0'; return ((shiftFlags & K_SHIFTFLAG) ? shiftedDigit[n] : keyCode); } if (keyCode >= 'A' && keyCode <= 'Z') { Shift = (shiftFlags & K_SHIFTFLAG); if (shiftFlags & (CAPITALFLAG)) Shift = !Shift; return (Shift ? keyCode : keyCode + 32); } if (keyCode >= VK_NUMPAD0 && keyCode <= VK_NUMPAD9) { if (!(shiftFlags & NUMLOCKFLAG)) return 0; return keyCode - (VK_NUMPAD0 - '0'); } Shift = (shiftFlags & K_SHIFTFLAG); switch (keyCode) { case VK_ACCENT: return Shift ? '~' : '`'; case VK_HYPHEN: return Shift ? '_' : '-'; case VK_EQUAL: return Shift ? '+' : '='; case VK_BKSLASH: case 0xE2: // I5332 return Shift ? '|' : 92; case VK_LBRKT: return Shift ? '{' : '['; case VK_RBRKT: return Shift ? '}' : ']'; case VK_COLON: return Shift ? ':' : ';'; case VK_QUOTE: return Shift ? '"' : 39; case VK_COMMA: return Shift ? '<' : ','; case VK_PERIOD: return Shift ? '>' : '.'; case VK_SLASH: return Shift ? '?' : '/'; case VK_SPACE: return ' '; } return 0; //keyCode; } void MapVirtualKeys(void) { for (int i = 0; i < 256; i++) { VKToChar[i][0] = MapVirtualKeys(i, 0); VKToChar[i][1] = MapVirtualKeys(i, K_SHIFTFLAG); CharToVK[VKToChar[i][0]] = i; CharToVK[VKToChar[i][1]] = i; CharToShift[VKToChar[i][0]] = 0; CharToShift[VKToChar[i][1]] = K_SHIFTFLAG; } }