/* Copyright: © SIL International. Description: This is an implementation of the LDML keyboard spec 3.0. Create Date: 5 Aug 2022 Authors: Marc Durdin (MD) */ #include #include "ldml/ldml_processor.hpp" #include "state.hpp" #include "kmx_file.h" #include "kmx/kmx_plus.h" #include "kmx/kmx_xstring.h" #include "ldml/keyboardprocessor_ldml.h" #include "kmx/kmx_file_validator.hpp" #include "debuglog.h" #include namespace { constexpr km_core_attr const engine_attrs = { 256, KM_CORE_LIB_CURRENT, KM_CORE_LIB_AGE, KM_CORE_LIB_REVISION, KM_CORE_TECH_LDML, "SIL International" }; } // using km::kbp::kmx::ShouldDebug; // for DebugLog namespace km { namespace kbp { ldml_processor::ldml_processor(path const & kb_path, const std::vector &data) : abstract_processor( keyboard_attributes(kb_path.stem(), KM_CORE_LMDL_PROCESSOR_VERSION, kb_path.parent(), {}) ), _valid(false), transforms(), bksp_transforms(), keys() { if(data.size() <= sizeof(kmx::COMP_KEYBOARD_EX)) { DebugLog("data.size %zu too small", data.size()); return; } // // Locate the structs here, but still retain ptrs to the raw structs. kmx::KMX_FileValidator* comp_keyboard = (kmx::KMX_FileValidator*)data.data(); // Perform the standard validation if(!comp_keyboard->VerifyKeyboard(data.size())) { DebugLog("VerifyKeyboard() returned false"); return; } kmx::kmx_plus kplus(comp_keyboard, data.size()); // slices to a COMP_KEYBOARD if(!kplus.is_valid()) { DebugLog("kmx_plus.is_valid is false"); return; } // Now, if we have keys, use them. if (kplus.key2 != nullptr) { // read all keys into array for (KMX_DWORD i=0; ikmapCount; i++) { std::u16string str; auto kmapEntry = kplus.key2Helper.getKmap(i); assert(kmapEntry != nullptr); // now look up the key auto keyEntry = kplus.key2Helper.getKeys(kmapEntry->key); assert(keyEntry != nullptr); // TODO-LDML: LDML_KEYS_KEY_FLAGS_NOTRANSFORM if (keyEntry->flags & LDML_KEYS_KEY_FLAGS_EXTEND) { if (nullptr == kplus.strs) { DebugLog("for keys: kplus.strs == nullptr"); // need a string table to get strings assert(false); return; } str = kplus.strs->get(keyEntry->to); } else { str = keyEntry->get_to_string(); } keys.add((km_core_virtual_key)kmapEntry->vkey, (uint16_t)kmapEntry->mod, str); } } // else: no keys! but still valid. Just, no keys. // load transforms if (kplus.tran != nullptr && kplus.tran->groupCount > 0) { transforms.reset(km::kbp::ldml::transforms::load(kplus, kplus.tran, kplus.tranHelper)); if (!transforms) { DebugLog("Failed to load tran transforms"); return; // failed to load } } // load bksp transforms if (kplus.bksp != nullptr && kplus.bksp->groupCount > 0) { bksp_transforms.reset(km::kbp::ldml::transforms::load(kplus, kplus.bksp, kplus.bkspHelper)); if (!bksp_transforms) { DebugLog("Failed to load bksp transforms"); return; // failed to load } } // Only valid if we reach here DebugLog("_valid = true"); _valid = true; } bool ldml_processor::is_kmxplus_file(path const & kb_path, std::vector& data) { // TODO-LDML: we should refactor all the core components to delegate file loading // to the Engine, which requires an API change, but this makes delivery // of keyboard files more flexible under more WASM. std::ifstream file(static_cast(kb_path), std::ios::binary | std::ios::ate); if(!file.good()) { return false; } const std::streamsize size = file.tellg(); if(size >= KMX_MAX_ALLOWED_FILE_SIZE) { return false; } file.seekg(0, std::ios::beg); data.resize((size_t)size); if(!file.read((char *) data.data(), size)) { return false; } file.close(); const kmx::PCOMP_KEYBOARD comp_keyboard = (kmx::PCOMP_KEYBOARD)data.data(); if(comp_keyboard->dwIdentifier != KMX_DWORD(FILEID_COMPILED)) { return false; } if(comp_keyboard->dwFileVersion < VERSION_160 || (comp_keyboard->dwFlags & KF_KMXPLUS) == 0) { return false; } // A KMXPlus file is in the buffer (although more validation is required and will // be done in the constructor) return true; } km_core_status ldml_processor::process_queued_actions( km_core_state *state ) { assert(state); if (!state) return KM_CORE_STATUS_INVALID_ARGUMENT; // TODO Implement return KM_CORE_STATUS_OK; } bool ldml_processor::queue_action( km_core_state * state, km_core_action_item const* action_item ) { assert(state); assert(action_item); if ((!state) || (!action_item)) return false; return false; } km_core_status ldml_processor::process_event( km_core_state *state, km_core_virtual_key vk, uint16_t modifier_state, uint8_t is_key_down, uint16_t /*event_flags*/ // TODO-LDML: unused... for now... ) { assert(state); if (!state) return KM_CORE_STATUS_INVALID_ARGUMENT; if (!is_key_down) { // TODO: Implement caps lock handling state->actions().clear(); state->actions().commit(); return KM_CORE_STATUS_OK; } try { // At the start of every process_event always clear the action_items state->actions().clear(); switch (vk) { // Special handling for backspace VK case KM_CORE_VKEY_BKSP: { if (!!bksp_transforms) { // TODO-LDML: process bksp // std::u16string outputString; // // don't bother if no backspace transforms! // // TODO-LDML: unroll ctxt into a str // std::u16string ctxtstr; // for (size_t i = 0; i < ctxt.size(); i++) { // ctxtstr.append(ctxt[i]); // } // const size_t matchedContext = transforms->apply(ctxtstr, outputString); } KMX_DWORD last_char = 0UL; // attempt to get the last char auto end = state->context().rbegin(); if(end != state->context().rend()) { if((*end).type == KM_CORE_CT_CHAR) { last_char = (*end).character; // TODO-LDML: markers! } } if (last_char == 0UL) { /* We couldn't find a character at end of context (context is empty), so we'll pass the backspace keystroke on to the app to process; the app might want to use backspace to move between contexts or delete a text box, etc. Or it might be a legacy app and we've had our caret dumped in somewhere unknown, so we will have to depend on the app to be sensible about backspacing because we know nothing. */ state->actions().push_backspace(KM_CORE_BT_UNKNOWN); } else { state->actions().push_backspace(KM_CORE_BT_CHAR, last_char); state->context().pop_back(); } } break; default: // all other VKs { // Look up the key const std::u16string key_str = keys.lookup(vk, modifier_state); if (key_str.empty()) { // no key was found, so pass the keystroke on to the Engine state->actions().push_invalidate_context(); state->actions().push_emit_keystroke(); break; // ----- commit and exit } process_key_string(state, key_str); } // end of processing a 'normal' vk } // end of switch // end of normal processing: commit and exit state->actions().commit(); } catch (std::bad_alloc &) { state->actions().clear(); return KM_CORE_STATUS_NO_MEM; } return KM_CORE_STATUS_OK; } void ldml_processor::process_key_string(km_core_state *state, const std::u16string &key_str) const { // found a string - push it into the context and actions // we convert it here instead of using the emit_text() overload // so that we don't have to reconvert it inside the transform code. std::u32string str32 = kmx::u16string_to_u32string(key_str); if (!transforms) { // No transforms: just emit the string. emit_text(state, str32); } else { // Process transforms here /** * a copy of the current/changed context, for transform use. * */ std::u32string ctxtstr; (void)context_to_string(state, ctxtstr); // add the newly added key output to ctxtstr ctxtstr.append(str32); // and normalize /** the output buffer for transforms */ std::u32string outputString; // apply the transform, get how much matched (at the end) const size_t matchedContext = transforms->apply(ctxtstr, outputString); if (matchedContext == 0) { // No match, just emit the original string emit_text(state, str32); } else { // We have a match. ctxtstr.resize(ctxtstr.length() - str32.length()); /** how many chars of the context we need to clear */ auto charsToDelete = matchedContext - str32.length(); /* we don't need to clear the output of the current key */ /** how many context items need to be removed */ size_t contextRemoved = 0; for (auto c = state->context().rbegin(); charsToDelete > 0 && c != state->context().rend(); c++, contextRemoved++) { /** last char of context */ km_core_usv lastCtx = ctxtstr.back(); uint8_t type = c->type; assert(type == KM_CORE_BT_CHAR || type == KM_CORE_BT_MARKER); if (type == KM_CORE_BT_CHAR) { // single char, drop it charsToDelete--; assert(c->character == lastCtx); ctxtstr.pop_back(); state->actions().push_backspace(KM_CORE_BT_CHAR, c->character); // Cause prior char to be removed } else if (type == KM_CORE_BT_MARKER) { // it's a marker, 'worth' 3 uchars assert(charsToDelete >= 3); assert(lastCtx == c->marker); // end of list charsToDelete -= 3; // pop off the three-part sentinel string ctxtstr.pop_back(); ctxtstr.pop_back(); ctxtstr.pop_back(); // push a special backspace to delete the marker state->actions().push_backspace(KM_CORE_BT_MARKER, c->marker); } } // now, pop the right number of context items for (size_t i = 0; i < contextRemoved; i++) { // we don't pop during the above loop because the iterator gets confused state->context().pop_back(); } // Now, add in the updated text. This will convert UC_SENTINEL, etc back to marker actions. emit_text(state, outputString); // If we needed it further. we could update ctxtstr here: // ctxtstr.append(outputString); // ... but it is no longer needed at this point. } // end of transform match } // end of processing transforms } km_core_attr const & ldml_processor::attributes() const { return engine_attrs; } km_core_keyboard_key * ldml_processor::get_key_list() const { km_core_keyboard_key* key_list = new km_core_keyboard_key(KM_CORE_KEYBOARD_KEY_LIST_END); return key_list; } km_core_keyboard_imx * ldml_processor::get_imx_list() const { km_core_keyboard_imx* imx_list = new km_core_keyboard_imx(KM_CORE_KEYBOARD_IMX_END); return imx_list; } km_core_context_item * ldml_processor::get_intermediate_context() { km_core_context_item *citems = new km_core_context_item(KM_CORE_CONTEXT_ITEM_END); return citems; } km_core_status ldml_processor::validate() const { return _valid ? KM_CORE_STATUS_OK : KM_CORE_STATUS_INVALID_KEYBOARD; } void ldml_processor::emit_text(km_core_state *state, const std::u16string &str) { const std::u32string str32 = kmx::u16string_to_u32string(str); emit_text(state, str32); } void ldml_processor::emit_text(km_core_state *state, const std::u32string &str) { for (auto it = str.begin(); it < str.end(); it++) { const auto ch = *it; // If we are at the start of a sequence: if (ch == LDML_UC_SENTINEL) { it++; // consume LDML_UC_SENTINEL // TODO-LDML: Might assert if a malformed sequence is included- "should not happen"? assert(it < str.end()); // verify that the next char is LDML_MARKER_CODE assert(*it == LDML_MARKER_CODE); it++; // consume LDML_MARKER_CODE assert(it < str.end()); const auto marker_no = *it; emit_marker(state, marker_no); } else { emit_text(state, ch); } } } void ldml_processor::emit_text(km_core_state *state, km_core_usv ch) { assert(ch != LDML_UC_SENTINEL); state->context().push_character(ch); state->actions().push_character(ch); } void ldml_processor::emit_marker(km_core_state *state, KMX_DWORD marker_no) { assert(km::kbp::kmx::is_valid_marker(marker_no)); state->actions().push_marker(marker_no); state->context().push_marker(marker_no); } size_t ldml_processor::context_to_string(km_core_state *state, std::u32string &str) { str.clear(); auto &cp = state->context(); size_t ctxlen = 0; // TODO-LDML: is this needed? uint8_t last_type = KM_CORE_BT_UNKNOWN; for (auto c = cp.rbegin(); c != cp.rend(); c++, ctxlen++) { last_type = c->type; if (last_type == KM_CORE_BT_CHAR) { str.insert(0, 1, c->character); } else if (last_type == KM_CORE_BT_MARKER) { assert(km::kbp::kmx::is_valid_marker(c->marker)); prepend_marker(str, c->marker); } else { break; } } return ctxlen; // consumed the entire context buffer. } } // namespace kbp } // namespace km