mirror of
https://github.com/keymanapp/keyman.git
synced 2026-08-10 10:55:33 +00:00
766c6992ed
When normalizing, we need to stop processing on an NFC boundary, not an NFD boundary, to support normalizations such as in Bengali, where appending `U+09D7` to a context of `U+0995 U+09C7` should result in `U+0995 U+09CC`. The specification is unclear on this; see https://unicode-org.atlassian.net/browse/CLDR-19218 This also updates the ldml keyboard unit test suite to support running in full NFC mode (used in all Engine implementations) as well retaining the NFD mode (now only used by the debugger). Side note: the Bengali normalization failure case was picked up by the improvements to the unit test suite, proving once again that good tests are so valuable. Fixes: #15491 Fixes: #15505 Follows: #15488 Relates-to: CLDR-19218
49 lines
1.3 KiB
C++
49 lines
1.3 KiB
C++
/*
|
|
Copyright: © SIL International.
|
|
Description: Normalization and Regex utilities
|
|
Create Date: 23 May 2024
|
|
Authors: Steven R. Loomis
|
|
*/
|
|
|
|
#pragma once
|
|
|
|
#include <string>
|
|
#include "keyman_core.h"
|
|
|
|
namespace km {
|
|
namespace core {
|
|
namespace util {
|
|
|
|
/** Normalize a u32string inplace to NFC. @return false on failure */
|
|
bool normalize_nfc(std::u32string &str);
|
|
|
|
/** Normalize a u16string inplace to NFC. @return false on failure */
|
|
bool normalize_nfc(std::u16string &str);
|
|
|
|
/** Normalize a u32string inplace to NFD. @return false on failure */
|
|
bool normalize_nfd(std::u32string &str);
|
|
|
|
/** Normalize a u16string inplace to NFD. @return false on failure */
|
|
bool normalize_nfd(std::u16string &str);
|
|
|
|
/** normalize src to dst in NFD. @return false on failure */
|
|
bool normalize_nfd(km_core_cu const * src, std::u16string &dst);
|
|
|
|
/** normalize (decompose) a single cp to string. @return false on failure */
|
|
bool normalize_nfd(km_core_usv cp, std::u32string &dst);
|
|
|
|
/** @return true if string is already NFD */
|
|
bool is_nfd(const std::u16string& str);
|
|
|
|
/** @return true if string is already NFD */
|
|
bool is_nfd(const std::u32string& str);
|
|
|
|
/** @return true if cp can interacts with prior chars */
|
|
bool has_nfc_boundary_before(km_core_usv cp);
|
|
|
|
/** convenience function, caller owns storage */
|
|
km_core_usv *string_to_usv(const std::u32string& src);
|
|
|
|
}
|
|
}
|
|
}
|