mirror of
https://github.com/keymanapp/keyman.git
synced 2026-11-03 09:37:47 +00:00
fix(core): fix 2 marker cases in ldml_event_state::emit_difference 🙀
- handle the case where old and new context strings are the same (i.e. no work to do). - fix pointer arithmetic error in #10356 - the special case where one marker is being replaced by another. The named regression test didn't actually hit this case. Fixes: bug(core): 'string too long' #11057
This commit is contained in:
Steven R. Loomis
parent
54e59e47bc
commit
18c6ecfc08
5 changed files with 288 additions and 5 deletions
|
|
@ -377,6 +377,10 @@ ldml_event_state::emit_difference(const std::u32string &old_ctxt, const std::u32
|
|||
// So the BBBBB needs to be removed and then CCC added.
|
||||
auto ctxt_prefix = mismatch(old_ctxt.begin(), old_ctxt.end(), new_ctxt.begin(), new_ctxt.end());
|
||||
|
||||
if(ctxt_prefix.first == old_ctxt.end() && ctxt_prefix.second == new_ctxt.end()) {
|
||||
return; // No mismatch. We can just exit, there's nothing to do.
|
||||
}
|
||||
|
||||
// handle a special case where we're simply changing from one marker to another.
|
||||
// Example:
|
||||
// 0. old_ctxtstr_changed ends with … U+FFFF U+0008 | U+0001 …
|
||||
|
|
@ -390,14 +394,14 @@ ldml_event_state::emit_difference(const std::u32string &old_ctxt, const std::u32
|
|||
// marker change.
|
||||
// We can detect this because the unchanged_prefix will end with u+FFFF U+0008
|
||||
//
|
||||
// Oh, and yes, test case 'regex-test-8a-0' hits this.
|
||||
// k_212* and k_213* hit this case.
|
||||
std::u32string common_prefix(old_ctxt.begin(), ctxt_prefix.first);
|
||||
if (common_prefix.length() >= 2) {
|
||||
auto iter = common_prefix.rbegin();
|
||||
if (*(iter++) == LDML_MARKER_CODE && *(iter++) == UC_SENTINEL) {
|
||||
// adjust the iterator so that the "U+FFFF U+0008" is not a part of the common prefix.
|
||||
ctxt_prefix.first -= 2;
|
||||
ctxt_prefix.second += 2;
|
||||
ctxt_prefix.first -= 2; // backup the 'mismatch' point to before the FFFF
|
||||
ctxt_prefix.second -= 2; // backup the 'mismatch' point to before the FFFF
|
||||
// Now, old_ctxtstr_changed and new_ctxtstr_changed will start with U+FFFF U+0008 …
|
||||
}
|
||||
}
|
||||
|
|
|
|||
23
core/tests/unit/ldml/keyboards/k_212_marker_11057-test.xml
Normal file
23
core/tests/unit/ldml/keyboards/k_212_marker_11057-test.xml
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE keyboardTest3 SYSTEM "../../../../../resources/standards-data/ldml-keyboards/45/dtd/ldmlKeyboardTest3.dtd">
|
||||
<keyboardTest3 conformsTo="techpreview">
|
||||
<info keyboard="k_212_marker_11057.xml" author="Team Keyboard" name="marker" />
|
||||
<tests name="regression-11057">
|
||||
<test name="caret-a">
|
||||
<keystroke key="caret" />
|
||||
<keystroke key="a" />
|
||||
<check result="A" />
|
||||
</test>
|
||||
<test name="umlaut-e">
|
||||
<keystroke key="umlaut" />
|
||||
<keystroke key="e" />
|
||||
<check result="E" />
|
||||
</test>
|
||||
<test name="crasher">
|
||||
<keystroke key="caret"/>
|
||||
<keystroke key="caret"/>
|
||||
<keystroke key="umlaut"/>
|
||||
<check result=""/>
|
||||
</test>
|
||||
</tests>
|
||||
</keyboardTest3>
|
||||
30
core/tests/unit/ldml/keyboards/k_212_marker_11057.xml
Normal file
30
core/tests/unit/ldml/keyboards/k_212_marker_11057.xml
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<keyboard3 xmlns="https://schemas.unicode.org/cldr/45/keyboard3" locale="en" conformsTo="45">
|
||||
<info author="srl295" indicator="🙀" layout="qwerty" name="k_212_regression test" />
|
||||
|
||||
<keys>
|
||||
<key id="caret" output="\m{caret}" />
|
||||
<key id="umlaut" output="\m{umlaut}" />
|
||||
</keys>
|
||||
|
||||
<layers formId="us">
|
||||
<layer modifiers="none" id="base">
|
||||
<row keys="caret umlaut" />
|
||||
<row keys="q w e r t y" />
|
||||
<row keys="a" />
|
||||
<row keys="z" />
|
||||
<row keys="space" />
|
||||
</layer>
|
||||
</layers>
|
||||
|
||||
<transforms type="simple">
|
||||
<transformGroup>
|
||||
<transform from="\m{caret}a" to="A"/>
|
||||
<transform from="\m{umlaut}e" to="E"/>
|
||||
<!-- catch-all: drop any marker that didn't otherwise match before a char -->
|
||||
<transform from="\m{.}(.)" to="$1" />
|
||||
<!-- also drop any doubled marker -->
|
||||
<transform from="\m{.}(\m{.})" to="$1" />
|
||||
</transformGroup>
|
||||
</transforms>
|
||||
</keyboard3>
|
||||
225
core/tests/unit/ldml/keyboards/k_213_marker_11057.xml
Normal file
225
core/tests/unit/ldml/keyboards/k_213_marker_11057.xml
Normal file
|
|
@ -0,0 +1,225 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<keyboard3 xmlns="https://schemas.unicode.org/cldr/45/keyboard3" locale="fr-t-k0-optimise"
|
||||
conformsTo="45">
|
||||
<!--
|
||||
|
||||
@@keys: [K_EQUAL][K_EQUAL][SHIFT K_EQUAL][K_A]
|
||||
@@expected: \u0071
|
||||
|
||||
the three marker keys all cancel each other out, and when the 'Q' (K_A) key is output,
|
||||
the marker is removed.
|
||||
|
||||
-->
|
||||
<version number="1.0.0" />
|
||||
<info author="Team Keyboard" name="crasher https://github.com/keymanapp/keyman/issues/11057" layout="AZERTY" indicator="FR" />
|
||||
|
||||
<displays>
|
||||
<!-- display for diacritics -->
|
||||
<display output="\u0300" display="${grave}" />
|
||||
<display output="\m{acute}" display="´" />
|
||||
<display output="\m{grave}" display="`" />
|
||||
<display output="\m{umlaut}" display="¨" />
|
||||
<display output="\m{caret}" display="^" />
|
||||
<display output="\m{tilde}" display="~" />
|
||||
|
||||
<!-- display for 'mode' keys -->
|
||||
<display keyId="mark-currency" display="\u{00A4}" />
|
||||
<display keyId="mark-greek" display="μ" />
|
||||
<display keyId="mark-euro" display="Eu" />
|
||||
</displays>
|
||||
|
||||
<keys>
|
||||
<import base="cldr" path="45/keys-Zyyy-punctuation.xml" />
|
||||
<import base="cldr" path="45/keys-Zyyy-currency.xml" />
|
||||
|
||||
<!-- deadkeys -->
|
||||
<key id="mark-acute" output="\m{acute}" />
|
||||
<key id="mark-breve" output="\m{breve}" />
|
||||
<key id="mark-caret" output="\m{caret}" />
|
||||
<key id="mark-caron" output="\m{caron}" />
|
||||
<key id="mark-cedilla" output="\m{cedilla}" />
|
||||
<key id="mark-circumflex" output="\m{circumflex}" />
|
||||
<key id="mark-comma" output="\m{comma}" />
|
||||
<key id="mark-currency" output="\m{currency}" />
|
||||
<key id="mark-dotabove" output="\m{dotabove}" />
|
||||
<key id="mark-dotbelow" output="\m{dotbelow}" />
|
||||
<key id="mark-doubleacute" output="\m{doubleacute}" />
|
||||
<key id="mark-doublegrave" output="\m{doublegrave}" />
|
||||
<key id="mark-euro" output="\m{euro}" />
|
||||
<key id="mark-grave" output="\m{grave}" />
|
||||
<key id="mark-macron" output="\m{macron}" />
|
||||
<key id="mark-ogonek" output="\m{ogonek}" />
|
||||
<key id="mark-ring" output="\m{ring}" />
|
||||
<key id="mark-greek" output="\m{greek}" />
|
||||
<key id="mark-solidus" output="\m{solidus}" />
|
||||
<key id="mark-stroke" output="\m{stroke}" />
|
||||
<key id="mark-submacron" output="\m{submacron}" />
|
||||
<key id="mark-tilde" output="\m{tilde}" />
|
||||
<key id="mark-umlaut" output="\m{umlaut}" />
|
||||
|
||||
|
||||
<!-- extra keys -->
|
||||
<key id="e-grave" output="è" />
|
||||
<key id="e-acute" output="é" />
|
||||
<key id="c-cedilla" output="ç" />
|
||||
<key id="a-acute" output="á" />
|
||||
<key id="a-grave" output="à" />
|
||||
<key id="e-caret" output="ê" />
|
||||
|
||||
<!-- extra symbols -->
|
||||
<key id="bullet" output="•" />
|
||||
<key id="umlaut" output="¨" />
|
||||
<key id="sub-2" output="₂" />
|
||||
<key id="super-2" output="²" longPressKeyIds="sub-2" />
|
||||
<key id="en-dash" output="–" />
|
||||
<key id="plus-minus" output="±" />
|
||||
<key id="vulgar-half" output="½" />
|
||||
<key id="vulgar-quarter" output="¼" />
|
||||
<key id="ellipsis" output="…" />
|
||||
<key id="open-apos" output="‘" />
|
||||
<key id="close-apos" output="’" />
|
||||
<key id="open-g" output="«" />
|
||||
<key id="close-g" output="»" />
|
||||
|
||||
<key id="theta" output="θ" /> <!-- "for Romani" -->
|
||||
<key id="Theta" output="ϴ" /> <!-- "for Romani" -->
|
||||
<key id="tm" output="™" />
|
||||
<key id="open-double" output="“" />
|
||||
<key id="close-double" output="”" />
|
||||
<key id="oe" output="œ" />
|
||||
<key id="Oe" output="Œ" />
|
||||
<key id="registered" output="®" />
|
||||
<key id="minus-sign" output="−" />
|
||||
<key id="ss" output="ß" />
|
||||
<key id="ininity" output="∞" />
|
||||
<key id="division" output="÷" />
|
||||
<key id="less-equal" output="≤" />
|
||||
<key id="greater-equal" output="≥" />
|
||||
<key id="copy" output="©" />
|
||||
<key id="inverse-question" output="¿" />
|
||||
<key id="inverse-bang" output="¡" />
|
||||
<key id="middle-dot" output="•" />
|
||||
<key id="A-grave" output="À" />
|
||||
<key id="E-acute" output="É" />
|
||||
<key id="E-caret" output="Ê" />
|
||||
<key id="em-dash" output="—" />
|
||||
<key id="open-single" output="’" />
|
||||
<key id="close-single" output="’" />
|
||||
<key id="u-grave" output="Ù" />
|
||||
<key id="radix" output="√" />
|
||||
<key id="C-cedilla" output="Ç" />
|
||||
<key id="not-equal" output="≠" />
|
||||
<key id="open-angle-quote" output="‹" />
|
||||
<key id="close-angle-quote" output="›" />
|
||||
|
||||
<key id="infinity" output="∞" />
|
||||
<key id="E-grave" output="È" />
|
||||
<key id="U-grave" output="Ù" />
|
||||
<key id="nb-hyphen" output="‑" />
|
||||
<key id="multiplication" output="×" />
|
||||
<key id="ezh" output="ʒ" />
|
||||
<key id="Ezh" output="Ʒ" />
|
||||
<key id="ae" output="æ" />
|
||||
<key id="Ae" output="Æ" />
|
||||
<key id="permille" output="‰" />
|
||||
<key id="dbl-dagger" output="‡" />
|
||||
<key id="dagger" output="†" />
|
||||
<key id="asym-equal" output="≃" />
|
||||
<key id="Ss" output="ẞ" />
|
||||
</keys>
|
||||
|
||||
<layers formId="iso">
|
||||
<layer modifiers="none">
|
||||
<row
|
||||
keys="at a-grave e-acute e-grave e-caret open-paren close-paren open-apos close-apos open-g close-g apos mark-caret" />
|
||||
<row keys="a z e r t y u i o p hyphen plus" />
|
||||
<row keys="q s d f g h j k l m slash asterisk" />
|
||||
<row keys="open-angle w x c v b n period comma colon semi-colon" />
|
||||
<row keys="space" />
|
||||
</layer>
|
||||
|
||||
<layer modifiers="shift">
|
||||
<row keys="hash 1 2 3 4 5 6 7 8 9 0 double-quote mark-umlaut" />
|
||||
<row keys="A Z E R T Y U I O P en-dash plus-minus" />
|
||||
<row keys="Q S D F G H J K L M backslash vulgar-half" />
|
||||
<row keys="close-angle W X C V B N question bang ellipsis equal" />
|
||||
<row keys="space" />
|
||||
</layer>
|
||||
<!-- layer is for AltGr - for now we use ctrl alt -->
|
||||
<layer modifiers="ctrl alt">
|
||||
<row
|
||||
keys="mark-breve section mark-acute mark-grave amp open-square close-square mark-macron underscore open-double close-double degree mark-caron" />
|
||||
<row
|
||||
keys="ae pound euro registered open-curly close-curly u-grave mark-dotabove oe percent minus-sign dagger" />
|
||||
<row
|
||||
keys="theta ss dollar mark-currency mark-greek mark-euro gap mark-solidus pipe infinity division multiplication" />
|
||||
<row
|
||||
keys="less-equal ezh copy c-cedilla mark-cedilla mark-stroke mark-tilde inverse-question inverse-bang middle-dot asym-equal" />
|
||||
<row keys="space" />
|
||||
</layer>
|
||||
|
||||
<layer modifiers="ctrl alt shift">
|
||||
<row
|
||||
keys="mark-circumflex A-grave E-acute E-grave E-caret mark-doubleacute mark-doublegrave gap em-dash open-angle-quote close-angle-quote mark-ring gap" />
|
||||
<row keys="Ae gap gap gap tm gap U-grave mark-dotbelow Oe permille nb-hyphen dbl-dagger" />
|
||||
<row keys="Theta Ss gap gap gap mark-submacron gap gap gap gap radix vulgar-quarter" />
|
||||
<row
|
||||
keys="greater-equal Ezh gap C-cedilla mark-ogonek gap gap gap mark-comma gap not-equal" />
|
||||
<row keys="space" />
|
||||
</layer>
|
||||
</layers>
|
||||
|
||||
<variables>
|
||||
<!-- spacing accents as string variables -->
|
||||
<string id="grave" value="`" />
|
||||
<string id="caret" value="^" />
|
||||
<string id="umlaut" value="¨" />
|
||||
<string id="tilde" value="~" />
|
||||
|
||||
<!-- sets representing vowels and accented vowels. -->
|
||||
<!-- There's an extra space between the lower and upper case for visual separation -->
|
||||
<!-- but, there are only 10 vowels in the following four sets -->
|
||||
<!-- plus space at end for the visible version -->
|
||||
<set id="vowel" value="a e i o u A E I O U" />
|
||||
|
||||
<!-- currency key from/to -->
|
||||
|
||||
<!-- greek from/to: deviating from spec, uppercase final sigma (X) is encoded as another Σ -->
|
||||
<set id="greekfrom" value="a z e r u i o p s d g h j k l m x b n A Z E R U I O P S D G H J K L M X B N" />
|
||||
<set id="greekto" value="α ζ ε ρ θ ι ο π σ δ γ η ξ κ λ μ ς β ν Α Ζ Ε Ρ Θ Ι Ο Π Σ Δ Γ Η Ξ Κ Λ Μ Σ Β Ν" />
|
||||
|
||||
<!-- euro key from/to -->
|
||||
<set id="eurofrom" value="a e t i o s d g j ‘ ’ « » ' E T I D G J 7 8 9 0"/>
|
||||
<set id="euroto" value="ª ə þ ı º ſ ð ŋ ij ‚ ‘ „ ‟ ’ Ə Þ İ Đ Ŋ IJ › ‹ » «"/>
|
||||
|
||||
<!-- scratchy
|
||||
«" />
|
||||
<key id="close-g" output="» -->
|
||||
</variables>
|
||||
|
||||
<transforms type="simple">
|
||||
<transformGroup>
|
||||
<!-- use sets for those that fit neatly into sets -->
|
||||
<transform from="\m{caret}($[vowel])" to="$1\u{0302}" />
|
||||
<transform from="\m{umlaut}($[vowel])" to="$1\u{0308}" />
|
||||
|
||||
<!-- curr/greek/euro layer for hardware -->
|
||||
<transform from="\m{greek}($[greekfrom])" to="$[1:greekto]" />
|
||||
<transform from="\m{euro}($[eurofrom])" to="$[1:euroto]" />
|
||||
|
||||
<!-- tilde is on a subset, not all vowels, + n -->
|
||||
<transform from="\m{tilde}a" to="ã" />
|
||||
<transform from="\m{tilde}A" to="Ã" />
|
||||
<transform from="\m{tilde}n" to="ñ" />
|
||||
<transform from="\m{tilde}N" to="Ñ" />
|
||||
<transform from="\m{tilde}o" to="õ" />
|
||||
<transform from="\m{tilde}O" to="Õ" />
|
||||
</transformGroup>
|
||||
<transformGroup>
|
||||
<!-- catch-all: drop any marker that didn't otherwise match before a char -->
|
||||
<transform from="\m{.}(.)" to="$1" />
|
||||
<!-- also drop any doubled marker -->
|
||||
<transform from="\m{.}(\m{.})" to="$1" />
|
||||
</transformGroup>
|
||||
</transforms>
|
||||
</keyboard3>
|
||||
|
|
@ -28,10 +28,10 @@ tests_without_testdata = [
|
|||
'k_100_keytest',
|
||||
'k_101_keytest',
|
||||
'k_102_keytest',
|
||||
|
||||
'k_213_marker_11057',
|
||||
]
|
||||
|
||||
# These tests have a k_001_tiny-test.xml file as well.
|
||||
# These tests have a *-test.xml file as well.
|
||||
tests_with_testdata = [
|
||||
'ldml_test',
|
||||
'k_001_tiny',
|
||||
|
|
@ -44,6 +44,7 @@ tests_with_testdata = [
|
|||
'k_201_reorder_esk',
|
||||
'k_210_marker',
|
||||
'k_211_marker_escape',
|
||||
'k_212_marker_11057',
|
||||
]
|
||||
|
||||
tests = tests_without_testdata
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue