From 558c8013ee1a9fd8c221e401bb68616baaaa6480 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Fri, 5 Feb 2021 16:56:21 +1100 Subject: [PATCH] feat(developer): improve BCP 47 canonicalization Ensures we get a canonical tag per langtags.json as far as we possibly can. This is a breaking change for the compiler as tags which were formerly regarded as canonical are no longer regarded that way. This mostly relates to script subtag but a secondary bug meant that some other tags would have lost data in the canonicalization process (because we did a lookup based only on the language subtag previously, which is a no-no). See keymanapp/keyboards#1452 for related work. --- .../TIKE/compile/ValidateKeyboardInfo.pas | 4 +- ...n.Developer.UI.UfrmSelectBCP47Language.pas | 2 +- ...er.UI.Project.UfrmNewProjectParameters.pas | 2 +- ...n.Developer.System.ValidateRepoChanges.pas | 463 ++++++++++++++++++ windows/src/developer/kmcomp/kmcomp.dpr | 3 +- windows/src/developer/kmcomp/kmcomp.dproj | 16 +- windows/src/developer/kmcomp/main.pas | 10 +- ...eyman.Developer.System.ProjectTemplate.pas | 2 +- .../keymankeyboardlanguagesfile.pas | 2 +- .../keymankeyboardlanguagesinstalled.pas | 8 +- .../kmcomapi/com/keyman_implementation.pas | 2 +- .../processes/keyboard/kpinstallkeyboard.pas | 4 +- .../global/delphi/general/CompilePackage.pas | 2 +- ...yman.System.CanonicalLanguageCodeUtils.pas | 34 +- ...man.System.PackageInfoRefreshKeyboards.pas | 4 +- ...System.PackageInfoRefreshLexicalModels.pas | 2 +- ....System.CanonicalLanguageCodeUtilsTest.pas | 3 + 17 files changed, 526 insertions(+), 37 deletions(-) create mode 100644 windows/src/developer/kmcomp/Keyman.Developer.System.ValidateRepoChanges.pas diff --git a/windows/src/developer/TIKE/compile/ValidateKeyboardInfo.pas b/windows/src/developer/TIKE/compile/ValidateKeyboardInfo.pas index 3a5a12f92a..c025d0b5ec 100644 --- a/windows/src/developer/TIKE/compile/ValidateKeyboardInfo.pas +++ b/windows/src/developer/TIKE/compile/ValidateKeyboardInfo.pas @@ -91,7 +91,7 @@ begin if not IsValid(False, msg) then Result := Failed(msg); - if not TCanonicalLanguageCodeUtils.IsCanonical(Tag, msg, False) then + if not TCanonicalLanguageCodeUtils.IsCanonical(Tag, msg, False, False) then Result := Failed(msg); finally Free; @@ -106,7 +106,7 @@ begin if not IsValid(False, msg) then Result := Failed(msg); - if not TCanonicalLanguageCodeUtils.IsCanonical(Tag, msg, False) then + if not TCanonicalLanguageCodeUtils.IsCanonical(Tag, msg, False, False) then Result := Failed(msg); finally Free; diff --git a/windows/src/developer/TIKE/dialogs/languages/Keyman.Developer.UI.UfrmSelectBCP47Language.pas b/windows/src/developer/TIKE/dialogs/languages/Keyman.Developer.UI.UfrmSelectBCP47Language.pas index 731243ea95..282f12bc84 100644 --- a/windows/src/developer/TIKE/dialogs/languages/Keyman.Developer.UI.UfrmSelectBCP47Language.pas +++ b/windows/src/developer/TIKE/dialogs/languages/Keyman.Developer.UI.UfrmSelectBCP47Language.pas @@ -157,7 +157,7 @@ var begin inherited; tag.Tag := TKMXFileLanguages.TranslateISO6393ToBCP47(cbLanguageTag.Text); - t := TCanonicalLanguageCodeUtils.FindBestTag(Tag.Tag, False); + t := TCanonicalLanguageCodeUtils.FindBestTag(Tag.Tag, False, False); if t <> '' then begin with TBCP47Tag.Create(t) do diff --git a/windows/src/developer/TIKE/project/Keyman.Developer.UI.Project.UfrmNewProjectParameters.pas b/windows/src/developer/TIKE/project/Keyman.Developer.UI.Project.UfrmNewProjectParameters.pas index d5573c93af..9d99c9d876 100644 --- a/windows/src/developer/TIKE/project/Keyman.Developer.UI.Project.UfrmNewProjectParameters.pas +++ b/windows/src/developer/TIKE/project/Keyman.Developer.UI.Project.UfrmNewProjectParameters.pas @@ -476,7 +476,7 @@ begin Tag := StrToken(Tags, ' '); BCP47Tag := TBCP47Tag.Create(Tag); try - BCP47Tag.Tag := TCanonicalLanguageCodeUtils.FindBestTag(BCP47Tag.Tag, False); + BCP47Tag.Tag := TCanonicalLanguageCodeUtils.FindBestTag(BCP47Tag.Tag, False, False); if BCP47Tag.IsValid(False) then begin Language := TPackageKeyboardLanguage.Create(pack); diff --git a/windows/src/developer/kmcomp/Keyman.Developer.System.ValidateRepoChanges.pas b/windows/src/developer/kmcomp/Keyman.Developer.System.ValidateRepoChanges.pas new file mode 100644 index 0000000000..8a5e844382 --- /dev/null +++ b/windows/src/developer/kmcomp/Keyman.Developer.System.ValidateRepoChanges.pas @@ -0,0 +1,463 @@ +unit Keyman.Developer.System.ValidateRepoChanges; + +interface + +uses + System.Classes, + System.JSON, + + kpsfile; + +type + TValidateRepoChanges = class + private + class var err_old, err_new: TStringList; + class var root: string; + class function Search(path, operation: string): Boolean; static; + class function CheckKeyboardInfo(name: string): Boolean; static; + class function CheckPackage(name: string): Boolean; static; + class function LoadKeyboardInfoFile(filename: string): TJSONObject; static; + class function GetLanguageCodesFromJson(root: TJSONObject; + langs: TStringList): Boolean; static; + class function GetLanguageCodesFromKps(kps: TKPSFile; + langs: TStringList): Boolean; static; + class function CompareKeyboardInfoScripts(name: string): Boolean; static; + public + class function Execute(path, operation: string): Boolean; + end; + +implementation + +uses + System.Character, + System.Generics.Collections, + System.SysUtils, + + BCP47Tag, + Keyman.System.KeyboardInfoFile, + Keyman.System.CanonicalLanguageCodeUtils, + Keyman.System.Standards.LangTagsRegistry, + TempFileManager, + Unicode, + utilexecute; + +{ TValidateRepoChanges } + +class function TValidateRepoChanges.Execute(path, operation: string): Boolean; +begin + if path = '' then path := 'c:\projects\keyman\keyboards'; + + path := IncludeTrailingPathDelimiter(path); + + root := path; + +// Result := CheckKeyboardInfo(path + 'release\k\kayan\kayan.keyboard_info'); + + err_old := TStringList.Create; + err_new := TStringList.Create; + try + Result := Search(path, operation); + err_old.SaveToFile('repo-check.old.txt'); + err_new.SaveToFile('repo-check.new.txt'); + finally + err_old.Free; + err_new.Free; + end; +end; + +class function TValidateRepoChanges.Search(path, operation: string): Boolean; +var + f: TSearchRec; +begin + if FindFirst(path + '*.keyboard_info', 0, f) = 0 then + begin + repeat + if operation = 'compare-bcp47-revisions' then CheckKeyboardInfo(path + f.Name) + else if operation = 'compare-bcp47-scripts' then CompareKeyboardInfoScripts(path + f.Name); + until FindNext(f) <> 0; + FindClose(f); + end; + if FindFirst(path + '*.kps', 0, f) = 0 then + begin + repeat + if operation = 'compare-bcp47-revisions' then CheckPackage(path + f.Name); + until FindNext(f) <> 0; + FindClose(f); + end; + if FindFirst(path + '*', faDirectory, f) = 0 then + begin + repeat + if ((f.Attr and faDirectory) = faDirectory) and (f.Name <> '.') and (f.Name <> '..') and not SameText(f.Name, 'build') then + Search(path + f.Name + '\', operation); + until FindNext(f) <> 0; + FindClose(f); + end; + + Result := True; +end; + +class function TValidateRepoChanges.LoadKeyboardInfoFile(filename: string): TJSONObject; +begin + with TStringStream.Create('', TEncoding.UTF8) do + try + LoadFromFile(filename); + Result := TJSONObject.ParseJsonValue(DataString) as TJSONObject; + finally + Free; + end; +end; + +class function TValidateRepoChanges.CheckKeyboardInfo(name: string): Boolean; +var + i: Integer; + json_new, json_old: TJSONObject; +// t: TTempFile; + relpath: string; + content: string; + ec: Integer; + lang_new: TStringList; + lang_old: TStringList; + langc_old: TArray; + langc_new: TArray; + found_error: Boolean; + j: Integer; + langc_old_prev: string; +begin + found_error := False; + relpath := name.Substring(root.Length).Replace('\', '/'); + + json_new := LoadKeyboardInfoFile(name); + if not Assigned(json_new) then + raise Exception.Create('Unable to load file '+name); + // Get previous revision from git + +// t := TTempFileManager.Get('.keyboard_info'); + + if not TUtilExecute.Console('git show "HEAD:'+relpath+'"', root, content, ec) then + RaiseLastOSError; + + if ec <> 0 then + raise Exception.Create('Unable to execute git show for '+name); + + if Copy(content, 1, 3) = string(UTF8Signature) then + Delete(content, 1, 3); + + json_old := TJSONObject.ParseJsonValue(content) as TJSONObject; + if not Assigned(json_old) then + raise Exception.Create('Unable to load old file '+name); + + lang_new := TStringList.Create; + lang_old := TStringList.Create; + try + if not GetLanguageCodesFromJson(json_new, lang_new) then + raise Exception.Create('Unable to get Language codes'); + if not GetLanguageCodesFromJson(json_old, lang_old) then + raise Exception.Create('Unable to get Language codes'); + + j := 0; + for i := 0 to lang_new.Count - 1 do + begin + repeat + if j >= lang_old.Count then + begin + if not found_error then + begin + writeln('Checking '+relpath); + found_error := True; + end; + writeln(' Mismatch in number of language codes'); + break; + end; + + langc_new := lang_new[i].Split([',']); + langc_old := lang_old[j].Split([',']); + Inc(j); + until langc_old[0] <> langc_old_prev; + langc_old_prev := langc_old[0]; + if j >= lang_old.Count then Break; + + if langc_new[0] <> langc_old[0] then + begin + if not found_error then + begin + writeln('Checking '+relpath); + found_error := True; + end; + writeln(Format(' New code %s [%s] does not match old code %s [%s]', [langc_new[1], langc_new[0], langc_old[1], langc_old[0]])); + end; + end; + + if found_error then + begin + err_old.Add('Checking '+relpath); + for i := 0 to lang_old.Count - 1 do + begin + langc_old := lang_old[i].Split([',']); + err_old.Add(Format(' %s [%s]', [langc_old[1], langc_old[0]])); + end; + err_old.Add(''); + + err_new.Add('Checking '+relpath); + for i := 0 to lang_new.Count - 1 do + begin + langc_new := lang_new[i].Split([',']); + err_new.Add(Format(' %s [%s]', [langc_new[1], langc_new[0]])); + end; + err_new.Add(''); + end; + finally + lang_new.Free; + lang_old.Free; + end; + + Result := True; +end; + +class function TValidateRepoChanges.GetLanguageCodesFromJson(root: TJSONObject; langs: TStringList): Boolean; +var + i: Integer; + a: TJSONArray; + + procedure AddLang(lang: string); + begin + langs.Add(TCanonicalLanguageCodeUtils.FindBestTag(lang, False, False)+','+lang); + end; + +begin + if root.Values[TKeyboardInfoFile.SLanguages] = nil then + Exit(False); + + if root.Values[TKeyboardInfoFile.SLanguages] is TJSONArray then + begin + a := root.Values[TKeyboardInfoFile.SLanguages] as TJSONArray; + for i := 0 to a.Count - 1 do + AddLang(a.Items[i].AsType); + end + else + begin + root := root.Values[TKeyboardInfoFile.SLanguages] as TJSONObject; + for i := 0 to root.Count - 1 do + AddLang(root.Pairs[i].JsonString.Value); + end; + + langs.Sort; + + Result := True; +end; + +class function TValidateRepoChanges.GetLanguageCodesFromKps(kps: TKPSFile; langs: TStringList): Boolean; +var + i: Integer; + j: Integer; + + procedure AddLang(lang: string); + begin + langs.Add(TCanonicalLanguageCodeUtils.FindBestTag(lang, False, False)+','+lang); + end; + +begin + for i := 0 to kps.Keyboards.Count - 1 do + for j := 0 to kps.Keyboards[i].Languages.Count - 1 do + AddLang(kps.Keyboards[i].Languages[j].ID); + + langs.Sort; + + Result := True; +end; + +class function TValidateRepoChanges.CheckPackage(name: string): Boolean; +var + i: Integer; + kps_new, kps_old: TKPSFile; +// t: TTempFile; + relpath: string; + content: string; + ec: Integer; + lang_new: TStringList; + lang_old: TStringList; + langc_old: TArray; + langc_new: TArray; + found_error: Boolean; + j: Integer; + langc_old_prev: string; +begin + found_error := False; + relpath := name.Substring(root.Length).Replace('\', '/'); + + kps_new := TKPSFile.Create; + kps_new.FileName := name; + kps_new.LoadXML; + + if not TUtilExecute.Console('git show "HEAD:'+relpath+'"', root, content, ec) then + RaiseLastOSError; + + if ec <> 0 then + raise Exception.Create('Unable to execute git show for '+name); + + if Copy(content, 1, 3) = string(UTF8Signature) then + Delete(content, 1, 3); + + kps_old := TKPSFile.Create; + kps_old.LoadXMLFromText(content); + + lang_new := TStringList.Create; + lang_old := TStringList.Create; + try + if not GetLanguageCodesFromKps(kps_new, lang_new) then + raise Exception.Create('Unable to get Language codes'); + if not GetLanguageCodesFromKps(kps_old, lang_old) then + raise Exception.Create('Unable to get Language codes'); + + j := 0; + for i := 0 to lang_new.Count - 1 do + begin + repeat + if j >= lang_old.Count then + begin + if not found_error then + begin + writeln('Checking '+relpath); + found_error := True; + end; + writeln(' Mismatch in number of language codes'); + break; + end; + + langc_new := lang_new[i].Split([',']); + langc_old := lang_old[j].Split([',']); + Inc(j); + until langc_old[0] <> langc_old_prev; + langc_old_prev := langc_old[0]; + if j >= lang_old.Count then Break; + + if langc_new[0] <> langc_old[0] then + begin + if not found_error then + begin + writeln('Checking '+relpath); + found_error := True; + end; + writeln(Format(' New code %s [%s] does not match old code %s [%s]', [langc_new[1], langc_new[0], langc_old[1], langc_old[0]])); + end; + end; + + if found_error then + begin + err_old.Add('Checking '+relpath); + for i := 0 to lang_old.Count - 1 do + begin + langc_old := lang_old[i].Split([',']); + err_old.Add(Format(' %s [%s]', [langc_old[1], langc_old[0]])); + end; + err_old.Add(''); + + err_new.Add('Checking '+relpath); + for i := 0 to lang_new.Count - 1 do + begin + langc_new := lang_new[i].Split([',']); + err_new.Add(Format(' %s [%s]', [langc_new[1], langc_new[0]])); + end; + err_new.Add(''); + end; + finally + lang_new.Free; + lang_old.Free; + end; + kps_old.Free; + kps_new.Free; + + Result := True; +end; + +class function TValidateRepoChanges.CompareKeyboardInfoScripts(name: string): Boolean; +var + i: Integer; + json: TJSONObject; + relpath: string; + script, base_script: string; + lang: TStringList; + base_lang_item, lang_item: TArray; + found_error: Boolean; + + procedure WriteHeader; + begin + if not found_error then + begin + writeln; + writeln('Checking '+relpath); + end; + found_error := True; + end; + + function GetScript(lang: string): string; + var + v: string; + LangTag: TLangTag; + BCP47: TBCP47Tag; + begin + if TLangTagsMap.AllTags.TryGetValue(lang, v) then + lang := v; + + if not TLangTagsMap.LangTags.TryGetValue(lang, LangTag) then + begin + BCP47 := TBCP47Tag.Create(lang); + try + Exit(BCP47.Script); + finally + BCP47.Free; + end; + end; + + Result := LangTag.script; + end; + +begin + found_error := False; + relpath := name.Substring(root.Length).Replace('\', '/'); + + json := LoadKeyboardInfoFile(name); + if not Assigned(json) then + raise Exception.Create('Unable to load file '+name); + + lang := TStringList.Create; + try + if not GetLanguageCodesFromJson(json, lang) then + raise Exception.Create('Unable to get Language codes'); + + if lang.Count = 0 then + begin + WriteHeader; + writeln(' Warning: no languages found'); + Exit(True); + end; + + base_lang_item := lang[0].Split([',']); + + // Lookup the tag first, canonicalize to the base tag for known tags + base_script := GetScript(base_lang_item[0]); + if base_script = '' then + begin + WriteHeader; + writeln(' Warning: could not identify tag '+base_lang_item[0]); + Exit(True); + end; + + for i := 1 to lang.Count - 1 do + begin + lang_item := lang[i].Split([',']); + script := GetScript(lang_item[0]); + if script <> base_script then + begin + WriteHeader; + writeln(Format(' Tag %s [%s] has script <%s>, which differs from base tag %s [%s], script <%s>', + [lang_item[1], lang_item[0], script, base_lang_item[1], base_lang_item[0], base_script])); + end; + end; + finally + lang.Free; + end; + + Result := True; +end; + +end. diff --git a/windows/src/developer/kmcomp/kmcomp.dpr b/windows/src/developer/kmcomp/kmcomp.dpr index ad3aa8a91e..426061a9d8 100644 --- a/windows/src/developer/kmcomp/kmcomp.dpr +++ b/windows/src/developer/kmcomp/kmcomp.dpr @@ -119,7 +119,8 @@ uses KeymanPaths in '..\..\global\delphi\general\KeymanPaths.pas', Keyman.System.CanonicalLanguageCodeUtils in '..\..\global\delphi\general\Keyman.System.CanonicalLanguageCodeUtils.pas', Keyman.System.Standards.LangTagsRegistry in '..\..\global\delphi\standards\Keyman.System.Standards.LangTagsRegistry.pas', - Keyman.Developer.System.Project.UrlRenderer in '..\TIKE\project\Keyman.Developer.System.Project.UrlRenderer.pas'; + Keyman.Developer.System.Project.UrlRenderer in '..\TIKE\project\Keyman.Developer.System.Project.UrlRenderer.pas', + Keyman.Developer.System.ValidateRepoChanges in 'Keyman.Developer.System.ValidateRepoChanges.pas'; {$R icons.RES} {$R version.res} diff --git a/windows/src/developer/kmcomp/kmcomp.dproj b/windows/src/developer/kmcomp/kmcomp.dproj index d0b93eeaef..d128f386fb 100644 --- a/windows/src/developer/kmcomp/kmcomp.dproj +++ b/windows/src/developer/kmcomp/kmcomp.dproj @@ -110,7 +110,7 @@ RELEASE;$(DCC_Define) - -sentry-client-test-exception + -validate-repo-changes C:\Projects\keyman\keyboards compare-bcp47-scripts true 2 1033 @@ -267,7 +267,18 @@ -
$R icons.RES
+
, + Sentry.Client in '..\..\ext\sentry\Sentry.Client.pas', + Sentry.Client.Console in '..\..\ext\sentry\Sentry.Client.Console.pas', + sentry in '..\..\ext\sentry\sentry.pas', + Keyman.System.KeymanSentryClient in '..\..\global\delphi\general\Keyman.System.KeymanSentryClient.pas', + KeymanPaths in '..\..\global\delphi\general\KeymanPaths.pas', + Keyman.System.CanonicalLanguageCodeUtils in '..\..\global\delphi\general\Keyman.System.CanonicalLanguageCodeUtils.pas', + Keyman.System.Standards.LangTagsRegistry in '..\..\global\delphi\standards\Keyman.System.Standards.LangTagsRegistry.pas', + Keyman.Developer.System.Project.UrlRenderer in '..\TIKE\project\Keyman.Developer.System.Project.UrlRenderer.pas', + Keyman.Developer.System.ValidateRepoChanges in 'Keyman.Developer.System.ValidateRepoChanges.pas'; + +{$R icons.RES
 @@ -277,6 +288,7 @@ + Cfg_2 Base diff --git a/windows/src/developer/kmcomp/main.pas b/windows/src/developer/kmcomp/main.pas index 0c51b2ccea..a3a65d36be 100644 --- a/windows/src/developer/kmcomp/main.pas +++ b/windows/src/developer/kmcomp/main.pas @@ -49,6 +49,7 @@ uses Keyman.Developer.System.Project.ProjectLog, Keyman.Developer.System.Project.ProjectLogConsole, + Keyman.Developer.System.ValidateRepoChanges, OnlineConstants, VersionInfo, compile, @@ -74,7 +75,7 @@ var FUpdateInstaller: Boolean; FInstallerMSI: string; FClean: Boolean; - FFullySilent: Boolean; + FValidateRepoChanges, FFullySilent: Boolean; FWarnAsError: Boolean; FCheckFilenameConventions: Boolean; FValidating: Boolean; @@ -98,6 +99,7 @@ begin FUpdateInstaller := False; FClean := False; FNologo := False; + FValidateRepoChanges := False; FWarnAsError := False; FCheckFilenameConventions := False; FValidating := False; @@ -121,6 +123,8 @@ begin s := LowerCase(ParamStr(i)); if s = '-nologo' then // I4706 FNologo := True + else if s = '-validate-repo-changes' then + FValidateRepoChanges := True else if s = '-s' then FSilent := True // I4706 else if s = '-ss' then // I4706 begin @@ -257,7 +261,9 @@ begin TProjectLogConsole.Create(FSilent, FFullySilent, hOutfile, FColorMode); - if FMerging then + if FValidateRepoChanges then + FError := not TValidateRepoChanges.Execute(FParamInfile, FParamOutfile) + else if FMerging then FError := not TMergeKeyboardInfo.Execute(FParamSourcePath, FParamInfile, FParamInfile2, FParamOutfile, FParamHelpLink, FMergingValidateIds, FSilent, TProjectLogConsole.Instance.Log) else if FValidating then FError := not TValidateKeyboardInfo.Execute(FParamInfile, FJsonSchemaPath, FParamDistribution, FSilent, TProjectLogConsole.Instance.Log) diff --git a/windows/src/developer/kmconvert/Keyman.Developer.System.ProjectTemplate.pas b/windows/src/developer/kmconvert/Keyman.Developer.System.ProjectTemplate.pas index 795fd468cd..1f2443f0f6 100644 --- a/windows/src/developer/kmconvert/Keyman.Developer.System.ProjectTemplate.pas +++ b/windows/src/developer/kmconvert/Keyman.Developer.System.ProjectTemplate.pas @@ -197,7 +197,7 @@ var bcp47tag := TBCP47Tag.Create(tag); try - bcp47tag.Tag := TCanonicalLanguageCodeUtils.FindBestTag(bcp47tag.Tag, False); + bcp47tag.Tag := TCanonicalLanguageCodeUtils.FindBestTag(bcp47tag.Tag, False, False); Result := Result + '"' + bcp47tag.Tag + '"'; finally bcp47tag.Free; diff --git a/windows/src/engine/kmcomapi/com/keyboardlanguages/keymankeyboardlanguagesfile.pas b/windows/src/engine/kmcomapi/com/keyboardlanguages/keymankeyboardlanguagesfile.pas index baee852bbf..263574412f 100644 --- a/windows/src/engine/kmcomapi/com/keyboardlanguages/keymankeyboardlanguagesfile.pas +++ b/windows/src/engine/kmcomapi/com/keyboardlanguages/keymankeyboardlanguagesfile.pas @@ -75,7 +75,7 @@ begin begin for i := 0 to APackageKeyboardLanguages.Count - 1 do begin - FCanonicalBCP47Tag := TCanonicalLanguageCodeUtils.FindBestTag(APackageKeyboardLanguages[i].ID, True); + FCanonicalBCP47Tag := TCanonicalLanguageCodeUtils.FindBestTag(APackageKeyboardLanguages[i].ID, True, True); if (FCanonicalBCP47Tag <> '') and (IndexOfBCP47Code(FCanonicalBCP47Tag) < 0) then FLanguages.Add(TKeymanKeyboardLanguageFile.Create(AContext, AOwner, FCanonicalBCP47Tag, 0, APackageKeyboardLanguages[i].Name)); diff --git a/windows/src/engine/kmcomapi/com/keyboardlanguages/keymankeyboardlanguagesinstalled.pas b/windows/src/engine/kmcomapi/com/keyboardlanguages/keymankeyboardlanguagesinstalled.pas index a7d3b96502..c9b6b75b9f 100644 --- a/windows/src/engine/kmcomapi/com/keyboardlanguages/keymankeyboardlanguagesinstalled.pas +++ b/windows/src/engine/kmcomapi/com/keyboardlanguages/keymankeyboardlanguagesinstalled.pas @@ -89,7 +89,7 @@ var ml: TMitigateWin10_1803.TMitigatedLanguage; begin // This adds an in-memory item to the array so that it can be installed - FCanonicalBCP47Tag := TCanonicalLanguageCodeUtils.FindBestTag(BCP47Tag, True); + FCanonicalBCP47Tag := TCanonicalLanguageCodeUtils.FindBestTag(BCP47Tag, True, True); if FCanonicalBCP47Tag = '' then Exit(nil); @@ -160,7 +160,7 @@ procedure TKeymanKeyboardLanguagesInstalled.DoRefresh; then FName := regLM.ReadString(SRegValue_LanguageProfileName) else FName := ''; - FCanonicalBCP47Tag := TCanonicalLanguageCodeUtils.FindBestTag(FLocale, True); + FCanonicalBCP47Tag := TCanonicalLanguageCodeUtils.FindBestTag(FLocale, True, True); if (FCanonicalBCP47Tag <> '') and (IndexOfBCP47Code(FCanonicalBCP47Tag) < 0) then begin FKeyboardLanguage := TKeymanKeyboardLanguageInstalled.Create(Context, FOwner, FCanonicalBCP47Tag, FLangID, FGUID, FName); @@ -237,7 +237,7 @@ procedure TKeymanKeyboardLanguagesInstalled.DoRefresh; reg.GetValueNames(FIDs); for i := 0 to FIDs.Count - 1 do begin - FCanonicalBCP47Tag := TCanonicalLanguageCodeUtils.FindBestTag(FIDs[i], True); + FCanonicalBCP47Tag := TCanonicalLanguageCodeUtils.FindBestTag(FIDs[i], True, True); if (FCanonicalBCP47Tag <> '') and not HasLanguage(FCanonicalBCP47Tag) then begin FName := reg.ReadString(FIDs[i]); @@ -290,7 +290,7 @@ var RegistrationRequired: WordBool; LangID: Integer; begin - Tag := TCanonicalLanguageCodeUtils.FindBestTag(BCP47Code, True); + Tag := TCanonicalLanguageCodeUtils.FindBestTag(BCP47Code, True, True); if (Tag = '') or (IndexOfBCP47Code(Tag) >= 0) then // Already installed, or invalid tag; should we warn? Exit; diff --git a/windows/src/engine/kmcomapi/com/keyman_implementation.pas b/windows/src/engine/kmcomapi/com/keyman_implementation.pas index d13d83b3eb..74766bad76 100644 --- a/windows/src/engine/kmcomapi/com/keyman_implementation.pas +++ b/windows/src/engine/kmcomapi/com/keyman_implementation.pas @@ -263,7 +263,7 @@ function TKeyman.GetCanonicalTag(const Tag: WideString): WideString; begin // We implement this here to avoid sharing standards datasets across // multiple executables - Result := TCanonicalLanguageCodeUtils.FindBestTag(Tag, True); + Result := TCanonicalLanguageCodeUtils.FindBestTag(Tag, True, True); end; function TKeyman.Get_AutoApply: WordBool; diff --git a/windows/src/engine/kmcomapi/processes/keyboard/kpinstallkeyboard.pas b/windows/src/engine/kmcomapi/processes/keyboard/kpinstallkeyboard.pas index 90eb9a4501..e9561b172a 100644 --- a/windows/src/engine/kmcomapi/processes/keyboard/kpinstallkeyboard.pas +++ b/windows/src/engine/kmcomapi/processes/keyboard/kpinstallkeyboard.pas @@ -397,7 +397,7 @@ begin for i := 0 to PackageLanguageMetadata.Count - 1 do begin - BCP47Tag := TCanonicalLanguageCodeUtils.FindBestTag(PackageLanguageMetadata[i].ID, True); + BCP47Tag := TCanonicalLanguageCodeUtils.FindBestTag(PackageLanguageMetadata[i].ID, True, True); if BCP47Tag <> '' then begin if TMitigateWin10_1803.IsMitigationRequired(BCP47Tag, ml) then @@ -430,7 +430,7 @@ begin FLanguageInstalled := False; for i := 0 to PackageLanguageMetadata.Count - 1 do begin - BCP47Tag := TCanonicalLanguageCodeUtils.FindBestTag(PackageLanguageMetadata[i].ID, True); + BCP47Tag := TCanonicalLanguageCodeUtils.FindBestTag(PackageLanguageMetadata[i].ID, True, True); if BCP47Tag <> '' then FLanguageInstalled := LegacyRegisterAndInstallLanguageProfile(BCP47Tag, kbdname, ki.KeyboardName, FIconFileName, PackageLanguageMetadata[i].Name); if FLanguageInstalled then diff --git a/windows/src/global/delphi/general/CompilePackage.pas b/windows/src/global/delphi/general/CompilePackage.pas index 75b1099470..84f937b3ea 100644 --- a/windows/src/global/delphi/general/CompilePackage.pas +++ b/windows/src/global/delphi/general/CompilePackage.pas @@ -466,7 +466,7 @@ begin for l in k.Languages do begin Tag := TBCP47Tag.Create(l.ID); - NewID := TCanonicalLanguageCodeUtils.FindBestTag(l.ID, False); + NewID := TCanonicalLanguageCodeUtils.FindBestTag(l.ID, False, False); if NewID = '' then begin // We don't have enough data to validate this tag diff --git a/windows/src/global/delphi/general/Keyman.System.CanonicalLanguageCodeUtils.pas b/windows/src/global/delphi/general/Keyman.System.CanonicalLanguageCodeUtils.pas index 5bea86de8c..2cb497a783 100644 --- a/windows/src/global/delphi/general/Keyman.System.CanonicalLanguageCodeUtils.pas +++ b/windows/src/global/delphi/general/Keyman.System.CanonicalLanguageCodeUtils.pas @@ -4,9 +4,9 @@ interface type TCanonicalLanguageCodeUtils = class - class function FindBestTag(const Tag: string; AddRegion: Boolean): string; - class function IsCanonical(const Tag: string; AddRegion: Boolean): Boolean; overload; - class function IsCanonical(const Tag: string; var Msg: string; AddRegion: Boolean): Boolean; overload; + class function FindBestTag(const Tag: string; AddRegion, AddScriptIfNotSuppressed: Boolean): string; + class function IsCanonical(const Tag: string; AddRegion, AddScriptIfNotSuppressed: Boolean): Boolean; overload; + class function IsCanonical(const Tag: string; var Msg: string; AddRegion, AddScriptIfNotSuppressed: Boolean): Boolean; overload; class function GetFullTagList(const Tag: string): TArray; end; @@ -24,7 +24,7 @@ uses /// This will canonicalize known tags, then apply rules to ensure script subtag /// is present if not suppressed, and add a default region if none given. /// -class function TCanonicalLanguageCodeUtils.FindBestTag(const Tag: string; AddRegion: Boolean): string; +class function TCanonicalLanguageCodeUtils.FindBestTag(const Tag: string; AddRegion, AddScriptIfNotSuppressed: Boolean): string; var t: TBCP47Tag; LangTag: TLangTag; @@ -37,6 +37,10 @@ begin if t.Tag = '' then Exit(''); + // Special case for IPA keyboards; otherwise we'd have und-Zyyy-fonipa + if (t.Language = 'und') and (t.Variant = 'fonipa') then + Exit('und-fonipa'); + // First, canonicalize any unnecessary ISO639-3 codes t.Language := TLanguageCodeUtils.TranslateISO6393ToBCP47(t.Language); @@ -47,22 +51,22 @@ begin t.Tag := Result; end; - if not TLangTagsMap.LangTags.TryGetValue(t.Language, LangTag) then + if not TLangTagsMap.LangTags.TryGetValue(t.Tag, LangTag) then begin - // Not a valid language subtag but perhaps it's a custom language - // We'll make no further assumptions + // Not a known tag but perhaps it's a custom language + // We'll make no further assumptions Exit(t.Tag); end; // Then, lookup the lang-script and see if there is a suppress-script - if SameText(t.Script, LangTag.script) and LangTag.suppress then - t.Script := '' // Or add the default script in if it is missing and not a suppress-script - else if (t.Script = '') and not LangTag.suppress then + if (t.Script = '') and not LangTag.suppress and AddScriptIfNotSuppressed then + // AddScriptIfNotSuppressed will generally be True for Windows scenarios; + // for other systems and for registry systems it will be False t.Script := LangTag.script; // Add the region if not specified - // For Windows scenarios, we'll want to add a region. For cross-platform, + // For Windows scenarios, we may want to add a region. For cross-platform, // we probably don't want to. if (t.Region = '') and AddRegion then t.Region := LangTag.region; @@ -73,9 +77,9 @@ begin end; end; -class function TCanonicalLanguageCodeUtils.IsCanonical(const Tag: string; AddRegion: Boolean): Boolean; +class function TCanonicalLanguageCodeUtils.IsCanonical(const Tag: string; AddRegion, AddScriptIfNotSuppressed: Boolean): Boolean; begin - Result := SameText(Tag, FindBestTag(Tag, AddRegion)); + Result := SameText(Tag, FindBestTag(Tag, AddRegion, AddScriptIfNotSuppressed)); end; /// @@ -117,11 +121,11 @@ begin end; class function TCanonicalLanguageCodeUtils.IsCanonical(const Tag: string; - var Msg: string; AddRegion: Boolean): Boolean; + var Msg: string; AddRegion, AddScriptIfNotSuppressed: Boolean): Boolean; var c: string; begin - c := FindBestTag(Tag, AddRegion); + c := FindBestTag(Tag, AddRegion, AddScriptIfNotSuppressed); Result := SameText(c, Tag); if not Result then begin diff --git a/windows/src/global/delphi/packages/Keyman.System.PackageInfoRefreshKeyboards.pas b/windows/src/global/delphi/packages/Keyman.System.PackageInfoRefreshKeyboards.pas index 49148bfb42..87b8102ba2 100644 --- a/windows/src/global/delphi/packages/Keyman.System.PackageInfoRefreshKeyboards.pas +++ b/windows/src/global/delphi/packages/Keyman.System.PackageInfoRefreshKeyboards.pas @@ -207,7 +207,7 @@ begin DoError(Format(SError_LanguageTagIsNotValid, [kbd.ID, lang.ID, msg]), plsError); Result := False; end - else if not TCanonicalLanguageCodeUtils.IsCanonical(tag, msg, False) then + else if not TCanonicalLanguageCodeUtils.IsCanonical(tag, msg, False, False) then begin DoError(Format(SWarning_LanguageTagIsNotCanonical, [kbd.ID, lang.ID, msg]), plsWarning); end; @@ -277,7 +277,7 @@ begin codes := TKMXFileLanguages.GetKMXFileBCP47Codes(f.FileName); for i := 0 to High(codes) do begin - t := TCanonicalLanguageCodeUtils.FindBestTag(codes[i], False); + t := TCanonicalLanguageCodeUtils.FindBestTag(codes[i], False, False); if t = '' then // We won't add codes that are unrecognised Continue; diff --git a/windows/src/global/delphi/packages/Keyman.System.PackageInfoRefreshLexicalModels.pas b/windows/src/global/delphi/packages/Keyman.System.PackageInfoRefreshLexicalModels.pas index 32b71ca6f4..5978c9adc0 100644 --- a/windows/src/global/delphi/packages/Keyman.System.PackageInfoRefreshLexicalModels.pas +++ b/windows/src/global/delphi/packages/Keyman.System.PackageInfoRefreshLexicalModels.pas @@ -117,7 +117,7 @@ begin DoError(Format(SError_LanguageTagIsNotValid, [lm.ID, lang.ID, msg]), plsError); Result := False; end - else if not TCanonicalLanguageCodeUtils.IsCanonical(tag, msg, False) then + else if not TCanonicalLanguageCodeUtils.IsCanonical(tag, msg, False, False) then begin DoError(Format(SWarning_LanguageTagIsNotCanonical, [lm.ID, lang.ID, msg]), plsWarning); diff --git a/windows/src/unit-tests/standards-data/Keyman.Test.System.CanonicalLanguageCodeUtilsTest.pas b/windows/src/unit-tests/standards-data/Keyman.Test.System.CanonicalLanguageCodeUtilsTest.pas index 5522f4dea7..3f1b5f2465 100644 --- a/windows/src/unit-tests/standards-data/Keyman.Test.System.CanonicalLanguageCodeUtilsTest.pas +++ b/windows/src/unit-tests/standards-data/Keyman.Test.System.CanonicalLanguageCodeUtilsTest.pas @@ -81,6 +81,9 @@ begin Assert.AreEqual('se-Latn-NO-fonipa', TCanonicalLanguageCodeUtils.FindBestTag('se-fonipa', True)); Assert.AreEqual('se-Latn-NO-fonipa', TCanonicalLanguageCodeUtils.FindBestTag('se-no-fonipa', True)); Assert.AreEqual('fr-FR-fonipa', TCanonicalLanguageCodeUtils.FindBestTag('fr-fonipa', True)); + + // az-Cyrl + Assert.AreEqual('az-Cyrl-RU', TCanonicalLanguageCodeUtils.FindBestTag('az-Cyrl', True)); end; initialization