feat(developer): improve BCP 47 canonicalization

Ensures we get a canonical tag per langtags.json as far as we possibly
can. This is a breaking change for the compiler as tags which were
formerly regarded as canonical are no longer regarded that way. This
mostly relates to script subtag but a secondary bug meant that some
other tags would have lost data in the canonicalization process
(because we did a lookup based only on the language subtag previously,
which is a no-no).

See keymanapp/keyboards#1452 for related work.

windows/src/developer/TIKE/compile/ValidateKeyboardInfo.pas

@@ -91,7 +91,7 @@ begin
         if not IsValid(False, msg) then
           Result := Failed(msg);
 
-        if not TCanonicalLanguageCodeUtils.IsCanonical(Tag, msg, False) then
+        if not TCanonicalLanguageCodeUtils.IsCanonical(Tag, msg, False, False) then
           Result := Failed(msg);
       finally
         Free;
@@ -106,7 +106,7 @@ begin
         if not IsValid(False, msg) then
           Result := Failed(msg);
 
-        if not TCanonicalLanguageCodeUtils.IsCanonical(Tag, msg, False) then
+        if not TCanonicalLanguageCodeUtils.IsCanonical(Tag, msg, False, False) then
           Result := Failed(msg);
       finally
         Free;

windows/src/developer/TIKE/dialogs/languages/Keyman.Developer.UI.UfrmSelectBCP47Language.pas

@@ -157,7 +157,7 @@ var
 begin
   inherited;
   tag.Tag := TKMXFileLanguages.TranslateISO6393ToBCP47(cbLanguageTag.Text);
-  t := TCanonicalLanguageCodeUtils.FindBestTag(Tag.Tag, False);
+  t := TCanonicalLanguageCodeUtils.FindBestTag(Tag.Tag, False, False);
   if t <> '' then
   begin
     with TBCP47Tag.Create(t) do

windows/src/developer/TIKE/project/Keyman.Developer.UI.Project.UfrmNewProjectParameters.pas

@@ -476,7 +476,7 @@ begin
     Tag := StrToken(Tags, ' ');
     BCP47Tag := TBCP47Tag.Create(Tag);
     try
-      BCP47Tag.Tag := TCanonicalLanguageCodeUtils.FindBestTag(BCP47Tag.Tag, False);
+      BCP47Tag.Tag := TCanonicalLanguageCodeUtils.FindBestTag(BCP47Tag.Tag, False, False);
       if BCP47Tag.IsValid(False) then
       begin
         Language := TPackageKeyboardLanguage.Create(pack);

windows/src/developer/kmcomp/Keyman.Developer.System.ValidateRepoChanges.pas

@@ -0,0 +1,463 @@
+unit Keyman.Developer.System.ValidateRepoChanges;
+
+interface
+
+uses
+  System.Classes,
+  System.JSON,
+
+  kpsfile;
+
+type
+  TValidateRepoChanges = class
+  private
+    class var err_old, err_new: TStringList;
+    class var root: string;
+    class function Search(path, operation: string): Boolean; static;
+    class function CheckKeyboardInfo(name: string): Boolean; static;
+    class function CheckPackage(name: string): Boolean; static;
+    class function LoadKeyboardInfoFile(filename: string): TJSONObject; static;
+    class function GetLanguageCodesFromJson(root: TJSONObject;
+      langs: TStringList): Boolean; static;
+    class function GetLanguageCodesFromKps(kps: TKPSFile;
+      langs: TStringList): Boolean; static;
+    class function CompareKeyboardInfoScripts(name: string): Boolean; static;
+  public
+    class function Execute(path, operation: string): Boolean;
+  end;
+
+implementation
+
+uses
+  System.Character,
+  System.Generics.Collections,
+  System.SysUtils,
+
+  BCP47Tag,
+  Keyman.System.KeyboardInfoFile,
+  Keyman.System.CanonicalLanguageCodeUtils,
+  Keyman.System.Standards.LangTagsRegistry,
+  TempFileManager,
+  Unicode,
+  utilexecute;
+
+{ TValidateRepoChanges }
+
+class function TValidateRepoChanges.Execute(path, operation: string): Boolean;
+begin
+  if path = '' then path := 'c:\projects\keyman\keyboards';
+
+  path := IncludeTrailingPathDelimiter(path);
+
+  root := path;
+
+//  Result := CheckKeyboardInfo(path + 'release\k\kayan\kayan.keyboard_info');
+
+  err_old := TStringList.Create;
+  err_new := TStringList.Create;
+  try
+    Result := Search(path, operation);
+    err_old.SaveToFile('repo-check.old.txt');
+    err_new.SaveToFile('repo-check.new.txt');
+  finally
+    err_old.Free;
+    err_new.Free;
+  end;
+end;
+
+class function TValidateRepoChanges.Search(path, operation: string): Boolean;
+var
+  f: TSearchRec;
+begin
+  if FindFirst(path + '*.keyboard_info', 0, f) = 0 then
+  begin
+    repeat
+      if operation = 'compare-bcp47-revisions' then CheckKeyboardInfo(path + f.Name)
+      else if operation = 'compare-bcp47-scripts' then CompareKeyboardInfoScripts(path + f.Name);
+    until FindNext(f) <> 0;
+    FindClose(f);
+  end;
+  if FindFirst(path + '*.kps', 0, f) = 0 then
+  begin
+    repeat
+      if operation = 'compare-bcp47-revisions' then CheckPackage(path + f.Name);
+    until FindNext(f) <> 0;
+    FindClose(f);
+  end;
+  if FindFirst(path + '*', faDirectory, f) = 0 then
+  begin
+    repeat
+      if ((f.Attr and faDirectory) = faDirectory) and (f.Name <> '.') and (f.Name <> '..') and not SameText(f.Name, 'build') then
+        Search(path + f.Name + '\', operation);
+    until FindNext(f) <> 0;
+    FindClose(f);
+  end;
+
+  Result := True;
+end;
+
+class function TValidateRepoChanges.LoadKeyboardInfoFile(filename: string): TJSONObject;
+begin
+  with TStringStream.Create('', TEncoding.UTF8) do
+  try
+    LoadFromFile(filename);
+    Result := TJSONObject.ParseJsonValue(DataString) as TJSONObject;
+  finally
+    Free;
+  end;
+end;
+
+class function TValidateRepoChanges.CheckKeyboardInfo(name: string): Boolean;
+var
+  i: Integer;
+  json_new, json_old: TJSONObject;
+//  t: TTempFile;
+  relpath: string;
+  content: string;
+  ec: Integer;
+  lang_new: TStringList;
+  lang_old: TStringList;
+  langc_old: TArray<string>;
+  langc_new: TArray<string>;
+  found_error: Boolean;
+  j: Integer;
+  langc_old_prev: string;
+begin
+  found_error := False;
+  relpath := name.Substring(root.Length).Replace('\', '/');
+
+  json_new := LoadKeyboardInfoFile(name);
+  if not Assigned(json_new) then
+    raise Exception.Create('Unable to load file '+name);
+  // Get previous revision from git
+
+//  t := TTempFileManager.Get('.keyboard_info');
+
+  if not TUtilExecute.Console('git show "HEAD:'+relpath+'"', root, content, ec) then
+    RaiseLastOSError;
+
+  if ec <> 0 then
+    raise Exception.Create('Unable to execute git show for '+name);
+
+  if Copy(content, 1, 3) = string(UTF8Signature) then
+    Delete(content, 1, 3);
+
+  json_old := TJSONObject.ParseJsonValue(content) as TJSONObject;
+  if not Assigned(json_old) then
+    raise Exception.Create('Unable to load old file '+name);
+
+  lang_new := TStringList.Create;
+  lang_old := TStringList.Create;
+  try
+    if not GetLanguageCodesFromJson(json_new, lang_new) then
+      raise Exception.Create('Unable to get Language codes');
+    if not GetLanguageCodesFromJson(json_old, lang_old) then
+      raise Exception.Create('Unable to get Language codes');
+
+    j := 0;
+    for i := 0 to lang_new.Count - 1 do
+    begin
+      repeat
+        if j >= lang_old.Count then
+        begin
+          if not found_error then
+          begin
+            writeln('Checking '+relpath);
+            found_error := True;
+          end;
+          writeln('  Mismatch in number of language codes');
+          break;
+        end;
+
+        langc_new := lang_new[i].Split([',']);
+        langc_old := lang_old[j].Split([',']);
+        Inc(j);
+      until langc_old[0] <> langc_old_prev;
+      langc_old_prev := langc_old[0];
+      if j >= lang_old.Count then Break;
+
+      if langc_new[0] <> langc_old[0] then
+      begin
+        if not found_error then
+        begin
+          writeln('Checking '+relpath);
+          found_error := True;
+        end;
+        writeln(Format('  New code %s [%s] does not match old code %s [%s]', [langc_new[1], langc_new[0], langc_old[1], langc_old[0]]));
+      end;
+    end;
+
+    if found_error then
+    begin
+      err_old.Add('Checking '+relpath);
+      for i := 0 to lang_old.Count - 1 do
+      begin
+        langc_old := lang_old[i].Split([',']);
+        err_old.Add(Format('  %s [%s]', [langc_old[1], langc_old[0]]));
+      end;
+      err_old.Add('');
+
+      err_new.Add('Checking '+relpath);
+      for i := 0 to lang_new.Count - 1 do
+      begin
+        langc_new := lang_new[i].Split([',']);
+        err_new.Add(Format('  %s [%s]', [langc_new[1], langc_new[0]]));
+      end;
+      err_new.Add('');
+    end;
+  finally
+    lang_new.Free;
+    lang_old.Free;
+  end;
+
+  Result := True;
+end;
+
+class function TValidateRepoChanges.GetLanguageCodesFromJson(root: TJSONObject; langs: TStringList): Boolean;
+var
+  i: Integer;
+  a: TJSONArray;
+
+  procedure AddLang(lang: string);
+  begin
+    langs.Add(TCanonicalLanguageCodeUtils.FindBestTag(lang, False, False)+','+lang);
+  end;
+
+begin
+  if root.Values[TKeyboardInfoFile.SLanguages] = nil then
+    Exit(False);
+
+  if root.Values[TKeyboardInfoFile.SLanguages] is TJSONArray then
+  begin
+    a := root.Values[TKeyboardInfoFile.SLanguages] as TJSONArray;
+    for i := 0 to a.Count - 1 do
+      AddLang(a.Items[i].AsType<string>);
+  end
+  else
+  begin
+    root := root.Values[TKeyboardInfoFile.SLanguages] as TJSONObject;
+    for i := 0 to root.Count - 1 do
+      AddLang(root.Pairs[i].JsonString.Value);
+  end;
+
+  langs.Sort;
+
+  Result := True;
+end;
+
+class function TValidateRepoChanges.GetLanguageCodesFromKps(kps: TKPSFile; langs: TStringList): Boolean;
+var
+  i: Integer;
+  j: Integer;
+
+  procedure AddLang(lang: string);
+  begin
+    langs.Add(TCanonicalLanguageCodeUtils.FindBestTag(lang, False, False)+','+lang);
+  end;
+
+begin
+  for i := 0 to kps.Keyboards.Count - 1 do
+    for j := 0 to kps.Keyboards[i].Languages.Count - 1 do
+      AddLang(kps.Keyboards[i].Languages[j].ID);
+
+  langs.Sort;
+
+  Result := True;
+end;
+
+class function TValidateRepoChanges.CheckPackage(name: string): Boolean;
+var
+  i: Integer;
+  kps_new, kps_old: TKPSFile;
+//  t: TTempFile;
+  relpath: string;
+  content: string;
+  ec: Integer;
+  lang_new: TStringList;
+  lang_old: TStringList;
+  langc_old: TArray<string>;
+  langc_new: TArray<string>;
+  found_error: Boolean;
+  j: Integer;
+  langc_old_prev: string;
+begin
+  found_error := False;
+  relpath := name.Substring(root.Length).Replace('\', '/');
+
+  kps_new := TKPSFile.Create;
+  kps_new.FileName := name;
+  kps_new.LoadXML;
+
+  if not TUtilExecute.Console('git show "HEAD:'+relpath+'"', root, content, ec) then
+    RaiseLastOSError;
+
+  if ec <> 0 then
+    raise Exception.Create('Unable to execute git show for '+name);
+
+  if Copy(content, 1, 3) = string(UTF8Signature) then
+    Delete(content, 1, 3);
+
+  kps_old := TKPSFile.Create;
+  kps_old.LoadXMLFromText(content);
+
+  lang_new := TStringList.Create;
+  lang_old := TStringList.Create;
+  try
+    if not GetLanguageCodesFromKps(kps_new, lang_new) then
+      raise Exception.Create('Unable to get Language codes');
+    if not GetLanguageCodesFromKps(kps_old, lang_old) then
+      raise Exception.Create('Unable to get Language codes');
+
+    j := 0;
+    for i := 0 to lang_new.Count - 1 do
+    begin
+      repeat
+        if j >= lang_old.Count then
+        begin
+          if not found_error then
+          begin
+            writeln('Checking '+relpath);
+            found_error := True;
+          end;
+          writeln('  Mismatch in number of language codes');
+          break;
+        end;
+
+        langc_new := lang_new[i].Split([',']);
+        langc_old := lang_old[j].Split([',']);
+        Inc(j);
+      until langc_old[0] <> langc_old_prev;
+      langc_old_prev := langc_old[0];
+      if j >= lang_old.Count then Break;
+
+      if langc_new[0] <> langc_old[0] then
+      begin
+        if not found_error then
+        begin
+          writeln('Checking '+relpath);
+          found_error := True;
+        end;
+        writeln(Format('  New code %s [%s] does not match old code %s [%s]', [langc_new[1], langc_new[0], langc_old[1], langc_old[0]]));
+      end;
+    end;
+
+    if found_error then
+    begin
+      err_old.Add('Checking '+relpath);
+      for i := 0 to lang_old.Count - 1 do
+      begin
+        langc_old := lang_old[i].Split([',']);
+        err_old.Add(Format('  %s [%s]', [langc_old[1], langc_old[0]]));
+      end;
+      err_old.Add('');
+
+      err_new.Add('Checking '+relpath);
+      for i := 0 to lang_new.Count - 1 do
+      begin
+        langc_new := lang_new[i].Split([',']);
+        err_new.Add(Format('  %s [%s]', [langc_new[1], langc_new[0]]));
+      end;
+      err_new.Add('');
+    end;
+  finally
+    lang_new.Free;
+    lang_old.Free;
+  end;
+  kps_old.Free;
+  kps_new.Free;
+
+  Result := True;
+end;
+
+class function TValidateRepoChanges.CompareKeyboardInfoScripts(name: string): Boolean;
+var
+  i: Integer;
+  json: TJSONObject;
+  relpath: string;
+  script, base_script: string;
+  lang: TStringList;
+  base_lang_item, lang_item: TArray<string>;
+  found_error: Boolean;
+
+  procedure WriteHeader;
+  begin
+    if not found_error then
+    begin
+      writeln;
+      writeln('Checking '+relpath);
+    end;
+    found_error := True;
+  end;
+
+  function GetScript(lang: string): string;
+  var
+    v: string;
+    LangTag: TLangTag;
+    BCP47: TBCP47Tag;
+  begin
+    if TLangTagsMap.AllTags.TryGetValue(lang, v) then
+      lang := v;
+
+    if not TLangTagsMap.LangTags.TryGetValue(lang, LangTag) then
+    begin
+      BCP47 := TBCP47Tag.Create(lang);
+      try
+        Exit(BCP47.Script);
+      finally
+        BCP47.Free;
+      end;
+    end;
+
+    Result := LangTag.script;
+  end;
+
+begin
+  found_error := False;
+  relpath := name.Substring(root.Length).Replace('\', '/');
+
+  json := LoadKeyboardInfoFile(name);
+  if not Assigned(json) then
+    raise Exception.Create('Unable to load file '+name);
+
+  lang := TStringList.Create;
+  try
+    if not GetLanguageCodesFromJson(json, lang) then
+      raise Exception.Create('Unable to get Language codes');
+
+    if lang.Count = 0 then
+    begin
+      WriteHeader;
+      writeln('  Warning: no languages found');
+      Exit(True);
+    end;
+
+    base_lang_item := lang[0].Split([',']);
+
+    // Lookup the tag first, canonicalize to the base tag for known tags
+    base_script := GetScript(base_lang_item[0]);
+    if base_script = '' then
+    begin
+      WriteHeader;
+      writeln('  Warning: could not identify tag '+base_lang_item[0]);
+      Exit(True);
+    end;
+
+    for i := 1 to lang.Count - 1 do
+    begin
+      lang_item := lang[i].Split([',']);
+      script := GetScript(lang_item[0]);
+      if script <> base_script then
+      begin
+        WriteHeader;
+        writeln(Format('  Tag %s [%s] has script <%s>, which differs from base tag %s [%s], script <%s>',
+          [lang_item[1], lang_item[0], script, base_lang_item[1], base_lang_item[0], base_script]));
+      end;
+    end;
+  finally
+    lang.Free;
+  end;
+
+  Result := True;
+end;
+
+end.

windows/src/developer/kmcomp/kmcomp.dpr

@@ -119,7 +119,8 @@ uses
   KeymanPaths in '..\..\global\delphi\general\KeymanPaths.pas',
   Keyman.System.CanonicalLanguageCodeUtils in '..\..\global\delphi\general\Keyman.System.CanonicalLanguageCodeUtils.pas',
   Keyman.System.Standards.LangTagsRegistry in '..\..\global\delphi\standards\Keyman.System.Standards.LangTagsRegistry.pas',
-  Keyman.Developer.System.Project.UrlRenderer in '..\TIKE\project\Keyman.Developer.System.Project.UrlRenderer.pas';
+  Keyman.Developer.System.Project.UrlRenderer in '..\TIKE\project\Keyman.Developer.System.Project.UrlRenderer.pas',
+  Keyman.Developer.System.ValidateRepoChanges in 'Keyman.Developer.System.ValidateRepoChanges.pas';
 
 {$R icons.RES}
 {$R version.res}

windows/src/developer/kmcomp/kmcomp.dproj

@@ -110,7 +110,7 @@
         <DCC_Define>RELEASE;$(DCC_Define)</DCC_Define>
     </PropertyGroup>
     <PropertyGroup Condition="'$(Cfg_1_Win32)'!=''">
-        <Debugger_RunParams>-sentry-client-test-exception</Debugger_RunParams>
+        <Debugger_RunParams>-validate-repo-changes C:\Projects\keyman\keyboards compare-bcp47-scripts</Debugger_RunParams>
         <DCC_AssertionsAtRuntime>true</DCC_AssertionsAtRuntime>
         <DCC_DebugInformation>2</DCC_DebugInformation>
         <VerInfo_Locale>1033</VerInfo_Locale>
@@ -267,7 +267,18 @@
         <DCCReference Include="..\..\global\delphi\lexicalmodels\Keyman.Developer.System.LexicalModelCompile.pas"/>
         <DCCReference Include="..\..\global\delphi\lexicalmodels\Keyman.System.LexicalModelUtils.pas"/>
         <DCCReference Include="Keyman.Developer.System.Project.ProjectLogConsole.pas">
-            <Form>$R icons.RES</Form>
+            <Form>,
+  Sentry.Client in &apos;..\..\ext\sentry\Sentry.Client.pas&apos;,
+  Sentry.Client.Console in &apos;..\..\ext\sentry\Sentry.Client.Console.pas&apos;,
+  sentry in &apos;..\..\ext\sentry\sentry.pas&apos;,
+  Keyman.System.KeymanSentryClient in &apos;..\..\global\delphi\general\Keyman.System.KeymanSentryClient.pas&apos;,
+  KeymanPaths in &apos;..\..\global\delphi\general\KeymanPaths.pas&apos;,
+  Keyman.System.CanonicalLanguageCodeUtils in &apos;..\..\global\delphi\general\Keyman.System.CanonicalLanguageCodeUtils.pas&apos;,
+  Keyman.System.Standards.LangTagsRegistry in &apos;..\..\global\delphi\standards\Keyman.System.Standards.LangTagsRegistry.pas&apos;,
+  Keyman.Developer.System.Project.UrlRenderer in &apos;..\TIKE\project\Keyman.Developer.System.Project.UrlRenderer.pas&apos;,
+  Keyman.Developer.System.ValidateRepoChanges in &apos;Keyman.Developer.System.ValidateRepoChanges.pas&apos;;
+
+{$R icons.RES</Form>
         </DCCReference>
         <DCCReference Include="..\..\ext\sentry\Sentry.Client.pas"/>
         <DCCReference Include="..\..\ext\sentry\Sentry.Client.Console.pas"/>
@@ -277,6 +288,7 @@
         <DCCReference Include="..\..\global\delphi\general\Keyman.System.CanonicalLanguageCodeUtils.pas"/>
         <DCCReference Include="..\..\global\delphi\standards\Keyman.System.Standards.LangTagsRegistry.pas"/>
         <DCCReference Include="..\TIKE\project\Keyman.Developer.System.Project.UrlRenderer.pas"/>
+        <DCCReference Include="Keyman.Developer.System.ValidateRepoChanges.pas"/>
         <BuildConfiguration Include="Debug">
             <Key>Cfg_2</Key>
             <CfgParent>Base</CfgParent>

windows/src/developer/kmcomp/main.pas

@@ -49,6 +49,7 @@ uses
 
   Keyman.Developer.System.Project.ProjectLog,
   Keyman.Developer.System.Project.ProjectLogConsole,
+  Keyman.Developer.System.ValidateRepoChanges,
   OnlineConstants,
   VersionInfo,
   compile,
@@ -74,7 +75,7 @@ var
   FUpdateInstaller: Boolean;
   FInstallerMSI: string;
   FClean: Boolean;
-  FFullySilent: Boolean;
+  FValidateRepoChanges, FFullySilent: Boolean;
   FWarnAsError: Boolean;
   FCheckFilenameConventions: Boolean;
   FValidating: Boolean;
@@ -98,6 +99,7 @@ begin
   FUpdateInstaller := False;
   FClean := False;
   FNologo := False;
+  FValidateRepoChanges := False;
   FWarnAsError := False;
   FCheckFilenameConventions := False;
   FValidating := False;
@@ -121,6 +123,8 @@ begin
     s := LowerCase(ParamStr(i));
     if s = '-nologo' then   // I4706
       FNologo := True
+    else if s = '-validate-repo-changes' then
+      FValidateRepoChanges := True
     else if s = '-s' then FSilent := True   // I4706
     else if s = '-ss' then   // I4706
     begin
@@ -257,7 +261,9 @@ begin
 
     TProjectLogConsole.Create(FSilent, FFullySilent, hOutfile, FColorMode);
 
-    if FMerging then
+    if FValidateRepoChanges then
+      FError := not TValidateRepoChanges.Execute(FParamInfile, FParamOutfile)
+    else if FMerging then
       FError := not TMergeKeyboardInfo.Execute(FParamSourcePath, FParamInfile, FParamInfile2, FParamOutfile, FParamHelpLink, FMergingValidateIds, FSilent, TProjectLogConsole.Instance.Log)
     else if FValidating then
       FError := not TValidateKeyboardInfo.Execute(FParamInfile, FJsonSchemaPath, FParamDistribution, FSilent, TProjectLogConsole.Instance.Log)

windows/src/developer/kmconvert/Keyman.Developer.System.ProjectTemplate.pas

@@ -197,7 +197,7 @@ var
 
       bcp47tag := TBCP47Tag.Create(tag);
       try
-        bcp47tag.Tag := TCanonicalLanguageCodeUtils.FindBestTag(bcp47tag.Tag, False);
+        bcp47tag.Tag := TCanonicalLanguageCodeUtils.FindBestTag(bcp47tag.Tag, False, False);
         Result := Result + '"' + bcp47tag.Tag + '"';
       finally
         bcp47tag.Free;

windows/src/engine/kmcomapi/com/keyboardlanguages/keymankeyboardlanguagesfile.pas

@@ -75,7 +75,7 @@ begin
   begin
     for i := 0 to APackageKeyboardLanguages.Count - 1 do
     begin
-      FCanonicalBCP47Tag := TCanonicalLanguageCodeUtils.FindBestTag(APackageKeyboardLanguages[i].ID, True);
+      FCanonicalBCP47Tag := TCanonicalLanguageCodeUtils.FindBestTag(APackageKeyboardLanguages[i].ID, True, True);
       if (FCanonicalBCP47Tag <> '') and (IndexOfBCP47Code(FCanonicalBCP47Tag) < 0) then
         FLanguages.Add(TKeymanKeyboardLanguageFile.Create(AContext, AOwner, FCanonicalBCP47Tag, 0,
           APackageKeyboardLanguages[i].Name));

windows/src/engine/kmcomapi/com/keyboardlanguages/keymankeyboardlanguagesinstalled.pas

@@ -89,7 +89,7 @@ var
   ml: TMitigateWin10_1803.TMitigatedLanguage;
 begin
   // This adds an in-memory item to the array so that it can be installed
-  FCanonicalBCP47Tag := TCanonicalLanguageCodeUtils.FindBestTag(BCP47Tag, True);
+  FCanonicalBCP47Tag := TCanonicalLanguageCodeUtils.FindBestTag(BCP47Tag, True, True);
   if FCanonicalBCP47Tag = '' then
     Exit(nil);
 
@@ -160,7 +160,7 @@ procedure TKeymanKeyboardLanguagesInstalled.DoRefresh;
               then FName := regLM.ReadString(SRegValue_LanguageProfileName)
               else FName := '';
 
-            FCanonicalBCP47Tag := TCanonicalLanguageCodeUtils.FindBestTag(FLocale, True);
+            FCanonicalBCP47Tag := TCanonicalLanguageCodeUtils.FindBestTag(FLocale, True, True);
             if (FCanonicalBCP47Tag <> '') and (IndexOfBCP47Code(FCanonicalBCP47Tag) < 0) then
             begin
               FKeyboardLanguage := TKeymanKeyboardLanguageInstalled.Create(Context, FOwner, FCanonicalBCP47Tag, FLangID, FGUID, FName);
@@ -237,7 +237,7 @@ procedure TKeymanKeyboardLanguagesInstalled.DoRefresh;
         reg.GetValueNames(FIDs);
         for i := 0 to FIDs.Count - 1 do
         begin
-          FCanonicalBCP47Tag := TCanonicalLanguageCodeUtils.FindBestTag(FIDs[i], True);
+          FCanonicalBCP47Tag := TCanonicalLanguageCodeUtils.FindBestTag(FIDs[i], True, True);
           if (FCanonicalBCP47Tag <> '') and not HasLanguage(FCanonicalBCP47Tag) then
           begin
             FName := reg.ReadString(FIDs[i]);
@@ -290,7 +290,7 @@ var
   RegistrationRequired: WordBool;
   LangID: Integer;
 begin
-  Tag := TCanonicalLanguageCodeUtils.FindBestTag(BCP47Code, True);
+  Tag := TCanonicalLanguageCodeUtils.FindBestTag(BCP47Code, True, True);
   if (Tag =  '') or (IndexOfBCP47Code(Tag) >= 0) then
     // Already installed, or invalid tag; should we warn?
     Exit;

windows/src/engine/kmcomapi/com/keyman_implementation.pas

@@ -263,7 +263,7 @@ function TKeyman.GetCanonicalTag(const Tag: WideString): WideString;
 begin
   // We implement this here to avoid sharing standards datasets across
   // multiple executables
-  Result := TCanonicalLanguageCodeUtils.FindBestTag(Tag, True);
+  Result := TCanonicalLanguageCodeUtils.FindBestTag(Tag, True, True);
 end;
 
 function TKeyman.Get_AutoApply: WordBool;

windows/src/engine/kmcomapi/processes/keyboard/kpinstallkeyboard.pas

@@ -397,7 +397,7 @@ begin
           for i := 0 to PackageLanguageMetadata.Count - 1 do
           begin
 
-            BCP47Tag := TCanonicalLanguageCodeUtils.FindBestTag(PackageLanguageMetadata[i].ID, True);
+            BCP47Tag := TCanonicalLanguageCodeUtils.FindBestTag(PackageLanguageMetadata[i].ID, True, True);
             if BCP47Tag <> '' then
             begin
               if TMitigateWin10_1803.IsMitigationRequired(BCP47Tag, ml) then
@@ -430,7 +430,7 @@ begin
         FLanguageInstalled := False;
         for i := 0 to PackageLanguageMetadata.Count - 1 do
         begin
-          BCP47Tag := TCanonicalLanguageCodeUtils.FindBestTag(PackageLanguageMetadata[i].ID, True);
+          BCP47Tag := TCanonicalLanguageCodeUtils.FindBestTag(PackageLanguageMetadata[i].ID, True, True);
           if BCP47Tag <> '' then
             FLanguageInstalled := LegacyRegisterAndInstallLanguageProfile(BCP47Tag, kbdname, ki.KeyboardName, FIconFileName, PackageLanguageMetadata[i].Name);
           if FLanguageInstalled then

windows/src/global/delphi/general/CompilePackage.pas

@@ -466,7 +466,7 @@ begin
     for l in k.Languages do
     begin
       Tag := TBCP47Tag.Create(l.ID);
-      NewID := TCanonicalLanguageCodeUtils.FindBestTag(l.ID, False);
+      NewID := TCanonicalLanguageCodeUtils.FindBestTag(l.ID, False, False);
       if NewID = '' then
       begin
         // We don't have enough data to validate this tag

windows/src/global/delphi/general/Keyman.System.CanonicalLanguageCodeUtils.pas

@@ -4,9 +4,9 @@ interface
 
 type
   TCanonicalLanguageCodeUtils = class
-    class function FindBestTag(const Tag: string; AddRegion: Boolean): string;
-    class function IsCanonical(const Tag: string; AddRegion: Boolean): Boolean; overload;
-    class function IsCanonical(const Tag: string; var Msg: string; AddRegion: Boolean): Boolean; overload;
+    class function FindBestTag(const Tag: string; AddRegion, AddScriptIfNotSuppressed: Boolean): string;
+    class function IsCanonical(const Tag: string; AddRegion, AddScriptIfNotSuppressed: Boolean): Boolean; overload;
+    class function IsCanonical(const Tag: string; var Msg: string; AddRegion, AddScriptIfNotSuppressed: Boolean): Boolean; overload;
     class function GetFullTagList(const Tag: string): TArray<string>;
   end;
 
@@ -24,7 +24,7 @@ uses
 ///  This will canonicalize known tags, then apply rules to ensure script subtag
 ///  is present if not suppressed, and add a default region if none given.
 ///</remarks>
-class function TCanonicalLanguageCodeUtils.FindBestTag(const Tag: string; AddRegion: Boolean): string;
+class function TCanonicalLanguageCodeUtils.FindBestTag(const Tag: string; AddRegion, AddScriptIfNotSuppressed: Boolean): string;
 var
   t: TBCP47Tag;
   LangTag: TLangTag;
@@ -37,6 +37,10 @@ begin
     if t.Tag = '' then
       Exit('');
 
+    // Special case for IPA keyboards; otherwise we'd have und-Zyyy-fonipa
+    if (t.Language = 'und') and (t.Variant = 'fonipa') then
+      Exit('und-fonipa');
+
     // First, canonicalize any unnecessary ISO639-3 codes
     t.Language := TLanguageCodeUtils.TranslateISO6393ToBCP47(t.Language);
 
@@ -47,22 +51,22 @@ begin
       t.Tag := Result;
     end;
 
-    if not TLangTagsMap.LangTags.TryGetValue(t.Language, LangTag) then
+    if not TLangTagsMap.LangTags.TryGetValue(t.Tag, LangTag) then
     begin
-       // Not a valid language subtag but perhaps it's a custom language
-       // We'll make no further assumptions
+      // Not a known tag but perhaps it's a custom language
+      // We'll make no further assumptions
       Exit(t.Tag);
     end;
 
     // Then, lookup the lang-script and see if there is a suppress-script
-    if SameText(t.Script, LangTag.script) and LangTag.suppress then
-      t.Script := ''
     // Or add the default script in if it is missing and not a suppress-script
-    else if (t.Script = '') and not LangTag.suppress then
+    if (t.Script = '') and not LangTag.suppress and AddScriptIfNotSuppressed then
+      // AddScriptIfNotSuppressed will generally be True for Windows scenarios;
+      // for other systems and for registry systems it will be False
       t.Script := LangTag.script;
 
     // Add the region if not specified
-    // For Windows scenarios, we'll want to add a region. For cross-platform,
+    // For Windows scenarios, we may want to add a region. For cross-platform,
     // we probably don't want to.
     if (t.Region = '') and AddRegion then
       t.Region := LangTag.region;
@@ -73,9 +77,9 @@ begin
   end;
 end;
 
-class function TCanonicalLanguageCodeUtils.IsCanonical(const Tag: string; AddRegion: Boolean): Boolean;
+class function TCanonicalLanguageCodeUtils.IsCanonical(const Tag: string; AddRegion, AddScriptIfNotSuppressed: Boolean): Boolean;
 begin
-  Result := SameText(Tag, FindBestTag(Tag, AddRegion));
+  Result := SameText(Tag, FindBestTag(Tag, AddRegion, AddScriptIfNotSuppressed));
 end;
 
 ///
@@ -117,11 +121,11 @@ begin
 end;
 
 class function TCanonicalLanguageCodeUtils.IsCanonical(const Tag: string;
-  var Msg: string; AddRegion: Boolean): Boolean;
+  var Msg: string; AddRegion, AddScriptIfNotSuppressed: Boolean): Boolean;
 var
   c: string;
 begin
-  c := FindBestTag(Tag, AddRegion);
+  c := FindBestTag(Tag, AddRegion, AddScriptIfNotSuppressed);
   Result := SameText(c, Tag);
   if not Result then
   begin

windows/src/global/delphi/packages/Keyman.System.PackageInfoRefreshKeyboards.pas

@@ -207,7 +207,7 @@ begin
           DoError(Format(SError_LanguageTagIsNotValid, [kbd.ID, lang.ID, msg]), plsError);
           Result := False;
         end
-        else if not TCanonicalLanguageCodeUtils.IsCanonical(tag, msg, False) then
+        else if not TCanonicalLanguageCodeUtils.IsCanonical(tag, msg, False, False) then
         begin
           DoError(Format(SWarning_LanguageTagIsNotCanonical, [kbd.ID, lang.ID, msg]), plsWarning);
         end;
@@ -277,7 +277,7 @@ begin
       codes := TKMXFileLanguages.GetKMXFileBCP47Codes(f.FileName);
       for i := 0 to High(codes) do
       begin
-        t := TCanonicalLanguageCodeUtils.FindBestTag(codes[i], False);
+        t := TCanonicalLanguageCodeUtils.FindBestTag(codes[i], False, False);
         if t = '' then
           // We won't add codes that are unrecognised
           Continue;

windows/src/global/delphi/packages/Keyman.System.PackageInfoRefreshLexicalModels.pas

@@ -117,7 +117,7 @@ begin
           DoError(Format(SError_LanguageTagIsNotValid, [lm.ID, lang.ID, msg]), plsError);
           Result := False;
         end
-        else if not TCanonicalLanguageCodeUtils.IsCanonical(tag, msg, False) then
+        else if not TCanonicalLanguageCodeUtils.IsCanonical(tag, msg, False, False) then
         begin
 
           DoError(Format(SWarning_LanguageTagIsNotCanonical, [lm.ID, lang.ID, msg]), plsWarning);

windows/src/unit-tests/standards-data/Keyman.Test.System.CanonicalLanguageCodeUtilsTest.pas

@@ -81,6 +81,9 @@ begin
   Assert.AreEqual('se-Latn-NO-fonipa', TCanonicalLanguageCodeUtils.FindBestTag('se-fonipa', True));
   Assert.AreEqual('se-Latn-NO-fonipa', TCanonicalLanguageCodeUtils.FindBestTag('se-no-fonipa', True));
   Assert.AreEqual('fr-FR-fonipa', TCanonicalLanguageCodeUtils.FindBestTag('fr-fonipa', True));
+
+  // az-Cyrl
+  Assert.AreEqual('az-Cyrl-RU', TCanonicalLanguageCodeUtils.FindBestTag('az-Cyrl', True));
 end;
 
 initialization