Merge pull request #4656 from keymanapp/feat/ios/auto-bundling-default-resources

feat(ios): auto-bundles the most recent version of default resources

ios/.gitignore

@@ -17,6 +17,9 @@ engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/keyman-sentr
 Keyman 2*
 Carthage
 
+# Default resource packages (which may be downloaded automatically via build-script
+engine/KMEI/KeymanEngine/resources/Keyman.bundle/Contents/Resources/*.kmp
+
 # Generated offline help
 keyman/Keyman/resources/OfflineHelp.bundle/Contents/Resources
 

ios/engine/KMEI/KeymanEngine/Classes/Constants.swift

@@ -69,20 +69,28 @@ public enum Key {
 
 public enum Defaults {
   private static let font = Font(family: "LatinWeb", source: ["DejaVuSans.ttf"], size: nil)
-  public static let keyboard = InstallableKeyboard(id: "sil_euro_latin",
-                                                   name: "EuroLatin (SIL)",
-                                                   languageID: "en",
-                                                   languageName: "English",
-                                                   version: "1.9.1",
-                                                   isRTL: false,
-                                                   font: font,
-                                                   oskFont: nil,
-                                                   isCustom: false)
-  public static let lexicalModel = InstallableLexicalModel(id: "nrc.en.mtnt",
-                                                   name: "English dictionary (MTNT)",
-                                                   languageID: "en",
-                                                   version: "0.1.4",
-                                                   isCustom: false)
+
+  public static let keyboardID = FullKeyboardID(keyboardID: "sil_euro_latin", languageID: "en")
+  public static let lexicalModelID = FullLexicalModelID(lexicalModelID: "nrc.en.mtnt", languageID: "en")
+
+  public static var keyboardPackage: KeyboardKeymanPackage = {
+    let bundledKMP = Resources.bundle.url(forResource: keyboardID.keyboardID, withExtension: ".kmp")!
+    return try! ResourceFileManager.shared.prepareKMPInstall(from: bundledKMP) as! KeyboardKeymanPackage
+  }()
+
+  public static let lexicalModelPackage: LexicalModelKeymanPackage = {
+    let bundledKMP = Resources.bundle.url(forResource: lexicalModelID.lexicalModelID, withExtension: ".model.kmp")!
+    return try! ResourceFileManager.shared.prepareKMPInstall(from: bundledKMP) as! LexicalModelKeymanPackage
+  }()
+
+  // Must be retrieved from their packages!
+  public static let keyboard: InstallableKeyboard = {
+    return keyboardPackage.findResource(withID: keyboardID)!
+  }()
+
+  public static let lexicalModel: InstallableLexicalModel = {
+    return lexicalModelPackage.findResource(withID: lexicalModelID)!
+  }()
 }
 
 public enum Resources {

ios/engine/KMEI/KeymanEngine/Classes/Migrations.swift

@@ -146,9 +146,16 @@ public enum Migrations {
 
   static func updateResources(storage: Storage) {
     var lastVersion = engineVersion
-    if (lastVersion ?? Version.fallback) >= Version.latestFeature {
-      // We're either current or have just been downgraded; no need to do modify resources.
-      // If it's a downgrade, it's near-certainly a testing environment.
+
+    // Will always seek to update default resources on version upgrades,
+    // even if only due to the build component of the version.
+    //
+    // This may make intentional downgrading of our default resources tedious,
+    // as there's (currently) no way to detect if a user intentionally did so before
+    // the app upgrade.
+    if lastVersion != nil, lastVersion! > Version.currentTagged {
+      // We've just been downgraded; no need to modify resources.
+      // If it's a downgrade, it's near-certainly a testing environment
       return
     }
 
@@ -266,7 +273,7 @@ public enum Migrations {
     }
 
     // Store the version we just upgraded to.
-    storage.userDefaults.lastEngineVersion = Version.latestFeature
+    storage.userDefaults.lastEngineVersion = Version.currentTagged
   }
 
   static func migrateUserDefaultsToStructs(storage: Storage) {

ios/engine/KMEI/KeymanEngine/Classes/Model/Version.swift

@@ -14,6 +14,11 @@ public class Version: NSObject, Comparable {
     case alpha
     case beta
     case stable
+
+    // In case we wish to do any tier-based comparisons.
+    public static func < (lhs: Version.Tier, rhs: Version.Tier) -> Bool {
+      return lhs.rawValue < rhs.rawValue
+    }
   }
 
   public static let fallback = Version("1.0")!

ios/engine/KMEI/KeymanEngine/Classes/Resource Management/ResourceFileManager.swift

@@ -137,7 +137,7 @@ public class ResourceFileManager {
    */
   public func prepareKMPInstall(from url: URL) throws -> KeymanPackage {
     // Once selected, start the standard install process.
-    log.info("Installing KMP from \(url)")
+    log.info("Opening KMP from \(url)")
 
     // Step 1: Copy it to a temporary location, making it a .zip in the process
     let cacheDirectory = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask)[0]

ios/engine/KMEI/KeymanEngineTests/MigrationTests.swift

@@ -63,8 +63,10 @@ class MigrationTests: XCTestCase {
 //  }
 
   func testSimpleEarly14Migration() {
-    // A case where the user only has SENCOTEN installed for both keyboard & lexical model.
-    // sil_euro_latin was explicitly removed by the user.
+    // A case where the user has both SENCOTEN installed for both keyboard & lexical model.
+    // Mirrors `testNoDefaultsEarly14Migration`, but where sil_euro_latin
+    // was not removed by the user; in contrast, the base resources
+    // should be auto-updated.
     TestUtils.Migrations.applyBundleToFileSystem(TestUtils.Migrations.simple_14)
     Migrations.migrate(storage: Storage.active)
     Migrations.updateResources(storage: Storage.active)
@@ -74,12 +76,17 @@ class MigrationTests: XCTestCase {
     let userLexicalModels = userDefaults.userLexicalModels ?? []
 
     XCTAssertEqual(userKeyboards.count, 2)
-    XCTAssertEqual(userLexicalModels.count, 2)
+    // Because there's a lexical model update, it installs the whole package.
+    // Not exactly ideal, but correct for pre-existing behavior, which
+    // installs the whole default package instead of a single language-code pairing.
+    XCTAssertEqual(userLexicalModels.count, 4)
 
-    let kbdSEU = userKeyboards.first(where: { $0.fullID == TestUtils.Keyboards.sil_euro_latin.fullID })
-    XCTAssertNotNil(kbdSEU)
-    // Enable once auto-updating
-    //XCTAssertGreaterThan(Version(kbdSEU!.version)!, Version(TestUtils.Keyboards.sil_euro_latin.version)!)
+    //[s]il_[e]uro_[l]atin
+    let kbdSEL = userKeyboards.first(where: { $0.fullID == TestUtils.Keyboards.sil_euro_latin.fullID })
+    XCTAssertNotNil(kbdSEL)
+    // Because there's a keyboard update (1.9.3 vs 1.9.1, at the time of writing).
+    // we expect a more recent version than the testing version.
+    XCTAssertGreaterThan(Version(kbdSEL!.version)!, Version(TestUtils.Keyboards.sil_euro_latin.version)!)
     XCTAssertTrue(userLexicalModels.contains(where: { $0.fullID == TestUtils.LexicalModels.mtnt.fullID }))
 
     XCTAssertTrue(userKeyboards.contains(where: { $0.fullID == TestUtils.Keyboards.fv_sencoten.fullID }))
@@ -122,7 +129,7 @@ class MigrationTests: XCTestCase {
 
     sil_euro_latin_kbds.forEach {
       XCTAssertEqual($0.packageID, "sil_euro_latin")
-      XCTAssertEqual($0.version, TestUtils.Keyboards.sil_euro_latin.version)
+      XCTAssertEqual($0.version, Defaults.keyboard.version)
     }
 
     let sil_euro_latin_package = ResourceFileManager.shared.installedPackages.first(where: { $0.id == "sil_euro_latin" }) as! KeyboardKeymanPackage
@@ -144,7 +151,7 @@ class MigrationTests: XCTestCase {
 
     let userDefaults = Storage.active.userDefaults
 
-    // SIL EuroLatin should be updated to 1.9.1.  The lexical model version should be unchanged.
+    // SIL EuroLatin should be updated to the currently-bundled version.  The lexical model version should be unchanged.
 
     let defaultKbd = userDefaults.userKeyboards![0]
     XCTAssertEqual(defaultKbd.id, Defaults.keyboard.id)

ios/kmbuild.sh

@@ -1,5 +1,11 @@
 #!/bin/sh
 
+# set -e: Terminate script if a command returns an error
+set -e
+# set -u: Terminate script if an unset variable is used
+set -u
+# set -x: Debugging use, print each statement
+# set -x
 
 ## START STANDARD BUILD SCRIPT INCLUDE
 # adjust relative paths as necessary
@@ -7,13 +13,12 @@ THIS_SCRIPT="$(greadlink -f "${BASH_SOURCE[0]}" 2>/dev/null || readlink -f "${BA
 . "$(dirname "$THIS_SCRIPT")/../resources/build/build-utils.sh"
 ## END STANDARD BUILD SCRIPT INCLUDE
 
-KEYMAN_MAC_BASE_PATH="$KEYMAN_ROOT/mac"
+# This script runs from its own folder
+cd "$(dirname "$THIS_SCRIPT")"
 
 # Include our resource functions; they're pretty useful!
 . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh"
-
-# This script runs from its own folder
-cd "$(dirname "$THIS_SCRIPT")"
+. "$KEYMAN_ROOT/resources/build/build-download-resources.sh"
 
 # Please note that this build script (understandably) assumes that it is running on Mac OS X.
 verify_on_mac
@@ -31,6 +36,7 @@ display_usage ( ) {
     echo "  -no-build               Cancels the build entirely.  Useful with 'build.sh -clean -no-build'."
     echo "  -upload-sentry          Uploads debug symbols, etc, to Sentry"
     echo "  -debug                  Sets the configuration to debug mode instead of release."
+    echo "  -download-resources     Download up-to-date versions of the engine's default resources from downloads.keyman.com."
 exit 1
 }
 
@@ -52,6 +58,7 @@ DO_ARCHIVE=true
 DO_CARTHAGE=true
 CLEAN_ONLY=false
 CONFIG=Release
+DO_KMP_DOWNLOADS=false
 
 # Parse args
 while [[ $# -gt 0 ]] ; do
@@ -68,7 +75,7 @@ while [[ $# -gt 0 ]] ; do
             DO_KEYMANAPP=false
             ;;
         -no-codesign)
-            CODE_SIGN="CODE_SIGN_IDENTITY= CODE_SIGNING_REQUIRED=NO $DEV_TEAM CODE_SIGN_ENTITLEMENTS= CODE_SIGNING_ALLOWED=NO"
+            CODE_SIGN="CODE_SIGN_IDENTITY= CODE_SIGNING_REQUIRED=NO ${DEV_TEAM:-} CODE_SIGN_ENTITLEMENTS= CODE_SIGNING_ALLOWED=NO"
             DO_ARCHIVE=false
             ;;
         -no-archive)
@@ -92,6 +99,9 @@ while [[ $# -gt 0 ]] ; do
         -debug)
             CONFIG=Debug
             ;;
+        -download-resources)
+            DO_KMP_DOWNLOADS=true
+            ;;
     esac
     shift # past argument
 done
@@ -101,6 +111,9 @@ KMEI_RESOURCES=engine/KMEI/KeymanEngine/resources
 BUNDLE_PATH=$KMEI_RESOURCES/Keyman.bundle/contents/resources
 KMW_SOURCE=../web/source
 
+DEFAULT_KBD_ID="sil_euro_latin"
+DEFAULT_LM_ID="nrc.en.mtnt"
+
 # Build product paths
 APP_BUNDLE_PATH=$BUILD_PATH/${CONFIG}-iphoneos/Keyman.app
 KEYBOARD_BUNDLE_PATH=$BUILD_PATH/${CONFIG}-iphoneos/SWKeyboard.appex
@@ -118,6 +131,7 @@ fi
 echo
 echo "KMW_SOURCE: $KMW_SOURCE"
 echo "DO_KMW_BUILD: $DO_KMW_BUILD"
+echo "DO_KMP_DOWNLOADS: $DO_KMP_DOWNLOADS"
 echo "CONFIGURATION: $CONFIG"
 echo
 
@@ -126,9 +140,10 @@ update_bundle ( ) {
         mkdir -p "$BUNDLE_PATH"
     fi
 
+    base_dir="$(pwd)"
+
     if [ $DO_KMW_BUILD = true ]; then
         echo Building KeymanWeb from $KMW_SOURCE
-        base_dir="$(pwd)"
 
         cd $KMW_SOURCE
 
@@ -168,6 +183,24 @@ update_bundle ( ) {
 
         cd "$base_dir"
     fi
+
+    # Our default resources are part of the bundle, so let's check on them.
+    if [ $DO_KMP_DOWNLOADS = true ]; then
+      echo_heading "Downloading up-to-date packages for default resources"
+
+      downloadKeyboardPackage "$DEFAULT_KBD_ID" "$base_dir/$BUNDLE_PATH/$DEFAULT_KBD_ID.kmp"
+      downloadModelPackage "$DEFAULT_LM_ID" "$base_dir/$BUNDLE_PATH/$DEFAULT_LM_ID.model.kmp"
+
+      echo "${SUCCESS_GREEN}Packages successfully updated${NORMAL}"
+
+      # If we aren't downloading resources, make sure copies of them already exist!
+    elif [ ! -f "$base_dir/$BUNDLE_PATH/$DEFAULT_KBD_ID.kmp" ]; then
+      fail "No run with -download-resources has been performed yet; the keyboard package is missing!"
+    elif [ ! -f "$base_dir/$BUNDLE_PATH/$DEFAULT_LM_ID.model.kmp" ]; then
+      fail "No run with -download-resources has been performed yet; the lexical model package is missing!"
+    else
+      warn "Reusing previously-downloaded packages for default resources"
+    fi
 }
 
 # First things first - update our dependencies.
@@ -188,7 +221,10 @@ echo "  * UPLOAD_SENTRY=$UPLOAD_SENTRY"
 echo
 echo "Building KMEI..."
 
-rm -r $BUILD_PATH/$CONFIG-universal 2>/dev/null
+if [ -d "$BUILD_PATH/$CONFIG-universal" ]; then
+  rm -r $BUILD_PATH/$CONFIG-universal
+fi
+
 xcodebuild $XCODEFLAGS_EXT $CODE_SIGN -scheme KME-universal \
            VERSION=$VERSION \
            VERSION_WITH_TAG=$VERSION_WITH_TAG \
@@ -212,8 +248,10 @@ if [ $DO_KEYMANAPP = true ]; then
   echo "Building Keyman app."
 
   # Provides a needed link for codesigning for our CI.
-  if ! [ -z "${DEVELOPMENT_TEAM}" ]; then
+  if ! [ -z "${DEVELOPMENT_TEAM+x}" ]; then
     DEV_TEAM="DEVELOPMENT_TEAM=${DEVELOPMENT_TEAM}"
+  else
+    DEV_TEAM=
   fi
 
   if [ $DO_ARCHIVE = false ]; then