diff --git a/web/testing/regression-tests/node_src/test-host.js b/web/testing/regression-tests/node_src/test-host.js
index d8dd0337b7..ad158e0c8f 100644
--- a/web/testing/regression-tests/node_src/test-host.js
+++ b/web/testing/regression-tests/node_src/test-host.js
@@ -53,11 +53,9 @@ module.exports = {
     let body = [];
 
     function finish(json) {
-      //console.log(json);
-
       response.setHeader('Content-Type', 'application/json');
       if(request.method !== 'POST' || !json || typeof json != 'object' || !json.id || !json.shortname || !json.compilerVersion || !json.engineVersion) {
-        console.error('/save-results Invalid request');
+        console.warn('/save-results Invalid request');
         response.writeHead(400);
         return response.end(JSON.stringify({result: "Invalid request"}));
       }
@@ -76,7 +74,7 @@ module.exports = {
           typeof(json.shortname) != 'string' || !json.shortname.match(/^[a-z]+$/) ||
           typeof(compilerVersion) != 'string' || !compilerVersion.match(/^((\d+)(\.\d+)*)|source$/) ||
           typeof(engineVersion) != 'string' || !engineVersion.match(/^((\d+)(\.\d+)*)|source$/)) {
-        console.error('/save-results Invalid request');
+        console.warn('/save-results Invalid request');
         response.writeHead(400);
         return response.end(JSON.stringify({result: "Invalid request"}));
       }
@@ -85,7 +83,7 @@ module.exports = {
       let base = path.join(KEYBOARDS_ROOT, json.shortname, json.id);
       if(!fs.existsSync(base)) {
         const msg = `Keyboard ${json.shortname}/${json.id} not found at ${base}`;
-        console.error('/save-results 404 '+msg);
+        console.warn('/save-results 404 '+msg);
         response.writeHead(404);
         return response.end(JSON.stringify({result: msg}));
       }
@@ -94,7 +92,7 @@ module.exports = {
         fs.mkdirSync(base);
         if(!fs.existsSync(base)) {
           const msg = `Could not create tests folder ${base}`;
-          console.error('/save-results 500 '+msg);
+          console.warn('/save-results 500 '+msg);
           response.writeHead(500);
           return response.end(JSON.stringify({result: msg}));
         }
diff --git a/web/testing/regression-tests/src/test-runner.js b/web/testing/regression-tests/src/test-runner.js
index 5f54799f87..f2ecc86643 100644
--- a/web/testing/regression-tests/src/test-runner.js
+++ b/web/testing/regression-tests/src/test-runner.js
@@ -159,6 +159,7 @@ var testRunner = {
       console.log('saving test '+locator);
       let json = testRunner.parseLocator(locator);
       if(!json.id) throw new Error('Invalid locator: '+locator);
+
       json.results = typeof results == 'string' ? JSON.parse(results) : results;
       json.engineVersion = keyman.build.toString();
       try {
@@ -285,9 +286,13 @@ var testRunner = {
         keyman.interface.processKeystroke(keyman.util.physicalDevice, com.keyman.text ? com.keyman.text.Processor.getOutputTarget(receiver) : receiver, e);
         this.keyboards[keyboardId].results[testId] = receiver.value;
       } catch(err) {
-        console.error(err);
+        console.warn(err.toString());
         this.keyboards[keyboardId].results[testId] = {error: err.message, filename: err.filename, lineno: err.lineno};
+        if(!knownFailures[keyboardId]) {
+          return false;
+        }
       }
+      return true;
     }
   },
 
diff --git a/web/testing/regression-tests/test-builder.js b/web/testing/regression-tests/test-builder.js
index 9033e75600..b757adf498 100644
--- a/web/testing/regression-tests/test-builder.js
+++ b/web/testing/regression-tests/test-builder.js
@@ -43,7 +43,7 @@ keyboards.forEach(function(keyboard) {
             .then(function() {
               return testRunner.loadTests(locator)
                 .then((shouldRun) => { if(shouldRun) { testRunner.runTests(id); } return shouldRun; })
-                .then((shouldSave) => { testRunner.saveTestResults(locator, (testRunner.keyboards[id] || {}).results); return shouldSave; });
+                .then((shouldSave) => testRunner.saveTestResults(locator, (testRunner.keyboards[id] || {}).results));
                 //.then(() => testRunner.runTests(id))
                 //.then(() => testRunner.saveTestResults(locator, testRunner.keyboards[id].results));
               });
diff --git a/web/testing/regression-tests/test.js b/web/testing/regression-tests/test.js
index 18898e5c82..9cd5560376 100644
--- a/web/testing/regression-tests/test.js
+++ b/web/testing/regression-tests/test.js
@@ -333,6 +333,15 @@ cleanKeyboards.then(() => {
 
         const testsFilename = path.join(KEYBOARDS_ROOT, keyboard.shortname, keyboard.id, 'tests', `${keyboard.id}.tests`);
         const testsJSON = JSON.parse(fs.readFileSync(testsFilename, 'utf8'));
+
+        // First, check the base result for errors
+        for(let k in baseResultJSON) {
+          if(typeof baseResultJSON[k] !== 'string') {
+            let input = `${testsJSON.inputTests[k].context ? `"${testsJSON.inputTests[k].context}" ` : ""}+ ${keyname(testsJSON.inputTests[k].modifier, testsJSON.inputTests[k].key)}`;
+            fail(`${keyboard.shortname}/${keyboard.id}[$k]: error in test: ${input}: ${baseResultJSON[k].error}`, 5);
+          }
+        }
+
         //console.log(baseResultFilename, baseResult);
         testedCompilerVersions.forEach((cv) => {
           testedEngineVersions.forEach((ev) => {