Overhaul the test runner

This allows us to catch up to recent changes in web-platform-tests/wpt#39203. To reduce the maintenance burden, this rewrites the test runner to be focused on, as much as possible, reusing upstream runner code. For three files this is not possible, and those are explicitly called out.

Author: domenic

scripts/get-latest-platform-tests.js

@@ -18,20 +18,22 @@ process.on("unhandledRejection", err => {
 // 1. Go to https://github.com/web-platform-tests/wpt/tree/master/url
 // 2. Press "y" on your keyboard to get a permalink
 // 3. Copy the commit hash
-const commitHash = "c95d15338d3d4380beffb8b1b8dda62a8d746ab9";
+const commitHash = "6d461b4ddb2f1b8d226ca6ae92e14bbd464731a5";
 
 const urlPrefix = `https://raw.githubusercontent.com/web-platform-tests/wpt/${commitHash}/url/`;
 const targetDir = path.resolve(__dirname, "..", "test", "web-platform-tests");
 
-(fs.rmSync || fs.rmdirSync)(targetDir, { recursive: true, force: true });
-fs.mkdirSync(path.resolve(targetDir, "resources"), { recursive: true });
-
-for (const file of [
+// These resources we download, but the test runner doesn't need to know about them.
+const resources = [
   "resources/percent-encoding.json",
   "resources/setters_tests.json",
   "resources/toascii.json",
   "resources/urltestdata.json",
-  "resources/IdnaTestV2.json",
+  "resources/IdnaTestV2.json"
+];
+
+// These tests we can download and run directly in /test/web-platform.js.
+exports.directlyRunnableTests = [
   "url-searchparams.any.js",
   "url-setters-stripping.any.js",
   "url-tojson.any.js",
@@ -46,8 +48,31 @@ for (const file of [
   "urlsearchparams-set.any.js",
   "urlsearchparams-sort.any.js",
   "urlsearchparams-stringifier.any.js"
-]) {
-  fetch(`${urlPrefix}${file}`).then(res => {
-    res.body.pipe(fs.createWriteStream(path.resolve(targetDir, file)));
-  });
+];
+
+// These tests need some special handling in /test/web-platform.js, since they need to be hooked up to their resource
+// files in a case-by-case way. We still download them, but they're in a separately-exported array so that the runner
+// can distinguish.
+exports.resourceDependentTests = [
+  "IdnaTestV2.window.js",
+  "url-constructor.any.js",
+  "url-origin.any.js",
+  "url-setters.any.js"
+];
+
+// These tests need their logic duplicated in /test/web-platform.js, because we can't easly shim them. They are not
+// downloaded, but we list them here so that it's easy to understand our categorization scheme.
+// - failure.html
+// - percent-encoding.window.js
+// - toascii.window.js
+
+if (require.main === module) {
+  (fs.rmSync || fs.rmdirSync)(targetDir, { recursive: true, force: true });
+  fs.mkdirSync(path.resolve(targetDir, "resources"), { recursive: true });
+
+  for (const file of [...resources, ...exports.directlyRunnableTests, ...exports.resourceDependentTests]) {
+    fetch(`${urlPrefix}${file}`).then(res => {
+      res.body.pipe(fs.createWriteStream(path.resolve(targetDir, file)));
+    });
+  }
 }

test/testharness.js

@@ -39,5 +39,10 @@ module.exports = {
 
   assert_unreached() {
     assert(false);
+  },
+
+  subsetTestByKey(key, testRunnerFunc, func, name) {
+    // Don't do any keying stuff.
+    testRunnerFunc(func, name);
   }
 };