add some example tests

Author: ChristopherChudzicki

.env.example

@@ -13,3 +13,4 @@ COURSE_BASE_URL=http://localhost:3000/courses
 FIELDS_HUGO_CONFIG_PATH=/path/to/ocw-hugo-projects/mit-fields/config.yaml
 FIELDS_CONTENT_PATH=/path/to/fields-site/
 VERBOSE=1
+WEBPACK_PORT=3001

README.md

@@ -168,6 +168,7 @@ An example environment file can be found at `.env.example`.  To further explain
 | `FIELDS_CONTENT_PATH` | `fields` | `/path/to/ocw-content-rc/philosophy` | A path to a Hugo site that will be rendered when running `npm run start:fields` |
 | `VERBOSE` | N/A | `0` | Used in `build_all_courses.sh`, if set to `1` this will print verbose output from the course builds to the console |
 | `DOWNLOAD` | N/A | `1` | Used in `npm run start:course`, if set to `0` this will not download course data from S3 and instead source `OCW_TEST_COURSE` from the specified `OCW_TO_HUGO_OUTPUT_DIR`. If not specified, it will default to 1 and try to download data from S3. |
+| `WEBPACK_PORT` | N/A | `3001` | Port used by Webpack Dev Server |
 | `WEBPACK_ANALYZE` | N/A | `true` | Used in webpack build. If set to `true`, a dependency analysis of the bundle will be included in the build output. |
 
 ### Running ocw-www
@@ -235,6 +236,9 @@ To customize your `fields` site:
  - Start the site with `npm run start:fields`
  - The site should be available at http://localhost:3000/
 
+### Writing Tests
+Most tests in OCW Hugo Themes should be written as e2e tests with Playwright. See [End to End Testing](./tests-e2e/README.md) for more.
+
 ### Miscellaneous commands
 
 - `WEBPACK_ANALYZE=true yarn run build:webpack`: This builds the project for production and should open a an analysis of the bundle in your web browser. 

course-v2/layouts/partials/course_detail.html

@@ -1,15 +1,15 @@
 <div class="card">
   <div class="card-body">
-    <div class="course-detail-section">
+    <section class="course-detail-section">
       {{ partial "course_description.html" (dict "context" .) }}
-    </div>
+    </section>
 
-    <div class="course-detail-section">
+    <section class="course-detail-section">
       {{ partial "course_info.html" (dict "context" . "inPanel" false) }}
-    </div>
+    </section>
 
-    <div class="course-detail-section">
+    <section class="course-detail-section">
       {{ partial "learning_resource_types.html" (dict "context" . "inPanel" false) }}
-    </div>
+    </section>
   </div>
 </div>

package.json

@@ -23,7 +23,7 @@
     "build:webpack": "cross-env NODE_ENV=production webpack --config base-theme/assets/webpack/webpack.prod.ts",
     "webpack:stats": "NODE_ENV=production webpack --config base-theme/assets/webpack/webpack.prod.ts --profile --json > stats.json",
     "webpack:analyze": "webpack-bundle-analyzer stats.json dist",
-    "fmt": "LOG_LEVEL= prettier-eslint --write --no-semi $PWD/'**/assets/**/*.js' $PWD/'**/assets/**/*.ts' $PWD/'**/assets/**/*.tsx' $PWD/'**/assets/**/*.scss'",
+    "fmt": "LOG_LEVEL= prettier-eslint --write --no-semi $PWD/'**/assets/**/*.js' $PWD/'**/assets/**/*.ts' $PWD/'**/assets/**/*.tsx' $PWD/'**/assets/**/*.scss' $PWD/'tests-e2e/**/*.ts' ",
     "fmt:check": "LOG_LEVEL= prettier-eslint --list-different --no-semi $PWD/'**/assets/**/*.js' $PWD/'**/assets/**/*.ts' $PWD/'**/assets/**/*.tsx' $PWD/'**/assets/**/*.scss'",
     "scss_lint": "node ./node_modules/sass-lint/bin/sass-lint.js --verbose --no-exit ./assets",
     "test": "jest",
@@ -59,6 +59,7 @@
     "@types/webpack-bundle-analyzer": "^4.6.0",
     "@types/webpack-dev-server": "^4.7.2",
     "@typescript-eslint/eslint-plugin": "^5.40.1",
+    "ansi-colors": "^4.1.3",
     "archiver": "^5.0.0",
     "assets-webpack-plugin": "^7.1.1",
     "babel-loader": "^8.2.5",
@@ -67,6 +68,7 @@
     "casual-browserify": "^1.5.19-2",
     "clean-webpack-plugin": "^3.0.0",
     "cli-progress": "^3.8.2",
+    "cli-table3": "^0.6.3",
     "concurrently": "^6.3.0",
     "copy-webpack-plugin": "^11.0.0",
     "core-js": "^3.20.2",

package_scripts/build.sh

@@ -4,8 +4,6 @@ set -euo pipefail
 
 source package_scripts/common.sh
 
-export STATIC_API_BASE_URL=https://ocw.mit.edu/
-
 # We should have 2 arguments; the path to the content and the config
 if [[ $# -lt 2 ]]; then
   echo "Usage: build.sh /path/to/site /path/to/config.yaml"

playwright.config.ts

@@ -1,9 +1,13 @@
-import type { PlaywrightTestConfig } from '@playwright/test'
+import { PlaywrightTestConfig } from '@playwright/test'
 import { devices } from '@playwright/test'
 import dotenv from 'dotenv'
 import * as path from "path"
+import * as envalid from "envalid"
 
 dotenv.config()
+const env = envalid.cleanEnv(process.env, {
+  WEBPACK_PORT: envalid.port()
+})
 
 /**
  * See https://playwright.dev/docs/test-configuration.
@@ -34,7 +38,7 @@ const config: PlaywrightTestConfig = {
   ],
   webServer: {
     command:             "yarn start:webpack",
-    url:                 "http://localhost:3001/static/css/main.css",
+    url:                 `http://localhost:${env.WEBPACK_PORT}/static/css/main.css`,
     reuseExistingServer: !process.env.CI,
   },
   globalSetup: path.resolve(__dirname, './tests-e2e/global-setup.ts'),

test-sites/ocw-ci-test-course/content/pages/shortcode-demos.md

@@ -7,4 +7,4 @@ uid: 7e1d1926-96b4-4f0c-b4b6-2f34ab7562f6
 ---
 ### Resource Link
 
-Check no extra spaces xxx{{% resource_link "38167987-b1af-4544-86a1-909a7cdb4f18" "First Test Page" %}}xxx
+Check no extra spaces xxx{{% resource_link "38167987-b1af-4544-86a1-909a7cdb4f18" "Resource link to First Test Page" %}}xxx

test-sites/ocw-ci-test-course/data/course.json

@@ -2,7 +2,8 @@
   "term": "Fall",
   "year": "2022",
   "level": [
-    "Graduate"
+    "Graduate",
+    "Undergraduate"
   ],
   "topics": [
     [
@@ -19,8 +20,11 @@
   "legacy_uid": "",
   "instructors": {
     "content": [
-      "a0af4753-e784-4e5e-b23b-1a5911b15283",
-      "144c1f31-44a3-23a9-7819-21a612f6574f"
+      "a16d0fec-adce-ec6a-10b1-d8153a7427e6",
+      "a32373bc-8c78-e37a-f79e-1f15686d1a9b",
+      "144c1f31-44a3-23a9-7819-21a612f6574f",
+      "0bb74f02-6e52-c6eb-1b90-fe7d782be397",
+      "c8b71de9-118e-b585-3e09-8cbe2bcfc6f2"
     ],
     "website": "ocw-www"
   },
@@ -29,9 +33,11 @@
     "website": "ocw-ci-test-course"
   },
   "course_title": "OCW CI Test Course",
-  "course_description": "This is a test course for use in CI pipelines.",
+  "course_description": "This is a test course for use in CI pipelines. This course can be found:\n\n- On Github at \u00a0[https://github.mit.edu/ocw-content-rc/ocw-ci-test-course](https://github.mit.edu/ocw-content-rc/ocw-ci-test-course)\n- On OCW Studio at [https://ocw-studio-rc.odl.mit.edu/sites/ocw-ci-test-course](https://ocw-studio-rc.odl.mit.edu/sites/ocw-ci-test-course)\n\nThe content of this course is additionally checked into the `ocw-hugo-themes` repository.\n\n\u00a0Recommended usage by OL Engineers:\n\n- Clone the github repository to your local.\n- When developing on `ocw-hugo-themes`, make edits in Studio as needed for the tests you want to write. Pull to local, and copy the files to `ocw-hugo-themes/test-sites/ocw-ci-test-course`.",
   "department_numbers": [
-    "8"
+    "8",
+    "6",
+    "18"
   ],
   "extra_course_numbers": "456",
   "primary_course_number": "123",

tests-e2e/README.md

@@ -0,0 +1,25 @@
+# End-to-end Testing
+
+This repository uses [Playwright](https://playwright.dev/) for end-to-end testing. Roughly, the Playwright tests build static assets with Webpack and Hugo then use headless browsers to manipulate the generated pages. We make assertions about the pages using Playwright's typescript library [`@playwright/test`](https://playwright.dev/docs/intro).
+
+In OCW Hugo Themes, **most tests should be e2e**. See [Why playwright?](#why-playwright) for more. 
+
+## FAQ
+
+### How to Run Playwright
+
+To run the tests:
+1. `yarn start:webpack` in one shell, and
+2. `yarn test:e2e` in another. *Or `yarn test:e2e --debug` to step through tests.
+
+Running `yarn test:e2e` by itself will also work. However, if you start a webpack server in a separate process first, Playwright will use that server instead rather than starting a new one, resulting in much less overhead if you need to re-run tests.
+
+### Is there a `--watch` mode for re-running tests?
+
+No. For VS Code users, a similar experience can be achieved with [Playwright for VS Code](https://marketplace.visualstudio.com/items?itemName=ms-playwright.playwright). Individual tests can be run from your editor with the click of a button.
+
+### How do I generate test content?
+The easiest way is probably through Studio. See [https://ocw-studio-rc.odl.mit.edu/sites/ocw-ci-test-course](https://ocw-studio-rc.odl.mit.edu/sites/ocw-ci-test-course/type/metadata/), for example. 
+
+### Why Playwright?
+Historically, we've used Jest and JSDom to test *some* aspects of OCW Hugo Themes. However, building any significant confidence in OCW Hugo Themes via Jest+JSDom has proven difficult for two reasons: (1) a large portion of the repo's logic is in Hugo templates, and (2) the output of Hugo is static HTML files, not JS modules. Neither of these issues is a challenge when testing Hugo's output via Playwright, and we get other benefits, too. (For example: ability to test features like infinite scroll that rely on layout, or the possibility to do visual testing.)

tests-e2e/global-setup.ts

@@ -1,51 +1,80 @@
-import type { ExecOptions } from "node:child_process"
-import http from "node:http"
-
+import { ExecOptions } from "node:child_process"
+import * as http from "node:http"
+import * as path from "node:path"
 import dotenv from "dotenv"
 import execShCb from "exec-sh"
 import * as envalid from "envalid"
-import { fromRoot, TEST_SITES } from "./util"
+import { TEST_SITES } from "./util"
 import handler from "serve-handler"
+import Table from "cli-table3"
+import * as color from "ansi-colors"
+
 
 dotenv.config()
 const env = envalid.cleanEnv(process.env, {
-  WEBPACK_PORT: envalid.port(),
+  WEBPACK_PORT: envalid.port()
 })
 
 const execSh = execShCb.promise
 
+/**
+ * Resolve a path relative to package root.
+ */
+const fromRoot = (...pathFromRoot: string[]) => {
+  return path.join(__dirname, "../", ...pathFromRoot)
+}
 interface HugoOptions {
   baseURL: string
   themesDir: string
   config: string
   destination: string
 }
 const hugo = (hugoOptions: HugoOptions, execOptions: ExecOptions) => {
-  const flags = Object.entries(hugoOptions).map(([key, value]) => `--${key}=${value}`).join(' ')
-  return execSh(`hugo ${flags} --verbose`, execOptions)
+  const flags = Object.entries(hugoOptions)
+    .map(([key, value]) => `--${key}=${value}`)
+    .join(" ")
+  return execSh(`yarn hugo ${flags} --verbose`, execOptions)
 }
 
 const buildSite = (name: string, configPath: string) => {
-  return hugo({
-    themesDir:   fromRoot("./"),
-    destination: fromRoot(`./test-sites/${name}/dist`),
-    baseURL:     `http://localhost:${env.WEBPACK_PORT}`,
-    config:      configPath
-  }, {
-    cwd: fromRoot(`./test-sites/${name}`)
-  })
+  return hugo(
+    {
+      themesDir:   fromRoot("./"),
+      destination: fromRoot(`./test-sites/${name}/dist`),
+      baseURL:     `http://localhost:${env.WEBPACK_PORT}`,
+      config:      configPath
+    },
+    {
+      cwd: fromRoot(`./test-sites/${name}`)
+    }
+  )
 }
 
 const setupTests = async () => {
+  const table = new Table({
+    head:  ["Site", "URL"],
+    style: { head: ["cyan", "bold"] }
+  })
+
   const sites = Object.values(TEST_SITES)
   await Promise.all(sites.map(site => buildSite(site.name, site.configPath)))
   sites.forEach(site => {
-    http.createServer((request, response) => {
-      return handler(request, response, {
-        public: fromRoot(`./test-sites/${site.name}/dist`)
+    http
+      .createServer((request, response) => {
+        return handler(request, response, {
+          public: fromRoot(`./test-sites/${site.name}/dist`)
+        })
       })
-    }).listen(site.port)
+      .listen(site.port)
+    table.push([site.name, `http://localhost:${site.port}`])
   })
+
+  console.log([
+    "\n",
+    color.bold("Now serving the following sites:\n"),
+    table.toString(),
+    "\n"
+  ].join(""))
 }
 
 export default setupTests