all: Refactor tests significantly

This lets us use different profiles for different tests. It also
lets us put all tests for a particular plugin in it's own directory
and makes it easier for users to see how to write tests for their
own plugins not hosted here.

Author: cassava

.github/workflows/build-cloe.yaml

@@ -54,6 +54,9 @@ jobs:
         run: |
           make -f Makefile.all export-vendor &&
           make ${{ matrix.package_target }}
+      - name: Build smoketest dependencies
+        run: |
+          make smoketest-deps
       - name: Run smoketests
         run: |
           make smoketest

.lvimrc

@@ -15,6 +15,12 @@ augroup rstsettings
     autocmd Filetype rst set tw=79 cc=80
 augroup end
 
+if &filetype == "json"
+  setlocal filetype=json5
+  syntax off
+  syntax on
+endif
+
 doau Filetype
 
 " Define AddPackageToPath function that can be used

Makefile

@@ -41,7 +41,7 @@ include Makefile.setup
 include Makefile.all
 
 # Workspace targets -----------------------------------------------------------
-.PHONY: lockfile status deploy sphinx doxygen docker-all docker-test docker-release purge-all smoketest
+.PHONY: lockfile status deploy sphinx doxygen docker-all docker-test docker-release purge-all smoketest smoketest-deps
 help::
 	echo "Available workspace targets:"
 	echo "  smoketest       to run BATS system tests"
@@ -98,11 +98,11 @@ doxygen:
 	mkdir -p ${BUILD_DIR}/doxygen
 	doxygen Doxyfile
 
-smoketest:
+smoketest-deps: smoketest-deps-select
+	# Call this target with WITH_VTD=1 to include VTD binding tests.
+
+smoketest: smoketest-select
 	# Call this target with WITH_VTD=1 to include VTD binding tests.
-	$(call print_header, "Running smoke tests...")
-	@${CLOE_LAUNCH} clean -P conantest.py
-	@\time -f "\nTotal smoketest time (real) = %e sec" bash -c "source tests/setup_testname.bash && bats tests"
 
 purge-all:
 	$(call print_header, "Removing all cloe Conan packages...")

Makefile.all

@@ -130,8 +130,8 @@ $(filter-out cloe, ${ALL_PKGS} ${ALL_VENDOR_PKGS}):
 # Re-define cloe target to use Makefile.package, and only run for targets
 # where it makes sense, since "cloe" is a Conan meta-package.
 cloe:
-	for case in export package package-outdated list purge clean; do \
-		if [[ "$$(echo '${MAKECMDGOALS}' | sed -re ${REGEX_TARGET})" == "$${case}" ]]; then \
+	for case in export package package-outdated list purge clean smoketest smoketest-deps; do \
+		if [ "$$(echo '${MAKECMDGOALS}' | sed -re ${REGEX_TARGET})" == "$${case}" ]; then \
 			${MAKE} -f Makefile.package CONAN_OPTIONS="${CONAN_OPTIONS}" $${case} || exit 1; \
 		fi \
 	done
@@ -262,6 +262,9 @@ $(call make_select_target, package-select,     "create selected packages with po
 $(call make_select_target, list-select,        "list selected package install files         [conan-cache]")
 $(call make_select_target, purge-select,       "remove selected packages from cache         [conan-cache]")
 
+$(call make_select_target, smoketest-select,        "run tests for selected packages             [conan-cache]")
+$(call make_select_target, smoketest-deps-select,   "build test deps for selected packages       [conan-cache]")
+
 help::
 	echo
 

Makefile.package

@@ -18,6 +18,10 @@
 PROJECT_ROOT     := $(dir $(abspath $(lastword ${MAKEFILE_LIST})))
 PROJECT_VERSION  := $(shell [ -r ${PROJECT_ROOT}/VERSION ] && cat ${PROJECT_ROOT}/VERSION || echo unknown)
 
+CLOE_ROOT   := ${PROJECT_ROOT}
+CLOE_LAUNCH := PYTHONPATH="${CLOE_ROOT}/cli" python3 -m cloe_launch
+
+SHELL := /bin/bash
 DATE := $(shell date +"%Y%m%d")
 TIMESTAMP := $(shell date -u +'%Y-%m-%dT%H:%M:%SZ')
 
@@ -115,18 +119,20 @@ endif
 
 # INFORMATIONAL TARGETS -------------------------------------------------------
 .DEFAULT: help
-.SILENT: help status parse-info parse-editable info-name info-version info-channel info-fqn info
-.PHONY: help status parse-info parse-editable info-name info-version info-channel info-fqn info
+.SILENT: help status parse-info parse-editable info-name info-version info-channel info-fqn info smoketest smoketest-deps
+.PHONY: help status parse-info parse-editable info-name info-version info-channel info-fqn info smoketest smoketest-deps
 help:: parse-info parse-editable
 	echo "Usage: make <target>"
 	echo
 	echo "The following targets define common operations with this package in"
 	echo "editable (local in-source) and uneditable (in the Conan cache) modes."
 	echo
 	echo "Available targets:"
-	echo "  help        to show this help"
-	echo "  status      to show status of package"
-	echo "  info        to show detailed package info"
+	echo "  help            to show this help"
+	echo "  status          to show status of package"
+	echo "  info            to show detailed package info"
+	echo "  smoketest-deps  to build smoketest dependencies for package"
+	echo "  smoketest       to run smoketests for package (requires built packages)"
 	echo
 	echo "  export            to export recipe and sources        [conan-cache]"
 	echo "  download          to download or create package       [conan-cache]"
@@ -218,6 +224,20 @@ info: parse-info
 		echo ${PACKAGE_INFO}; \
 	fi
 
+smoketest:
+	@for profile in tests/profile_*.py; do \
+		test -f "$${profile}" || continue; \
+		printf "Running BATS tests with profile: $${profile}\n\n"; \
+		SHELL=/bin/bash ${CLOE_LAUNCH} shell -P "$${profile}" -- -c "source ${CLOE_ROOT}/tests/setup_testname.bash; bats tests"; \
+	done
+
+smoketest-deps:
+	@for profile in tests/profile_*.py; do \
+		test -f "$${profile}" || continue; \
+		echo "Building dependencies for profile: $${profile}"; \
+		${CLOE_LAUNCH} prepare -P "$${profile}"; \
+	done
+
 # CONFIGURATION TARGETS -------------------------------------------------------
 .PHONY: editable uneditable
 

dist/docker/Dockerfile.archlinux

@@ -67,7 +67,7 @@ RUN --mount=type=cache,target=/ccache \
 # Build Cloe.
 COPY . /cloe
 ARG PROJECT_VERSION=unknown
-ARG PACKAGE_TARGET=package-auto
+ARG PACKAGE_TARGET="package-auto smoketest-deps"
 RUN --mount=type=cache,target=/ccache \
     --mount=type=secret,target=/root/setup.sh,id=setup,mode=0400 \
     if [ -r /root/setup.sh ]; then . /root/setup.sh; fi && \

dist/docker/Dockerfile.bionic

@@ -68,7 +68,7 @@ RUN --mount=type=cache,target=/ccache \
 # Build Cloe.
 COPY . /cloe
 ARG PROJECT_VERSION=unknown
-ARG PACKAGE_TARGET=package-auto
+ARG PACKAGE_TARGET="package-auto smoketest-deps"
 RUN --mount=type=cache,target=/ccache \
     --mount=type=secret,target=/root/setup.sh,id=setup,mode=0400 \
     if [ -r /root/setup.sh ]; then . /root/setup.sh; fi && \

dist/docker/Dockerfile.focal

@@ -85,7 +85,7 @@ RUN --mount=type=cache,target=/ccache \
 # Build Cloe.
 COPY . /cloe
 ARG PROJECT_VERSION=unknown
-ARG PACKAGE_TARGET=package-auto
+ARG PACKAGE_TARGET="package-auto smoketest-deps"
 RUN --mount=type=cache,target=/ccache \
     --mount=type=secret,target=/root/setup.sh,id=setup,mode=0400 \
     if [ -r /root/setup.sh ]; then . /root/setup.sh; fi && \

dist/docker/Dockerfile.xenial

@@ -120,7 +120,7 @@ RUN --mount=type=cache,target=/ccache \
 # Build Cloe.
 COPY . /cloe
 ARG PROJECT_VERSION=unknown
-ARG PACKAGE_TARGET=package-auto
+ARG PACKAGE_TARGET="package-auto smoketest-deps"
 RUN --mount=type=cache,target=/ccache \
     --mount=type=secret,target=/root/setup.sh,id=setup,mode=0400 \
     if [ -r /root/setup.sh ]; then . /root/setup.sh; fi && \

engine/tests/config_nop_infinite.json

@@ -0,0 +1,45 @@
+{
+  "version": "4",
+  "include": [
+    "controller_virtue.json",
+    "controller_basic.json"
+  ],
+  "simulators": [
+    {
+      "binding": "nop"
+    }
+  ],
+  "vehicles": [
+    {
+      "name": "default",
+      "from": {
+        "simulator": "nop",
+        "index": 0
+      },
+      "components": {
+        "cloe::speedometer": {
+          "binding": "speedometer",
+          "name": "default_speed",
+          "from": "cloe::gndtruth_ego_sensor"
+        },
+        "cloe::default_world_sensor": {
+          "binding": "noisy_object_sensor",
+          "name": "noisy_object_sensor",
+          "from": "cloe::default_world_sensor",
+          "args": {
+            "noise": [
+              {
+                "target": "translation",
+                "distribution": {
+                  "binding": "normal",
+                  "mean": 0.0,
+                  "std_deviation": 0.3
+                }
+              }
+            ]
+          }
+        }
+      }
+    }
+  ]
+}