From f6198dc629a510a8d843d8e23409a93183f16f8f Mon Sep 17 00:00:00 2001 From: Michael Boulton Date: Sat, 15 Nov 2025 16:24:41 +0000 Subject: [PATCH 1/9] feat: add generic backends --- MANIFEST.in | 1 + example/custom_backend/__init__.py | 81 +++++ example/custom_backend/jsonschema.yaml | 39 +++ example/custom_backend/pyproject.toml | 10 + example/custom_backend/run_tests.sh | 15 + .../test_file_touched.tavern.yaml | 9 + pyproject.toml | 2 +- tavern/_core/plugins.py | 6 +- tavern/_core/pytest/config.py | 2 +- tavern/_core/pytest/util.py | 51 ++- tavern/_core/schema/files.py | 9 +- tavern/_core/schema/tests.jsonschema.yaml | 300 ------------------ tavern/_core/strict_util.py | 24 +- tavern/_plugins/grpc/jsonschema.yaml | 53 ++++ tavern/_plugins/mqtt/jsonschema.yaml | 90 +++++- tavern/_plugins/rest/jsonschema.yaml | 180 +++++++++++ tavern/_plugins/rest/tavernhook.py | 11 + tavern/response.py | 17 +- tox-integration.ini | 1 + 19 files changed, 586 insertions(+), 315 deletions(-) create mode 100644 example/custom_backend/__init__.py create mode 100644 example/custom_backend/jsonschema.yaml create mode 100644 example/custom_backend/pyproject.toml create mode 100755 example/custom_backend/run_tests.sh create mode 100644 example/custom_backend/test_file_touched.tavern.yaml create mode 100644 tavern/_plugins/rest/jsonschema.yaml diff --git a/MANIFEST.in b/MANIFEST.in index 05b2580c1..0f4d4bfe8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,5 @@ include tavern/_core/schema/tests.jsonschema.yaml include tavern/_plugins/mqtt/jsonschema.yaml +include tavern/_plugins/rest/jsonschema.yaml include tavern/_plugins/grpc/schema.yaml include LICENSE diff --git a/example/custom_backend/__init__.py b/example/custom_backend/__init__.py new file mode 100644 index 000000000..dc9aeceea --- /dev/null +++ b/example/custom_backend/__init__.py @@ -0,0 +1,81 @@ +import logging +import pathlib +from collections.abc import Iterable +from os.path import abspath, dirname, join +from typing import Any, Optional, Union + +import box +import yaml + +from tavern._core import exceptions +from tavern._core.pytest.config import TestConfig +from tavern.request import BaseRequest +from tavern.response import BaseResponse + + +class Session: + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + + +class Request(BaseRequest): + def __init__( + self, session: Any, rspec: dict, test_block_config: TestConfig + ) -> None: + self.session = session + + self._request_vars = rspec + + @property + def request_vars(self) -> box.Box: + return self._request_vars + + def run(self): + pathlib.Path(self._request_vars["filename"]).touch() + + +class Response(BaseResponse): + def verify(self, response): + if not pathlib.Path(self.expected["filename"]).exists(): + raise exceptions.BadSchemaError( + f"Expected file '{self.expected['filename']}' does not exist" + ) + + return {} + + def __init__( + self, + client, + name: str, + expected: TestConfig, + test_block_config: TestConfig, + ) -> None: + super().__init__(name, expected, test_block_config) + + +logger: logging.Logger = logging.getLogger(__name__) + +session_type = Session + +request_type = Request +request_block_name = "touch_file" + + +verifier_type = Response +response_block_name = "file_exists" + + +def get_expected_from_request( + response_block: Union[dict, Iterable[dict]], + test_block_config: TestConfig, + session: Session, +) -> Optional[dict]: + return response_block + + +schema_path: str = join(abspath(dirname(__file__)), "jsonschema.yaml") +with open(schema_path, encoding="utf-8") as schema_file: + schema = yaml.load(schema_file, Loader=yaml.SafeLoader) diff --git a/example/custom_backend/jsonschema.yaml b/example/custom_backend/jsonschema.yaml new file mode 100644 index 000000000..511c27e59 --- /dev/null +++ b/example/custom_backend/jsonschema.yaml @@ -0,0 +1,39 @@ +$schema: "http://json-schema.org/draft-07/schema#" + +title: file touch schema +description: Schema for touching files + +### + +definitions: + touch_file: + type: object + description: touch a file + additionalProperties: false + required: + - filename + + properties: + filename: + type: string + description: Name of file to touch + + file_exists: + type: object + description: name of file which should exist + additionalProperties: false + required: + - filename + + properties: + filename: + type: string + description: Name of file to check for + + stage: + properties: + touch_file: + $ref: "#/definitions/touch_file" + + file_exists: + $ref: "#/definitions/file_exists" diff --git a/example/custom_backend/pyproject.toml b/example/custom_backend/pyproject.toml new file mode 100644 index 000000000..2edd54f41 --- /dev/null +++ b/example/custom_backend/pyproject.toml @@ -0,0 +1,10 @@ +[project] +name = "custom-backend" +version = "0.1.0" +description = "Add your description here" +readme = "README.md" +requires-python = ">=3.14" +dependencies = [] + +[project.entry-points.tavern_file] +custom_backend = "custom_backend" \ No newline at end of file diff --git a/example/custom_backend/run_tests.sh b/example/custom_backend/run_tests.sh new file mode 100755 index 000000000..b4413e522 --- /dev/null +++ b/example/custom_backend/run_tests.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -ex + +if [ ! -d ".venv" ]; then + uv venv +fi +. .venv/bin/activate + +uv pip install -e . 'tavern @ ../..' + +PYTHONPATH=. tavern-ci \ + --tavern-extra-backends=file=custom_backend \ + test_file_touched.tavern.yaml \ + --debug "$@" --stdout \ No newline at end of file diff --git a/example/custom_backend/test_file_touched.tavern.yaml b/example/custom_backend/test_file_touched.tavern.yaml new file mode 100644 index 000000000..d0891db66 --- /dev/null +++ b/example/custom_backend/test_file_touched.tavern.yaml @@ -0,0 +1,9 @@ +--- +test_name: Test file touched + +stages: + - name: Touch file and check it exists + touch_file: + filename: hello.txt + file_exists: + filename: hello.txt \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 9b0a4a2aa..01418e894 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -254,4 +254,4 @@ cmd = "uv lock" runner = "uv-venv-lock-runner" skip_missing_interpreters = true isolated_build = true -base_python = "3.11" \ No newline at end of file +base_python = "3.11" diff --git a/tavern/_core/plugins.py b/tavern/_core/plugins.py index c2ca409fa..ea68472e3 100644 --- a/tavern/_core/plugins.py +++ b/tavern/_core/plugins.py @@ -127,11 +127,13 @@ def _load_plugins(self, test_block_config: TestConfig) -> list[_Plugin]: plugins = [] def enabled(current_backend, ext): - return ( + is_enabled = ( ext.name == test_block_config.tavern_internal.backends[current_backend] ) + logger.debug(f"is {ext.name} enabled? {is_enabled}") + return is_enabled - for backend in test_block_config.backends(): + for backend in test_block_config.tavern_internal.backends.keys(): logger.debug("loading backend for %s", backend) namespace = f"tavern_{backend}" diff --git a/tavern/_core/pytest/config.py b/tavern/_core/pytest/config.py index 6f70a2f8b..bfe007c31 100644 --- a/tavern/_core/pytest/config.py +++ b/tavern/_core/pytest/config.py @@ -57,7 +57,7 @@ def backends() -> list[str]: if has_module("paho.mqtt"): available_backends.append("mqtt") - if has_module("grpc"): + if has_module("grpc") and has_module("grpc_reflection"): available_backends.append("grpc") logger.debug(f"available request backends: {available_backends}") diff --git a/tavern/_core/pytest/util.py b/tavern/_core/pytest/util.py index c9c4a0366..848672e18 100644 --- a/tavern/_core/pytest/util.py +++ b/tavern/_core/pytest/util.py @@ -5,6 +5,7 @@ import pytest +from tavern._core import exceptions from tavern._core.dict_util import format_keys, get_tavern_box from tavern._core.general import load_global_config from tavern._core.pytest.config import TavernInternalConfig, TestConfig @@ -71,6 +72,13 @@ def add_parser_options(parser_addoption, with_defaults: bool = True) -> None: default=False, action="store_true", ) + parser_addoption( + "--tavern-extra-backends", + help="list of extra backends to register", + default="", + type=str, + action="store", + ) def add_ini_options(parser: pytest.Parser) -> None: @@ -123,6 +131,12 @@ def add_ini_options(parser: pytest.Parser) -> None: type="bool", default=False, ) + parser.addini( + "tavern-extra-backends", + help="list of extra backends to register", + type="args", + default=[], + ) def load_global_cfg(pytest_config: pytest.Config) -> TestConfig: @@ -177,11 +191,26 @@ def _load_global_cfg(pytest_config: pytest.Config) -> TestConfig: def _load_global_backends(pytest_config: pytest.Config) -> dict[str, Any]: """Load which backend should be used""" - return { + backends: dict[str, str | None] = { b: get_option_generic(pytest_config, f"tavern-{b}-backend", None) for b in TestConfig.backends() } + extra_backends: list[str] = get_option_generic( + pytest_config, "tavern-extra-backends", [] + ) + for backend in extra_backends: + split = backend.split("=", 1) + if len(split) != 2: + raise exceptions.BadSchemaError( + f"extra backends must be in the form 'name=value', got {backend}" + ) + + key, value = split + backends[key] = value + + return backends + def _load_global_strictness(pytest_config: pytest.Config) -> StrictLevel: """Load the global 'strictness' setting""" @@ -214,11 +243,23 @@ def get_option_generic( use = default # Middle priority - if pytest_config.getini(ini_flag) is not None: - use = pytest_config.getini(ini_flag) + if ini := pytest_config.getini(ini_flag): + if isinstance(default, list): + if isinstance(ini, list): + use.extend(ini) # type:ignore + else: + raise ValueError( + f"Expected list for {ini_flag} option, got {ini} of type {type(ini)}" + ) + else: + use = ini # Top priority - if pytest_config.getoption(cli_flag) is not None: - use = pytest_config.getoption(cli_flag) + if cli := pytest_config.getoption(cli_flag): + if isinstance(default, list): + cli = cli.split(",") + use.extend(cli) # type:ignore + else: + use = cli return use diff --git a/tavern/_core/schema/files.py b/tavern/_core/schema/files.py index 3d9429101..85eedad62 100644 --- a/tavern/_core/schema/files.py +++ b/tavern/_core/schema/files.py @@ -5,6 +5,7 @@ import tempfile from collections.abc import Mapping +import box import pykwalify import yaml from pykwalify import core @@ -53,7 +54,13 @@ def _load_schema_with_plugins(self, schema_filename: str) -> dict: # Don't require a schema logger.debug("No schema defined for %s", p.name) else: - base_schema["properties"].update(plugin_schema.get("properties", {})) + for key in ["properties", "definitions"]: + if key not in plugin_schema: + continue + + value = box.Box(plugin_schema[key]) + value.merge_update(base_schema[key]) + base_schema[key] = value self._loaded[mangled] = base_schema return self._loaded[mangled] diff --git a/tavern/_core/schema/tests.jsonschema.yaml b/tavern/_core/schema/tests.jsonschema.yaml index 6611cb0f0..63608d10e 100644 --- a/tavern/_core/schema/tests.jsonschema.yaml +++ b/tavern/_core/schema/tests.jsonschema.yaml @@ -64,284 +64,6 @@ definitions: items: $ref: "#/definitions/stage" - http_request: - type: object - additionalProperties: false - description: HTTP request to perform as part of stage - - required: - - url - - properties: - url: - description: URL to make request to - oneOf: - - type: string - - type: object - properties: - "$ext": - $ref: "#/definitions/verify_block" - - cert: - description: Certificate to use - either a path to a certificate and key in one file, or a two item list containing the certificate and key separately - oneOf: - - type: string - - type: array - minItems: 2 - maxItems: 2 - items: - type: string - - auth: - description: Authorisation to use for request - a list containing username and password - type: array - minItems: 2 - maxItems: 2 - items: - type: string - - verify: - description: Whether to verify the server's certificates - oneOf: - - type: boolean - default: false - - type: string - - method: - description: HTTP method to use for request - default: GET - type: string - - follow_redirects: - type: boolean - description: Whether to follow redirects from 3xx responses - default: false - - stream: - type: boolean - description: Whether to stream the download from the request - default: false - - cookies: - type: array - description: Which cookies to use in the request - - items: - oneOf: - - type: string - - type: object - - json: - description: JSON body to send in request body - $ref: "#/definitions/any_json" - - params: - description: Query parameters - type: object - - headers: - description: Headers for request - type: object - - data: - description: Form data to send in request - oneOf: - - type: object - - type: string - - timeout: - description: How long to wait for requests to time out - oneOf: - - type: number - - type: array - minItems: 2 - maxItems: 2 - items: - type: number - - file_body: - type: string - description: Path to a file to upload as the request body - - files: - oneOf: - - type: object - - type: array - description: Files to send as part of the request - - clear_session_cookies: - description: Whether to clear sesion cookies before running this request - type: boolean - - mqtt_publish: - type: object - description: Publish MQTT message - additionalProperties: false - - properties: - topic: - type: string - description: Topic to publish on - - payload: - type: string - description: Raw payload to post - - json: - description: JSON payload to post - $ref: "#/definitions/any_json" - - qos: - type: integer - description: QoS level to use for request - default: 0 - - retain: - type: boolean - description: Whether the message should be retained - default: false - - mqtt_response: - type: object - additionalProperties: false - description: Expected MQTT response - - properties: - unexpected: - type: boolean - description: Receiving this message fails the test - - topic: - type: string - description: Topic message should be received on - - payload: - description: Expected raw payload in response - oneOf: - - type: number - - type: integer - - type: string - - type: boolean - - json: - description: Expected JSON payload in response - $ref: "#/definitions/any_json" - - timeout: - type: number - description: How long to wait for response to arrive - - qos: - type: integer - description: QoS level that message should be received on - minimum: 0 - maximum: 2 - - verify_response_with: - oneOf: - - $ref: "#/definitions/verify_block" - - type: array - items: - $ref: "#/definitions/verify_block" - - save: - type: object - description: Which objects to save from the response - - grpc_request: - type: object - required: - - service - properties: - host: - type: string - - service: - type: string - - body: - type: object - - json: - type: object - - retain: - type: boolean - - grpc_response: - type: object - properties: - status: - oneOf: - - type: string - - type: integer - - details: - type: object - - proto_body: - type: object - - timeout: - type: number - - verify_response_with: - oneOf: - - $ref: "#/definitions/verify_block" - - type: array - items: - $ref: "#/definitions/verify_block" - - http_response: - type: object - additionalProperties: false - description: Expected HTTP response - - properties: - strict: - $ref: "#/definitions/strict_block" - - status_code: - description: Status code(s) to match - oneOf: - - type: integer - - type: array - minItems: 1 - items: - type: integer - - cookies: - type: array - description: Cookies expected to be returned - uniqueItems: true - minItems: 1 - - items: - type: string - - json: - description: Expected JSON response - $ref: "#/definitions/any_json" - - redirect_query_params: - description: Query parameters parsed from the 'location' of a redirect - type: object - - verify_response_with: - oneOf: - - $ref: "#/definitions/verify_block" - - type: array - items: - $ref: "#/definitions/verify_block" - - headers: - description: Headers expected in response - type: object - - save: - type: object - description: Which objects to save from the response - stage_ref: type: object description: Reference to another stage from an included config file @@ -408,28 +130,6 @@ definitions: type: string description: Name of this stage - mqtt_publish: - $ref: "#/definitions/mqtt_publish" - - mqtt_response: - oneOf: - - $ref: "#/definitions/mqtt_response" - - type: array - items: - $ref: "#/definitions/mqtt_response" - - request: - $ref: "#/definitions/http_request" - - response: - $ref: "#/definitions/http_response" - - grpc_request: - $ref: "#/definitions/grpc_request" - - grpc_response: - $ref: "#/definitions/grpc_response" - ### type: object diff --git a/tavern/_core/strict_util.py b/tavern/_core/strict_util.py index 88a48a9fa..3b87179d1 100644 --- a/tavern/_core/strict_util.py +++ b/tavern/_core/strict_util.py @@ -58,10 +58,28 @@ def is_on(self) -> bool: def validate_and_parse_option(key: str) -> StrictOption: + """Parse and validate a strict option configuration string. + + Args: + key: String in format "section[:setting]" where: + section: One of "json", "headers", or "redirect_query_params" + setting: Optional "on", "off" or "list_any_order" + + Returns: + StrictOption containing the parsed section and setting + + Raises: + InvalidConfigurationException: If the key format is invalid + """ regex = re.compile( - "(?P
{sections})(:(?P{switches}))?".format( - sections="|".join(valid_keys), switches="|".join(valid_switches) - ) + r""" + (?P
{sections}) # The section name (json/headers/redirect_query_params) + (?: # Optional non-capturing group for setting + : # Literal colon separator + (?P{switches}) # The setting value (on/off/list_any_order) + )? # End optional group + """.format(sections="|".join(valid_keys), switches="|".join(valid_switches)), + re.X, ) match = regex.fullmatch(key) diff --git a/tavern/_plugins/grpc/jsonschema.yaml b/tavern/_plugins/grpc/jsonschema.yaml index f18c1131d..008c58766 100644 --- a/tavern/_plugins/grpc/jsonschema.yaml +++ b/tavern/_plugins/grpc/jsonschema.yaml @@ -9,6 +9,59 @@ additionalProperties: false required: - grpc +definitions: + grpc_request: + type: object + required: + - service + properties: + host: + type: string + + service: + type: string + + body: + type: object + + json: + type: object + + retain: + type: boolean + + grpc_response: + type: object + properties: + status: + oneOf: + - type: string + - type: integer + + details: + type: object + + proto_body: + type: object + + timeout: + type: number + + verify_response_with: + oneOf: + - $ref: "#/definitions/verify_block" + - type: array + items: + $ref: "#/definitions/verify_block" + + stage: + properties: + grpc_request: + $ref: "#/definitions/grpc_request" + + grpc_response: + $ref: "#/definitions/grpc_response" + properties: grpc: type: object diff --git a/tavern/_plugins/mqtt/jsonschema.yaml b/tavern/_plugins/mqtt/jsonschema.yaml index 7d1fb7edf..de9629408 100644 --- a/tavern/_plugins/mqtt/jsonschema.yaml +++ b/tavern/_plugins/mqtt/jsonschema.yaml @@ -1,7 +1,7 @@ $schema: "http://json-schema.org/draft-07/schema#" title: Paho MQTT schema -description: Schema for paho-mqtt connection +description: Schema for paho-mqtt connection and requests/responses ### @@ -10,6 +10,94 @@ additionalProperties: false required: - paho-mqtt +definitions: + mqtt_publish: + type: object + description: Publish MQTT message + additionalProperties: false + + properties: + topic: + type: string + description: Topic to publish on + + payload: + type: string + description: Raw payload to post + + json: + description: JSON payload to post + $ref: "#/definitions/any_json" + + qos: + type: integer + description: QoS level to use for request + default: 0 + + retain: + type: boolean + description: Whether the message should be retained + default: false + + mqtt_response: + type: object + additionalProperties: false + description: Expected MQTT response + + properties: + unexpected: + type: boolean + description: Receiving this message fails the test + + topic: + type: string + description: Topic message should be received on + + payload: + description: Expected raw payload in response + oneOf: + - type: number + - type: integer + - type: string + - type: boolean + + json: + description: Expected JSON payload in response + $ref: "#/definitions/any_json" + + timeout: + type: number + description: How long to wait for response to arrive + + qos: + type: integer + description: QoS level that message should be received on + minimum: 0 + maximum: 2 + + verify_response_with: + oneOf: + - $ref: "#/definitions/verify_block" + - type: array + items: + $ref: "#/definitions/verify_block" + + save: + type: object + description: Which objects to save from the response + + stage: + properties: + mqtt_publish: + $ref: "#/definitions/mqtt_publish" + + mqtt_response: + oneOf: + - $ref: "#/definitions/mqtt_response" + - type: array + items: + $ref: "#/definitions/mqtt_response" + properties: paho-mqtt: type: object diff --git a/tavern/_plugins/rest/jsonschema.yaml b/tavern/_plugins/rest/jsonschema.yaml new file mode 100644 index 000000000..21ed563aa --- /dev/null +++ b/tavern/_plugins/rest/jsonschema.yaml @@ -0,0 +1,180 @@ +$schema: "http://json-schema.org/draft-07/schema#" + +title: REST schema +description: Schema for REST requests + +### + +definitions: + http_request: + type: object + additionalProperties: false + description: HTTP request to perform as part of stage + + required: + - url + + properties: + url: + description: URL to make request to + oneOf: + - type: string + - type: object + properties: + "$ext": + $ref: "#/definitions/verify_block" + + cert: + description: Certificate to use - either a path to a certificate and key in one file, or a two item list containing the certificate and key separately + oneOf: + - type: string + - type: array + minItems: 2 + maxItems: 2 + items: + type: string + + auth: + description: Authorisation to use for request - a list containing username and password + type: array + minItems: 2 + maxItems: 2 + items: + type: string + + verify: + description: Whether to verify the server's certificates + oneOf: + - type: boolean + default: false + - type: string + + method: + description: HTTP method to use for request + default: GET + type: string + + follow_redirects: + type: boolean + description: Whether to follow redirects from 3xx responses + default: false + + stream: + type: boolean + description: Whether to stream the download from the request + default: false + + cookies: + type: array + description: Which cookies to use in the request + + items: + oneOf: + - type: string + - type: object + + json: + description: JSON body to send in request body + $ref: "#/definitions/any_json" + + params: + description: Query parameters + type: object + + headers: + description: Headers for request + type: object + + data: + description: Form data to send in request + oneOf: + - type: object + - type: string + + timeout: + description: How long to wait for requests to time out + oneOf: + - type: number + - type: array + minItems: 2 + maxItems: 2 + items: + type: number + + file_body: + type: string + description: Path to a file to upload as the request body + + files: + oneOf: + - type: object + - type: array + description: Files to send as part of the request + + clear_session_cookies: + description: Whether to clear sesion cookies before running this request + type: boolean + + http_response: + type: object + additionalProperties: false + description: Expected HTTP response + + properties: + strict: + $ref: "#/definitions/strict_block" + + status_code: + description: Status code(s) to match + oneOf: + - type: integer + - type: array + minItems: 1 + items: + type: integer + + cookies: + type: array + description: Cookies expected to be returned + uniqueItems: true + minItems: 1 + + items: + type: string + + json: + description: Expected JSON response + $ref: "#/definitions/any_json" + + redirect_query_params: + description: Query parameters parsed from the 'location' of a redirect + type: object + + verify_response_with: + oneOf: + - $ref: "#/definitions/verify_block" + - type: array + items: + $ref: "#/definitions/verify_block" + + headers: + description: Headers expected in response + type: object + + save: + type: object + description: Which objects to save from the response + + stage: + type: object + description: One stage in a test + additionalProperties: false + required: + - name + + properties: + request: + $ref: "#/definitions/http_request" + + response: + $ref: "#/definitions/http_response" diff --git a/tavern/_plugins/rest/tavernhook.py b/tavern/_plugins/rest/tavernhook.py index 8f6a56e2b..881e45c8a 100644 --- a/tavern/_plugins/rest/tavernhook.py +++ b/tavern/_plugins/rest/tavernhook.py @@ -1,6 +1,8 @@ import logging +from os.path import abspath, dirname, join import requests +import yaml from tavern._core import exceptions from tavern._core.dict_util import format_keys @@ -19,6 +21,8 @@ class TavernRestPlugin(PluginHelperBase): request_type = RestRequest request_block_name = "request" + schema: dict + @staticmethod def get_expected_from_request( response_block: dict, test_block_config: TestConfig, session @@ -33,3 +37,10 @@ def get_expected_from_request( verifier_type = RestResponse response_block_name = "response" + + +schema_path: str = join(abspath(dirname(__file__)), "jsonschema.yaml") +with open(schema_path, encoding="utf-8") as schema_file: + schema = yaml.load(schema_file, Loader=yaml.SafeLoader) + +TavernRestPlugin.schema = schema diff --git a/tavern/response.py b/tavern/response.py index 81d3bebbe..b077f2f45 100644 --- a/tavern/response.py +++ b/tavern/response.py @@ -23,6 +23,21 @@ def indent_err_text(err: str) -> str: @dataclasses.dataclass class BaseResponse: + """Base for all response verifiers. + + Subclasses must have an __init__ method like: + + def __init__( + self, + client: Any, + name: str, + expected: TestConfig, + test_block_config: TestConfig, + ) -> None: + super().__init__(name, expected, test_block_config) + # ...other setup + """ + name: str expected: Any test_block_config: TestConfig @@ -45,7 +60,7 @@ def _adderr(self, msg: str, *args, e=None) -> None: self.errors += [(msg % args)] @abstractmethod - def verify(self, response): + def verify(self, response) -> Mapping: """Verify response against expected values and returns any values that we wanted to save for use in future requests diff --git a/tox-integration.ini b/tox-integration.ini index 8f1bf0347..8b89d7495 100644 --- a/tox-integration.ini +++ b/tox-integration.ini @@ -18,6 +18,7 @@ changedir = hooks: example/hooks generic: tests/integration noextra: tests/integration + custom_backend: example/custom_backend deps = flask allure-pytest From 63f1d455d6a05e82e089a1ed57d0e213f3c229fe Mon Sep 17 00:00:00 2001 From: Michael Boulton Date: Sat, 15 Nov 2025 18:06:11 +0000 Subject: [PATCH 2/9] feat(tests): Add new test cases for custom backend validations - Added test case to verify failure when a non-existent file is touched and validated (`some_other_file.txt`). - Added test case to validate failure for an invalid `touch_file` schema with a nonexistent field. - Added test case to validate failure for an invalid `file_exists` response schema with a nonexistent field. This enhances validation coverage for the custom backend by introducing negative test scenarios. --- .../test_file_touched.tavern.yaml | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/example/custom_backend/test_file_touched.tavern.yaml b/example/custom_backend/test_file_touched.tavern.yaml index d0891db66..ed2311fef 100644 --- a/example/custom_backend/test_file_touched.tavern.yaml +++ b/example/custom_backend/test_file_touched.tavern.yaml @@ -6,4 +6,41 @@ stages: touch_file: filename: hello.txt file_exists: - filename: hello.txt \ No newline at end of file + filename: hello.txt + +--- +test_name: Test file touched - should fail because file doesn't exist + +marks: + - xfail + +stages: + - name: Touch file that doesn't exist + touch_file: + filename: some_other_file.txt + file_exists: + filename: nonexistent_file.txt + +--- +test_name: Test with invalid schema - should fail + +_xfail: verify + +stages: + - name: Test invalid touch_file schema + touch_file: + nonexistent_field: some_value + file_exists: + filename: hello.txt + +--- +test_name: Test with invalid response schema - should fail + +_xfail: verify + +stages: + - name: Test invalid file_exists schema + touch_file: + filename: hello.txt + file_exists: + nonexistent_field: some_value \ No newline at end of file From f8c2d83a3628bfdd7f03846b71b98cc77c9af590 Mon Sep 17 00:00:00 2001 From: Michael Boulton Date: Sat, 15 Nov 2025 18:12:19 +0000 Subject: [PATCH 3/9] refactor(custom_backend): Rename and reorganize custom backend as my_tavern_plugin - Moved `example/custom_backend` to `example/custom_backend/my_tavern_plugin` for clearer structure. - Renamed `custom_backend` to `my_tavern_plugin` across all files, including `pyproject.toml`, test files, and paths. - Updated `tox-integration.ini` to reflect the new plugin path. - Adjusted `run_tests.sh` to use the updated plugin and test file paths. - Renamed and relocated `jsonschema.yaml` to match the new plugin directory. These changes improve project organization and better align naming conventions with intended functionality. --- example/custom_backend/my_tavern_plugin/__init__.py | 0 .../custom_backend/{ => my_tavern_plugin}/jsonschema.yaml | 0 .../{__init__.py => my_tavern_plugin/plugin.py} | 0 example/custom_backend/pyproject.toml | 4 ++-- example/custom_backend/run_tests.sh | 6 +++--- .../{ => tests}/test_file_touched.tavern.yaml | 0 tox-integration.ini | 1 - 7 files changed, 5 insertions(+), 6 deletions(-) create mode 100644 example/custom_backend/my_tavern_plugin/__init__.py rename example/custom_backend/{ => my_tavern_plugin}/jsonschema.yaml (100%) rename example/custom_backend/{__init__.py => my_tavern_plugin/plugin.py} (100%) rename example/custom_backend/{ => tests}/test_file_touched.tavern.yaml (100%) diff --git a/example/custom_backend/my_tavern_plugin/__init__.py b/example/custom_backend/my_tavern_plugin/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/example/custom_backend/jsonschema.yaml b/example/custom_backend/my_tavern_plugin/jsonschema.yaml similarity index 100% rename from example/custom_backend/jsonschema.yaml rename to example/custom_backend/my_tavern_plugin/jsonschema.yaml diff --git a/example/custom_backend/__init__.py b/example/custom_backend/my_tavern_plugin/plugin.py similarity index 100% rename from example/custom_backend/__init__.py rename to example/custom_backend/my_tavern_plugin/plugin.py diff --git a/example/custom_backend/pyproject.toml b/example/custom_backend/pyproject.toml index 2edd54f41..7a661c823 100644 --- a/example/custom_backend/pyproject.toml +++ b/example/custom_backend/pyproject.toml @@ -1,5 +1,5 @@ [project] -name = "custom-backend" +name = "my_tavern_plugin" version = "0.1.0" description = "Add your description here" readme = "README.md" @@ -7,4 +7,4 @@ requires-python = ">=3.14" dependencies = [] [project.entry-points.tavern_file] -custom_backend = "custom_backend" \ No newline at end of file +my_tavern_plugin = "my_tavern_plugin.plugin" \ No newline at end of file diff --git a/example/custom_backend/run_tests.sh b/example/custom_backend/run_tests.sh index b4413e522..c3d563976 100755 --- a/example/custom_backend/run_tests.sh +++ b/example/custom_backend/run_tests.sh @@ -10,6 +10,6 @@ fi uv pip install -e . 'tavern @ ../..' PYTHONPATH=. tavern-ci \ - --tavern-extra-backends=file=custom_backend \ - test_file_touched.tavern.yaml \ - --debug "$@" --stdout \ No newline at end of file + --tavern-extra-backends=file=my_tavern_plugin \ + --debug "$@" --stdout \ + tests \ No newline at end of file diff --git a/example/custom_backend/test_file_touched.tavern.yaml b/example/custom_backend/tests/test_file_touched.tavern.yaml similarity index 100% rename from example/custom_backend/test_file_touched.tavern.yaml rename to example/custom_backend/tests/test_file_touched.tavern.yaml diff --git a/tox-integration.ini b/tox-integration.ini index 8b89d7495..8f1bf0347 100644 --- a/tox-integration.ini +++ b/tox-integration.ini @@ -18,7 +18,6 @@ changedir = hooks: example/hooks generic: tests/integration noextra: tests/integration - custom_backend: example/custom_backend deps = flask allure-pytest From d5033329df158f61c53c62324ffec1dd495f66e3 Mon Sep 17 00:00:00 2001 From: Michael Boulton Date: Sat, 15 Nov 2025 18:21:00 +0000 Subject: [PATCH 4/9] docs(custom_backend): Add README for custom backend plugin example - Created `README.md` in `example/custom_backend` to provide an overview of the custom backend plugin. - Documented the implementation details, including `Request`, `Response`, `Session` classes, and supporting functions. - Added instructions for configuring the plugin in `pyproject.toml` and `pytest.ini`. - Included an example test case showcasing usage of `touch_file` and `file_exists` stages. This README serves as a guide for creating and using a custom backend plugin with Tavern. --- example/custom_backend/README.md | 58 ++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 example/custom_backend/README.md diff --git a/example/custom_backend/README.md b/example/custom_backend/README.md new file mode 100644 index 000000000..c86044776 --- /dev/null +++ b/example/custom_backend/README.md @@ -0,0 +1,58 @@ +# Tavern Custom Backend Plugin + +This example demonstrates how to create a custom backend plugin for Tavern, a pytest plugin for API testing. The custom +backend allows you to extend Tavern's functionality with your own request/response handling logic. + +## Overview + +This example plugin implements a simple file touch/verification system: + +- `touch_file` stage: Creates or updates a file timestamp (similar to the Unix `touch` command) +- `file_exists` stage: Verifies that a specified file exists + +## Implementation Details + +This example includes: + +- `Request` class: Extends `tavern.request.BaseRequest` and implements the `request_vars` property and `run()` method +- `Response` class: Extends `tavern.response.BaseResponse` and implements the `verify()` method +- `Session` class: Context manager for maintaining any state +- `get_expected_from_request` function: Optional function to generate expected response from request +- `jsonschema.yaml`: Schema validation for request/response objects +- `schema_path`: Path to the schema file for validation + +## Entry Point Configuration + +In your project's `pyproject.toml`, configure the plugin entry point: + +```toml +[project.entry-points.'tavern.plugins.backends'] +your_backend_name = 'your.package.path:your_backend_module' +``` + +Then when running tests, specify the extra backend: + +```bash +pytest --tavern-extra-backends=your_backend_name=your.package.path:your_backend_module +``` + +Or in your `pytest.ini`: + +```ini +[tool:pytest] +tavern-extra-backends = your_backend_name=your.package.path:your_backend_module +``` + +## Example Test + +```yaml +--- +test_name: Test file touched + +stages: + - name: Touch file and check it exists + touch_file: + filename: hello.txt + file_exists: + filename: hello.txt +``` From fb5422758ae0f07fee72548eaa2b80b8f87751a9 Mon Sep 17 00:00:00 2001 From: Michael Boulton Date: Sat, 15 Nov 2025 18:24:35 +0000 Subject: [PATCH 5/9] docs(custom_backend): Update README with custom backend loading explanation - Added details about why Tavern needs the `tavern-extra-backends` flag for custom backends. - Explained how Tavern defaults to loading only "grpc", "http", and "mqtt" backends and how the flag helps register a custom backend with Tavern. - Included a reference to `stevedore` for plugin loading for better clarity. --- example/custom_backend/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/example/custom_backend/README.md b/example/custom_backend/README.md index c86044776..cb314af8d 100644 --- a/example/custom_backend/README.md +++ b/example/custom_backend/README.md @@ -43,6 +43,10 @@ Or in your `pytest.ini`: tavern-extra-backends = your_backend_name=your.package.path:your_backend_module ``` +This is because Tavern by default only tries to load "grpc", "http" and "mqtt" backends. The flag registers the custom +backend with Tavern, which can then tell [stevedore](https://github.com/openstack/stevedore) to load the plugin from +the entrypoint. + ## Example Test ```yaml From 9ef0d5964cf46215ff6fb7b9b020bc582c14d1bb Mon Sep 17 00:00:00 2001 From: Michael Boulton Date: Sat, 15 Nov 2025 18:24:42 +0000 Subject: [PATCH 6/9] docs(custom_backend): Update plugin description and Python version in pyproject.toml - Updated the `description` field in `pyproject.toml` to include more details about the custom generic plugin's functionality. - Adjusted the `requires-python` version from `>=3.14` to `>=3.12` for compatibility. --- example/custom_backend/pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example/custom_backend/pyproject.toml b/example/custom_backend/pyproject.toml index 7a661c823..7d5be92b5 100644 --- a/example/custom_backend/pyproject.toml +++ b/example/custom_backend/pyproject.toml @@ -1,9 +1,9 @@ [project] name = "my_tavern_plugin" version = "0.1.0" -description = "Add your description here" +description = "A custom 'generic' plugin for tavern that touches files and checks if they are created." readme = "README.md" -requires-python = ">=3.14" +requires-python = ">=3.12" dependencies = [] [project.entry-points.tavern_file] From ac8f99b21497096bcf87ddc0e652e783b2493e93 Mon Sep 17 00:00:00 2001 From: Michael Boulton Date: Sat, 15 Nov 2025 18:24:54 +0000 Subject: [PATCH 7/9] docs(plugin): Add docstrings to `Session` and `Request` classes - Added a docstring to `Session` class to describe its purpose and context manager protocol implementation. - Added a docstring to `Request` class to explain its functionality, specifically touching a file during a request operation. --- example/custom_backend/my_tavern_plugin/plugin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/example/custom_backend/my_tavern_plugin/plugin.py b/example/custom_backend/my_tavern_plugin/plugin.py index dc9aeceea..5f7b8a72b 100644 --- a/example/custom_backend/my_tavern_plugin/plugin.py +++ b/example/custom_backend/my_tavern_plugin/plugin.py @@ -14,6 +14,7 @@ class Session: + """No-op session, but must implement the context manager protocol""" def __enter__(self): pass @@ -22,6 +23,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): class Request(BaseRequest): + """Touches a file when the 'request' is made""" def __init__( self, session: Any, rspec: dict, test_block_config: TestConfig ) -> None: From a56de6b008c324f7c1c49f49d065143298eb769d Mon Sep 17 00:00:00 2001 From: Michael Boulton Date: Fri, 21 Nov 2025 14:48:30 +0000 Subject: [PATCH 8/9] feat(core): support default backend selection for custom plugins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Updated the example README to show the new `tavern-extra-backends` syntax, allowing a backend name without an explicit implementation and documenting entry‑point naming rules. - Enhanced the example `run_tests.sh` script with three test runs: using the default backend, specifying a custom implementation, and exercising the error case when a non‑existent implementation is requested. - Refactored plugin discovery in `tavern/_core/plugins.py`: - Added handling for backends with no explicit implementation (treated as default). - Tracked discovered plugins per backend and raise a `PluginLoadError` if multiple plugins are enabled for the same backend. - Updated the stevedore check function to use the new logic. - Modified extra backend parsing in `tavern/_core/pytest/util.py`: - Now accepts both `name` (default backend) and `name=value` forms. - Stores `None` for default backends and provides clearer error messages for malformed input. - Minor formatting improvements in configuration docs. --- example/custom_backend/README.md | 24 ++++++++++++--------- example/custom_backend/run_tests.sh | 12 ++++++++++- tavern/_core/plugins.py | 33 +++++++++++++++++++++++------ tavern/_core/pytest/util.py | 26 +++++++++++++++-------- 4 files changed, 69 insertions(+), 26 deletions(-) diff --git a/example/custom_backend/README.md b/example/custom_backend/README.md index cb314af8d..c968ac362 100644 --- a/example/custom_backend/README.md +++ b/example/custom_backend/README.md @@ -26,26 +26,30 @@ This example includes: In your project's `pyproject.toml`, configure the plugin entry point: ```toml -[project.entry-points.'tavern.plugins.backends'] -your_backend_name = 'your.package.path:your_backend_module' +[project.entry-points.tavern_your_backend_name] +my_implementation = 'your.package.path:your_backend_module' ``` Then when running tests, specify the extra backend: ```bash -pytest --tavern-extra-backends=your_backend_name=your.package.path:your_backend_module +pytest --tavern-extra-backends=your_backend_name +# Or, to specify an implementation to override the project entrypoint: +pytest --tavern-extra-backends=your_backend_name=my_other_implementation ``` -Or in your `pytest.ini`: +Or the equivalent in pyproject.toml or pytest.ini. Note: -```ini -[tool:pytest] -tavern-extra-backends = your_backend_name=your.package.path:your_backend_module -``` +- The entry point name should start with `tavern_`. +- The key of the entrypoint is just a name of the implementation and can be anything. +- The `--tavern-extra-backends` flag should *not* be prefixed with `tavern_`. +- If Tavern detects multiple entrypoints for a backend, it will raise an error. In this case, you must use the second + form to specify which implementation of the backend to use. This is similar to the build-in `--tavern-http-backend` + flag. This is because Tavern by default only tries to load "grpc", "http" and "mqtt" backends. The flag registers the custom -backend with Tavern, which can then tell [stevedore](https://github.com/openstack/stevedore) to load the plugin from -the entrypoint. +backend with Tavern, which can then tell [stevedore](https://github.com/openstack/stevedore) to load the plugin from the +entrypoint. ## Example Test diff --git a/example/custom_backend/run_tests.sh b/example/custom_backend/run_tests.sh index c3d563976..c6a703117 100755 --- a/example/custom_backend/run_tests.sh +++ b/example/custom_backend/run_tests.sh @@ -9,7 +9,17 @@ fi uv pip install -e . 'tavern @ ../..' +PYTHONPATH=. tavern-ci \ + --tavern-extra-backends=file \ + --debug "$@" --stdout \ + tests + PYTHONPATH=. tavern-ci \ --tavern-extra-backends=file=my_tavern_plugin \ --debug "$@" --stdout \ - tests \ No newline at end of file + tests + +PYTHONPATH=. tavern-ci \ + --tavern-extra-backends=file=i_dont_exist \ + --debug "$@" --stdout \ + tests diff --git a/tavern/_core/plugins.py b/tavern/_core/plugins.py index ea68472e3..41a6ff936 100644 --- a/tavern/_core/plugins.py +++ b/tavern/_core/plugins.py @@ -125,12 +125,27 @@ def _load_plugins(self, test_block_config: TestConfig) -> list[_Plugin]: """ plugins = [] + discovered_plugins: dict[str, list[str]] = {} + + def is_plugin_backend_enabled(current_backend, ext): + if test_block_config.tavern_internal.backends[current_backend] is None: + # Use whatever default - will raise an error if >1 is discovered + is_enabled = True + logger.debug(f"Using default backend for {ext.name}") + else: + is_enabled = ( + ext.name + == test_block_config.tavern_internal.backends[current_backend] + ) + logger.debug( + f"Is {current_backend} for {ext.name} enabled? {is_enabled}" + ) + + if is_enabled: + if current_backend not in discovered_plugins: + discovered_plugins[current_backend] = [] + discovered_plugins[current_backend].append(ext.name) - def enabled(current_backend, ext): - is_enabled = ( - ext.name == test_block_config.tavern_internal.backends[current_backend] - ) - logger.debug(f"is {ext.name} enabled? {is_enabled}") return is_enabled for backend in test_block_config.tavern_internal.backends.keys(): @@ -140,7 +155,7 @@ def enabled(current_backend, ext): manager = stevedore.EnabledExtensionManager( namespace=namespace, - check_func=partial(enabled, backend), + check_func=partial(is_plugin_backend_enabled, backend), verify_requirements=True, on_load_failure_callback=plugin_load_error, ) @@ -155,6 +170,12 @@ def enabled(current_backend, ext): plugins.extend(manager.extensions) + for plugin, enabled in discovered_plugins.items(): + if len(enabled) > 1: + raise exceptions.PluginLoadError( + f"Multiple plugins enabled for '{plugin}' backend: {enabled}" + ) + return plugins diff --git a/tavern/_core/pytest/util.py b/tavern/_core/pytest/util.py index 848672e18..6016b177c 100644 --- a/tavern/_core/pytest/util.py +++ b/tavern/_core/pytest/util.py @@ -93,13 +93,19 @@ def add_ini_options(parser: pytest.Parser) -> None: default=[], ) parser.addini( - "tavern-http-backend", help="Which http backend to use", default="requests" + "tavern-http-backend", + help="Which http backend to use", + default="requests", ) parser.addini( - "tavern-mqtt-backend", help="Which mqtt backend to use", default="paho-mqtt" + "tavern-mqtt-backend", + help="Which mqtt backend to use", + default="paho-mqtt", ) parser.addini( - "tavern-grpc-backend", help="Which grpc backend to use", default="grpc" + "tavern-grpc-backend", + help="Which grpc backend to use", + default="grpc", ) parser.addini( "tavern-strict", @@ -200,15 +206,17 @@ def _load_global_backends(pytest_config: pytest.Config) -> dict[str, Any]: pytest_config, "tavern-extra-backends", [] ) for backend in extra_backends: - split = backend.split("=", 1) - if len(split) != 2: + split = backend.split("=") + if len(split) == 1: + backends[split[0]] = None + elif len(split) == 2: + key, value = split + backends[key] = value + else: raise exceptions.BadSchemaError( - f"extra backends must be in the form 'name=value', got {backend}" + f"extra backends must be in the form 'name' or 'name=value', got '{backend}'" ) - key, value = split - backends[key] = value - return backends From 11e0052fa48c8a74e1b26c09f70e00236d3e7f9d Mon Sep 17 00:00:00 2001 From: Michael Boulton Date: Fri, 21 Nov 2025 14:51:08 +0000 Subject: [PATCH 9/9] feat(plugins): Add docstring to `is_plugin_backend_enabled` function - Added a detailed docstring to the `is_plugin_backend_enabled` function in `tavern/_core/plugins.py`. - Clarified the purpose of the function, its arguments, and its return value. - Enhanced readability by documenting the function's role in checking backend enablement based on configuration. - Introduced the `stevedore.extension.Extension` type hint for better type clarity. --- tavern/_core/plugins.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tavern/_core/plugins.py b/tavern/_core/plugins.py index 41a6ff936..403f5db87 100644 --- a/tavern/_core/plugins.py +++ b/tavern/_core/plugins.py @@ -11,6 +11,7 @@ from typing import Any, Optional, Protocol import stevedore +import stevedore.extension from tavern._core import exceptions from tavern._core.dict_util import format_keys @@ -127,7 +128,21 @@ def _load_plugins(self, test_block_config: TestConfig) -> list[_Plugin]: plugins = [] discovered_plugins: dict[str, list[str]] = {} - def is_plugin_backend_enabled(current_backend, ext): + def is_plugin_backend_enabled( + current_backend: str, ext: stevedore.extension.Extension + ) -> bool: + """Checks if a plugin backend is enabled based on configuration. + + If no specific backend is configured, defaults to enabled. + Adds enabled plugins to discovered_plugins tracking dictionary. + + Args: + current_backend: The backend being checked (e.g. 'http', 'mqtt') + ext: The stevedore extension object representing the plugin + + Returns: + Whether the plugin backend is enabled + """ if test_block_config.tavern_internal.backends[current_backend] is None: # Use whatever default - will raise an error if >1 is discovered is_enabled = True