diff --git a/.gitignore b/.gitignore index 3b14cd8c953f7..f401f3686d831 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,8 @@ junit_*.xml perf.data* **/*.rej **/*.orig +**/*.profraw +**/*.profdata # This is un-orthodox, but adding it to the repo would "tie" it to each users # local version of nixpkgs. This way, we can all use the flake and have a diff --git a/bin/ci-builder b/bin/ci-builder index f8c09eb8397b5..4921c248ab205 100755 --- a/bin/ci-builder +++ b/bin/ci-builder @@ -60,7 +60,7 @@ arch_go=$(arch_go "$arch_gcc") cid_file=ci/builder/.${channel%%-*}.cidfile -rust_components=rustc,cargo,rust-std-$arch_gcc-unknown-linux-gnu +rust_components=rustc,cargo,rust-std-$arch_gcc-unknown-linux-gnu,llvm-tools-preview if [[ $rust_version = nightly ]]; then rust_components+=,miri-preview else diff --git a/ci/builder/Dockerfile b/ci/builder/Dockerfile index e06f6b5494f7c..92cb7525cc678 100644 --- a/ci/builder/Dockerfile +++ b/ci/builder/Dockerfile @@ -158,7 +158,8 @@ RUN mkdir rust \ && cargo install --root /usr/local --version "=0.9.44" cargo-nextest \ && `: Until https://github.com/est31/cargo-udeps/pull/147 is released in cargo-udeps` \ && cargo install --root /usr/local --git https://github.com/est31/cargo-udeps --rev=b84d478ef3efd7264dba8c15c31a50c6399dc5bb --locked cargo-udeps --features=vendored-openssl \ - && cargo install --root /usr/local --version "=0.2.15" --no-default-features --features=s3,openssl/vendored sccache + && cargo install --root /usr/local --version "=0.2.15" --no-default-features --features=s3,openssl/vendored sccache \ + && cargo install --root /usr/local --version "=0.3.6" cargo-binutils # Link the system lld into the cross-compiling sysroot. diff --git a/ci/nightly/coverage.py b/ci/nightly/coverage.py index a2a0bf07ec9ba..5fdf61652a3bf 100644 --- a/ci/nightly/coverage.py +++ b/ci/nightly/coverage.py @@ -19,48 +19,70 @@ pipeline.template.yml and the docstring on `trim_pipeline` below. """ +import subprocess import sys from pathlib import Path -import yaml - import materialize.cli.mzcompose def main() -> int: - with open(Path(__file__).parent.parent / "test" / "pipeline.template.yml") as f: - pipeline = yaml.safe_load(f) - - tests = [] - - for step in pipeline["steps"]: - for plugin in step.get("plugins", []): - for plugin_name, plugin_config in plugin.items(): - if plugin_name == "./ci/plugins/mzcompose": - tests.append((plugin_config["composition"], plugin_config["run"])) + # Just the fast sqllogictests, for now + tests = [("sqllogictest", "default")] for (composition, workflow) in tests: - print(f"==> Running workflow {workflow} in {composition}") materialize.cli.mzcompose.main( [ - "run", - workflow, - "--coverage", "--find", composition, + "--coverage", + "run", + workflow, ] ) materialize.cli.mzcompose.main( [ - "down", - "-v", - "--coverage", "--find", composition, + "down", + "-v", ] ) - # TODO: gather and combine coverage information. + # NB: mzcompose _munge_services() sets LLVM_PROFILE_FILE, so that + # output will go to a special coverage volume, but as a + # conesquence we can't really distinguish between sqllogictest and + # clusterd's output. + subprocess.run( + [ + "rust-profdata", + "merge", + "-sparse", + *Path("test/sqllogictest/coverage").glob("sqllogictest*.profraw"), + "-o", + "sqllogictest.profdata", + ] + ) + with open("coverage-sqllogictest.json", "w") as out: + subprocess.run( + [ + "rust-cov", + "export", + "./target-xcompile/x86_64-unknown-linux-gnu/release/sqllogictest", + "--instr-profile=sqllogictest.profdata", + ], + stdout=out, + ) + with open("coverage-clusterd.json", "w") as out: + subprocess.run( + [ + "rust-cov", + "export", + "./target-xcompile/x86_64-unknown-linux-gnu/release/clusterd", + "--instr-profile=sqllogictest.profdata", + ], + stdout=out, + ) return 0 diff --git a/ci/nightly/pipeline.yml b/ci/nightly/pipeline.yml index 3116ef3bfd4c1..1f410bdc12148 100644 --- a/ci/nightly/pipeline.yml +++ b/ci/nightly/pipeline.yml @@ -97,10 +97,10 @@ steps: - id: coverage label: Code coverage timeout_in_minutes: 240 - command: bin/ci-builder run nightly bin/pyactivate -m ci.nightly.coverage + command: bin/ci-builder run stable bin/pyactivate -m ci.nightly.coverage + artifact_paths: coverage-*.json agents: queue: linux-x86_64 - skip: Disabled due to persistent OOMs when linking - id: kafka-matrix label: Kafka smoke test against previous Kafka versions diff --git a/misc/python/materialize/cli/run.py b/misc/python/materialize/cli/run.py index 3026cb4709f14..964c2e1ac0e7a 100644 --- a/misc/python/materialize/cli/run.py +++ b/misc/python/materialize/cli/run.py @@ -114,6 +114,11 @@ def main() -> int: help="Disables the limited codesigning we do on macOS to support Instruments", action="store_true", ) + parser.add_argument( + "--coverage", + help="Build with coverage", + action="store_true", + ) args = parser.parse_intermixed_args() # Handle `+toolchain` like rustup. @@ -262,6 +267,8 @@ def _build(args: argparse.Namespace, extra_programs: list[str] = []) -> int: if args.tokio_console: features += ["tokio-console"] env["RUSTFLAGS"] = env.get("RUSTFLAGS", "") + " --cfg=tokio_unstable" + if args.coverage: + env["RUSTFLAGS"] = env.get("RUSTFLAGS", "") + " -Cinstrument-coverage" if args.features: features.extend(args.features.split(",")) if features: diff --git a/misc/python/materialize/mzbuild.py b/misc/python/materialize/mzbuild.py index 02adf20675c25..08f6f0cdae9bf 100644 --- a/misc/python/materialize/mzbuild.py +++ b/misc/python/materialize/mzbuild.py @@ -231,14 +231,8 @@ def __init__(self, rd: RepositoryDetails, path: Path, config: Dict[str, Any]): self.channel = None if rd.coverage: self.rustflags += [ - "-Zinstrument-coverage", - # Nix generates some unresolved symbols that -Zinstrument-coverage - # somehow causes the linker to complain about, so just disable - # warnings about unresolved symbols and hope it all works out. - # See: https://github.com/nix-rust/nix/issues/1116 - "-Clink-arg=-Wl,--warn-unresolved-symbols", + "-Cinstrument-coverage", ] - self.channel = "nightly" if len(self.bins) == 0 and len(self.examples) == 0: raise ValueError("mzbuild config is missing pre-build target") diff --git a/misc/python/materialize/mzcompose/__init__.py b/misc/python/materialize/mzcompose/__init__.py index f4fbe4fa5fd4d..9172f518925e6 100644 --- a/misc/python/materialize/mzcompose/__init__.py +++ b/misc/python/materialize/mzcompose/__init__.py @@ -185,16 +185,19 @@ def _munge_services( if "allow_host_ports" in config: config.pop("allow_host_ports") - if self.repo.rd.coverage: + coverage_volume = "./coverage:/coverage" + if self.repo.rd.coverage and coverage_volume not in config.get( + "volumes", [] + ): # Emit coverage information to a file in a directory that is # bind-mounted to the "coverage" directory on the host. We # inject the configuration to all services for simplicity, but # this only have an effect if the service runs instrumented Rust # binaries. + config.setdefault("volumes", []).append(coverage_volume) config.setdefault("environment", []).append( - f"LLVM_PROFILE_FILE=/coverage/{name}-%m.profraw" + f"LLVM_PROFILE_FILE=/coverage/{name}-%p-%9m.profraw" ) - config.setdefault("volumes", []).append("./coverage:/coverage") # Determine mzbuild specs and inject them into services accordingly. deps = self.repo.resolve_dependencies(images)