Re-enable coverage

.gitignore

@@ -29,6 +29,8 @@ junit_*.xml
 perf.data*
 **/*.rej
 **/*.orig
+**/*.profraw
+**/*.profdata
 
 # This is un-orthodox, but adding it to the repo would "tie" it to each users
 # local version of nixpkgs. This way, we can all use the flake and have a

bin/ci-builder

@@ -60,7 +60,7 @@ arch_go=$(arch_go "$arch_gcc")
 
 cid_file=ci/builder/.${channel%%-*}.cidfile
 
-rust_components=rustc,cargo,rust-std-$arch_gcc-unknown-linux-gnu
+rust_components=rustc,cargo,rust-std-$arch_gcc-unknown-linux-gnu,llvm-tools-preview
 if [[ $rust_version = nightly ]]; then
     rust_components+=,miri-preview
 else

ci/builder/Dockerfile

@@ -158,7 +158,8 @@ RUN mkdir rust \
     && cargo install --root /usr/local --version "=0.9.44" cargo-nextest \
     && `: Until https://github.com/est31/cargo-udeps/pull/147 is released in cargo-udeps` \
     && cargo install --root /usr/local --git https://github.com/est31/cargo-udeps --rev=b84d478ef3efd7264dba8c15c31a50c6399dc5bb --locked  cargo-udeps --features=vendored-openssl \
-    && cargo install --root /usr/local --version "=0.2.15" --no-default-features --features=s3,openssl/vendored sccache
+    && cargo install --root /usr/local --version "=0.2.15" --no-default-features --features=s3,openssl/vendored sccache \
+    && cargo install --root /usr/local --version "=0.3.6" cargo-binutils
 
 # Link the system lld into the cross-compiling sysroot.
 

ci/nightly/coverage.py

@@ -19,48 +19,70 @@
 pipeline.template.yml and the docstring on `trim_pipeline` below.
 """
 
+import subprocess
 import sys
 from pathlib import Path
 
-import yaml
-
 import materialize.cli.mzcompose
 
 
 def main() -> int:
-    with open(Path(__file__).parent.parent / "test" / "pipeline.template.yml") as f:
-        pipeline = yaml.safe_load(f)
-
-    tests = []
-
-    for step in pipeline["steps"]:
-        for plugin in step.get("plugins", []):
-            for plugin_name, plugin_config in plugin.items():
-                if plugin_name == "./ci/plugins/mzcompose":
-                    tests.append((plugin_config["composition"], plugin_config["run"]))
+    # Just the fast sqllogictests, for now
+    tests = [("sqllogictest", "default")]
 
     for (composition, workflow) in tests:
-        print(f"==> Running workflow {workflow} in {composition}")
         materialize.cli.mzcompose.main(
             [
-                "run",
-                workflow,
-                "--coverage",
                 "--find",
                 composition,
+                "--coverage",
+                "run",
+                workflow,
             ]
         )
         materialize.cli.mzcompose.main(
             [
-                "down",
-                "-v",
-                "--coverage",
                 "--find",
                 composition,
+                "down",
+                "-v",
             ]
         )
 
-    # TODO: gather and combine coverage information.
+    # NB: mzcompose _munge_services() sets LLVM_PROFILE_FILE, so that
+    # output will go to a special coverage volume, but as a
+    # conesquence we can't really distinguish between sqllogictest and
+    # clusterd's output.
+    subprocess.run(
+        [
+            "rust-profdata",
+            "merge",
+            "-sparse",
+            *Path("test/sqllogictest/coverage").glob("sqllogictest*.profraw"),
+            "-o",
+            "sqllogictest.profdata",
+        ]
+    )
+    with open("coverage-sqllogictest.json", "w") as out:
+        subprocess.run(
+            [
+                "rust-cov",
+                "export",
+                "./target-xcompile/x86_64-unknown-linux-gnu/release/sqllogictest",
+                "--instr-profile=sqllogictest.profdata",
+            ],
+            stdout=out,
+        )
+    with open("coverage-clusterd.json", "w") as out:
+        subprocess.run(
+            [
+                "rust-cov",
+                "export",
+                "./target-xcompile/x86_64-unknown-linux-gnu/release/clusterd",
+                "--instr-profile=sqllogictest.profdata",
+            ],
+            stdout=out,
+        )
 
     return 0
 

ci/nightly/pipeline.yml

@@ -97,10 +97,10 @@ steps:
   - id: coverage
     label: Code coverage
     timeout_in_minutes: 240
-    command: bin/ci-builder run nightly bin/pyactivate -m ci.nightly.coverage
+    command: bin/ci-builder run stable bin/pyactivate -m ci.nightly.coverage
+    artifact_paths: coverage-*.json
     agents:
       queue: linux-x86_64
-    skip: Disabled due to persistent OOMs when linking
 
   - id: kafka-matrix
     label: Kafka smoke test against previous Kafka versions

misc/python/materialize/cli/run.py

@@ -114,6 +114,11 @@ def main() -> int:
         help="Disables the limited codesigning we do on macOS to support Instruments",
         action="store_true",
     )
+    parser.add_argument(
+        "--coverage",
+        help="Build with coverage",
+        action="store_true",
+    )
     args = parser.parse_intermixed_args()
 
     # Handle `+toolchain` like rustup.
@@ -262,6 +267,8 @@ def _build(args: argparse.Namespace, extra_programs: list[str] = []) -> int:
     if args.tokio_console:
         features += ["tokio-console"]
         env["RUSTFLAGS"] = env.get("RUSTFLAGS", "") + " --cfg=tokio_unstable"
+    if args.coverage:
+        env["RUSTFLAGS"] = env.get("RUSTFLAGS", "") + " -Cinstrument-coverage"
     if args.features:
         features.extend(args.features.split(","))
     if features:

misc/python/materialize/mzbuild.py

@@ -231,14 +231,8 @@ def __init__(self, rd: RepositoryDetails, path: Path, config: Dict[str, Any]):
         self.channel = None
         if rd.coverage:
             self.rustflags += [
-                "-Zinstrument-coverage",
-                # Nix generates some unresolved symbols that -Zinstrument-coverage
-                # somehow causes the linker to complain about, so just disable
-                # warnings about unresolved symbols and hope it all works out.
-                # See: https://github.com/nix-rust/nix/issues/1116
-                "-Clink-arg=-Wl,--warn-unresolved-symbols",
+                "-Cinstrument-coverage",
             ]
-            self.channel = "nightly"
         if len(self.bins) == 0 and len(self.examples) == 0:
             raise ValueError("mzbuild config is missing pre-build target")
 

misc/python/materialize/mzcompose/__init__.py

@@ -185,16 +185,19 @@ def _munge_services(
             if "allow_host_ports" in config:
                 config.pop("allow_host_ports")
 
-            if self.repo.rd.coverage:
+            coverage_volume = "./coverage:/coverage"
+            if self.repo.rd.coverage and coverage_volume not in config.get(
+                "volumes", []
+            ):
                 # Emit coverage information to a file in a directory that is
                 # bind-mounted to the "coverage" directory on the host. We
                 # inject the configuration to all services for simplicity, but
                 # this only have an effect if the service runs instrumented Rust
                 # binaries.
+                config.setdefault("volumes", []).append(coverage_volume)
                 config.setdefault("environment", []).append(
-                    f"LLVM_PROFILE_FILE=/coverage/{name}-%m.profraw"
+                    f"LLVM_PROFILE_FILE=/coverage/{name}-%p-%9m.profraw"
                 )
-                config.setdefault("volumes", []).append("./coverage:/coverage")
 
         # Determine mzbuild specs and inject them into services accordingly.
         deps = self.repo.resolve_dependencies(images)