diff --git a/site/src/github/comparison_summary.rs b/site/src/github/comparison_summary.rs
index e3f607645..76798b91f 100644
--- a/site/src/github/comparison_summary.rs
+++ b/site/src/github/comparison_summary.rs
@@ -176,14 +176,18 @@ async fn summarize_run(
     let inst_comparison =
         calculate_metric_comparison(ctxt, &commit, Metric::InstructionsUser).await?;
 
-    let errors = if !inst_comparison.newly_failed_benchmarks.is_empty() {
+    let has_broken_benchmarks = !inst_comparison.newly_failed_benchmarks.is_empty();
+    let errors = if has_broken_benchmarks {
         let benchmarks = inst_comparison
             .newly_failed_benchmarks
             .keys()
             .map(|benchmark| format!("- {benchmark}"))
             .collect::<Vec<_>>()
             .join("\n");
-        format!("\n**Warning ⚠**: The following benchmark(s) failed to build:\n{benchmarks}\n")
+        let alert_row = ":exclamation: ".repeat(5);
+        format!(
+            "\n{alert_row}\n**Warning :warning:**: The following benchmark(s) failed to build:\n{benchmarks}\n{alert_row}\n"
+        )
     } else {
         String::new()
     };
@@ -206,7 +210,9 @@ async fn summarize_run(
         &mut message,
         "### Overall result: {}{}\n",
         overall_result,
-        if is_regression {
+        if has_broken_benchmarks {
+            " - BENCHMARK(S) FAILED"
+        } else if is_regression {
             " - ACTION NEEDED"
         } else {
             " - no action needed"