jaegertracing#5608 Improve SPM e2e test with test for error rate (jaegertracing#6991)

## Which problem is this PR solving?
- part of jaegertracing#5608

## Description of the changes
- Add smoke test for /metrics/errors
  - validate correct service name
  - validate correct labels (case groupByOperation=true)

## How was this change tested?
- run `scripts/e2e/spm.sh` locally

## Checklist
- [x] I have read
https://github.com/jaegertracing/jaeger/blob/master/CONTRIBUTING_GUIDELINES.md
- [x] I have signed all commits
- [x] I have added unit tests for the new functionality
- [x] I have run lint and test steps successfully
  - for `jaeger`: `make lint test`
  - for `jaeger-ui`: `npm run lint` and `npm run test`

---------

Signed-off-by: pipiland <[email protected]>

Author: pipiland2612

scripts/e2e/spm.sh

@@ -76,53 +76,102 @@ validate_service_metrics() {
     local fiveMinutes=300000
     local oneMinute=60000
     local fifteenSec=15000 # Prometheus is also configured to scrape every 15sec.
+
     # When endTs=(blank) the server will default it to now().
     local url="http://localhost:16686/api/metrics/calls?service=${service}&endTs=&lookback=${fiveMinutes}&step=${fifteenSec}&ratePer=${oneMinute}"
     response=$(curl -s "$url")
-    service_name=$(echo "$response" | jq -r 'if .metrics and .metrics[0] then .metrics[0].labels[] | select(.name=="service_name") | .value else empty end')
-    if [ "$service_name" != "$service" ]; then
-      echo "⏳ No metrics found for service '$service'"
+    if ! assert_service_name_equals "$response" "$service" ; then
       return 1
     fi
-    # Store the values in an array
-    mapfile -t metric_points < <(echo "$response" | jq -r '.metrics[0].metricPoints[].gaugeValue.doubleValue')
-    echo "Metric datapoints found for service '$service': " "${metric_points[@]}"
-    # Check that atleast some values are non-zero after the threshold
-    local non_zero_count=0
+
+    # Check that at least some values are non-zero after the threshold
+    local non_zero_count
+    non_zero_count=$(count_non_zero_metrics_point "$response")
     local expected_non_zero_count=4
-    local zero_count=0
+    local zero_count
+    zero_count=$(count_zero_metrics_point "$response")
     local expected_max_zero_count=4
-    for value in "${metric_points[@]}"; do
-      if [[ $(echo "$value > 0.0" | bc) == "1" ]]; then
-        non_zero_count=$((non_zero_count + 1))
-      else
-        zero_count=$((zero_count + 1))
-      fi
+    echo "⏳ Metrics data points found: ${zero_count} zero, ${non_zero_count} non-zero"
 
-      if [[ $zero_count -gt $expected_max_zero_count ]]; then
-        echo "❌ ERROR: Zero values crossing threshold limit not expected (Threshold limit - '$expected_max_zero_count')"
-        return 1
-      fi
-    done
-    if [ $non_zero_count -lt $expected_non_zero_count ]; then
+    if [[ $zero_count -gt $expected_max_zero_count ]]; then
+      echo "❌ ERROR: Zero values crossing threshold limit not expected (Threshold limit - '$expected_max_zero_count')"
+      return 1
+    fi
+    if [[ $non_zero_count -lt $expected_non_zero_count ]]; then
       echo "⏳ Expecting at least 4 non-zero data points"
       return 1
     fi
 
      # Validate if labels are correct
     local url="http://localhost:16686/api/metrics/calls?service=${service}&groupByOperation=true&endTs=&lookback=${fiveMinutes}&step=${fifteenSec}&ratePer=${oneMinute}"
+    response=$(curl -s "$url")
+    if ! assert_labels_set_equals "$response" "operation service_name" ; then
+      return 1
+    fi
 
-    local labels
-    labels=$(curl -s "$url" | jq -r '.metrics[0].labels[].name' | sort | tr '\n' ' ')
-    local exp_labels="operation service_name "
+    ### Validate Errors Rate metrics
+    local url="http://localhost:16686/api/metrics/errors?service=${service}&endTs=&lookback=${fiveMinutes}&step=${fifteenSec}&ratePer=${oneMinute}"
+    response=$(curl -s "$url")
+    if ! assert_service_name_equals "$response" "$service" ; then
+      return 1
+    fi
 
-    if [[ "$labels" != "$exp_labels" ]]; then
-      echo "❌ ERROR: Obtained labels: '$labels' are not same as expected labels: '$exp_labels'"
+    local url="http://localhost:16686/api/metrics/errors?service=${service}&groupByOperation=true&endTs=&lookback=${fiveMinutes}&step=${fifteenSec}&ratePer=${oneMinute}"
+    response=$(curl -s "$url")
+    if ! assert_labels_set_equals "$response" "operation service_name" ; then
       return 1
     fi
+
+    non_zero_count=$(count_non_zero_metrics_point "$response")
+    local services_with_error="driver frontend ui redis"
+    if [[ "$services_with_error" =~ $service ]]; then # the service is in the list
+      if [[ $non_zero_count == "0" ]]; then
+        echo "❌ ERROR: expect service $service to have positive errors rate"
+        return 1
+      fi
+    else
+      if [[ $non_zero_count != "0" ]]; then
+        echo "❌ ERROR: expect service $service to have 0 errors, but have $non_zero_count data points with positive errors"
+        return 1
+      fi
+    fi
+
+
     return 0
 }
 
+assert_service_name_equals() {
+  local response=$1
+  local expected=$2
+  service_name=$(echo "$response" | jq -r 'if .metrics and .metrics[0] then .metrics[0].labels[] | select(.name=="service_name") | .value else empty end')
+  if [[ "$service_name" != "$expected" ]]; then
+    echo "❌ ERROR: Obtained service_name: '$service_name' are not same as expected: '$expected'"
+    return 1
+  fi
+  return 0
+}
+
+assert_labels_set_equals() {
+  local response=$1
+  local expected="$2 " # need one extra space due to how labels is computed
+
+  labels=$(echo "$response" | jq -r '.metrics[0].labels[].name' | sort | tr '\n' ' ')
+
+  if [[ "$labels" != "$expected" ]]; then
+    echo "❌ ERROR: Obtained labels: '$labels' are not same as expected labels: '$expected'"
+    return 1
+  fi
+  return 0
+}
+
+count_zero_metrics_point() {
+  echo "$1" | jq -r '[.metrics[0].metricPoints[].gaugeValue.doubleValue | select(. == 0)] | length'
+}
+
+count_non_zero_metrics_point() {
+  echo "$1" | jq -r '[.metrics[0].metricPoints[].gaugeValue.doubleValue | select(. != 0)] | length'
+}
+
 check_spm() {
   local wait_seconds=10
   local successful_service=0