From b87a2ece50f893b4d851c55f92c315583a816dd2 Mon Sep 17 00:00:00 2001 From: Dani Louca Date: Tue, 28 Oct 2025 23:28:28 -0400 Subject: [PATCH 1/2] Add a new discovery bundle for weaviate vector search engine Signed-off-by: Dani Louca --- .chloggen/weaviate-discovery.yaml | 16 ++ .github/workflows/integration-test.yml | 2 +- Makefile | 8 + .../metadata/receivers/weaviate.yaml | 97 ++++++++ .../receivers/weaviate.discovery.yaml | 91 ++++++++ docker/docker-compose.yml | 8 + .../receivers/weaviate.discovery.yaml | 87 +++++++ .../discovery/generated_bundledfs.go | 1 + .../discoveryreceiver/generated_metadata.go | 24 ++ .../receivers/weaviate/testdata/expected.yaml | 221 ++++++++++++++++++ .../weaviate/testdata/otlp_exporter.yaml | 13 ++ .../weaviate/weaviate_discovery_test.go | 99 ++++++++ tests/testutils/golden.go | 13 ++ tests/testutils/testcase.go | 6 +- 14 files changed, 683 insertions(+), 3 deletions(-) create mode 100644 .chloggen/weaviate-discovery.yaml create mode 100644 cmd/discoverybundler/metadata/receivers/weaviate.yaml create mode 100644 cmd/otelcol/config/collector/config.d.linux/receivers/weaviate.discovery.yaml create mode 100644 internal/confmapprovider/discovery/bundle.d/receivers/weaviate.discovery.yaml create mode 100644 tests/receivers/weaviate/testdata/expected.yaml create mode 100644 tests/receivers/weaviate/testdata/otlp_exporter.yaml create mode 100644 tests/receivers/weaviate/weaviate_discovery_test.go diff --git a/.chloggen/weaviate-discovery.yaml b/.chloggen/weaviate-discovery.yaml new file mode 100644 index 0000000000..58325ec588 --- /dev/null +++ b/.chloggen/weaviate-discovery.yaml @@ -0,0 +1,16 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. crosslink) +component: discovery + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add a new discovery bundle for weaviate vector search engine + +# One or more tracking issues related to the change +issues: [6884] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: The discovery rule targets Weaviate instances exposing Prometheus metrics on the default port 2112. \ No newline at end of file diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 1cb44c645d..dc087f7d8e 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -313,7 +313,7 @@ jobs: id: get-matrix run: | includes="" - for service in "apache" "jmx/cassandra" "kafkametrics" "mongodb" "nginx" "envoy" "oracledb" "mysql" "redis"; do + for service in "apache" "jmx/cassandra" "kafkametrics" "mongodb" "nginx" "envoy" "oracledb" "mysql" "redis" "weaviate"; do for arch in "amd64" "arm64"; do if [ "$service" = "mongodb" ]; then # tests for mongo "6.0" and "7.0" are flaky, skipping for now diff --git a/Makefile b/Makefile index bb7561c99c..a6ff932aa3 100644 --- a/Makefile +++ b/Makefile @@ -165,6 +165,14 @@ integration-test-nginx-discovery-with-cover: integration-test-redis-discovery: @make integration-test-target TARGET='discovery_integration_redis' +.PHONY: integration-test-weaviate-discovery-with-cover +integration-test-weaviate-discovery-with-cover: + @make integration-test-cover-target TARGET='discovery_integration_weaviate' + +.PHONY: integration-test-weaviate-discovery +integration-test-weaviate-discovery: + @make integration-test-target TARGET='discovery_integration_weaviate' + .PHONY: integration-test-redis-discovery-with-cover integration-test-redis-discovery-with-cover: @make integration-test-cover-target TARGET='discovery_integration_redis' diff --git a/cmd/discoverybundler/metadata/receivers/weaviate.yaml b/cmd/discoverybundler/metadata/receivers/weaviate.yaml new file mode 100644 index 0000000000..0f35afc471 --- /dev/null +++ b/cmd/discoverybundler/metadata/receivers/weaviate.yaml @@ -0,0 +1,97 @@ +receiver_id: prometheus/weaviate +service_type: weaviate +properties_tmpl: | + enabled: true + rule: + docker_observer: type == "container" and port == 2112 and any([name, image, command], {# matches "(?i)weaviate"}) and not (command matches "splunk.discovery") + k8s_observer: type == "port" and port == 2112 and pod.name matches "(?i)weaviate" + config: + default: + config: + scrape_configs: + - job_name: 'weaviate' + metrics_path: /metrics + scrape_interval: 10s + static_configs: + - targets: ['`endpoint`'] + metric_relabel_configs: + - source_labels: [__name__] + action: keep + regex: "(async_operations_running|\ + batch_delete_durations_ms|\ + batch_durations_ms|\ + index_queue_delete_duration_ms|\ + index_queue_paused|\ + index_queue_preload_count|\ + index_queue_preload_duration_ms|\ + index_queue_push_duration_ms|\ + index_queue_search_duration_ms|\ + index_queue_size|\ + index_queue_stale_count|\ + index_queue_vectors_dequeued|\ + index_queue_wait_duration_ms|\ + lsm_active_segments|\ + lsm_bloom_filter_duration_ms|\ + lsm_segment_count|\ + lsm_segment_objects|\ + lsm_segment_size|\ + object_count|\ + objects_durations_ms|\ + requests_total| + startup_diskio_throughput|\ + startup_durations_ms|\ + vector_index_durations_ms|\ + vector_index_maintenance_durations_ms|\ + vector_index_operations|\ + vector_index_size|\ + vector_index_tombstone_cleaned|\ + vector_index_tombstone_cleanup_threads|\ + vector_index_tombstones|\ + weaviate_build_info|\ + weaviate_runtime_config_hash|\ + weaviate_runtime_config_last_load_success|\ + weaviate_schema_collections|\ + weaviate_schema_shards|\ + weaviate_internal_sample_memberlist_queue_broadcasts|\ + weaviate_internal_timer_memberlist_gossip|\ + weaviate_internal_counter_raft_apply|\ + weaviate_internal_counter_raft_state_candidate|\ + weaviate_internal_counter_raft_state_follower|\ + weaviate_internal_counter_raft_state_leader|\ + weaviate_internal_counter_raft_transition_heartbeat_timeout|\ + weaviate_internal_gauge_raft_commitNumLogs|\ + weaviate_internal_gauge_raft_leader_dispatchNumLogs|\ + weaviate_internal_gauge_raft_leader_oldestLogAge|\ + weaviate_internal_gauge_raft_peers|\ + weaviate_internal_sample_raft_boltdb_logBatchSize|\ + weaviate_internal_sample_raft_boltdb_logSize|\ + weaviate_internal_sample_raft_boltdb_logsPerBatch|\ + weaviate_internal_sample_raft_boltdb_writeCapacity|\ + weaviate_internal_sample_raft_thread_fsm_saturation|\ + weaviate_internal_sample_raft_thread_main_saturation|\ + weaviate_internal_timer_raft_boltdb_getLog|\ + weaviate_internal_timer_raft_boltdb_storeLogs|\ + weaviate_internal_timer_raft_commitTime|\ + weaviate_internal_timer_raft_fsm_apply|\ + weaviate_internal_timer_raft_fsm_enqueue|\ + weaviate_internal_timer_raft_leader_dispatchLog|\ + weaviate_usage_gcs_operations_total|\ + weaviate_usage_gcs_operation_latency_seconds|\ + weaviate_usage_gcs_resource_count|\ + weaviate_usage_gcs_uploaded_file_size_bytes|\ + weaviate_usage_s3_operations_total|\ + weaviate_usage_s3_operation_latency_seconds|\ + weaviate_usage_s3_resource_count|\ + weaviate_usage_s3_uploaded_file_size_bytes)" +status: + metrics: + - status: successful + strict: weaviate_build_info + message: weaviate prometheus receiver is working! + statements: + - status: failed + regexp: "connection refused" + message: The container is not serving http connections. + - status: failed + regexp: "dial tcp: lookup" + message: Unable to resolve weaviate prometheus tcp endpoint \ No newline at end of file diff --git a/cmd/otelcol/config/collector/config.d.linux/receivers/weaviate.discovery.yaml b/cmd/otelcol/config/collector/config.d.linux/receivers/weaviate.discovery.yaml new file mode 100644 index 0000000000..fbd7877058 --- /dev/null +++ b/cmd/otelcol/config/collector/config.d.linux/receivers/weaviate.discovery.yaml @@ -0,0 +1,91 @@ +##################################################################################### +# This file is generated by the Splunk Distribution of the OpenTelemetry Collector. # +# # +# It reflects the default configuration bundled in the Collector executable for use # +# in discovery mode (--discovery) and is provided for reference or customization. # +# Please note that any changes made to this file will need to be reconciled during # +# upgrades of the Collector. # +##################################################################################### +# prometheus/weaviate: +# enabled: true +# rule: +# docker_observer: type == "container" and port == 2112 and any([name, image, command], {# matches "(?i)weaviate"}) and not (command matches "splunk.discovery") +# k8s_observer: type == "port" and port == 2112 and pod.name matches "(?i)weaviate" +# config: +# default: +# config: +# scrape_configs: +# - job_name: 'weaviate' +# metrics_path: /metrics +# scrape_interval: 10s +# static_configs: +# - targets: ['`endpoint`'] +# metric_relabel_configs: +# - source_labels: [__name__] +# action: keep +# regex: "(async_operations_running|\ +# batch_delete_durations_ms|\ +# batch_durations_ms|\ +# index_queue_delete_duration_ms|\ +# index_queue_paused|\ +# index_queue_preload_count|\ +# index_queue_preload_duration_ms|\ +# index_queue_push_duration_ms|\ +# index_queue_search_duration_ms|\ +# index_queue_size|\ +# index_queue_stale_count|\ +# index_queue_vectors_dequeued|\ +# index_queue_wait_duration_ms|\ +# lsm_active_segments|\ +# lsm_bloom_filter_duration_ms|\ +# lsm_segment_count|\ +# lsm_segment_objects|\ +# lsm_segment_size|\ +# object_count|\ +# objects_durations_ms|\ +# requests_total| +# startup_diskio_throughput|\ +# startup_durations_ms|\ +# vector_index_durations_ms|\ +# vector_index_maintenance_durations_ms|\ +# vector_index_operations|\ +# vector_index_size|\ +# vector_index_tombstone_cleaned|\ +# vector_index_tombstone_cleanup_threads|\ +# vector_index_tombstones|\ +# weaviate_build_info|\ +# weaviate_runtime_config_hash|\ +# weaviate_runtime_config_last_load_success|\ +# weaviate_schema_collections|\ +# weaviate_schema_shards|\ +# weaviate_internal_sample_memberlist_queue_broadcasts|\ +# weaviate_internal_timer_memberlist_gossip|\ +# weaviate_internal_counter_raft_apply|\ +# weaviate_internal_counter_raft_state_candidate|\ +# weaviate_internal_counter_raft_state_follower|\ +# weaviate_internal_counter_raft_state_leader|\ +# weaviate_internal_counter_raft_transition_heartbeat_timeout|\ +# weaviate_internal_gauge_raft_commitNumLogs|\ +# weaviate_internal_gauge_raft_leader_dispatchNumLogs|\ +# weaviate_internal_gauge_raft_leader_oldestLogAge|\ +# weaviate_internal_gauge_raft_peers|\ +# weaviate_internal_sample_raft_boltdb_logBatchSize|\ +# weaviate_internal_sample_raft_boltdb_logSize|\ +# weaviate_internal_sample_raft_boltdb_logsPerBatch|\ +# weaviate_internal_sample_raft_boltdb_writeCapacity|\ +# weaviate_internal_sample_raft_thread_fsm_saturation|\ +# weaviate_internal_sample_raft_thread_main_saturation|\ +# weaviate_internal_timer_raft_boltdb_getLog|\ +# weaviate_internal_timer_raft_boltdb_storeLogs|\ +# weaviate_internal_timer_raft_commitTime|\ +# weaviate_internal_timer_raft_fsm_apply|\ +# weaviate_internal_timer_raft_fsm_enqueue|\ +# weaviate_internal_timer_raft_leader_dispatchLog|\ +# weaviate_usage_gcs_operations_total|\ +# weaviate_usage_gcs_operation_latency_seconds|\ +# weaviate_usage_gcs_resource_count|\ +# weaviate_usage_gcs_uploaded_file_size_bytes|\ +# weaviate_usage_s3_operations_total|\ +# weaviate_usage_s3_operation_latency_seconds|\ +# weaviate_usage_s3_resource_count|\ +# weaviate_usage_s3_uploaded_file_size_bytes)" diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index ffba52f730..0c914588ba 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -249,6 +249,14 @@ services: ports: - "6379:6379" # Kafka test, requires multiple Kafka containers and Zookeeper: + weaviate: + image: cr.weaviate.io/semitechnologies/weaviate:1.33.2 + profiles: + - integration-test-weaviate-discovery + ports: + - "2112:2112" + environment: + PROMETHEUS_MONITORING_ENABLED: "true" zookeeper: image: zookeeper:3.5 profiles: diff --git a/internal/confmapprovider/discovery/bundle.d/receivers/weaviate.discovery.yaml b/internal/confmapprovider/discovery/bundle.d/receivers/weaviate.discovery.yaml new file mode 100644 index 0000000000..837389cdac --- /dev/null +++ b/internal/confmapprovider/discovery/bundle.d/receivers/weaviate.discovery.yaml @@ -0,0 +1,87 @@ +############################################################################################## +# Do not edit manually! # +# All changes must be made to associated .yaml metadata file before running 'make bundle.d'. # +############################################################################################## +prometheus/weaviate: + enabled: true + rule: + docker_observer: type == "container" and port == 2112 and any([name, image, command], {# matches "(?i)weaviate"}) and not (command matches "splunk.discovery") + k8s_observer: type == "port" and port == 2112 and pod.name matches "(?i)weaviate" + config: + default: + config: + scrape_configs: + - job_name: 'weaviate' + metrics_path: /metrics + scrape_interval: 10s + static_configs: + - targets: ['`endpoint`'] + metric_relabel_configs: + - source_labels: [__name__] + action: keep + regex: "(async_operations_running|\ + batch_delete_durations_ms|\ + batch_durations_ms|\ + index_queue_delete_duration_ms|\ + index_queue_paused|\ + index_queue_preload_count|\ + index_queue_preload_duration_ms|\ + index_queue_push_duration_ms|\ + index_queue_search_duration_ms|\ + index_queue_size|\ + index_queue_stale_count|\ + index_queue_vectors_dequeued|\ + index_queue_wait_duration_ms|\ + lsm_active_segments|\ + lsm_bloom_filter_duration_ms|\ + lsm_segment_count|\ + lsm_segment_objects|\ + lsm_segment_size|\ + object_count|\ + objects_durations_ms|\ + requests_total| + startup_diskio_throughput|\ + startup_durations_ms|\ + vector_index_durations_ms|\ + vector_index_maintenance_durations_ms|\ + vector_index_operations|\ + vector_index_size|\ + vector_index_tombstone_cleaned|\ + vector_index_tombstone_cleanup_threads|\ + vector_index_tombstones|\ + weaviate_build_info|\ + weaviate_runtime_config_hash|\ + weaviate_runtime_config_last_load_success|\ + weaviate_schema_collections|\ + weaviate_schema_shards|\ + weaviate_internal_sample_memberlist_queue_broadcasts|\ + weaviate_internal_timer_memberlist_gossip|\ + weaviate_internal_counter_raft_apply|\ + weaviate_internal_counter_raft_state_candidate|\ + weaviate_internal_counter_raft_state_follower|\ + weaviate_internal_counter_raft_state_leader|\ + weaviate_internal_counter_raft_transition_heartbeat_timeout|\ + weaviate_internal_gauge_raft_commitNumLogs|\ + weaviate_internal_gauge_raft_leader_dispatchNumLogs|\ + weaviate_internal_gauge_raft_leader_oldestLogAge|\ + weaviate_internal_gauge_raft_peers|\ + weaviate_internal_sample_raft_boltdb_logBatchSize|\ + weaviate_internal_sample_raft_boltdb_logSize|\ + weaviate_internal_sample_raft_boltdb_logsPerBatch|\ + weaviate_internal_sample_raft_boltdb_writeCapacity|\ + weaviate_internal_sample_raft_thread_fsm_saturation|\ + weaviate_internal_sample_raft_thread_main_saturation|\ + weaviate_internal_timer_raft_boltdb_getLog|\ + weaviate_internal_timer_raft_boltdb_storeLogs|\ + weaviate_internal_timer_raft_commitTime|\ + weaviate_internal_timer_raft_fsm_apply|\ + weaviate_internal_timer_raft_fsm_enqueue|\ + weaviate_internal_timer_raft_leader_dispatchLog|\ + weaviate_usage_gcs_operations_total|\ + weaviate_usage_gcs_operation_latency_seconds|\ + weaviate_usage_gcs_resource_count|\ + weaviate_usage_gcs_uploaded_file_size_bytes|\ + weaviate_usage_s3_operations_total|\ + weaviate_usage_s3_operation_latency_seconds|\ + weaviate_usage_s3_resource_count|\ + weaviate_usage_s3_uploaded_file_size_bytes)" diff --git a/internal/confmapprovider/discovery/generated_bundledfs.go b/internal/confmapprovider/discovery/generated_bundledfs.go index 3562c2a5e9..45bdd146c1 100644 --- a/internal/confmapprovider/discovery/generated_bundledfs.go +++ b/internal/confmapprovider/discovery/generated_bundledfs.go @@ -34,4 +34,5 @@ import "embed" //go:embed bundle.d/receivers/rabbitmq.discovery.yaml //go:embed bundle.d/receivers/redis.discovery.yaml //go:embed bundle.d/receivers/sqlserver.discovery.yaml +//go:embed bundle.d/receivers/weaviate.discovery.yaml var BundledFS embed.FS diff --git a/internal/receiver/discoveryreceiver/generated_metadata.go b/internal/receiver/discoveryreceiver/generated_metadata.go index 8fb7e6f7e3..83610fb253 100644 --- a/internal/receiver/discoveryreceiver/generated_metadata.go +++ b/internal/receiver/discoveryreceiver/generated_metadata.go @@ -457,4 +457,28 @@ var receiverMetaMap = map[string]ReceiverMeta{ }, }, }, + "prometheus/weaviate": { + ServiceType: "weaviate", + Status: Status{ + Metrics: []Match{ + { + Status: "successful", + Strict: "weaviate_build_info", + Message: "weaviate prometheus receiver is working!", + }, + }, + Statements: []Match{ + { + Status: "failed", + Regexp: "connection refused", + Message: "The container is not serving http connections.", + }, + { + Status: "failed", + Regexp: "dial tcp: lookup", + Message: "Unable to resolve weaviate prometheus tcp endpoint", + }, + }, + }, + }, } diff --git a/tests/receivers/weaviate/testdata/expected.yaml b/tests/receivers/weaviate/testdata/expected.yaml new file mode 100644 index 0000000000..cb4f006da9 --- /dev/null +++ b/tests/receivers/weaviate/testdata/expected.yaml @@ -0,0 +1,221 @@ +resourceMetrics: + - resource: + attributes: + - key: container.image.name + value: + stringValue: cr.weaviate.io/semitechnologies/weaviate + - key: container.name + value: + stringValue: unruffled_roentgen + - key: discovery.endpoint.id + value: + stringValue: 529a617433bd52f15f65c5e3e1e387a55031d57386070042fd8222a3c21483e1:2112 + - key: server.address + value: + stringValue: 172.17.0.3 + - key: server.port + value: + stringValue: "2112" + - key: service.instance.id + value: + stringValue: 172.17.0.3:2112 + - key: service.name + value: + stringValue: weaviate + - key: url.scheme + value: + stringValue: http + scopeMetrics: + - metrics: + - description: Number of collections per node + gauge: + dataPoints: + - asDouble: 0 + attributes: + - key: nodeID + value: + stringValue: 529a617433bd + timeUnixNano: "1000000" + metadata: + - key: prometheus.type + value: + stringValue: gauge + name: weaviate_schema_collections + - description: weaviate_internal_counter_raft_state_candidate + metadata: + - key: prometheus.type + value: + stringValue: counter + name: weaviate_internal_counter_raft_state_candidate + sum: + aggregationTemporality: 2 + dataPoints: + - asDouble: 1 + startTimeUnixNano: "1000000" + timeUnixNano: "1000000" + isMonotonic: true + - description: weaviate_internal_gauge_raft_leader_oldestLogAge + gauge: + dataPoints: + - asDouble: 0 + timeUnixNano: "1000000" + metadata: + - key: prometheus.type + value: + stringValue: gauge + name: weaviate_internal_gauge_raft_leader_oldestLogAge + - description: The scraping was successful + gauge: + dataPoints: + - asDouble: 1 + timeUnixNano: "1000000" + metadata: + - key: prometheus.type + value: + stringValue: gauge + name: up + - description: A metric with a constant '1' value labeled by version, revision, branch, goversion from which weaviate was built, and the goos and goarch for the build. + gauge: + dataPoints: + - asDouble: 1 + attributes: + - key: branch + value: + stringValue: v1.33.2 + - key: goarch + value: + stringValue: arm64 + - key: goos + value: + stringValue: linux + - key: goversion + value: + stringValue: go1.24.9 + - key: revision + value: + stringValue: 7c65630 + - key: tags + value: + stringValue: unknown + - key: version + value: + stringValue: 1.33.2 + timeUnixNano: "1000000" + metadata: + - key: prometheus.type + value: + stringValue: gauge + name: weaviate_build_info + - description: Duration of the scrape + gauge: + dataPoints: + - asDouble: 0.019609458 + timeUnixNano: "1000000" + metadata: + - key: prometheus.type + value: + stringValue: gauge + name: scrape_duration_seconds + unit: s + - description: The number of samples the target exposed + gauge: + dataPoints: + - asDouble: 240 + timeUnixNano: "1000000" + metadata: + - key: prometheus.type + value: + stringValue: gauge + name: scrape_samples_scraped + - description: The number of samples remaining after metric relabeling was applied + gauge: + dataPoints: + - asDouble: 46 + timeUnixNano: "1000000" + metadata: + - key: prometheus.type + value: + stringValue: gauge + name: scrape_samples_post_metric_relabeling + - description: The approximate number of new series in this scrape + gauge: + dataPoints: + - asDouble: 46 + timeUnixNano: "1000000" + metadata: + - key: prometheus.type + value: + stringValue: gauge + name: scrape_series_added + - description: weaviate_internal_counter_raft_state_follower + metadata: + - key: prometheus.type + value: + stringValue: counter + name: weaviate_internal_counter_raft_state_follower + sum: + aggregationTemporality: 2 + dataPoints: + - asDouble: 1 + startTimeUnixNano: "1000000" + timeUnixNano: "1000000" + isMonotonic: true + - description: weaviate_internal_counter_raft_transition_heartbeat_timeout + metadata: + - key: prometheus.type + value: + stringValue: counter + name: weaviate_internal_counter_raft_transition_heartbeat_timeout + sum: + aggregationTemporality: 2 + dataPoints: + - asDouble: 1 + startTimeUnixNano: "1000000" + timeUnixNano: "1000000" + isMonotonic: true + - description: weaviate_internal_gauge_raft_leader_dispatchNumLogs + gauge: + dataPoints: + - asDouble: 1 + timeUnixNano: "1000000" + metadata: + - key: prometheus.type + value: + stringValue: gauge + name: weaviate_internal_gauge_raft_leader_dispatchNumLogs + - description: weaviate_internal_gauge_raft_peers + gauge: + dataPoints: + - asDouble: 1 + timeUnixNano: "1000000" + metadata: + - key: prometheus.type + value: + stringValue: gauge + name: weaviate_internal_gauge_raft_peers + - description: weaviate_internal_counter_raft_state_leader + metadata: + - key: prometheus.type + value: + stringValue: counter + name: weaviate_internal_counter_raft_state_leader + sum: + aggregationTemporality: 2 + dataPoints: + - asDouble: 1 + startTimeUnixNano: "1000000" + timeUnixNano: "1000000" + isMonotonic: true + - description: weaviate_internal_gauge_raft_commitNumLogs + gauge: + dataPoints: + - asDouble: 1 + timeUnixNano: "1000000" + metadata: + - key: prometheus.type + value: + stringValue: gauge + name: weaviate_internal_gauge_raft_commitNumLogs + scope: + name: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver + version: latest diff --git a/tests/receivers/weaviate/testdata/otlp_exporter.yaml b/tests/receivers/weaviate/testdata/otlp_exporter.yaml new file mode 100644 index 0000000000..f3c8ea0b57 --- /dev/null +++ b/tests/receivers/weaviate/testdata/otlp_exporter.yaml @@ -0,0 +1,13 @@ +exporters: + otlp: + endpoint: "${OTLP_ENDPOINT}" + tls: + insecure: true + +service: + telemetry: + logs: + level: debug + pipelines: + metrics: + exporters: [otlp] \ No newline at end of file diff --git a/tests/receivers/weaviate/weaviate_discovery_test.go b/tests/receivers/weaviate/weaviate_discovery_test.go new file mode 100644 index 0000000000..460c5ebc6c --- /dev/null +++ b/tests/receivers/weaviate/weaviate_discovery_test.go @@ -0,0 +1,99 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build discovery_integration_weaviate + +package test + +import ( + "fmt" + "path/filepath" + "testing" + "time" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden" + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest/pmetrictest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/pdata/pmetric" + + "github.com/signalfx/splunk-otel-collector/tests/testutils" +) + +const ( + testDataDir = "testdata" + nodeID = "5245ra617433bd" +) + +func TestWeaviateDockerObserver(t *testing.T) { + testutils.SkipIfNotContainerTest(t) + dockerSocketProxy, err := testutils.CreateDockerSocketProxy(t) + require.NoError(t, err) + + tc := testutils.NewTestcase(t) + defer tc.PrintLogsOnFailure() + defer tc.ShutdownOTLPReceiverSink() + _, shutdown := tc.SplunkOtelCollectorContainer("otlp_exporter.yaml", func(collector testutils.Collector) testutils.Collector { + cc := collector.(*testutils.CollectorContainer) + return cc.WithEnv(map[string]string{ + "SPLUNK_DISCOVERY_LOG_LEVEL": "debug", + }).WithArgs( + "--discovery", + "--set", `splunk.discovery.extensions.k8s_observer.enabled=false`, + "--set", `splunk.discovery.extensions.host_observer.enabled=false`, + "--set", fmt.Sprintf("splunk.discovery.extensions.docker_observer.config.endpoint=tcp://%s", dockerSocketProxy.ContainerEndpoint), + ) + }) + defer shutdown() + + var actualMetrics pmetric.Metrics + expectedMetricsFile := filepath.Join(testDataDir, "expected.yaml") + expected, err := golden.ReadMetrics(expectedMetricsFile) + require.NoError(t, err) + require.EventuallyWithT(t, func(tt *assert.CollectT) { + if len(tc.OTLPReceiverSink.AllMetrics()) == 0 { + assert.Fail(tt, "No metrics collected") + return + } + actualMetrics = tc.OTLPReceiverSink.AllMetrics()[len(tc.OTLPReceiverSink.AllMetrics())-1] + require.NotNil(t, actualMetrics) + }, 60*time.Second, 1*time.Second) + + testutils.MaybeUpdateExpectedMetricsResults(t, expectedMetricsFile, &actualMetrics) + err = pmetrictest.CompareMetrics(expected, actualMetrics, + pmetrictest.IgnoreResourceAttributeValue("service.instance.id"), + pmetrictest.IgnoreResourceAttributeValue("net.host.port"), + pmetrictest.IgnoreResourceAttributeValue("net.host.name"), + pmetrictest.IgnoreResourceAttributeValue("server.address"), + pmetrictest.IgnoreResourceAttributeValue("container.name"), + pmetrictest.IgnoreResourceAttributeValue("server.port"), + pmetrictest.IgnoreResourceAttributeValue("service.name"), + pmetrictest.IgnoreResourceAttributeValue("service_instance_id"), + pmetrictest.IgnoreResourceAttributeValue("service_version"), + pmetrictest.IgnoreResourceAttributeValue("discovery.endpoint.id"), + pmetrictest.IgnoreMetricAttributeValue("service_version"), + pmetrictest.IgnoreMetricAttributeValue("service_instance_id"), + pmetrictest.IgnoreResourceAttributeValue("server.address"), + pmetrictest.ChangeDatapointAttributeValue("nodeID", func(string) string { return nodeID }), + pmetrictest.IgnoreTimestamp(), + pmetrictest.IgnoreStartTimestamp(), + pmetrictest.IgnoreMetricDataPointsOrder(), + pmetrictest.IgnoreScopeMetricsOrder(), + pmetrictest.IgnoreScopeVersion(), + pmetrictest.IgnoreResourceMetricsOrder(), + pmetrictest.IgnoreMetricsOrder(), + pmetrictest.IgnoreMetricValues(), + ) + assert.NoError(t, err) +} diff --git a/tests/testutils/golden.go b/tests/testutils/golden.go index d2fb696dbb..f9f7a2f505 100644 --- a/tests/testutils/golden.go +++ b/tests/testutils/golden.go @@ -24,6 +24,8 @@ import ( "testing" "time" + "go.opentelemetry.io/collector/pdata/pmetric" + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest/pmetrictest" "github.com/stretchr/testify/assert" @@ -173,3 +175,14 @@ func RunMetricsCollectionTest(t *testing.T, configFile string, expectedFilePath require.NoError(t, golden.WriteMetrics(t, outputPath, actual)) } } + +func MaybeUpdateExpectedMetricsResults(t *testing.T, file string, metrics *pmetric.Metrics) { + if shouldUpdateExpectedResults() { + require.NoError(t, golden.WriteMetrics(t, file, *metrics)) + t.Logf("Wrote updated expected metric results to %s", file) + } +} + +var shouldUpdateExpectedResults = func() bool { + return os.Getenv("UPDATE_EXPECTED_RESULTS") == "true" +} diff --git a/tests/testutils/testcase.go b/tests/testutils/testcase.go index 387c72fe78..e3effc18d6 100644 --- a/tests/testutils/testcase.go +++ b/tests/testutils/testcase.go @@ -142,8 +142,10 @@ func (t *Testcase) SplunkOtelCollectorContainer(configFilename string, builders port := strings.Split(t.OTLPEndpointForCollector, ":")[1] t.OTLPEndpointForCollector = fmt.Sprintf("host.docker.internal:%s", port) - port = strings.Split(t.HECEndpointForCollector, ":")[1] - t.HECEndpointForCollector = fmt.Sprintf("host.docker.internal:%s", port) + if t.HECEndpointForCollector != "" { + port = strings.Split(t.HECEndpointForCollector, ":")[1] + t.HECEndpointForCollector = fmt.Sprintf("host.docker.internal:%s", port) + } } var c Collector From b2cd5bd0e1acedd48cccb65e0cd78d060581a931 Mon Sep 17 00:00:00 2001 From: Dani Louca Date: Wed, 29 Oct 2025 18:16:51 -0400 Subject: [PATCH 2/2] Fix Integration test Signed-off-by: Dani Louca --- cmd/discoverybundler/metadata/receivers/weaviate.yaml | 2 +- .../config.d.linux/receivers/weaviate.discovery.yaml | 2 +- .../discovery/bundle.d/receivers/weaviate.discovery.yaml | 2 +- tests/receivers/weaviate/weaviate_discovery_test.go | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/discoverybundler/metadata/receivers/weaviate.yaml b/cmd/discoverybundler/metadata/receivers/weaviate.yaml index 0f35afc471..fc1b5dcbcb 100644 --- a/cmd/discoverybundler/metadata/receivers/weaviate.yaml +++ b/cmd/discoverybundler/metadata/receivers/weaviate.yaml @@ -37,7 +37,7 @@ properties_tmpl: | lsm_segment_size|\ object_count|\ objects_durations_ms|\ - requests_total| + requests_total|\ startup_diskio_throughput|\ startup_durations_ms|\ vector_index_durations_ms|\ diff --git a/cmd/otelcol/config/collector/config.d.linux/receivers/weaviate.discovery.yaml b/cmd/otelcol/config/collector/config.d.linux/receivers/weaviate.discovery.yaml index fbd7877058..415866f37c 100644 --- a/cmd/otelcol/config/collector/config.d.linux/receivers/weaviate.discovery.yaml +++ b/cmd/otelcol/config/collector/config.d.linux/receivers/weaviate.discovery.yaml @@ -43,7 +43,7 @@ # lsm_segment_size|\ # object_count|\ # objects_durations_ms|\ -# requests_total| +# requests_total|\ # startup_diskio_throughput|\ # startup_durations_ms|\ # vector_index_durations_ms|\ diff --git a/internal/confmapprovider/discovery/bundle.d/receivers/weaviate.discovery.yaml b/internal/confmapprovider/discovery/bundle.d/receivers/weaviate.discovery.yaml index 837389cdac..f7ae789d3e 100644 --- a/internal/confmapprovider/discovery/bundle.d/receivers/weaviate.discovery.yaml +++ b/internal/confmapprovider/discovery/bundle.d/receivers/weaviate.discovery.yaml @@ -39,7 +39,7 @@ prometheus/weaviate: lsm_segment_size|\ object_count|\ objects_durations_ms|\ - requests_total| + requests_total|\ startup_diskio_throughput|\ startup_durations_ms|\ vector_index_durations_ms|\ diff --git a/tests/receivers/weaviate/weaviate_discovery_test.go b/tests/receivers/weaviate/weaviate_discovery_test.go index 460c5ebc6c..13523c7680 100644 --- a/tests/receivers/weaviate/weaviate_discovery_test.go +++ b/tests/receivers/weaviate/weaviate_discovery_test.go @@ -33,7 +33,7 @@ import ( const ( testDataDir = "testdata" - nodeID = "5245ra617433bd" + maskValue = "xyz" ) func TestWeaviateDockerObserver(t *testing.T) { @@ -67,7 +67,7 @@ func TestWeaviateDockerObserver(t *testing.T) { return } actualMetrics = tc.OTLPReceiverSink.AllMetrics()[len(tc.OTLPReceiverSink.AllMetrics())-1] - require.NotNil(t, actualMetrics) + require.NotNil(tt, actualMetrics) }, 60*time.Second, 1*time.Second) testutils.MaybeUpdateExpectedMetricsResults(t, expectedMetricsFile, &actualMetrics) @@ -84,8 +84,8 @@ func TestWeaviateDockerObserver(t *testing.T) { pmetrictest.IgnoreResourceAttributeValue("discovery.endpoint.id"), pmetrictest.IgnoreMetricAttributeValue("service_version"), pmetrictest.IgnoreMetricAttributeValue("service_instance_id"), - pmetrictest.IgnoreResourceAttributeValue("server.address"), - pmetrictest.ChangeDatapointAttributeValue("nodeID", func(string) string { return nodeID }), + pmetrictest.ChangeDatapointAttributeValue("nodeID", func(string) string { return maskValue }), + pmetrictest.ChangeDatapointAttributeValue("goarch", func(string) string { return maskValue }), pmetrictest.IgnoreTimestamp(), pmetrictest.IgnoreStartTimestamp(), pmetrictest.IgnoreMetricDataPointsOrder(),