Add a new discovery bundle for weaviate vector search engine

.chloggen/weaviate-discovery.yaml

@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. crosslink)
+component: discovery
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add a new discovery bundle for weaviate vector search engine
+
+# One or more tracking issues related to the change
+issues: [6884]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: The discovery rule targets Weaviate instances exposing Prometheus metrics on the default port 2112.

.github/workflows/integration-test.yml

@@ -315,7 +315,7 @@ jobs:
         id: get-matrix
         run: |
           includes=""
-          for service in "apache" "jmx/cassandra" "kafkametrics" "mongodb" "nginx" "envoy" "oracledb" "mysql" "redis"; do
+          for service in "apache" "jmx/cassandra" "kafkametrics" "mongodb" "nginx" "envoy" "oracledb" "mysql" "redis" "weaviate"; do
             for arch in "amd64" "arm64"; do
               if [ "$service" = "mongodb" ]; then
                 # tests for mongo "6.0" and "7.0" are flaky, skipping for now

Makefile

@@ -163,6 +163,14 @@ integration-test-nginx-discovery-with-cover:
 integration-test-redis-discovery:
 	@make integration-test-target TARGET='discovery_integration_redis'
 
+.PHONY: integration-test-weaviate-discovery-with-cover
+integration-test-weaviate-discovery-with-cover:
+	@make integration-test-cover-target TARGET='discovery_integration_weaviate'
+
+.PHONY: integration-test-weaviate-discovery
+integration-test-weaviate-discovery:
+	@make integration-test-target TARGET='discovery_integration_weaviate'
+
 .PHONY: integration-test-redis-discovery-with-cover
 integration-test-redis-discovery-with-cover:
 	@make integration-test-cover-target TARGET='discovery_integration_redis'

cmd/discoverybundler/metadata/receivers/weaviate.yaml

@@ -0,0 +1,97 @@
+receiver_id: prometheus/weaviate
+service_type: weaviate
+properties_tmpl: |
+  enabled: true
+  rule:
+    docker_observer: type == "container" and port == 2112 and any([name, image, command], {# matches "(?i)weaviate"}) and not (command matches "splunk.discovery")
+    k8s_observer: type == "port" and port == 2112 and pod.name matches "(?i)weaviate"
+  config:
+    default:
+      config:
+        scrape_configs:
+          - job_name: 'weaviate'
+            metrics_path: /metrics
+            scrape_interval: 10s
+            static_configs:
+              - targets: ['`endpoint`']
+            metric_relabel_configs:
+              - source_labels: [__name__]
+                action: keep
+                regex: "(async_operations_running|\
+                batch_delete_durations_ms|\
+                batch_durations_ms|\
+                index_queue_delete_duration_ms|\
+                index_queue_paused|\
+                index_queue_preload_count|\
+                index_queue_preload_duration_ms|\
+                index_queue_push_duration_ms|\
+                index_queue_search_duration_ms|\
+                index_queue_size|\
+                index_queue_stale_count|\
+                index_queue_vectors_dequeued|\
+                index_queue_wait_duration_ms|\
+                lsm_active_segments|\
+                lsm_bloom_filter_duration_ms|\
+                lsm_segment_count|\
+                lsm_segment_objects|\
+                lsm_segment_size|\
+                object_count|\
+                objects_durations_ms|\
+                requests_total|\
+                startup_diskio_throughput|\
+                startup_durations_ms|\
+                vector_index_durations_ms|\
+                vector_index_maintenance_durations_ms|\
+                vector_index_operations|\
+                vector_index_size|\
+                vector_index_tombstone_cleaned|\
+                vector_index_tombstone_cleanup_threads|\
+                vector_index_tombstones|\
+                weaviate_build_info|\
+                weaviate_runtime_config_hash|\
+                weaviate_runtime_config_last_load_success|\
+                weaviate_schema_collections|\
+                weaviate_schema_shards|\
+                weaviate_internal_sample_memberlist_queue_broadcasts|\
+                weaviate_internal_timer_memberlist_gossip|\
+                weaviate_internal_counter_raft_apply|\
+                weaviate_internal_counter_raft_state_candidate|\
+                weaviate_internal_counter_raft_state_follower|\
+                weaviate_internal_counter_raft_state_leader|\
+                weaviate_internal_counter_raft_transition_heartbeat_timeout|\
+                weaviate_internal_gauge_raft_commitNumLogs|\
+                weaviate_internal_gauge_raft_leader_dispatchNumLogs|\
+                weaviate_internal_gauge_raft_leader_oldestLogAge|\
+                weaviate_internal_gauge_raft_peers|\
+                weaviate_internal_sample_raft_boltdb_logBatchSize|\
+                weaviate_internal_sample_raft_boltdb_logSize|\
+                weaviate_internal_sample_raft_boltdb_logsPerBatch|\
+                weaviate_internal_sample_raft_boltdb_writeCapacity|\
+                weaviate_internal_sample_raft_thread_fsm_saturation|\
+                weaviate_internal_sample_raft_thread_main_saturation|\
+                weaviate_internal_timer_raft_boltdb_getLog|\
+                weaviate_internal_timer_raft_boltdb_storeLogs|\
+                weaviate_internal_timer_raft_commitTime|\
+                weaviate_internal_timer_raft_fsm_apply|\
+                weaviate_internal_timer_raft_fsm_enqueue|\
+                weaviate_internal_timer_raft_leader_dispatchLog|\
+                weaviate_usage_gcs_operations_total|\
+                weaviate_usage_gcs_operation_latency_seconds|\
+                weaviate_usage_gcs_resource_count|\
+                weaviate_usage_gcs_uploaded_file_size_bytes|\
+                weaviate_usage_s3_operations_total|\
+                weaviate_usage_s3_operation_latency_seconds|\
+                weaviate_usage_s3_resource_count|\
+                weaviate_usage_s3_uploaded_file_size_bytes)"
+status:
+  metrics:
+    - status: successful
+      strict: weaviate_build_info
+      message: weaviate prometheus receiver is working!
+  statements:
+    - status: failed
+      regexp: "connection refused"
+      message: The container is not serving http connections.
+    - status: failed
+      regexp: "dial tcp: lookup"
+      message: Unable to resolve weaviate prometheus tcp endpoint

cmd/otelcol/config/collector/config.d.linux/receivers/weaviate.discovery.yaml

@@ -0,0 +1,91 @@
+#####################################################################################
+# This file is generated by the Splunk Distribution of the OpenTelemetry Collector. #
+#                                                                                   #
+# It reflects the default configuration bundled in the Collector executable for use #
+# in discovery mode (--discovery) and is provided for reference or customization.   #
+# Please note that any changes made to this file will need to be reconciled during  #
+# upgrades of the Collector.                                                        #
+#####################################################################################
+# prometheus/weaviate:
+#   enabled: true
+#   rule:
+#     docker_observer: type == "container" and port == 2112 and any([name, image, command], {# matches "(?i)weaviate"}) and not (command matches "splunk.discovery")
+#     k8s_observer: type == "port" and port == 2112 and pod.name matches "(?i)weaviate"
+#   config:
+#     default:
+#       config:
+#         scrape_configs:
+#           - job_name: 'weaviate'
+#             metrics_path: /metrics
+#             scrape_interval: 10s
+#             static_configs:
+#               - targets: ['`endpoint`']
+#             metric_relabel_configs:
+#               - source_labels: [__name__]
+#                 action: keep
+#                 regex: "(async_operations_running|\
+#                 batch_delete_durations_ms|\
+#                 batch_durations_ms|\
+#                 index_queue_delete_duration_ms|\
+#                 index_queue_paused|\
+#                 index_queue_preload_count|\
+#                 index_queue_preload_duration_ms|\
+#                 index_queue_push_duration_ms|\
+#                 index_queue_search_duration_ms|\
+#                 index_queue_size|\
+#                 index_queue_stale_count|\
+#                 index_queue_vectors_dequeued|\
+#                 index_queue_wait_duration_ms|\
+#                 lsm_active_segments|\
+#                 lsm_bloom_filter_duration_ms|\
+#                 lsm_segment_count|\
+#                 lsm_segment_objects|\
+#                 lsm_segment_size|\
+#                 object_count|\
+#                 objects_durations_ms|\
+#                 requests_total|\
+#                 startup_diskio_throughput|\
+#                 startup_durations_ms|\
+#                 vector_index_durations_ms|\
+#                 vector_index_maintenance_durations_ms|\
+#                 vector_index_operations|\
+#                 vector_index_size|\
+#                 vector_index_tombstone_cleaned|\
+#                 vector_index_tombstone_cleanup_threads|\
+#                 vector_index_tombstones|\
+#                 weaviate_build_info|\
+#                 weaviate_runtime_config_hash|\
+#                 weaviate_runtime_config_last_load_success|\
+#                 weaviate_schema_collections|\
+#                 weaviate_schema_shards|\
+#                 weaviate_internal_sample_memberlist_queue_broadcasts|\
+#                 weaviate_internal_timer_memberlist_gossip|\
+#                 weaviate_internal_counter_raft_apply|\
+#                 weaviate_internal_counter_raft_state_candidate|\
+#                 weaviate_internal_counter_raft_state_follower|\
+#                 weaviate_internal_counter_raft_state_leader|\
+#                 weaviate_internal_counter_raft_transition_heartbeat_timeout|\
+#                 weaviate_internal_gauge_raft_commitNumLogs|\
+#                 weaviate_internal_gauge_raft_leader_dispatchNumLogs|\
+#                 weaviate_internal_gauge_raft_leader_oldestLogAge|\
+#                 weaviate_internal_gauge_raft_peers|\
+#                 weaviate_internal_sample_raft_boltdb_logBatchSize|\
+#                 weaviate_internal_sample_raft_boltdb_logSize|\
+#                 weaviate_internal_sample_raft_boltdb_logsPerBatch|\
+#                 weaviate_internal_sample_raft_boltdb_writeCapacity|\
+#                 weaviate_internal_sample_raft_thread_fsm_saturation|\
+#                 weaviate_internal_sample_raft_thread_main_saturation|\
+#                 weaviate_internal_timer_raft_boltdb_getLog|\
+#                 weaviate_internal_timer_raft_boltdb_storeLogs|\
+#                 weaviate_internal_timer_raft_commitTime|\
+#                 weaviate_internal_timer_raft_fsm_apply|\
+#                 weaviate_internal_timer_raft_fsm_enqueue|\
+#                 weaviate_internal_timer_raft_leader_dispatchLog|\
+#                 weaviate_usage_gcs_operations_total|\
+#                 weaviate_usage_gcs_operation_latency_seconds|\
+#                 weaviate_usage_gcs_resource_count|\
+#                 weaviate_usage_gcs_uploaded_file_size_bytes|\
+#                 weaviate_usage_s3_operations_total|\
+#                 weaviate_usage_s3_operation_latency_seconds|\
+#                 weaviate_usage_s3_resource_count|\
+#                 weaviate_usage_s3_uploaded_file_size_bytes)"

docker/docker-compose.yml

@@ -249,6 +249,14 @@ services:
     ports:
       - "6379:6379"
   # Kafka test, requires multiple Kafka containers and Zookeeper:
+  weaviate:
+    image: cr.weaviate.io/semitechnologies/weaviate:1.33.2
+    profiles:
+      - integration-test-weaviate-discovery
+    ports:
+      - "2112:2112"
+    environment:
+      PROMETHEUS_MONITORING_ENABLED: "true"
   zookeeper:
     image: zookeeper:3.5
     profiles:

internal/confmapprovider/discovery/bundle.d/receivers/weaviate.discovery.yaml

@@ -0,0 +1,87 @@
+##############################################################################################
+#                               Do not edit manually!                                        #
+# All changes must be made to associated .yaml metadata file before running 'make bundle.d'. #
+##############################################################################################
+prometheus/weaviate:
+  enabled: true
+  rule:
+    docker_observer: type == "container" and port == 2112 and any([name, image, command], {# matches "(?i)weaviate"}) and not (command matches "splunk.discovery")
+    k8s_observer: type == "port" and port == 2112 and pod.name matches "(?i)weaviate"
+  config:
+    default:
+      config:
+        scrape_configs:
+          - job_name: 'weaviate'
+            metrics_path: /metrics
+            scrape_interval: 10s
+            static_configs:
+              - targets: ['`endpoint`']
+            metric_relabel_configs:
+              - source_labels: [__name__]
+                action: keep
+                regex: "(async_operations_running|\
+                batch_delete_durations_ms|\
+                batch_durations_ms|\
+                index_queue_delete_duration_ms|\
+                index_queue_paused|\
+                index_queue_preload_count|\
+                index_queue_preload_duration_ms|\
+                index_queue_push_duration_ms|\
+                index_queue_search_duration_ms|\
+                index_queue_size|\
+                index_queue_stale_count|\
+                index_queue_vectors_dequeued|\
+                index_queue_wait_duration_ms|\
+                lsm_active_segments|\
+                lsm_bloom_filter_duration_ms|\
+                lsm_segment_count|\
+                lsm_segment_objects|\
+                lsm_segment_size|\
+                object_count|\
+                objects_durations_ms|\
+                requests_total|\
+                startup_diskio_throughput|\
+                startup_durations_ms|\
+                vector_index_durations_ms|\
+                vector_index_maintenance_durations_ms|\
+                vector_index_operations|\
+                vector_index_size|\
+                vector_index_tombstone_cleaned|\
+                vector_index_tombstone_cleanup_threads|\
+                vector_index_tombstones|\
+                weaviate_build_info|\
+                weaviate_runtime_config_hash|\
+                weaviate_runtime_config_last_load_success|\
+                weaviate_schema_collections|\
+                weaviate_schema_shards|\
+                weaviate_internal_sample_memberlist_queue_broadcasts|\
+                weaviate_internal_timer_memberlist_gossip|\
+                weaviate_internal_counter_raft_apply|\
+                weaviate_internal_counter_raft_state_candidate|\
+                weaviate_internal_counter_raft_state_follower|\
+                weaviate_internal_counter_raft_state_leader|\
+                weaviate_internal_counter_raft_transition_heartbeat_timeout|\
+                weaviate_internal_gauge_raft_commitNumLogs|\
+                weaviate_internal_gauge_raft_leader_dispatchNumLogs|\
+                weaviate_internal_gauge_raft_leader_oldestLogAge|\
+                weaviate_internal_gauge_raft_peers|\
+                weaviate_internal_sample_raft_boltdb_logBatchSize|\
+                weaviate_internal_sample_raft_boltdb_logSize|\
+                weaviate_internal_sample_raft_boltdb_logsPerBatch|\
+                weaviate_internal_sample_raft_boltdb_writeCapacity|\
+                weaviate_internal_sample_raft_thread_fsm_saturation|\
+                weaviate_internal_sample_raft_thread_main_saturation|\
+                weaviate_internal_timer_raft_boltdb_getLog|\
+                weaviate_internal_timer_raft_boltdb_storeLogs|\
+                weaviate_internal_timer_raft_commitTime|\
+                weaviate_internal_timer_raft_fsm_apply|\
+                weaviate_internal_timer_raft_fsm_enqueue|\
+                weaviate_internal_timer_raft_leader_dispatchLog|\
+                weaviate_usage_gcs_operations_total|\
+                weaviate_usage_gcs_operation_latency_seconds|\
+                weaviate_usage_gcs_resource_count|\
+                weaviate_usage_gcs_uploaded_file_size_bytes|\
+                weaviate_usage_s3_operations_total|\
+                weaviate_usage_s3_operation_latency_seconds|\
+                weaviate_usage_s3_resource_count|\
+                weaviate_usage_s3_uploaded_file_size_bytes)"

internal/confmapprovider/discovery/generated_bundledfs.go

@@ -34,4 +34,5 @@ import "embed"
 //go:embed bundle.d/receivers/rabbitmq.discovery.yaml
 //go:embed bundle.d/receivers/redis.discovery.yaml
 //go:embed bundle.d/receivers/sqlserver.discovery.yaml
+//go:embed bundle.d/receivers/weaviate.discovery.yaml
 var BundledFS embed.FS