[exporter/clickhouse] Upgrade trace table (#34245)

SpencerTorres · dmitryax · web-flow · commit 1620fc64c8d3 · 2024-09-25T16:09:14.000-07:00
This PR updates the default `otel_traces` table schema.

There are no breaking changes, this schema is compatible with the
previous `INSERT` statement. This does not affect existing users, only
new users that have `create_schema` enabled in the config.

Notable changes:
- Optimized ORDER BY to use a 32 bit DateTime without TraceId
- Reduced DateTime64(9) to DateTime on the trace timestamp table
- TTLs now use Date instead of DateTime since parts will only be dropped
on the partition date

No major column changes for now, but there were some small performance
benefits observed with these changes.

As always, it is best to edit these tables to fit your needs and query
patterns. I expect further changes in the future as we continue to
optimize this table for a generalized use case.

**Testing:**
Integration tests are disabled, but I enabled them locally to test that
this change works as intended.
Unit tests are passing as well.

**Documentation:**
- Updated README
- Example DDL has been updated, and now includes the trace timestamp
table and materialized view

---------

Co-authored-by: Dmitrii Anoshin &lt;anoshindx@gmail.com&gt;
diff --git a/.chloggen/clickhouseexporter_update_trace_table.yaml b/.chloggen/clickhouseexporter_update_trace_table.yaml
@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: clickhouseexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Updated the default trace table
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [34245]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: Reduced data types, improved partitioning and time range queries
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []
diff --git a/exporter/clickhouseexporter/README.md b/exporter/clickhouseexporter/README.md
@@ -128,7 +128,7 @@ Limit 100;
 - Find spans with specific attribute.
 
 ```sql
-SELECT Timestamp as log_time,
+SELECT Timestamp,
        TraceId,
        SpanId,
        ParentSpanId,
@@ -156,7 +156,7 @@ WITH
     '391dae938234560b16bb63f51501cb6f' as trace_id,
     (SELECT min(Start) FROM otel_traces_trace_id_ts WHERE TraceId = trace_id) as start,
     (SELECT max(End) + 1 FROM otel_traces_trace_id_ts WHERE TraceId = trace_id) as end
-SELECT Timestamp as log_time,
+SELECT Timestamp,
        TraceId,
        SpanId,
        ParentSpanId,
@@ -180,7 +180,7 @@ Limit 100;
 - Find spans is error.
 
 ```sql
-SELECT Timestamp as log_time,
+SELECT Timestamp,
        TraceId,
        SpanId,
        ParentSpanId,
@@ -196,15 +196,15 @@ SELECT Timestamp as log_time,
        toString(Links.TraceId)
 FROM otel_traces
 WHERE ServiceName = 'clickhouse-exporter'
-  AND StatusCode = 'STATUS_CODE_ERROR'
+  AND StatusCode = 'Error'
   AND Timestamp >= NOW() - INTERVAL 1 HOUR
 Limit 100;
 ```
 
 - Find slow spans.
 
 ```sql
-SELECT Timestamp as log_time,
+SELECT Timestamp,
        TraceId,
        SpanId,
        ParentSpanId,
diff --git a/exporter/clickhouseexporter/example/default_ddl/traces.sql b/exporter/clickhouseexporter/example/default_ddl/traces.sql
@@ -1,40 +1,63 @@
 -- Default Trace table DDL
 
 CREATE TABLE IF NOT EXISTS otel_traces (
-     Timestamp DateTime64(9) CODEC(Delta, ZSTD(1)),
-     TraceId String CODEC(ZSTD(1)),
-     SpanId String CODEC(ZSTD(1)),
-     ParentSpanId String CODEC(ZSTD(1)),
-     TraceState String CODEC(ZSTD(1)),
-     SpanName LowCardinality(String) CODEC(ZSTD(1)),
-     SpanKind LowCardinality(String) CODEC(ZSTD(1)),
-     ServiceName LowCardinality(String) CODEC(ZSTD(1)),
-     ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
-     ScopeName String CODEC(ZSTD(1)),
-     ScopeVersion String CODEC(ZSTD(1)),
-     SpanAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
-     Duration Int64 CODEC(ZSTD(1)),
-     StatusCode LowCardinality(String) CODEC(ZSTD(1)),
-     StatusMessage String CODEC(ZSTD(1)),
-     Events Nested (
-         Timestamp DateTime64(9),
-         Name LowCardinality(String),
-         Attributes Map(LowCardinality(String), String)
-     ) CODEC(ZSTD(1)),
-     Links Nested (
-         TraceId String,
-         SpanId String,
-         TraceState String,
-         Attributes Map(LowCardinality(String), String)
-     ) CODEC(ZSTD(1)),
-     INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
-     INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-     INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-     INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-     INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-     INDEX idx_duration Duration TYPE minmax GRANULARITY 1
+    Timestamp DateTime64(9) CODEC(Delta, ZSTD(1)),
+    TraceId String CODEC(ZSTD(1)),
+    SpanId String CODEC(ZSTD(1)),
+    ParentSpanId String CODEC(ZSTD(1)),
+    TraceState String CODEC(ZSTD(1)),
+    SpanName LowCardinality(String) CODEC(ZSTD(1)),
+    SpanKind LowCardinality(String) CODEC(ZSTD(1)),
+    ServiceName LowCardinality(String) CODEC(ZSTD(1)),
+    ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    ScopeName String CODEC(ZSTD(1)),
+    ScopeVersion String CODEC(ZSTD(1)),
+    SpanAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    Duration UInt64 CODEC(ZSTD(1)),
+    StatusCode LowCardinality(String) CODEC(ZSTD(1)),
+    StatusMessage String CODEC(ZSTD(1)),
+    Events Nested (
+        Timestamp DateTime64(9),
+        Name LowCardinality(String),
+        Attributes Map(LowCardinality(String), String)
+    ) CODEC(ZSTD(1)),
+    Links Nested (
+        TraceId String,
+        SpanId String,
+        TraceState String,
+        Attributes Map(LowCardinality(String), String)
+    ) CODEC(ZSTD(1)),
+    INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_duration Duration TYPE minmax GRANULARITY 1
 ) ENGINE = MergeTree()
-TTL toDateTime("Timestamp") + toIntervalDay(180)
 PARTITION BY toDate(Timestamp)
-ORDER BY (ServiceName, SpanName, toUnixTimestamp(Timestamp), TraceId)
+ORDER BY (ServiceName, SpanName, toDateTime(Timestamp))
+TTL toDate(Timestamp) + toIntervalDay(180)
+SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;
+
+
+CREATE TABLE IF NOT EXISTS otel_traces_trace_id_ts (
+     TraceId String CODEC(ZSTD(1)),
+     Start DateTime CODEC(Delta, ZSTD(1)),
+     End DateTime CODEC(Delta, ZSTD(1)),
+     INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1
+) ENGINE = MergeTree()
+PARTITION BY toDate(Start)
+ORDER BY (TraceId, Start)
+TTL toDate(Start) + toIntervalDay(180)
 SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;
+
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS otel_traces_trace_id_ts_mv
+TO otel_traces_trace_id_ts
+AS SELECT
+    TraceId,
+    min(Timestamp) as Start,
+    max(Timestamp) as End
+FROM otel_traces
+WHERE TraceId != ''
+GROUP BY TraceId;
diff --git a/exporter/clickhouseexporter/exporter_traces.go b/exporter/clickhouseexporter/exporter_traces.go
@@ -163,42 +163,42 @@ const (
 	// language=ClickHouse SQL
 	createTracesTableSQL = `
 CREATE TABLE IF NOT EXISTS %s %s (
-     Timestamp DateTime64(9) CODEC(Delta, ZSTD(1)),
-     TraceId String CODEC(ZSTD(1)),
-     SpanId String CODEC(ZSTD(1)),
-     ParentSpanId String CODEC(ZSTD(1)),
-     TraceState String CODEC(ZSTD(1)),
-     SpanName LowCardinality(String) CODEC(ZSTD(1)),
-     SpanKind LowCardinality(String) CODEC(ZSTD(1)),
-     ServiceName LowCardinality(String) CODEC(ZSTD(1)),
-     ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
-     ScopeName String CODEC(ZSTD(1)),
-     ScopeVersion String CODEC(ZSTD(1)),
-     SpanAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
-     Duration Int64 CODEC(ZSTD(1)),
-     StatusCode LowCardinality(String) CODEC(ZSTD(1)),
-     StatusMessage String CODEC(ZSTD(1)),
-     Events Nested (
-         Timestamp DateTime64(9),
-         Name LowCardinality(String),
-         Attributes Map(LowCardinality(String), String)
-     ) CODEC(ZSTD(1)),
-     Links Nested (
-         TraceId String,
-         SpanId String,
-         TraceState String,
-         Attributes Map(LowCardinality(String), String)
-     ) CODEC(ZSTD(1)),
-     INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
-     INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-     INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-     INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-     INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-     INDEX idx_duration Duration TYPE minmax GRANULARITY 1
+	Timestamp DateTime64(9) CODEC(Delta, ZSTD(1)),
+	TraceId String CODEC(ZSTD(1)),
+	SpanId String CODEC(ZSTD(1)),
+	ParentSpanId String CODEC(ZSTD(1)),
+	TraceState String CODEC(ZSTD(1)),
+	SpanName LowCardinality(String) CODEC(ZSTD(1)),
+	SpanKind LowCardinality(String) CODEC(ZSTD(1)),
+	ServiceName LowCardinality(String) CODEC(ZSTD(1)),
+	ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+	ScopeName String CODEC(ZSTD(1)),
+	ScopeVersion String CODEC(ZSTD(1)),
+	SpanAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+	Duration UInt64 CODEC(ZSTD(1)),
+	StatusCode LowCardinality(String) CODEC(ZSTD(1)),
+	StatusMessage String CODEC(ZSTD(1)),
+	Events Nested (
+		Timestamp DateTime64(9),
+		Name LowCardinality(String),
+		Attributes Map(LowCardinality(String), String)
+	) CODEC(ZSTD(1)),
+	Links Nested (
+		TraceId String,
+		SpanId String,
+		TraceState String,
+		Attributes Map(LowCardinality(String), String)
+	) CODEC(ZSTD(1)),
+	INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
+	INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_duration Duration TYPE minmax GRANULARITY 1
 ) ENGINE = %s
-%s
 PARTITION BY toDate(Timestamp)
-ORDER BY (ServiceName, SpanName, toUnixTimestamp(Timestamp), TraceId)
+ORDER BY (ServiceName, SpanName, toDateTime(Timestamp))
+%s
 SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;
 `
 	// language=ClickHouse SQL
@@ -253,26 +253,27 @@ SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;
 
 const (
 	createTraceIDTsTableSQL = `
-create table IF NOT EXISTS %s_trace_id_ts %s (
+CREATE TABLE IF NOT EXISTS %s_trace_id_ts %s (
      TraceId String CODEC(ZSTD(1)),
-     Start DateTime64(9) CODEC(Delta, ZSTD(1)),
-     End DateTime64(9) CODEC(Delta, ZSTD(1)),
+     Start DateTime CODEC(Delta, ZSTD(1)),
+     End DateTime CODEC(Delta, ZSTD(1)),
      INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1
 ) ENGINE = %s
+PARTITION BY toDate(Start)
+ORDER BY (TraceId, Start)
 %s
-ORDER BY (TraceId, toUnixTimestamp(Start))
-SETTINGS index_granularity=8192;
+SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;
 `
 	createTraceIDTsMaterializedViewSQL = `
 CREATE MATERIALIZED VIEW IF NOT EXISTS %s_trace_id_ts_mv %s
 TO %s.%s_trace_id_ts
 AS SELECT
-TraceId,
-min(Timestamp) as Start,
-max(Timestamp) as End
+	TraceId,
+	min(Timestamp) as Start,
+	max(Timestamp) as End
 FROM
 %s.%s
-WHERE TraceId!=''
+WHERE TraceId != ''
 GROUP BY TraceId;
 `
 )
@@ -282,10 +283,10 @@ func createTracesTable(ctx context.Context, cfg *Config, db *sql.DB) error {
 		return fmt.Errorf("exec create traces table sql: %w", err)
 	}
 	if _, err := db.ExecContext(ctx, renderCreateTraceIDTsTableSQL(cfg)); err != nil {
-		return fmt.Errorf("exec create traceIDTs table sql: %w", err)
+		return fmt.Errorf("exec create traceID timestamp table sql: %w", err)
 	}
 	if _, err := db.ExecContext(ctx, renderTraceIDTsMaterializedViewSQL(cfg)); err != nil {
-		return fmt.Errorf("exec create traceIDTs view sql: %w", err)
+		return fmt.Errorf("exec create traceID timestamp view sql: %w", err)
 	}
 	return nil
 }
@@ -295,12 +296,12 @@ func renderInsertTracesSQL(cfg *Config) string {
 }
 
 func renderCreateTracesTableSQL(cfg *Config) string {
-	ttlExpr := generateTTLExpr(cfg.TTL, "toDateTime(Timestamp)")
+	ttlExpr := generateTTLExpr(cfg.TTL, "toDate(Timestamp)")
 	return fmt.Sprintf(createTracesTableSQL, cfg.TracesTableName, cfg.clusterString(), cfg.tableEngineString(), ttlExpr)
 }
 
 func renderCreateTraceIDTsTableSQL(cfg *Config) string {
-	ttlExpr := generateTTLExpr(cfg.TTL, "toDateTime(Start)")
+	ttlExpr := generateTTLExpr(cfg.TTL, "toDate(Start)")
 	return fmt.Sprintf(createTraceIDTsTableSQL, cfg.TracesTableName, cfg.clusterString(), cfg.tableEngineString(), ttlExpr)
 }
 
diff --git a/exporter/clickhouseexporter/integration_test.go b/exporter/clickhouseexporter/integration_test.go
@@ -95,7 +95,6 @@ func verifyExportLog(t *testing.T, logExporter *logsExporter) {
 
 	type log struct {
 		Timestamp          string            `db:"Timestamp"`
-		TimestampDate      string            `db:"TimestampDate"`
 		TimestampTime      string            `db:"TimestampTime"`
 		TraceID            string            `db:"TraceId"`
 		SpanID             string            `db:"SpanId"`
@@ -117,7 +116,6 @@ func verifyExportLog(t *testing.T, logExporter *logsExporter) {
 
 	expectLog := log{
 		Timestamp:         "2023-12-25T09:53:49Z",
-		TimestampDate:     "2023-12-25T00:00:00Z",
 		TimestampTime:     "2023-12-25T09:53:49Z",
 		TraceID:           "01020300000000000000000000000000",
 		SpanID:            "0102030000000000",
