Merge branch 'main' into add-new-oracledb-metrics

Author: atoulme

.chloggen/sqlserver-top-query.yaml

@@ -0,0 +1,31 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: sqlserverreceiver
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Support query-level log collection
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [36462]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  Added top query (most CPU time consumed) collection. The query will gather the queries took most of the time during the last
+  query interval and report related metrics. The number of queries can be configured. This will enable user to have better
+  understanding on what is going on with the database. This enhancement empowers users to not only monitor but also actively 
+  manage and optimize their MSSQL database performance based on real usage patterns.
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

.github/CODEOWNERS

@@ -298,7 +298,7 @@ receiver/solacereceiver/                                         @open-telemetry
 receiver/splunkenterprisereceiver/                               @open-telemetry/collector-contrib-approvers @shalper2 @MovieStoreGuy @greatestusername
 receiver/splunkhecreceiver/                                      @open-telemetry/collector-contrib-approvers @atoulme
 receiver/sqlqueryreceiver/                                       @open-telemetry/collector-contrib-approvers @dmitryax @crobert-1
-receiver/sqlserverreceiver/                                      @open-telemetry/collector-contrib-approvers @StefanKurek
+receiver/sqlserverreceiver/                                      @open-telemetry/collector-contrib-approvers @StefanKurek @sincejune
 receiver/sshcheckreceiver/                                       @open-telemetry/collector-contrib-approvers @nslaughter
 receiver/statsdreceiver/                                         @open-telemetry/collector-contrib-approvers @jmacd @dmitryax
 receiver/syslogreceiver/                                         @open-telemetry/collector-contrib-approvers @djaglowski @andrzej-stencel

cmd/githubgen/allowlist.txt

@@ -1,5 +1,4 @@
 abhishek-at-cloudwerx
-AkhigbeEromo
 cemdk
 driverpt
 dsimil

receiver/sqlserverreceiver/README.md

@@ -3,16 +3,18 @@
 <!-- status autogenerated section -->
 | Status        |           |
 | ------------- |-----------|
-| Stability     | [beta]: metrics   |
+| Stability     | [development]: logs   |
+|               | [beta]: metrics   |
 | Distributions | [contrib] |
 | Issues        | [![Open issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aopen%20label%3Areceiver%2Fsqlserver%20&label=open&color=orange&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aopen+is%3Aissue+label%3Areceiver%2Fsqlserver) [![Closed issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aclosed%20label%3Areceiver%2Fsqlserver%20&label=closed&color=blue&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aclosed+is%3Aissue+label%3Areceiver%2Fsqlserver) |
-| [Code Owners](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#becoming-a-code-owner)    | [@StefanKurek](https://www.github.com/StefanKurek) \| Seeking more code owners! |
+| [Code Owners](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#becoming-a-code-owner)    | [@StefanKurek](https://www.github.com/StefanKurek), [@sincejune](https://www.github.com/sincejune) \| Seeking more code owners! |
 
+[development]: https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/component-stability.md#development
 [beta]: https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/component-stability.md#beta
 [contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib
 <!-- end autogenerated section -->
 
-The `sqlserver` receiver grabs metrics about a Microsoft SQL Server instance. The receiver works by either using the
+The `sqlserver` receiver grabs metrics/logs about a Microsoft SQL Server instance. The receiver works by either using the
 Windows Performance Counters, or by directly connecting to the instance and querying it. Windows Performance Counters
 are only available when running on Windows.
 
@@ -36,6 +38,14 @@ Windows-specific options:
 - `computer_name` (optional): The computer name identifies the SQL Server name or IP address of the computer being monitored.
   If specified, `instance_name` is also required to be defined. This option is ignored in non-Windows environments.
 
+Top-Query collection specific options (only useful when top-query collection are enabled):
+- `lookback_time` (optional, example = `60`, default = `2 * collection_interval`): The time window (in second) in which to query for top queries.
+  - Queries that were finished execution outside the lookback window are not included in the collection. Increasing the lookback window (in seconds) will be useful for capturing long-running queries.
+- `max_query_sample_count` (optional, example = `5000`, default = `1000`): The maximum number of records to fetch in a single run.
+- `top_query_count`: (optional, example = `100`, default = `200`): The maximum number of active queries to report (to the next consumer) in a single run.
+- `enabled`: (optional, default = `false`): Enable collection of top queries.
+  - e.g. `sqlserver` receiver will fetch 1000 (value: `max_query_sample_count`) queries from database and report the top 200 (value: `top_query_count`) which used the most CPU time.
+
 Example:
 
 ```yaml
@@ -50,7 +60,7 @@ Example:
         port: 1433
 ```
 
-When a named instance is used on Windows, a computer name and a instance name must be specified.
+When a named instance is used on Windows, a computer name and an instance name must be specified.
 Example with named instance:
 
 ```yaml
@@ -68,10 +78,31 @@ Example with named instance:
 
 The full list of settings exposed for this receiver are documented in [config.go](./config.go) with detailed sample configurations in [testdata/config.yaml](./testdata/config.yaml).
 
+Top query collection enabled:
+```yaml
+    receivers:
+      sqlserver:
+        collection_interval: 5s
+        username: sa
+        password: securepassword
+        server: 0.0.0.0
+        port: 1433
+        top_query_collection:
+          enabled: true
+          lookback_time: 60
+          max_query_sample_count: 1000
+          top_query_count: 200
+          
+```
 ## Metrics
 
 Details about the metrics produced by this receiver can be found in [documentation.md](./documentation.md)
 
+
+## Logs 
+
+Details about the logs produced by this receiver can be found in [logs-documentation.md](./logs-documentation.md)
+
 ## Known issues
 SQL Server docker users may run into an issue that the collector fails to parse certificate from server due to `x509: negative serial number`. That's because we adopted Go `1.23` starting from contrib `v0.121.0`:
 > Before Go 1.23, ParseCertificate accepted certificates with negative serial numbers.

receiver/sqlserverreceiver/config.go

@@ -12,10 +12,28 @@ import (
 	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/sqlserverreceiver/internal/metadata"
 )
 
+type TopQueryCollection struct {
+	// Enabled enables the collection of the top queries by the execution time.
+	// It will collect the top N queries based on totalElapsedTimeDiffs during the last collection interval.
+	// The query statement will also be reported, hence, it is not ideal to send it as a metric. Hence
+	// we are reporting them as logs.
+	// The `N` is configured via `TopQueryCount`
+	Enabled             bool `mapstructure:"enabled"`
+	LookbackTime        uint `mapstructure:"lookback_time"`
+	MaxQuerySampleCount uint `mapstructure:"max_query_sample_count"`
+	TopQueryCount       uint `mapstructure:"top_query_count"`
+}
+
 // Config defines configuration for a sqlserver receiver.
 type Config struct {
 	scraperhelper.ControllerConfig `mapstructure:",squash"`
 	metadata.MetricsBuilderConfig  `mapstructure:",squash"`
+	// EnableTopQueryCollection enables the collection of the top queries by the execution time.
+	// It will collect the top N queries based on totalElapsedTimeDiffs during the last collection interval.
+	// The query statement will also be reported, hence, it is not ideal to send it as a metric. Hence
+	// we are reporting them as logs.
+	// The `N` is configured via `TopQueryCount`
+	TopQueryCollection `mapstructure:"top_query_collection"`
 
 	InstanceName string `mapstructure:"instance_name"`
 	ComputerName string `mapstructure:"computer_name"`
@@ -33,6 +51,14 @@ func (cfg *Config) Validate() error {
 		return err
 	}
 
+	if cfg.TopQueryCollection.MaxQuerySampleCount > 10000 {
+		return errors.New("`max_query_sample_count` must be between 0 and 10000")
+	}
+
+	if cfg.TopQueryCount > cfg.TopQueryCollection.MaxQuerySampleCount {
+		return errors.New("`top_query_count` must be less than or equal to `max_query_sample_count`")
+	}
+
 	if !directDBConnectionEnabled(cfg) {
 		if cfg.Server != "" || cfg.Username != "" || string(cfg.Password) != "" {
 			return errors.New("Found one or more of the following configuration options set: [server, port, username, password]. " +

receiver/sqlserverreceiver/config_test.go

@@ -64,6 +64,29 @@ func TestValidate(t *testing.T) {
 			},
 			expectedSuccess: true,
 		},
+		{
+			desc: "config with invalid MaxQuerySampleCount value",
+			cfg: &Config{
+				MetricsBuilderConfig: metadata.DefaultMetricsBuilderConfig(),
+				ControllerConfig:     scraperhelper.NewDefaultControllerConfig(),
+				TopQueryCollection: TopQueryCollection{
+					MaxQuerySampleCount: 100000,
+				},
+			},
+			expectedSuccess: false,
+		},
+		{
+			desc: "config with invalid TopQueryCount value",
+			cfg: &Config{
+				MetricsBuilderConfig: metadata.DefaultMetricsBuilderConfig(),
+				ControllerConfig:     scraperhelper.NewDefaultControllerConfig(),
+				TopQueryCollection: TopQueryCollection{
+					MaxQuerySampleCount: 100,
+					TopQueryCount:       200000,
+				},
+			},
+			expectedSuccess: false,
+		},
 	}
 
 	for _, tc := range testCases {
@@ -122,6 +145,10 @@ func TestLoadConfig(t *testing.T) {
 		}
 		expected.ComputerName = "CustomServer"
 		expected.InstanceName = "CustomInstance"
+		expected.Enabled = true
+		expected.LookbackTime = 60
+		expected.TopQueryCount = 200
+		expected.TopQueryCollection.MaxQuerySampleCount = 1000
 
 		sub, err := cm.Sub("sqlserver/named")
 		require.NoError(t, err)

receiver/sqlserverreceiver/factory.go

@@ -4,28 +4,44 @@
 package sqlserverreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/sqlserverreceiver"
 
 import (
+	"context"
 	"database/sql"
 	"errors"
 	"fmt"
 	"time"
 
+	lru "github.com/hashicorp/golang-lru/v2"
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/receiver"
 	"go.opentelemetry.io/collector/scraper"
 	"go.opentelemetry.io/collector/scraper/scraperhelper"
+	"go.uber.org/zap"
 
 	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/sqlquery"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/sqlserverreceiver/internal/metadata"
 )
 
 var errConfigNotSQLServer = errors.New("config was not a sqlserver receiver config")
 
+// newCache creates a new cache with the given size.
+// If the size is less or equal to 0, it will be set to 1.
+// It will never return an error.
+func newCache(size int) *lru.Cache[string, int64] {
+	if size <= 0 {
+		size = 1
+	}
+	// lru will only returns error when the size is less than 0
+	cache, _ := lru.New[string, int64](size)
+	return cache
+}
+
 // NewFactory creates a factory for SQL Server receiver.
 func NewFactory() receiver.Factory {
 	return receiver.NewFactory(
 		metadata.Type,
 		createDefaultConfig,
-		receiver.WithMetrics(createMetricsReceiver, metadata.MetricsStability))
+		receiver.WithMetrics(createMetricsReceiver, metadata.MetricsStability),
+		receiver.WithLogs(createLogsReceiver, metadata.LogsStability))
 }
 
 func createDefaultConfig() component.Config {
@@ -34,6 +50,12 @@ func createDefaultConfig() component.Config {
 	return &Config{
 		ControllerConfig:     cfg,
 		MetricsBuilderConfig: metadata.DefaultMetricsBuilderConfig(),
+		TopQueryCollection: TopQueryCollection{
+			Enabled:             false,
+			LookbackTime:        uint(2 * cfg.CollectionInterval / time.Second),
+			MaxQuerySampleCount: 1000,
+			TopQueryCount:       200,
+		},
 	}
 }
 
@@ -63,6 +85,22 @@ func setupQueries(cfg *Config) []string {
 	return queries
 }
 
+func setupLogQueries(cfg *Config) ([]string, []error) {
+	var queries []string
+	var errs []error
+
+	if cfg.Enabled {
+		q, err := getSQLServerQueryTextAndPlanQuery(cfg.InstanceName, cfg.MaxQuerySampleCount, cfg.LookbackTime)
+		if err != nil {
+			errs = append(errs, err)
+		} else {
+			queries = append(queries, q)
+		}
+	}
+
+	return queries, errs
+}
+
 func directDBConnectionEnabled(config *Config) bool {
 	return config.Server != "" &&
 		config.Username != "" &&
@@ -97,12 +135,71 @@ func setupSQLServerScrapers(params receiver.Settings, cfg *Config) []*sqlServerS
 	for i, query := range queries {
 		id := component.NewIDWithName(metadata.Type, fmt.Sprintf("query-%d: %s", i, query))
 
+		// lru only returns error when the size is less than 0
+		cache := newCache(1)
+
+		sqlServerScraper := newSQLServerScraper(id, query,
+			sqlquery.TelemetryConfig{},
+			dbProviderFunc,
+			sqlquery.NewDbClient,
+			params,
+			cfg,
+			cache)
+
+		scrapers = append(scrapers, sqlServerScraper)
+	}
+
+	return scrapers
+}
+
+// SQL Server scraper creation is split out into a separate method for the sake of testing.
+func setupSQLServerLogsScrapers(params receiver.Settings, cfg *Config) []*sqlServerScraperHelper {
+	if !directDBConnectionEnabled(cfg) {
+		params.Logger.Info("No direct connection will be made to the SQL Server: Configuration doesn't include some options.")
+		return nil
+	}
+
+	queries, errs := setupLogQueries(cfg)
+	if len(errs) > 0 {
+		params.Logger.Error("Failed to template queries in SQLServer receiver: Configuration might not be correct.", zap.Error(errors.Join(errs...)))
+		return nil
+	}
+
+	if len(queries) == 0 {
+		params.Logger.Info("No direct connection will be made to the SQL Server: No logs are enabled requiring it.")
+		return nil
+	}
+
+	queryTextAndPlanQuery, err := getSQLServerQueryTextAndPlanQuery(cfg.InstanceName, cfg.MaxQuerySampleCount, cfg.LookbackTime)
+	if err != nil {
+		params.Logger.Error("Failed to template needed queries in SQLServer receiver: Configuration might not be correct.", zap.Error(err))
+		return nil
+	}
+
+	// TODO: Test if this needs to be re-defined for each scraper
+	// This should be tested when there is more than one query being made.
+	dbProviderFunc := func() (*sql.DB, error) {
+		return sql.Open("sqlserver", getDBConnectionString(cfg))
+	}
+
+	var scrapers []*sqlServerScraperHelper
+	for i, query := range queries {
+		id := component.NewIDWithName(metadata.Type, fmt.Sprintf("logs-query-%d: %s", i, query))
+
+		cache := newCache(1)
+
+		if query == queryTextAndPlanQuery {
+			// we have 8 metrics in this query and multiple 2 to allow to cache more queries.
+			cache = newCache(int(cfg.MaxQuerySampleCount * 8 * 2))
+		}
+
 		sqlServerScraper := newSQLServerScraper(id, query,
 			sqlquery.TelemetryConfig{},
 			dbProviderFunc,
 			sqlquery.NewDbClient,
 			params,
-			cfg)
+			cfg,
+			cache)
 
 		scrapers = append(scrapers, sqlServerScraper)
 	}
@@ -132,6 +229,32 @@ func setupScrapers(params receiver.Settings, cfg *Config) ([]scraperhelper.Contr
 	return opts, nil
 }
 
+// Note: This method will fail silently if there is no work to do. This is an acceptable use case
+// as this receiver can still get information on Windows from performance counters without a direct
+// connection. Messages will be logged at the INFO level in such cases.
+func setupLogsScrapers(params receiver.Settings, cfg *Config) ([]scraperhelper.ControllerOption, error) {
+	sqlServerScrapers := setupSQLServerLogsScrapers(params, cfg)
+
+	var opts []scraperhelper.ControllerOption
+	for _, sqlScraper := range sqlServerScrapers {
+		s, err := scraper.NewLogs(sqlScraper.ScrapeLogs,
+			scraper.WithStart(sqlScraper.Start),
+			scraper.WithShutdown(sqlScraper.Shutdown))
+		if err != nil {
+			return nil, err
+		}
+
+		opt := scraperhelper.AddFactoryWithConfig(
+			scraper.NewFactory(metadata.Type, nil,
+				scraper.WithLogs(func(context.Context, scraper.Settings, component.Config) (scraper.Logs, error) {
+					return s, nil
+				}, component.StabilityLevelAlpha)), nil)
+		opts = append(opts, opt)
+	}
+
+	return opts, nil
+}
+
 func isDatabaseIOQueryEnabled(metrics *metadata.MetricsConfig) bool {
 	if metrics.SqlserverDatabaseLatency.Enabled ||
 		metrics.SqlserverDatabaseOperations.Enabled ||