[oracle] Missing total.bytes field in tablespace datastream

CHANGELOG.next.asciidoc

@@ -176,6 +176,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix issue where beats may report incorrect metrics for its own process when running inside a container {pull}39627[39627]
 - Fix for MySQL/Performance - Query failure for MySQL versions below v8.0.1, for performance metric `quantile_95`. {pull}38710[38710]
 - Normalize AWS RDS CPU Utilization values before making the metadata API call. {pull}39664[39664]
+- Fix query logic for temp and non-temp tablespaces in Oracle module. {issue}38051[38051] {pull}39787[39787]
 
 *Osquerybeat*
 

x-pack/metricbeat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json

@@ -4,7 +4,7 @@
         "kibanaSavedObjectMeta": {
             "searchSourceJSON": {}
         },
-        "title": "Tablespace Total Size [Metricbeat Oracle]",
+        "title": "Tablespace Max Total Size [Metricbeat Oracle]",
         "uiStateJSON": {},
         "version": 1,
         "visState": {
@@ -24,13 +24,40 @@
                         "fill": 0.5,
                         "formatter": "bytes",
                         "id": "61ca57f1-469d-11e7-af02-69e470af7417",
-                        "label": "Tablespace total size",
+                        "label": "Tablespace max total size",
                         "line_width": 1,
                         "metrics": [
                             {
-                                "field": "oracle.tablespace.space.total.bytes",
+                                "agg_with": "avg",
+                                "field": "oracle.tablespace.space.used.bytes",
                                 "id": "61ca57f2-469d-11e7-af02-69e470af7417",
-                                "type": "avg"
+                                "type": "max"
+                            },
+                            {
+                                "agg_with": "avg",
+                                "colors": [
+                                    "#68BC00"
+                                ],
+                                "field": "oracle.tablespace.space.free.bytes",
+                                "id": "e04e8f40-24cd-4066-b12c-da0db0ff73d4",
+                                "type": "max"
+                            },
+                            {
+                                "id": "2cf57800-8b54-41fa-a877-159b49699a50",
+                                "script": "params.used_bytes + params.free_bytes",
+                                "type": "math",
+                                "variables": [
+                                    {
+                                        "field": "61ca57f2-469d-11e7-af02-69e470af7417",
+                                        "id": "631a44d5-d18a-4743-bea0-6f61930fd65f",
+                                        "name": "used_bytes"
+                                    },
+                                    {
+                                        "field": "e04e8f40-24cd-4066-b12c-da0db0ff73d4",
+                                        "id": "c255d24c-3a29-4879-b999-77af43d97c6b",
+                                        "name": "free_bytes"
+                                    }
+                                ]
                             }
                         ],
                         "point_size": 1,

x-pack/metricbeat/module/oracle/tablespace/data.go

@@ -16,8 +16,9 @@ import (
 
 // extract is the E of a ETL processing. Gets the data files, used/free space and temp free space data that is fetch
 // by doing queries to Oracle
-func (m *MetricSet) extract(ctx context.Context, extractor tablespaceExtractMethods) (out *extractedData, err error) {
-	out = &extractedData{}
+func (m *MetricSet) extract(ctx context.Context, extractor tablespaceExtractMethods) (*extractedData, error) {
+	out := &extractedData{}
+	var err error
 
 	if out.dataFiles, err = extractor.dataFilesData(ctx); err != nil {
 		return nil, fmt.Errorf("error getting data_files: %w", err)
@@ -31,23 +32,23 @@ func (m *MetricSet) extract(ctx context.Context, extractor tablespaceExtractMeth
 		return nil, fmt.Errorf("error getting free space data: %w", err)
 	}
 
-	return
+	return out, nil
 }
 
 // transform is the T of an ETL (refer to the 'extract' method above if you need to see the origin). Transforms the data
 // to create a Kibana/Elasticsearch friendly JSON. Data from Oracle is pretty fragmented by design so a lot of data
 // was necessary. Data is organized by Tablespace entity (Tablespaces might contain one or more data files)
-func (m *MetricSet) transform(in *extractedData) (out map[string]mapstr.M) {
-	out = make(map[string]mapstr.M, 0)
+func (m *MetricSet) transform(in *extractedData) map[string]mapstr.M {
+	out := make(map[string]mapstr.M, 0)
 
-	for _, dataFile := range in.dataFiles {
-		m.addDataFileData(&dataFile, out)
+	for i := range in.dataFiles {
+		m.addDataFileData(&in.dataFiles[i], out)
 	}
 
 	m.addUsedAndFreeSpaceData(in.freeSpace, out)
 	m.addTempFreeSpaceData(in.tempFreeSpace, out)
 
-	return
+	return out
 }
 
 func (m *MetricSet) extractAndTransform(ctx context.Context) ([]mb.Event, error) {
@@ -78,7 +79,7 @@ func (m *MetricSet) addTempFreeSpaceData(tempFreeSpaces []tempFreeSpace, out map
 		name := val.(string)
 		if name == "TEMP" {
 			for _, tempFreeSpaceTable := range tempFreeSpaces {
-				oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.total.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.TablespaceSize})
+				oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.total.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.TotalSpaceBytes})
 				oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.used.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.UsedSpaceBytes})
 				oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.free.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.FreeSpace})
 			}
@@ -101,6 +102,7 @@ func (m *MetricSet) addUsedAndFreeSpaceData(freeSpaces []usedAndFreeSpace, out m
 				if name == freeSpaceTable.TablespaceName {
 					oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.free.bytes", &oracle.Int64Value{NullInt64: freeSpaceTable.TotalFreeBytes})
 					oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.used.bytes", &oracle.Int64Value{NullInt64: freeSpaceTable.TotalUsedBytes})
+					oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.total.bytes", &oracle.Int64Value{NullInt64: freeSpaceTable.TotalSpaceBytes})
 				}
 			}
 		}

x-pack/metricbeat/module/oracle/tablespace/data_test.go

@@ -12,13 +12,13 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-var expectedResults = []string{`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux01.dbf","online_status":"ONLINE","size":{"bytes":9999990,"free":{"bytes":99999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`,
-	`{"data_file":{"id":181,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux02.dbf","online_status":"ONLINE","size":{"bytes":9999991,"free":{"bytes":99999995},"max":{"bytes":9999995}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`,
-	`{"data_file":{"id":182,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux03.dbf","online_status":"ONLINE","size":{"bytes":9999992,"free":{"bytes":99999996},"max":{"bytes":9999996}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`,
-	`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/system01.dbf","online_status":"ONLINE","size":{"bytes":999990,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSTEM","space":{"free":{"bytes":9990},"used":{"bytes":9991}}}`,
+var expectedResults = []string{`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux01.dbf","online_status":"ONLINE","size":{"bytes":9999990,"free":{"bytes":99999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`,
+	`{"data_file":{"id":181,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux02.dbf","online_status":"ONLINE","size":{"bytes":9999991,"free":{"bytes":99999995},"max":{"bytes":9999995}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`,
+	`{"data_file":{"id":182,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux03.dbf","online_status":"ONLINE","size":{"bytes":9999992,"free":{"bytes":99999996},"max":{"bytes":9999996}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`,
+	`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/system01.dbf","online_status":"ONLINE","size":{"bytes":999990,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSTEM","space":{"free":{"bytes":9990},"total":{"bytes":99999},"used":{"bytes":9991}}}`,
 	`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/temp012017-03-02_07-54-38-075-AM.dbf","online_status":"ONLINE","size":{"bytes":999991,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"TEMP","space":{"free":{"bytes":99999},"total":{"bytes":99999},"used":{"bytes":99999}}}`,
-	`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/undotbs01.dbf","online_status":"ONLINE","size":{"bytes":999992,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"UNDOTBS1","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`,
-	`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/users01.dbf","online_status":"ONLINE","size":{"bytes":999993,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"USERS","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`}
+	`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/undotbs01.dbf","online_status":"ONLINE","size":{"bytes":999992,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"UNDOTBS1","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`,
+	`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/users01.dbf","online_status":"ONLINE","size":{"bytes":999993,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"USERS","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`}
 
 var notExpectedEvents = []string{`{}`, `{"foo":"bar"}`}
 

x-pack/metricbeat/module/oracle/tablespace/mocks_test.go

@@ -77,16 +77,16 @@ func (h happyDataFiles) dataFilesData(_ context.Context) ([]dataFile, error) {
 type happyTempFreeSpaceData struct{}
 
 func (happyTempFreeSpaceData) tempFreeSpaceData(_ context.Context) ([]tempFreeSpace, error) {
-	return []tempFreeSpace{{TablespaceName: "TEMP", TablespaceSize: sql.NullInt64{Valid: true, Int64: 99999}, UsedSpaceBytes: sql.NullInt64{Valid: true, Int64: 99999}, FreeSpace: sql.NullInt64{Int64: 99999, Valid: true}}}, nil
+	return []tempFreeSpace{{TablespaceName: "TEMP", TotalSpaceBytes: sql.NullInt64{Valid: true, Int64: 99999}, UsedSpaceBytes: sql.NullInt64{Valid: true, Int64: 99999}, FreeSpace: sql.NullInt64{Int64: 99999, Valid: true}}}, nil
 }
 
 type happyFreeSpaceData struct{}
 
 func (happyFreeSpaceData) usedAndFreeSpaceData(_ context.Context) ([]usedAndFreeSpace, error) {
 	return []usedAndFreeSpace{
-		{TablespaceName: "SYSTEM", TotalFreeBytes: sql.NullInt64{Int64: 9990, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}},
-		{TablespaceName: "SYSAUX", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}},
-		{TablespaceName: "UNDOTBS1", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}},
-		{TablespaceName: "USERS", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}},
+		{TablespaceName: "SYSTEM", TotalFreeBytes: sql.NullInt64{Int64: 9990, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}},
+		{TablespaceName: "SYSAUX", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}},
+		{TablespaceName: "UNDOTBS1", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}},
+		{TablespaceName: "USERS", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}},
 	}, nil
 }

x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go

@@ -11,31 +11,24 @@ import (
 )
 
 type tempFreeSpace struct {
-	TablespaceName string
-	TablespaceSize sql.NullInt64
-	UsedSpaceBytes sql.NullInt64
-	FreeSpace      sql.NullInt64
-}
-
-func (d *tempFreeSpace) hash() string {
-	return d.TablespaceName
-}
-
-func (d *tempFreeSpace) eventKey() string {
-	return d.TablespaceName
+	TablespaceName  string
+	TotalSpaceBytes sql.NullInt64
+	UsedSpaceBytes  sql.NullInt64
+	FreeSpace       sql.NullInt64
 }
 
 func (e *tablespaceExtractor) tempFreeSpaceData(ctx context.Context) ([]tempFreeSpace, error) {
-	rows, err := e.db.QueryContext(ctx, "SELECT TABLESPACE_NAME, TABLESPACE_SIZE, ALLOCATED_SPACE, FREE_SPACE FROM DBA_TEMP_FREE_SPACE")
+	rows, err := e.db.QueryContext(ctx, `SELECT t.TABLESPACE_NAME, (SELECT SUM(BYTES) FROM DBA_DATA_FILES) + (SELECT SUM(BYTES) FROM DBA_TEMP_FILES) AS TOTAL_SUM, t.ALLOCATED_SPACE, t.FREE_SPACE FROM DBA_TEMP_FREE_SPACE t `)
 	if err != nil {
 		return nil, fmt.Errorf("error executing query: %w", err)
 	}
+	defer rows.Close()
 
 	results := make([]tempFreeSpace, 0)
 
 	for rows.Next() {
 		dest := tempFreeSpace{}
-		if err = rows.Scan(&dest.TablespaceName, &dest.TablespaceSize, &dest.UsedSpaceBytes, &dest.FreeSpace); err != nil {
+		if err = rows.Scan(&dest.TablespaceName, &dest.TotalSpaceBytes, &dest.UsedSpaceBytes, &dest.FreeSpace); err != nil {
 			return nil, err
 		}
 		results = append(results, dest)

x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go

@@ -11,30 +11,24 @@ import (
 )
 
 type usedAndFreeSpace struct {
-	TablespaceName string
-	TotalFreeBytes sql.NullInt64
-	TotalUsedBytes sql.NullInt64
-}
-
-func (d *usedAndFreeSpace) hash() string {
-	return d.TablespaceName
-}
-
-func (d *usedAndFreeSpace) eventKey() string {
-	return d.TablespaceName
+	TablespaceName  string
+	TotalSpaceBytes sql.NullInt64
+	TotalFreeBytes  sql.NullInt64
+	TotalUsedBytes  sql.NullInt64
 }
 
 func (e *tablespaceExtractor) usedAndFreeSpaceData(ctx context.Context) ([]usedAndFreeSpace, error) {
-	rows, err := e.db.QueryContext(ctx, "SELECT b.tablespace_name, tbs_size used, a.free_space free FROM (SELECT tablespace_name, sum(bytes) AS free_space FROM dba_free_space GROUP BY tablespace_name) a, (SELECT tablespace_name, sum(bytes) AS tbs_size FROM dba_data_files GROUP BY tablespace_name) b WHERE a.tablespace_name(+)=b.tablespace_name")
+	rows, err := e.db.QueryContext(ctx, `SELECT b.tablespace_name, (b.tbs_size - NVL(a.free_space, 0)) AS used, NVL(a.free_space, 0) AS free, (SELECT SUM(bytes) FROM DBA_DATA_FILES) + (SELECT SUM(bytes) FROM DBA_TEMP_FILES) AS total_sum FROM (SELECT tablespace_name, SUM(bytes) AS free_space FROM DBA_FREE_SPACE GROUP BY tablespace_name) a RIGHT JOIN (SELECT tablespace_name, SUM(bytes) AS tbs_size FROM DBA_DATA_FILES GROUP BY tablespace_name) b ON a.tablespace_name = b.tablespace_name`)
 	if err != nil {
 		return nil, fmt.Errorf("error executing query: %w", err)
 	}
+	defer rows.Close()
 
 	results := make([]usedAndFreeSpace, 0)
 
 	for rows.Next() {
 		dest := usedAndFreeSpace{}
-		if err = rows.Scan(&dest.TablespaceName, &dest.TotalUsedBytes, &dest.TotalFreeBytes); err != nil {
+		if err = rows.Scan(&dest.TablespaceName, &dest.TotalUsedBytes, &dest.TotalFreeBytes, &dest.TotalSpaceBytes); err != nil {
 			return nil, err
 		}
 		results = append(results, dest)