Makes addon generation more generic, adds more tests, and allows for

multiple addons to be included in splunk startup

Author: hughesjj

packaging/technical-addon/cmd/modinput_config_generator/README.md

@@ -105,4 +105,17 @@ func main() {
 }
 ```
 
-Alternatively, if you need platform specific code, you may use `//go:build` flags, providing a file for every needed variation of code.
+Alternatively, if you need platform specific code, you may use `//go:build` flags, providing a file for every needed variation of code.
+
+# Debugging a container from testcommon.StartSplunk
+1. Change `autoremove` to false in the hostmodifierconfig
+2. Likely add a time.Sleep before the require.NoError check but after the container start
+3. In a terminal, run `docker container ls --all`
+4. Run `docker exec -it <container-id> /bin/bash`
+5. Inspect/debug a TA as normal, ex looking into `/opt/splunk/etc/apps/Sample_Addon` or `/opt/splunk/var/log/splunkd.log`
+
+To get the modular input in XML form, you can use the following command, replacing `Sample_Addon` with the addon name of your choice
+
+```bash
+/opt/splunk/bin/splunk cmd splunkd print-modinput-config Sample_Addon Sample_Addon://Sample_Addon	 
+```

packaging/technical-addon/cmd/modinput_config_generator/generator.go

@@ -16,6 +16,7 @@ package main
 
 import (
 	"fmt"
+	"io/fs"
 	"log"
 	"os"
 	"path/filepath"
@@ -28,25 +29,44 @@ import (
 
 const generatedFileName = "modinput_config.go"
 
-const configTemplate = `// Code generated by runner_config_generator; DO NOT EDIT.
+const configTemplate = `// Code generated by modinput_config_generator; DO NOT EDIT.
 package main
 
 import (
 	"github.com/splunk/splunk-technical-addon/internal/modularinput"
 )
 
-type {{ toPascal .SchemaName }}ModularInput struct {
-	SchemaName 		string
-	ModularInputs 	map[string]modularinput.ModInput
+const SchemaName = "{{ $.SchemaName }}";
+
+type {{ toPascal .SchemaName }}ModInput struct {
+	Value string
+	Name string
+}
+
+type {{ toPascal .SchemaName }}ModularInputs struct {
+	{{- range $name, $inputConfig := .ModularInputs }}
+	{{ toPascal $name }} {{ toPascal $.SchemaName }}ModInput
+	{{- end }}
+}
+
+func Get{{ toPascal .SchemaName }}ModularInputs(mip *modularinput.ModinputProcessor) *{{ toPascal .SchemaName }}ModularInputs {
+	return &{{ toPascal .SchemaName }}ModularInputs{
+		{{- range $name, $inputConfig := .ModularInputs }}
+		{{ toPascal $name }}: {{ toPascal $.SchemaName }}ModInput{
+			Value: mip.ModularInputs["{{ $name }}"].Value,
+			Name: "{{ $name }}",
+		},
+		{{- end }}
+	}
 }
 
 // GetDefault{{ toPascal .SchemaName }}ModularInputs returns the embedded modular input configuration
-func GetDefault{{ toPascal .SchemaName }}ModularInputs() {{ toPascal .SchemaName }}ModularInput {
-	return {{ toPascal .SchemaName }}ModularInput{
+func GetDefault{{ toPascal .SchemaName }}ModularInputs() modularinput.GenericModularInput {
+	return modularinput.GenericModularInput{
 		SchemaName: "{{ $.SchemaName}}",
-		ModularInputs: map[string]modularinput.ModInput{
+		ModularInputs: map[string]*modularinput.ModInput{
 			{{- range $name, $inputConfig := .ModularInputs }}
-			"{{ $name }}": modularinput.ModInput{
+			"{{ $name }}": &modularinput.ModInput{
 				Config: {{ printf "%#v" $inputConfig }},
 				{{- if $inputConfig.Default }}
 				Value: "{{ $inputConfig.Default }}",
@@ -81,45 +101,54 @@ func generateModinputConfig(config *modularinput.TemplateData, outDir string) er
 	}
 	defer outputFile.Close()
 
-	if err := tmpl.Execute(outputFile, config); err != nil {
+	if err = tmpl.Execute(outputFile, config); err != nil {
 		return fmt.Errorf("failed to execute template: %w", err)
 	}
 
 	log.Printf("Generated code: %s\n", outputPath)
 	return nil
 }
 
-// Given a yaml file specifying the modular inputs and a .tmpl file, generates a new inputs.conf.spec file and inputs.conf file
-func generateTaModInputConfs(config *modularinput.TemplateData, addonSourceDir string, outDir string) error {
-	// handle README/inputs.conf.spec
-	specTemplatePath := filepath.Join(addonSourceDir, "assets", "inputs.conf.spec.tmpl")
-	if _, err := os.Stat(specTemplatePath); os.IsNotExist(err) {
-		return fmt.Errorf("template file not found: %s", specTemplatePath)
-	}
-	specOutputDir := filepath.Join(outDir, "README")
-	if err := os.MkdirAll(specOutputDir, 0755); err != nil {
-		return fmt.Errorf("failed to create output directory: %w", err)
-	}
-	specOutputPath := filepath.Join(specOutputDir, "inputs.conf.spec")
-	if err := modularinput.RenderTemplate(specTemplatePath, specOutputPath, config); err != nil {
-		return err
-	}
-	log.Printf("Generated: %s\n", specOutputPath)
-
-	// Handle default/inputs.conf
-	defaultConfTemplatePath := filepath.Join(addonSourceDir, "assets", "inputs.conf.tmpl")
-	if _, err := os.Stat(defaultConfTemplatePath); os.IsNotExist(err) {
-		return fmt.Errorf("template file not found: %s", defaultConfTemplatePath)
+// Given a yaml file specifying the modular inputs, renders all templates (.tmpl) under assets directory to same relative
+// file tree in addon
+func generateTaModInputConfs(config *modularinput.TemplateData, addonSourceDir string, buildDir string) error {
+	assetsDir := filepath.Join(addonSourceDir, "assets")
+	templateSuffix := ".tmpl"
+	outDir := filepath.Join(buildDir, config.SchemaName)
+	err := os.MkdirAll(outDir, 0755)
+	if err != nil {
+		return fmt.Errorf("failed create output dir %s: %w", outDir, err)
 	}
-	defaultConfOutputDir := filepath.Join(outDir, "default")
-	if err := os.MkdirAll(defaultConfOutputDir, 0755); err != nil {
-		return fmt.Errorf("failed to create output directory: %w", err)
+	err = os.CopyFS(outDir, os.DirFS(assetsDir))
+	if err != nil {
+		return fmt.Errorf("failed to copy from %s to %s: %w", assetsDir, outDir, err)
 	}
-	defaultConfOutputPath := filepath.Join(defaultConfOutputDir, "inputs.conf")
-	if err := modularinput.RenderTemplate(defaultConfTemplatePath, defaultConfOutputPath, config); err != nil {
-		return err
+	err = filepath.WalkDir(outDir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if !d.IsDir() && strings.HasSuffix(d.Name(), templateSuffix) {
+			// Render new file to same path, chopping off the .tmpl suffix
+			relPath, err := filepath.Rel(outDir, path)
+			if err != nil {
+				return err
+			}
+			outputPath := filepath.Join(outDir, filepath.Dir(relPath), d.Name()[:len(d.Name())-len(templateSuffix)])
+			if err = modularinput.RenderTemplate(path, outputPath, config); err != nil {
+				return fmt.Errorf("could not render template: %w", err)
+			}
+			log.Printf("Generated template: %s\n", outputPath)
+			err = os.Remove(path)
+			if err != nil {
+				return err
+			}
+			log.Printf("Removed template: %s\n", path)
+		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("failed to render contents from %s to %s: %w", assetsDir, outDir, err)
 	}
-	log.Printf("Generated: %s\n", defaultConfOutputPath)
 
 	return nil
 }

packaging/technical-addon/cmd/modinput_config_generator/generator_test.go

@@ -18,30 +18,32 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"fmt"
 	"io"
-	"os"
 	"path/filepath"
-	"strings"
 	"testing"
 	"time"
 
-	"github.com/docker/docker/api/types/container"
-	"github.com/docker/docker/api/types/mount"
-	"github.com/google/go-cmp/cmp"
 	"github.com/splunk/splunk-technical-addon/internal/packaging"
 	"github.com/splunk/splunk-technical-addon/internal/testcommon"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"github.com/testcontainers/testcontainers-go"
 	"github.com/testcontainers/testcontainers-go/wait"
-	"go.uber.org/zap"
 )
 
 type ExampleOutput struct {
+	Flags   []string
+	EnvVars []string
+
+	SplunkHome   string
+	TaHome       string
+	PlatformHome string
+
+	EverythingSet              string
+	MinimalSet                 string
+	MinimalSetRequired         string
+	UnaryFlagWithEverythingSet string
+
 	Platform string
-	Flags    []string
-	EnvVars  []string
 }
 
 func TestPascalization(t *testing.T) {
@@ -78,11 +80,14 @@ func TestRunner(t *testing.T) {
 	ctx := context.Background()
 	addonPath := filepath.Join(t.TempDir(), "Sample_Addon.tgz")
 
-	buildDir := testcommon.GetBuildDir()
+	buildDir := packaging.GetBuildDir()
 	require.NotEmpty(t, buildDir)
 	err := packaging.PackageAddon(filepath.Join(buildDir, "Sample_Addon"), addonPath)
 	require.NoError(t, err)
-	tc := startSplunk(t, addonPath)
+	tc := testcommon.StartSplunk(t, testcommon.SplunkStartOpts{
+		AddonPaths:   []string{addonPath},
+		WaitStrategy: wait.ForExec([]string{"sudo", "stat", "/opt/splunk/var/log/splunk/Sample_Addon.log"}).WithStartupTimeout(time.Minute * 4),
+	})
 
 	// Check Schema
 	code, output, err := tc.Exec(ctx, []string{"sudo", "/opt/splunk/bin/splunk", "btool", "check", "--debug"})
@@ -98,10 +103,12 @@ func TestRunner(t *testing.T) {
 	require.NoError(t, err)
 	read, err = io.ReadAll(output)
 	assert.NoError(t, err)
-	expectedJSON := `{"Flags":["--test-flag","$SPLUNK_OTEL_TA_HOME/local/access_token","--test-flag"],"EnvVars":["EVERYTHING_SET=$SPLUNK_OTEL_TA_HOME/local/access_token","UNARY_FLAG_WITH_EVERYTHING_SET=$SPLUNK_OTEL_TA_HOME/local/access_token"],"Platform":"linux"}`
+	expectedJSON := `{"Flags":["--test-flag","/opt/splunk/etc/apps/Sample_Addon/local/access_token","--test-flag"],"EnvVars":["EVERYTHING_SET=/opt/splunk/etc/apps/Sample_Addon/local/access_token","UNARY_FLAG_WITH_EVERYTHING_SET=/opt/splunk/etc/apps/Sample_Addon/local/access_token"], "SplunkHome":"/opt/splunk/etc", "TaHome":"/opt/splunk/etc/apps/Sample_Addon", "PlatformHome":"/opt/splunk/etc/apps/Sample_Addon/linux_x86_64", "EverythingSet":"/opt/splunk/etc/apps/Sample_Addon/local/access_token", "MinimalSet":"", "MinimalSetRequired":"", "UnaryFlagWithEverythingSet":"/opt/splunk/etc/apps/Sample_Addon/local/access_token","Platform":"linux"}`
 	i := bytes.Index(read, []byte("Sample output:"))
 	unmarshalled := &ExampleOutput{}
-	require.NoError(t, json.Unmarshal(read[i+len("Sample output:"):], unmarshalled))
+	dec := json.NewDecoder(bytes.NewReader(read[i+len("Sample output:"):]))
+	dec.DisallowUnknownFields()
+	require.NoError(t, dec.Decode(unmarshalled))
 	expected := &ExampleOutput{}
 	require.NoError(t, json.Unmarshal([]byte(expectedJSON), expected))
 	assert.EqualValues(t, expected, unmarshalled)
@@ -110,7 +117,7 @@ func TestRunner(t *testing.T) {
 }
 
 func TestRunnerConfigGeneration(t *testing.T) {
-	sourceDir, err := testcommon.GetSourceDir()
+	sourceDir, err := packaging.GetSourceDir()
 	require.NoError(t, err)
 	sourceDir = filepath.Join(sourceDir, "cmd", "modinput_config_generator", "internal", "testdata")
 	tests := []struct {
@@ -138,102 +145,33 @@ func TestRunnerConfigGeneration(t *testing.T) {
 }
 
 func TestInputsConfGeneration(t *testing.T) {
-	sourceDir, err := testcommon.GetSourceDir()
+	sourceDir, err := packaging.GetSourceDir()
 	require.NoError(t, err)
 	sourceDir = filepath.Join(sourceDir, "cmd", "modinput_config_generator", "internal", "testdata")
 	tests := []struct {
 		testSchemaName   string
 		sampleYamlPath   string
 		outDir           string
-		sourceDir        string
+		addonSourceDir   string
 		expectedSpecPath string
 		shouldError      bool
 	}{
 		{
 			testSchemaName: "Sample_Addon",
 			outDir:         t.TempDir(),
-			sourceDir:      filepath.Join(sourceDir, "pkg/sample_addon"),
-			sampleYamlPath: filepath.Join(sourceDir, "pkg/sample_addon/runner/modular-inputs.yaml"),
+			addonSourceDir: filepath.Join(sourceDir, "pkg", "sample_addon"),
+			sampleYamlPath: filepath.Join(sourceDir, "pkg", "sample_addon", "runner", "modular-inputs.yaml"),
 		},
 	}
 
 	for _, tc := range tests {
 		t.Run(tc.testSchemaName, func(tt *testing.T) {
 			config, err := loadYaml(tc.sampleYamlPath, tc.testSchemaName)
 			assert.NoError(tt, err)
-			err = generateTaModInputConfs(config, tc.sourceDir, tc.outDir)
+			err = generateTaModInputConfs(config, tc.addonSourceDir, tc.outDir)
 			assert.NoError(tt, err)
-			assertFilesMatch(tt, filepath.Join("internal", "testdata", "pkg", "sample_addon", "expected", "inputs.conf"), filepath.Join(tc.outDir, "default", "inputs.conf"))
-			assertFilesMatch(tt, filepath.Join("internal", "testdata", "pkg", "sample_addon", "expected", "inputs.conf.spec"), filepath.Join(tc.outDir, "README", "inputs.conf.spec"))
+			testcommon.AssertFilesMatch(tt, filepath.Join("internal", "testdata", "pkg", "sample_addon", "expected", "inputs.conf"), filepath.Join(tc.outDir, tc.testSchemaName, "default", "inputs.conf"))
+			testcommon.AssertFilesMatch(tt, filepath.Join("internal", "testdata", "pkg", "sample_addon", "expected", "inputs.conf.spec"), filepath.Join(tc.outDir, tc.testSchemaName, "README", "inputs.conf.spec"))
 		})
 	}
 }
-
-func assertFilesMatch(tt *testing.T, expectedPath string, actualPath string) {
-	require.FileExists(tt, actualPath)
-	require.FileExists(tt, expectedPath)
-	expected, err := os.ReadFile(expectedPath)
-	if err != nil {
-		tt.Fatalf("Failed to read expected file: %v", err)
-	}
-
-	actual, err := os.ReadFile(actualPath)
-	if err != nil {
-		tt.Fatalf("Failed to read actual file: %v", err)
-	}
-
-	if diff := cmp.Diff(string(expected), string(actual)); diff != "" {
-		tt.Errorf("File contents mismatch (-expected +actual)\npaths: (%s, %s):\n%s", expectedPath, actualPath, diff)
-	}
-}
-
-func startSplunk(t *testing.T, taPath string) testcontainers.Container {
-	logger, err := zap.NewProduction()
-	if err != nil {
-		panic(err)
-	}
-	conContext := context.Background()
-	addonLocation := fmt.Sprintf("/tmp/local-tas/%v", filepath.Base(taPath))
-
-	req := testcontainers.ContainerRequest{
-		Image: "splunk/splunk:9.1.2",
-		HostConfigModifier: func(c *container.HostConfig) {
-			c.NetworkMode = "host"
-			c.Mounts = append(c.Mounts, mount.Mount{
-				Source: filepath.Dir(taPath),
-				Target: filepath.Dir(addonLocation),
-				Type:   mount.TypeBind,
-			})
-		},
-		Env: map[string]string{
-			"SPLUNK_START_ARGS": "--accept-license",
-			"SPLUNK_PASSWORD":   "Chang3d!",
-			"SPLUNK_APPS_URL":   addonLocation,
-		},
-		WaitingFor: wait.ForAll(
-			wait.NewHTTPStrategy("/en-US/account/login").WithPort("8000"),
-			wait.ForExec([]string{"sudo", "stat", "/opt/splunk/var/log/splunk/Sample_Addon.log"}),
-		).WithDeadline(4*time.Minute + 20*time.Second).WithStartupTimeoutDefault(4 * time.Minute),
-		LogConsumerCfg: &testcontainers.LogConsumerConfig{
-			Consumers: []testcontainers.LogConsumer{&testLogConsumer{t: t}},
-		},
-	}
-
-	tc, err := testcontainers.GenericContainer(conContext, testcontainers.GenericContainerRequest{
-		ContainerRequest: req,
-		Started:          true,
-	})
-	if err != nil {
-		logger.Info("Error while creating container")
-		panic(err)
-	}
-	return tc
-}
-
-type testLogConsumer struct {
-	t *testing.T
-}
-
-func (l *testLogConsumer) Accept(log testcontainers.Log) {
-	l.t.Log(log.LogType + ": " + strings.TrimSpace(string(log.Content)))
-}

packaging/technical-addon/cmd/modinput_config_generator/internal/testdata/pkg/sample_addon/README.md

@@ -0,0 +1 @@
+This addon is used both as an example for how to author addons, and additionally is used as a test for our framework of doing such.

packaging/technical-addon/cmd/modinput_config_generator/internal/testdata/pkg/sample_addon/assets/inputs.conf.spec.tmpl renamed to packaging/technical-addon/cmd/modinput_config_generator/internal/testdata/pkg/sample_addon/assets/README/inputs.conf.spec.tmpl

@@ -0,0 +1,28 @@
+######################################################
+#
+# {{ .SchemaName }}
+#
+# Copyright (C) 2005-2020 Splunk Inc. All Rights Reserved.
+#
+######################################################
+
+[install]
+state = enabled
+is_configured = false
+build = 0
+
+[ui]
+is_visible = false
+label = Sample Addon for testing
+
+[launcher]
+author = Splunk, Inc.
+description = Use this add on as a reference/example for creating your own and to test the generation logic
+version = {{ .Version }}
+
+[package]
+id = {{ .SchemaName }}
+
+[id]
+name = {{ .SchemaName }}
+version = {{ .Version }}