Merge pull request #625 from splunk/develop

alishamayor · web-flow · commit e41cf20e6c7a · 2021-05-11T15:37:11.000-07:00
Release/8.2.0
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -8,27 +8,6 @@ executors:
 
 jobs:
   splunk-ansible-test:
-    executor: circleci_xlarge
-    steps:
-      - checkout
-      - run:
-          name: Run linting
-          command: make lint
-      - run:
-          name: Run py3k linting
-          command: make py3k-lint
-      - run:
-          name: Run small tests
-          command: make small-tests
-      - run:
-          name: Run large (molecule) tests
-          command: make large-tests
-      - store_artifacts:
-          path: tests/results
-          destination: tests/results
-      - store_test_results:
-          path: tests/results
-  py3k-splunk-ansible-test:
     executor: circleci_xlarge
     steps:
       - checkout
@@ -57,4 +36,3 @@ workflows:
   run_tests:
     jobs:
       - splunk-ansible-test
-      - py3k-splunk-ansible-test
diff --git a/docs/ADVANCED.md b/docs/ADVANCED.md
@@ -124,6 +124,7 @@ Splunk-Ansible ships with an inventory script in `inventory/environ.py`. The scr
 | SPLUNK_DEFAULTS_HTTP_MAX_TIMEOUT | When fetching a remote `default.yml`, specify the timeout of the request | no | no | no |
 | SPLUNK_DEFAULTS_HTTP_MAX_RETRIES | When fetching a remote `default.yml`, the number of retries to make. For unlimited retries, use `-1` | no | no | no |
 | SPLUNK_DEFAULTS_HTTP_MAX_DELAY | When fetching a remote `default.yml`, specify the delay between each retry | no | no | no |
+| SPLUNK_DEFAULTS_HTTP_AUTH_HEADER | When fetching a remote `default.yml`, set the `Authorization` header | no | no | no |
 | SPLUNK_ANSIBLE_PRE_TASKS | Pass in a comma-separated list of local paths or remote URLs to Ansible playbooks that will be executed before `site.yml`. Must include the protocol, i.e. it must match the regex `^(http\|https\|file)://.*` | no | no | no |
 | SPLUNK_ANSIBLE_POST_TASKS | Pass in a comma-separated list of local paths or remote URLs to Ansible playbooks that will be executed after `site.yml`. Must include the protocol, i.e. it must match the regex `^(http\|https\|file)://.*` | no | no | no |
 | SPLUNK_ANSIBLE_ENV | Pass in a comma-separated list of "key=value" pairs that will be mapped to environment variables used during `site.yml` execution. These variables are also available in ansible pre/post playbooks and can be referenced as `hostvars['localhost'].ansible_environment['key']` | no | no | no |
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
@@ -2,11 +2,14 @@
 
 ## Navigation
 
+* [8.2.0](#820)
+* [8.1.4](#814)
 * [8.1.3](#813)
 * [8.1.2](#812)
 * [8.1.1](#811)
 * [8.1.0.1](#8101)
 * [8.1.0](#810)
+* [8.0.9](#809)
 * [8.0.8](#808)
 * [8.0.7](#807)
 * [8.0.6.1](#8061)
@@ -47,6 +50,28 @@
 
 ---
 
+## 8.2.0
+
+#### What's New?
+* Added support for setting `clientName` in `deploymentclient.conf`
+    * `splunk.deployment_client.name` in `default.yml`
+    * `SPLUNK_DEPLOYMENT_CLIENT_NAME` environment variable
+
+#### Changes
+* Bugfixes
+
+---
+
+## 8.1.4
+
+#### What's New?
+Syncing with latest codebase - currently up to sync with 8.2.0.
+
+#### Changes
+* See [8.2.0](#820) changes above.
+
+---
+
 ## 8.1.3
 
 #### Changes
@@ -91,6 +116,16 @@
 
 ---
 
+## 8.0.9
+
+#### What's New?
+Syncing with latest codebase - currently up to sync with 8.2.0.
+
+#### Changes
+* See [8.2.0](#820) changes above.
+
+---
+
 ## 8.0.8
 
 #### What's New?
diff --git a/docs/advanced/default.yml.spec.md b/docs/advanced/default.yml.spec.md
@@ -173,6 +173,14 @@ splunk:
   * Hostname of Splunk Enterprise deployer instance. May be overridden using SPLUNK_DEPLOYER_URL environment variable.
   * Default: null
 
+  deployment_client: <dict>
+  * Deployment client object that configures `deployment-client` stanza of `deploymentclient.conf` file.
+  * Default: null
+
+    name: null
+    * Client name for deployment client. May be overridden using SPLUNK_DEPLOYMENT_CLIENT_NAME environment variable.
+    * Default: null
+
   search_head_captain_url: null
   * Hostname of Splunk Enterprise search head cluster captain instance. May be overridden using SPLUNK_SEARCH_HEAD_CAPTAIN_URL environment variable.
   * Default: null
diff --git a/inventory/environ.py b/inventory/environ.py
@@ -558,6 +558,10 @@ def getUFSplunkVariables(vars_scope):
         vars_scope["splunk"]["before_start_cmd"] = os.environ.get("SPLUNK_BEFORE_START_CMD").split(",")
     if os.environ.get("SPLUNK_CMD"):
         vars_scope["splunk"]["cmd"] = os.environ.get("SPLUNK_CMD").split(",")
+    if os.environ.get("SPLUNK_DEPLOYMENT_CLIENT_NAME"):
+        vars_scope["splunk"]["deployment_client"] = {
+            "name": os.environ.get("SPLUNK_DEPLOYMENT_CLIENT_NAME")
+        }
 
 def getRandomString():
     """
@@ -636,9 +640,14 @@ def mergeDefaultsFromURL(vars_scope, url, headers=None, verify=False):
     """
     if not url:
         return vars_scope
+    if not headers:
+        headers = {}
     max_retries = int(os.environ.get("SPLUNK_DEFAULTS_HTTP_MAX_RETRIES", vars_scope["config"].get("max_retries")))
     max_delay = int(os.environ.get("SPLUNK_DEFAULTS_HTTP_MAX_DELAY", vars_scope["config"].get("max_delay")))
     max_timeout = int(os.environ.get("SPLUNK_DEFAULTS_HTTP_MAX_TIMEOUT", vars_scope["config"].get("max_timeout")))
+    auth = os.environ.get("SPLUNK_DEFAULTS_HTTP_AUTH_HEADER")
+    if auth:
+        headers["Authorization"] = auth
     unlimited_retries = (max_retries == -1)
     current_retry = 0
     while True:
diff --git a/inventory/splunk_defaults_linux.yml b/inventory/splunk_defaults_linux.yml
@@ -109,6 +109,8 @@ splunk:
     auxiliary_cluster_masters: []
     search_head_captain_url:
     deployer_url:
+    deployment_client:
+        name:
     connection_timeout: 0
     enable_service: False
     disable_popups: False
diff --git a/inventory/splunk_defaults_windows.yml b/inventory/splunk_defaults_windows.yml
@@ -101,6 +101,8 @@ splunk:
     auxiliary_cluster_masters: []
     search_head_captain_url:
     deployer_url:
+    deployment_client:
+        name:
     connection_timeout: 180
     enable_service: False
     disable_popups: False
diff --git a/inventory/splunkforwarder_defaults_linux.yml b/inventory/splunkforwarder_defaults_linux.yml
@@ -101,6 +101,8 @@ splunk:
     auxiliary_cluster_masters: []
     search_head_captain_url:
     deployer_url:
+    deployment_client:
+        name:
     connection_timeout: 0
     enable_service: False
     disable_popups: False
diff --git a/inventory/splunkforwarder_defaults_windows.yml b/inventory/splunkforwarder_defaults_windows.yml
@@ -101,6 +101,8 @@ splunk:
     auxiliary_cluster_masters: []
     search_head_captain_url:
     deployer_url:
+    deployment_client:
+        name:
     connection_timeout: 180
     enable_service: False
     disable_popups: False
diff --git a/roles/splunk_common/tasks/install_splunk_yum.yml b/roles/splunk_common/tasks/install_splunk_yum.yml
@@ -1,5 +1,10 @@
 # added the environment section due to bug:
 # https://github.com/ansible/ansible/issues/18979
+- name: Import Splunk key
+  rpm_key:
+    key: https://docs.splunk.com/images/6/6b/SplunkPGPKey.pub
+    state: present
+
 - name: Install Splunk (yum)
   yum:
     name: "{{ splunk.build_location }}"
diff --git a/roles/splunk_common/tasks/set_as_deployment_client.yml b/roles/splunk_common/tasks/set_as_deployment_client.yml
@@ -19,3 +19,21 @@
     - Restart the splunkd service
   ignore_errors: yes
   no_log: "{{ hide_password }}"
+
+- name: Set deployment client name
+  ini_file:
+    owner: "{{ splunk.user }}"
+    group: "{{ splunk.group }}"
+    dest: "{{ splunk.home }}/etc/system/local/deploymentclient.conf"
+    section: deployment-client
+    option: "clientName"
+    value: "{{ splunk.deployment_client.name }}"
+  become: yes
+  become_user: "{{ splunk.user }}"
+  notify:
+    - Restart the splunkd service
+  ignore_errors: yes
+  no_log: "{{ hide_password }}"
+  when:
+    - splunk.deployment_client is defined and splunk.deployment_client
+    - splunk.deployment_client.name is defined and splunk.deployment_client.name
diff --git a/roles/splunk_common/vars/main.yml b/roles/splunk_common/vars/main.yml
@@ -1,7 +1,7 @@
 ---
-default_apps: [ "legacy", "splunk_gdi", "SplunkForwarder", "SplunkLightForwarder", "gettingstarted",
-                "alert_webhook", "splunk_httpinput", "introspection_generator_addon", "user-prefs",
-                "appsbrowser", "framework", "learned", "launcher", "alert_logevent", "sample_app",
+default_apps: [ "legacy", "splunk_gdi", "SplunkForwarder", "SplunkLightForwarder", "gettingstarted", "splunk-dashboard-studio",
+                "alert_webhook", "splunk_httpinput", "introspection_generator_addon", "user-prefs", "splunk_essentials_8_2",
+                "appsbrowser", "framework", "learned", "launcher", "alert_logevent", "sample_app", "python_upgrade_readiness_app",
                 "splunk_instrumentation", "search", "splunk_archiver", "splunk_monitoring_console", "splunk_rapid_diag",
                 "_splunk_config", "splunk_enterprise_on_docker", "splunk_forwarder_on_docker", "journald_input",
                 "splunk_metrics_workspace", "splunk_internal_metrics", "splunk_telemetry", "splunk_secure_gateway" ]
diff --git a/tests/requirements.txt b/tests/requirements.txt
@@ -4,10 +4,9 @@ pylint
 mock
 docker
 requests
-molecule==3.0.2
+molecule==3.0.8
 coverage
 caniusepython3
-ansible==2.9.7
+ansible==2.10.7
 ansible-lint>=4.2.0
 testinfra
-sh==1.12.14
diff --git a/tests/small/test_environ.py b/tests/small/test_environ.py
@@ -7,6 +7,7 @@
 import os
 import sys
 import pytest
+import requests
 from mock import MagicMock, patch, mock_open
 
 FILE_DIR = os.path.dirname(os.path.realpath(__file__))
@@ -912,28 +913,31 @@ def test_getDFS(default_yml, os_env, output):
     assert type(vars_scope["splunk"]["dfs"]["spark_master_webui_port"]) == int
     assert vars_scope["splunk"]["dfs"] == output
 
-@pytest.mark.parametrize(("os_env", "deployment_server", "add", "before_start_cmd", "cmd"),
+@pytest.mark.parametrize(("os_env", "deployment_server", "deployment_client", "add", "before_start_cmd", "cmd"),
                          [
-                            ({}, None, None, None, None),
+                            ({}, None, None, None, None, None),
                             # Check environment variable parameters
-                            ({"SPLUNK_DEPLOYMENT_SERVER": ""}, None, None, None, None),
-                            ({"SPLUNK_DEPLOYMENT_SERVER": "something"}, "something", None, None, None),
-                            ({"SPLUNK_ADD": ""}, None, None, None, None),
-                            ({"SPLUNK_ADD": "echo 1"}, None, ["echo 1"], None, None),
-                            ({"SPLUNK_ADD": "echo 1,echo 2"}, None, ["echo 1", "echo 2"], None, None),
-                            ({"SPLUNK_BEFORE_START_CMD": ""}, None, None, None, None),
-                            ({"SPLUNK_BEFORE_START_CMD": "echo 1"}, None, None, ["echo 1"], None),
-                            ({"SPLUNK_BEFORE_START_CMD": "echo 1,echo 2"}, None, None, ["echo 1", "echo 2"], None),
-                            ({"SPLUNK_CMD": ""}, None, None, None, None),
-                            ({"SPLUNK_CMD": "echo 1"}, None, None, None, ["echo 1"]),
-                            ({"SPLUNK_CMD": "echo 1,echo 2"}, None, None, None, ["echo 1", "echo 2"]),
+                            ({"SPLUNK_DEPLOYMENT_SERVER": ""}, None, None, None, None, None),
+                            ({"SPLUNK_DEPLOYMENT_SERVER": "something"}, "something", None, None, None, None),
+                            ({"SPLUNK_DEPLOYMENT_CLIENT_NAME": ""}, None, None, None, None, None),
+                            ({"SPLUNK_DEPLOYMENT_CLIENT_NAME": "client_name"}, None, {"name": "client_name"}, None, None, None),
+                            ({"SPLUNK_ADD": ""}, None, None, None, None, None),
+                            ({"SPLUNK_ADD": "echo 1"}, None, None, ["echo 1"], None, None),
+                            ({"SPLUNK_ADD": "echo 1,echo 2"}, None, None, ["echo 1", "echo 2"], None, None),
+                            ({"SPLUNK_BEFORE_START_CMD": ""}, None, None, None, None, None),
+                            ({"SPLUNK_BEFORE_START_CMD": "echo 1"}, None, None, None, ["echo 1"], None),
+                            ({"SPLUNK_BEFORE_START_CMD": "echo 1,echo 2"}, None, None, None, ["echo 1", "echo 2"], None),
+                            ({"SPLUNK_CMD": ""}, None, None, None, None, None),
+                            ({"SPLUNK_CMD": "echo 1"}, None, None, None, None, ["echo 1"]),
+                            ({"SPLUNK_CMD": "echo 1,echo 2"}, None, None, None, None, ["echo 1", "echo 2"]),
                          ]
                         )
-def test_getUFSplunkVariables(os_env, deployment_server, add, before_start_cmd, cmd):
+def test_getUFSplunkVariables(os_env, deployment_server, deployment_client, add, before_start_cmd, cmd):
     vars_scope = {"splunk": {}}
     with patch("os.environ", new=os_env):
         environ.getUFSplunkVariables(vars_scope)
     assert vars_scope["splunk"].get("deployment_server") == deployment_server
+    assert vars_scope["splunk"].get("deployment_client") == deployment_client
     assert vars_scope["splunk"].get("add") == add
     assert vars_scope["splunk"].get("before_start_cmd") == before_start_cmd
     assert vars_scope["splunk"].get("cmd") == cmd
@@ -1059,9 +1063,47 @@ def test_mergeDefaults_url_with_req_params(key):
             else:
                 mock_merge_url.assert_called_with(config, "http://website/default.yml", None, False)
 
-@pytest.mark.skip(reason="TODO")
-def test_mergeDefaultsFromURL():
-    pass
+@pytest.mark.parametrize(("vars_scope", "content", "os_env", "headers", "verify"),
+    [
+        # Check dicts
+        ({"config": {"max_retries": 3, "max_delay": 4, "max_timeout": 5}}, "helloworld", {}, None, False),
+        # Change max_timeout
+        ({"config": {"max_retries": 3, "max_delay": 4, "max_timeout": 11}}, "helloworld", {}, None, False),
+        # Enable verify
+        ({"config": {"max_retries": 3, "max_delay": 4, "max_timeout": 5}}, "helloworld", {}, None, True),
+        # Exercise bytes content
+        ({"config": {"max_retries": 3, "max_delay": 4, "max_timeout": 5}}, b"helloworld", {}, None, False),
+        # Exercise various headers
+        ({"config": {"max_retries": 3, "max_delay": 4, "max_timeout": 5}}, "helloworld", {}, {}, False),
+        ({"config": {"max_retries": 3, "max_delay": 4, "max_timeout": 5}}, "helloworld", {}, {"HELLO": "WORLD"}, False),
+        ({"config": {"max_retries": 3, "max_delay": 4, "max_timeout": 5}}, "helloworld", {}, {"A": "B", "C": "D"}, False),
+        # Exercise OS env vars with headers
+        ({"config": {"max_retries": 3, "max_delay": 4, "max_timeout": 5}}, "helloworld", {"one": "two"}, {}, False),
+        ({"config": {"max_retries": 3, "max_delay": 4, "max_timeout": 5}}, "helloworld", {"SPLUNK_DEFAULTS_HTTP_AUTH_HEADER": "Bearer xyz"}, {"HELLO": "WORLD"}, False),
+    ]
+)
+def test_mergeDefaultsFromURL(vars_scope, content, os_env, headers, verify):
+    # Mock response
+    mock_response = MagicMock()
+    mock_response.content = "helloworld"
+    mock_response.status_code = 200
+    mock_response.raise_for_status = MagicMock()
+    # Invoke function
+    with patch("os.environ", new=os_env):
+        with patch("environ.merge_dict") as mock_merge:
+            with patch("environ.requests.get") as mock_get:
+                mock_get.return_value = mock_response
+                result = environ.mergeDefaultsFromURL(vars_scope, "http://website", headers, verify)
+    # Check headers and parameters send to GET call
+    expected_headers = {}
+    if headers:
+        expected_headers.update(headers)
+    if os_env and "SPLUNK_DEFAULTS_HTTP_AUTH_HEADER" is os_env:
+        expected_headers["Authorization"] = os_env["SPLUNK_DEFAULTS_HTTP_AUTH_HEADER"]
+    mock_get.assert_called_once()
+    mock_get.assert_called_with("http://website", headers=expected_headers, timeout=vars_scope["config"]["max_timeout"], verify=verify)
+    mock_merge.assert_called_once()
+    mock_merge.assert_called_with(vars_scope, "helloworld")
 
 @pytest.mark.parametrize(("file", "file_exists", "merge_called"),
             [
