Merge pull request #806 from splunk/develop

Changes for UDS and 9.0.9 release

Author: adityapinglesf

.gitignore

@@ -23,3 +23,4 @@ test*.xml
 **/*.xml
 .pytest_cache
 version.txt
+env

default.yml

@@ -0,0 +1,78 @@
+hide_password: false
+retry_delay: 3
+retry_num: 60
+wait_for_splunk_retry_num: 60
+shc_sync_retry_num: 60
+splunk_home_ownership_enforcement: true
+
+config:
+  baked: default.yml
+  defaults_dir: /tmp/defaults
+  env:
+    headers: null
+    var: SPLUNK_DEFAULTS_URL
+    verify: true
+  host:
+    headers: null
+    url: null
+    verify: true
+  max_delay: 60
+  max_retries: 3
+  max_timeout: 1200
+
+splunk:
+  role: splunk_standalone
+  enable_tcp_mode: false
+  upgrade: false
+  build_location: /tmp/splunk.tgz
+  apps_location: null
+  license_uri: null
+  admin_user: admin
+  app_paths:
+    default: /opt/splunk/etc/apps
+    deployment: /opt/splunk/etc/deployment-apps
+    httpinput: /opt/splunk/etc/apps/splunk_httpinput
+    idxc: /opt/splunk/etc/master-apps
+    shc: /opt/splunk/etc/shcluster/apps
+  enable_service: false
+  exec: /opt/splunk/bin/splunk
+  group: splunk
+  hec:
+    enable: True
+    ssl: True
+    port: 8088
+    token: 4a8a737d-5452-426c-a6f7-106dca4e813f
+  home: /opt/splunk
+  http_enableSSL: 0
+  http_enableSSL_cert: null
+  http_enableSSL_privKey: null
+  http_enableSSL_privKey_password: null
+  http_port: 8000
+  idxc:
+    enable: false
+    label: idxc_label
+    replication_factor: 3
+    replication_port: 9887
+    search_factor: 3
+    secret: dmwHG97SpM+GzeGPUELwr7xXowSAVmLW
+  ignore_license: false
+  license_download_dest: /tmp/splunk.lic
+  opt: /opt
+  password: helloworld
+  pid: /opt/splunk/var/run/splunk/splunkd.pid
+  s2s_enable: true
+  s2s_port: 9997
+  search_head_captain_url: null
+  secret: null
+  splunk_http_enabled: True
+  shc:
+    enable: false
+    label: shc_label
+    replication_factor: 3
+    replication_port: 9887
+    secret: EpcUlTUHMSOhdjRZb3QqPYf9Lf7L991c
+  smartstore: null
+  svc_port: 8089
+  tar_dir: splunk
+  user: splunk
+  wildcard_license: false

docs/CHANGELOG.md

@@ -8,6 +8,7 @@
 * [9.1.1](#911)
 * [9.1.0.2](#9102)
 * [9.1.0.1](#9101)
+* [9.0.9](#909)
 * [9.0.8](#908)
 * [9.0.7](#907)
 * [9.0.6](#906)
@@ -140,6 +141,14 @@
 
 ---
 
+## 9.0.9
+
+#### Changes
+* Support for latest major Splunk release
+* Documentation updates 
+
+---
+
 ## 9.0.8
 
 #### Changes

inventory/environ.py

@@ -108,6 +108,7 @@ def getDefaultVars():
     environment variables to return a consolidated inventory object
     """
     defaultVars = loadDefaults()
+    defaultVars["splunk"]["role"] = os.environ.get('SPLUNK_ROLE', defaultVars["splunk"].get("role") or "splunk_standalone")
     overrideEnvironmentVars(defaultVars)
     getAnsibleContext(defaultVars)
     getASan(defaultVars)
@@ -124,7 +125,6 @@ def getDefaultVars():
     getSplunkdSSL(defaultVars)
     getDistributedTopology(defaultVars)
     getLicenses(defaultVars)
-    defaultVars["splunk"]["role"] = os.environ.get('SPLUNK_ROLE', defaultVars["splunk"].get("role") or "splunk_standalone")
     # Determine DMC settings
     defaultVars["dmc_forwarder_monitoring"] = os.environ.get('DMC_FORWARDER_MONITORING', False)
     defaultVars["dmc_asset_interval"] = os.environ.get('DMC_ASSET_INTERVAL', '3,18,33,48 * * * *')
@@ -133,7 +133,6 @@ def getDefaultVars():
     if os.environ.get("SPLUNK_HOME_OWNERSHIP_ENFORCEMENT", "").lower() == "false":
         defaultVars["splunk_home_ownership_enforcement"] = False
     # Determine password visibility
-    defaultVars["hide_password"] = False
     if os.environ.get("HIDE_PASSWORD", "").lower() == "true":
         defaultVars["hide_password"] = True
     # Determine SHC preferred captaincy
@@ -145,6 +144,7 @@ def getDefaultVars():
     getJava(defaultVars)
     getSplunkBuild(defaultVars)
     getSplunkbaseToken(defaultVars)
+    getSplunkBuildAuth(defaultVars)
     getSplunkApps(defaultVars)
     getSplunkAppsLocal(defaultVars)
     getLaunchConf(defaultVars)
@@ -357,6 +357,13 @@ def getSplunkbaseToken(vars_scope):
         splunkbase_token = re.search("<id>(.*)</id>", output, re.IGNORECASE)
         vars_scope["splunkbase_token"] = splunkbase_token.group(1) if splunkbase_token else None
 
+def getSplunkBuildAuth(vars_scope):
+    """
+    Load username and password to be used in basic auth when fetching splunk build or apps
+    """
+    vars_scope["splunk"]["artifact_auth_user"] = os.environ.get("ARTIFACTORY_USER", vars_scope["splunk"].get("artifact_auth_user"))
+    vars_scope["splunk"]["artifact_auth_pass"] = os.environ.get("ARTIFACTORY_TOKEN", vars_scope["splunk"].get("artifact_auth_pass"))
+
 def getSplunkApps(vars_scope):
     """
     Determine the set of Splunk apps to install as union of defaults.yml and environment variables
@@ -566,6 +573,7 @@ def overrideEnvironmentVars(vars_scope):
     vars_scope["cert_prefix"] = os.environ.get("SPLUNK_CERT_PREFIX", vars_scope.get("cert_prefix", "https"))
     vars_scope["splunk"]["root_endpoint"] = os.environ.get('SPLUNK_ROOT_ENDPOINT', vars_scope["splunk"]["root_endpoint"])
     vars_scope["splunk"]["svc_port"] = os.environ.get('SPLUNK_SVC_PORT', vars_scope["splunk"]["svc_port"])
+    vars_scope["splunk"]["splunk_http_enabled"] = os.environ.get('ENABLE_TCP_MODE', vars_scope["splunk"]["enable_tcp_mode"])
     vars_scope["splunk"]["s2s"]["port"] = int(os.environ.get('SPLUNK_S2S_PORT', vars_scope["splunk"]["s2s"]["port"]))
     vars_scope["splunk"]["enable_service"] = os.environ.get('SPLUNK_ENABLE_SERVICE', vars_scope["splunk"]["enable_service"])
     vars_scope["splunk"]["service_name"] = os.environ.get('SPLUNK_SERVICE_NAME', vars_scope["splunk"]["service_name"])
@@ -574,6 +582,8 @@ def overrideEnvironmentVars(vars_scope):
     vars_scope["splunk"]["kvstore"]["port"] = os.environ.get('SPLUNK_KVSTORE_PORT', vars_scope["splunk"]["kvstore"]["port"])
     vars_scope["splunk"]["connection_timeout"] = int(os.environ.get('SPLUNK_CONNECTION_TIMEOUT', vars_scope["splunk"]["connection_timeout"]))
 
+    if vars_scope["splunk"]["splunk_http_enabled"] == "false" and "forwarder" not in vars_scope["splunk"]["role"].lower():
+        vars_scope["splunk"]["splunk_http_enabled"] = "true"
     # Set set_search_peers to False to disable peering to indexers when creating multisite topology
     if os.environ.get("SPLUNK_SET_SEARCH_PEERS", "").lower() == "false":
         vars_scope["splunk"]["set_search_peers"] = False

inventory/splunk_defaults_linux.yml

@@ -4,6 +4,7 @@ ansible_post_tasks:
 ansible_environment: {}
 retry_delay: 6
 retry_num: 60
+hide_password: false
 wait_for_splunk_retry_num: 60
 shc_sync_retry_num: 60
 
@@ -23,6 +24,7 @@ config:
         verify: True
 
 splunk:
+    enable_tcp_mode: False
     build_location:
     allow_upgrade: True
     tar_dir: "splunk"
@@ -31,6 +33,7 @@ splunk:
     user: "splunk"
     group: "splunk"
     exec: !!python/object/apply:os.path.join [*home, "bin", "splunk"]
+    uds_socket_path_url: "%2Fopt%2Fsplunkforwarder%2Fvar%2Frun%2Fsplunk%2Fcli.socket"
     pid: !!python/object/apply:os.path.join [*home, "var", "run", "splunk", "splunkd.pid"]
     admin_user: "admin"
     root_endpoint:

inventory/splunk_defaults_windows.yml

@@ -4,6 +4,7 @@ ansible_post_tasks:
 ansible_environment: {}
 retry_delay: 10
 retry_num: 60
+hide_password: false
 wait_for_splunk_retry_num: 150
 shc_sync_retry_num: 300
 
@@ -23,6 +24,7 @@ config:
         verify: True
 
 splunk:
+    enable_tcp_mode: False
     build_location:
     allow_upgrade: True
     tar_dir: "splunk"

inventory/splunkforwarder_defaults_linux.yml

@@ -4,6 +4,7 @@ ansible_post_tasks:
 ansible_environment: {}
 retry_delay: 6
 retry_num: 60
+hide_password: false
 wait_for_splunk_retry_num: 60
 shc_sync_retry_num: 60
 
@@ -24,6 +25,7 @@ config:
 
 splunk:
     build_location:
+    enable_tcp_mode: False
     allow_upgrade: True
     tar_dir: "splunkforwarder"
     opt: &opt "/opt"

inventory/splunkforwarder_defaults_windows.yml

@@ -4,6 +4,7 @@ ansible_post_tasks:
 ansible_environment: {}
 retry_delay: 10
 retry_num: 60
+hide_password: false
 wait_for_splunk_retry_num: 150
 shc_sync_retry_num: 300
 
@@ -24,6 +25,7 @@ config:
 
 splunk:
     build_location:
+    enable_tcp_mode: False
     allow_upgrade: True
     tar_dir: "splunkforwarder"
     opt: &opt "/opt"

library/splunk_api.py

@@ -0,0 +1,147 @@
+#!/usr/bin/python
+
+from ansible.module_utils.basic import AnsibleModule
+import os
+import requests
+import requests_unixsocket
+import json
+
+UDS_SOCKET_PATH = "/opt/splunkforwarder/var/run/splunk/cli.socket"
+UDS_SOCKET_PATH_URL = "%2Fopt%2Fsplunkforwarder%2Fvar%2Frun%2Fsplunk%2Fcli.socket"
+
+def supports_uds():
+    return os.path.exists(UDS_SOCKET_PATH)
+
+def api_call_tcp(cert_prefix, method, endpoint, username, password, svc_port, payload=None, headers=None, verify=False, status_code=None, timeout=None, body_format=None):
+    if not cert_prefix or cert_prefix not in ['http', 'https']:
+      cert_prefix = 'https'
+    if not svc_port:
+      svc_port = 8089
+    url = "{}://127.0.0.1:{}{}".format(cert_prefix, svc_port, endpoint)
+    if headers is None:
+        headers = {}
+    headers['Content-Type'] = 'application/json'
+    auth = (username, password)
+
+    data = None
+    if payload and body_format and body_format == "form-urlencoded":
+      data = payload
+    elif payload:
+      data = json.dumps(payload)
+
+    session = requests.Session()
+    # Disable SSL verification for the session
+    session.verify = False
+
+    response = None
+    excep_str = "No Exception"
+    try:
+      if data is None:
+              response = session.request(method, url, headers=headers, auth=auth, verify=verify, timeout=timeout)
+      else:
+              response = session.request(method, url, headers=headers, auth=auth, data=data, verify=verify, timeout=timeout)
+      if status_code is not None and response.status_code not in status_code:
+          raise ValueError("API call for {} and data as {} failed with status code {}: {}".format(url, payload, response.status_code, response.text))
+    except Exception as e:
+      excep_str = "URL: {}; data: {}, exception: {}".format(url, data, e)
+      cwd = os.getcwd()
+    return response, excep_str
+
+def api_call_uds(method, endpoint, username, password, svc_port, payload=None, headers=None, verify=False, status_code=None, timeout=None, body_format=None):
+    url = "http+unix://{}{}".format(UDS_SOCKET_PATH_URL,endpoint)
+    if headers is None:
+        headers = {}
+    headers['Content-Type'] = 'application/json'
+    auth = (username, password)
+
+    session = requests_unixsocket.Session()
+    # Disable SSL verification for the session
+    session.verify = False
+
+    data = None
+    if payload and body_format and body_format == "form-urlencoded":
+      data = payload
+    elif payload:
+      data = json.dumps(payload)
+
+    excep_str = "No Exception"
+    response = None
+    try:
+      if data is None:
+              response = session.request(method, url, headers=headers, auth=auth, verify=verify, timeout=timeout)
+      else:
+              response = session.request(method, url, headers=headers, auth=auth, data=data, verify=verify, timeout=timeout)
+      if status_code is not None and response.status_code not in status_code:
+        raise ValueError("API call for {} and data as {} failed with status code {}: {}".format(url, payload, response.status_code, response.text))
+    except Exception as e:
+      excep_str = "{}".format(e)
+    return response, excep_str
+
+def main():
+    module_args = dict(
+        method=dict(type='str', required=True),
+        url=dict(type='str', required=True),
+        username=dict(type='str', required=True),
+        password=dict(type='str', required=True, no_log=True),
+        cert_prefix=dict(type='str', required=False),
+        body=dict(type='dict', required=False),
+        body_format=dict(type='str', required=False),
+        headers=dict(type='dict', required=False),
+        verify=dict(type='bool', required=False),
+        return_content=dict(type='bool', required=False),
+        use_proxy=dict(type='str', required=False),
+        status_code=dict(type='list', required=False),
+        timeout=dict(type='int', required=False),
+        svc_port=dict(type='int', required=False)
+    )
+
+    module = AnsibleModule(
+        argument_spec=module_args,
+        supports_check_mode=True
+    )
+
+    if module.check_mode:
+        module.exit_json(changed=False)
+
+    method = module.params['method']
+    endpoint = module.params['url']
+    username = module.params['username']
+    password = module.params['password']
+    cert_prefix = module.params.get('cert_prefix', 'http')
+    body_format = module.params.get('body_format', None)
+    payload = module.params.get('body', None)
+    headers = module.params.get('headers', None)
+    verify = module.params.get('verify', False)
+    status_code = module.params.get('status_code', None)
+    timeout = module.params.get('timeout', None)
+    svc_port = module.params.get('svc_port', 8089)
+    return_content = module.params.get('return_content', False)
+    use_proxy = module.params.get('use_proxy', "no")
+
+    if status_code:
+      status_code = [int(x) for x in status_code]
+    if not use_proxy:
+      use_proxy = 'no'
+    if not return_content:
+      return_content = False
+
+    s = "{}{}{}{}{}{}{}{}{}".format(method, endpoint, username, password, svc_port, payload, headers, verify, status_code, timeout)
+    if supports_uds():
+        response, excep_str = api_call_uds(method, endpoint, username, password, svc_port, payload, headers, verify, status_code, timeout, body_format)
+    else:
+        response, excep_str = api_call_tcp(cert_prefix, method, endpoint, username, password, svc_port, payload, headers, verify, status_code, timeout, body_format)
+
+    if response is not None and ((status_code and response.status_code in status_code) or (status_code is None and response.status_code >= 200 and response.status_code < 300)):
+        try:
+          content = response.json()
+        except:
+          content = response.text
+        module.exit_json(changed=True, status = response.status_code, json=content,excep_str=excep_str)
+    else:
+        if response is None:
+          module.fail_json(msg="{};;; failed with NO RESPONSE and EXCEP_STR as {}".format(s, excep_str))
+        else:
+          module.fail_json(msg="{};;; AND excep_str: {}, failed with status code {}: {}".format(s, excep_str, response.status_code, response.text))
+
+if __name__ == '__main__':
+    main()