Merge pull request #7 from g-n-a-d/sharding

add_sharding

Author: ntk148v

ansible/roles/prometheus/defaults/main.yml

@@ -40,6 +40,9 @@ prometheus_web_config:
   http_server_config: {}
   basic_auth_users: {}
 
+# Prometheus customized arguments
+prometheus_enable_target_sharding: false
+
 # Configuration file options
 # --------------------------
 prometheus_global:

ansible/roles/prometheus/library/inject_sharding_config.py

@@ -0,0 +1,64 @@
+from ansible.module_utils.basic import AnsibleModule
+import yaml
+import json
+import os
+
+def load_config_file(path):
+    _, ext = os.path.splitext(path)
+    if ext in ['.yaml', '.yml']:
+        with open(path, 'r') as f:
+            return yaml.safe_load(f)
+    elif ext == '.json':
+        with open(path, 'r') as f:
+            return json.load(f)
+
+def save_config_file(data, path):
+    _, ext = os.path.splitext(path)
+    if ext in ['.yaml', '.yml']:
+        with open(path, 'w') as f:
+            yaml.dump(data, f, default_flow_style=False)
+    elif ext == '.json':
+        with open(path, 'w') as f:
+            json.dump(data, f, indent=2, ensure_ascii=False)
+
+def add_sharding(config, modulus, hash_value):
+    for job in config.get('scrape_configs', []):
+        job['relabel_configs'] = [
+            {
+                'source_labels': ['__address__'],
+                'modulus': modulus,
+                'target_label': '__tmp_hash__',
+                'action': 'hashmod'
+            },
+            {
+                'source_labels': ['__tmp_hash__'],
+                'regex': hash_value,
+                'action': 'keep'
+            }
+        ]
+    return config
+
+def main():
+    module = AnsibleModule(
+        argument_spec=dict(
+            source=dict(type='str', required=True),
+            modulus=dict(type='int', required=True),
+            hash_value=dict(type='int', required=True)
+        )
+    )
+
+    source = module.params['source']
+    modulus = module.params['modulus']
+    hash_value = module.params['hash_value']
+
+    config = load_config_file(source)
+    if not isinstance(config, dict):
+        module.exit_json(changed=False)
+        return
+    sharded_config = add_sharding(config, modulus, hash_value)
+    save_config_file(sharded_config, source)
+
+    module.exit_json(changed=True)
+
+if __name__ == '__main__':
+    main()

ansible/roles/prometheus/tasks/config.yml

@@ -85,8 +85,44 @@
     - "{{ ansitheus_custom_config_dir }}/prometheus/{{ inventory_hostname }}/prometheus.yml"
     - "{{ ansitheus_custom_config_dir }}/prometheus/prometheus.yml"
     - "{{ role_path }}/templates/prometheus.yml.j2"
+
+- name: Shard target configurations
+  inject_sharding_config:
+    source: "{{ prometheus_config_dir }}/prometheus.yml"
+    modulus: "{{ groups['prometheus'] | length }}"
+    hash_value: "{{ groups['prometheus'].index(inventory_hostname) }}"
+  when: prometheus_enable_target_sharding | bool
+
+- name: Find file_sd configurations on prometheus hosts
+  find:
+    paths: "{{ prometheus_config_dir }}/file_sd"
+    patterns: "*.yml,*.yaml,*.json"
+    use_regex: false
+  register: host_file_sd
+  when:
+    - prometheus_enable_target_sharding | bool
+    - prometheus_file_sd is defined
+    - prometheus_file_sd.files | length > 0
+
+- name: Shard target configurations file_sd
+  inject_sharding_config:
+    source: "{{ item.path }}"
+    modulus: "{{ groups['prometheus'] | length }}"
+    hash_value: "{{ groups['prometheus'].index(inventory_hostname) }}"
+  loop: "{{ host_file_sd.files }}"
+  when:
+    - prometheus_enable_target_sharding | bool
+    - host_file_sd is defined
+    - host_file_sd.files | length > 0
+
+- name: Validate prometheus config
+  meta: noop
   notify:
     - Validate prometheus config
+
+- name: Reload prometheus config
+  meta: noop
+  notify:
     - Reload prometheus config
 
 - name: Check prometheus containers

docs/target_sharding_for_prometheus.md

@@ -0,0 +1,49 @@
+# Target sharding for multi-node promnetheus deployment
+
+## 1. Hashmod
+
+`hashmod` is a built-in metrics filtering mechanism of Prometheus. In a multi-node Prometheus setup, we can use `hashmod` to distribute the target endpoints accross Promwetheus nodes, helping to reduce the workload of Prometheus instance on every nodes.
+
+### 1.1 How it works
+
+Each metric scraped by Prometheus has its own labels and values, `hashmod` uses these values to compute a `hash_value` for each metric given a defined `modulus` as described in [Hashing and Sharding on Label Values](https://training.promlabs.com/training/relabeling/writing-relabeling-rules/hashing-and-sharding-on-label-values/).
+
+`hasmod` can be configured inside `relabel_configs` field in Prometheus configuration file `prometheus.yml` or service discovery files, depending on specific setup.
+
+Example configuration:
+```yaml
+relabel_configs:
+- action: hashmod
+  modulus: 2
+  source_labels:
+  - __address__
+  target_label: __tmp_hash__
+```
+
+The `hash_value` is assigned to the `__tmp_hash__` label after performing hashmod on the value of `__address__` label.
+
+### 1.2 How we distribute targets
+
+During Ansible execution, the number of Prometheus nodes and the index of each node is determined based on the inventory provided. We set `modulus = <number-of-prometheus-nodes>` and `regex = <index-of-current-node>` to assign targets to a Prometheus node only when the `hash_value` of that target matches the index of that node.
+
+```yaml
+relabel_configs:
+- action: hashmod
+  modulus: <number-of-prometheus-nodes>
+  source_labels:
+  - __address__
+  target_label: __tmp_hash__
+- action: keep
+  regex: <index-of-current-node>
+  source_labels:
+  - __tmp_hash__
+```
+
+**Notes**  
+We use *special insternal label* `__address__` and `__tmp_hash__`, which cause Prometheus to filter all metrics belonging to a target at once, as these labels are assigned to all metrics scraped from that target. Therefore, this allow filtering before scraping. The targets whose metrics are to be dropped will not be scraped.
+
+## 2. Setup
+
+> Sharding should only be enabled if Prometheus nodes are running in remote-write mode. It is incompatible with Prometheus running in HA mode, as metrics are distributed across Prometheus instances, and the master Prometheus node may not always contain the desired metrics.
+
+To enable sharding, set variable `prometheus_enable_target_sharding: true` in ansible configuration file. 

etc/ansitheus/config.yml

@@ -42,6 +42,12 @@ enable_fluentd: "no"
 enable_openstack_exporter: "no"
 enable_nginx_exporter: "no"
 
+##################
+# Prometheus options
+##################
+
+prometheus_enable_target_sharding: true
+
 ##################
 # Port mappings
 #################