Merge pull request #136 from vvanholl/es_5.6.0.0

Es 5.6.0.0

Author: vvanholl

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-image: gradle:4.10.2-jdk10
+image: gradle:4.10.2-jdk11
 
 variables:
   GRADLE_OPTS: "-Dorg.gradle.daemon=false"

README.md

@@ -26,6 +26,7 @@ It collects all relevant metrics and makes them available to Prometheus via the
 
 | Elasticsearch  | Plugin         | Release date |
 | -------------- | -------------- | ------------ |
+| 6.5.0          | 6.5.0.0        | Nov 19, 2018 |
 | 6.4.3          | 6.4.3.0        | Nov 07, 2018 |
 | 6.4.2          | 6.4.2.0        | Oct 08, 2018 |
 | 6.4.1          | 6.4.1.0        | Sep 24, 2018 |
@@ -102,7 +103,7 @@ It collects all relevant metrics and makes them available to Prometheus via the
 ## Install
 
 - Since Elasticsearch 6.0.0 :
-    `./bin/elasticsearch-plugin install -b https://distfiles.compuscene.net/elasticsearch/elasticsearch-prometheus-exporter-6.4.3.0.zip`
+    `./bin/elasticsearch-plugin install -b https://distfiles.compuscene.net/elasticsearch/elasticsearch-prometheus-exporter-6.5.0.0.zip`
 
 - On Elasticsearch 5.x.x :
     `./bin/elasticsearch-plugin install -b https://distfiles.compuscene.net/elasticsearch/elasticsearch-prometheus-exporter-5.6.12.0.zip`

build.gradle

@@ -8,7 +8,7 @@ buildscript {
     }
 
     dependencies {
-        classpath "org.elasticsearch.gradle:build-tools:6.4.3" // How can we use ${versions.elasticsearch} here ???
+        classpath "org.elasticsearch.gradle:build-tools:6.5.0" // How can we use ${versions.elasticsearch} here ???
         classpath group: 'javax.xml.bind', name: 'jaxb-api', version: '2.3.0'
         classpath group: 'com.sun.xml.bind', name: 'jaxb-core', version: '2.3.0'
         classpath group: 'com.sun.xml.bind', name: 'jaxb-impl', version: '2.3.0'

gradle.properties

@@ -1,6 +1,6 @@
 group = org.elasticsearch.plugin.prometheus
 
-version = 6.4.3.1-SNAPSHOT
+version = 6.5.0.0
 
 pluginName = prometheus-exporter
 pluginClassname = org.elasticsearch.plugin.prometheus.PrometheusExporterPlugin

src/main/java/org/compuscene/metrics/prometheus/PrometheusMetricsCatalog.java

@@ -17,9 +17,10 @@
 
 package org.compuscene.metrics.prometheus;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.rest.prometheus.RestPrometheusMetricsAction;
+
 import java.io.IOException;
 import java.io.StringWriter;
 import java.io.Writer;
@@ -35,7 +36,7 @@
  * A class that describes a Prometheus metrics catalog.
  */
 public class PrometheusMetricsCatalog {
-    private static final Logger logger = ESLoggerFactory.getLogger(RestPrometheusMetricsAction.class.getSimpleName());
+    private static final Logger logger = LogManager.getLogger(RestPrometheusMetricsAction.class);
 
     private String clusterName;
     private String nodeName;

src/main/java/org/compuscene/metrics/prometheus/PrometheusMetricsCollector.java

@@ -39,8 +39,11 @@
 import org.elasticsearch.script.ScriptStats;
 import org.elasticsearch.threadpool.ThreadPoolStats;
 import org.elasticsearch.transport.TransportStats;
+
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+
 import io.prometheus.client.Summary;
 
 /**
@@ -619,6 +622,7 @@ private void updateThreadPoolMetrics(ThreadPoolStats tps) {
         }
     }
 
+    @SuppressWarnings("checkstyle:LineLength")
     private void registerIngestMetrics() {
         catalog.registerNodeGauge("ingest_total_count", "Ingestion total number");
         catalog.registerNodeGauge("ingest_total_time_seconds", "Ingestion total time in seconds");
@@ -629,22 +633,40 @@ private void registerIngestMetrics() {
         catalog.registerNodeGauge("ingest_pipeline_total_time_seconds", "Ingestion total time in seconds", "pipeline");
         catalog.registerNodeGauge("ingest_pipeline_total_current", "Ingestion total current", "pipeline");
         catalog.registerNodeGauge("ingest_pipeline_total_failed_count", "Ingestion total failed", "pipeline");
+
+        catalog.registerNodeGauge("ingest_pipeline_processor_total_count", "Ingestion total number", "pipeline", "processor");
+        catalog.registerNodeGauge("ingest_pipeline_processor_total_time_seconds", "Ingestion total time in seconds", "pipeline", "processor");
+        catalog.registerNodeGauge("ingest_pipeline_processor_total_current", "Ingestion total current", "pipeline", "processor");
+        catalog.registerNodeGauge("ingest_pipeline_processor_total_failed_count", "Ingestion total failed", "pipeline", "processor");
     }
 
+    @SuppressWarnings("checkstyle:LineLength")
     private void updateIngestMetrics(IngestStats is) {
         if (is != null) {
             catalog.setNodeGauge("ingest_total_count", is.getTotalStats().getIngestCount());
             catalog.setNodeGauge("ingest_total_time_seconds", is.getTotalStats().getIngestTimeInMillis() / 1000.0);
             catalog.setNodeGauge("ingest_total_current", is.getTotalStats().getIngestCurrent());
             catalog.setNodeGauge("ingest_total_failed_count", is.getTotalStats().getIngestFailedCount());
 
-            for (Map.Entry<String, IngestStats.Stats> entry : is.getStatsPerPipeline().entrySet()) {
-                String pipeline = entry.getKey();
-                catalog.setNodeGauge("ingest_pipeline_total_count", entry.getValue().getIngestCount(), pipeline);
-                catalog.setNodeGauge("ingest_pipeline_total_time_seconds", entry.getValue().getIngestTimeInMillis() / 1000.0,
+            for (IngestStats.PipelineStat st : is.getPipelineStats()) {
+                String pipeline = st.getPipelineId();
+                catalog.setNodeGauge("ingest_pipeline_total_count", st.getStats().getIngestCount(), pipeline);
+                catalog.setNodeGauge("ingest_pipeline_total_time_seconds", st.getStats().getIngestTimeInMillis() / 1000.0,
                         pipeline);
-                catalog.setNodeGauge("ingest_pipeline_total_current", entry.getValue().getIngestCurrent(), pipeline);
-                catalog.setNodeGauge("ingest_pipeline_total_failed_count", entry.getValue().getIngestFailedCount(), pipeline);
+                catalog.setNodeGauge("ingest_pipeline_total_current", st.getStats().getIngestCurrent(), pipeline);
+                catalog.setNodeGauge("ingest_pipeline_total_failed_count", st.getStats().getIngestFailedCount(), pipeline);
+
+                List<IngestStats.ProcessorStat> pss = is.getProcessorStats().get(pipeline);
+                if (pss != null) {
+                    for (IngestStats.ProcessorStat ps : pss) {
+                        String processor = ps.getName();
+                        catalog.setNodeGauge("ingest_pipeline_processor_total_count", ps.getStats().getIngestCount(), pipeline, processor);
+                        catalog.setNodeGauge("ingest_pipeline_processor_total_time_seconds", ps.getStats().getIngestTimeInMillis() / 1000.0,
+                                pipeline, processor);
+                        catalog.setNodeGauge("ingest_pipeline_processor_total_current", ps.getStats().getIngestCurrent(), pipeline, processor);
+                        catalog.setNodeGauge("ingest_pipeline_processor_total_failed_count", ps.getStats().getIngestFailedCount(), pipeline, processor);
+                    }
+                }
             }
         }
     }

src/main/java/org/elasticsearch/plugin/prometheus/PrometheusExporterPlugin.java

@@ -17,6 +17,7 @@
 
 package org.elasticsearch.plugin.prometheus;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.compuscene.metrics.prometheus.PrometheusMetricsCollector;
 import org.elasticsearch.action.ActionRequest;
@@ -25,13 +26,13 @@
 import org.elasticsearch.action.TransportNodePrometheusMetricsAction;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.*;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.rest.prometheus.RestPrometheusMetricsAction;
+
 import java.util.Arrays;
 import java.util.List;
 import java.util.function.Supplier;
@@ -40,7 +41,7 @@
  * Prometheus Exporter plugin main class.
  */
 public class PrometheusExporterPlugin extends Plugin implements ActionPlugin {
-    private static final Logger logger = Loggers.getLogger(PrometheusExporterPlugin.class);
+    private static final Logger logger = LogManager.getLogger(PrometheusExporterPlugin.class);
 
     public PrometheusExporterPlugin() {
         logger.info("starting Prometheus exporter plugin");