Use es-index-cleaner golang implementation

Signed-off-by: Pavol Loffay <[email protected]>

Author: pavolloffay

.github/workflows/ci-elasticsearch.yml

@@ -9,6 +9,11 @@ on:
 jobs:
   elasticsearch:
     runs-on: ubuntu-latest
+    services:
+      registry:
+        image: registry:2
+        ports:
+          - 5000:5000
     strategy:
       matrix:
         version:
@@ -41,5 +46,14 @@ jobs:
     - name: Install tools
       run: make install-ci
 
+    - uses: docker/setup-qemu-action@v1
+
+    - uses: docker/setup-buildx-action@v1
+      with:
+        driver-opts: network=host
+
     - name: Run elasticsearch integration tests
       run: bash scripts/es-integration-test.sh ${{ matrix.version.distribution }} ${{ matrix.version.image }}
+      env:
+        DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+        QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }}

Makefile

@@ -115,7 +115,6 @@ grpc-plugin-storage-integration-test:
 .PHONY: test-compile-es-scripts
 test-compile-es-scripts:
 	docker run --rm -v ${PWD}:/tmp/jaeger python:3-alpine3.11 /usr/local/bin/python -m py_compile /tmp/jaeger/plugin/storage/es/esRollover.py
-	docker run --rm -v ${PWD}:/tmp/jaeger python:3-alpine3.11 /usr/local/bin/python -m py_compile /tmp/jaeger/plugin/storage/es/esCleaner.py
 
 .PHONY: index-cleaner-integration-test
 index-cleaner-integration-test: docker-images-elastic
@@ -216,6 +215,10 @@ build-esmapping-generator:
 build-esmapping-generator-linux:
 	 GOOS=linux GOARCH=amd64 $(GOBUILD) -o ./plugin/storage/es/esmapping-generator ./cmd/esmapping-generator/main.go
 
+.PHONY: build-es-index-cleaner
+build-es-index-cleaner:
+	$(GOBUILD) -o ./cmd/es-index-cleaner/es-index-cleaner-$(GOOS)-$(GOARCH) ./cmd/es-index-cleaner/main.go
+
 .PHONY: docker-hotrod
 docker-hotrod:
 	GOOS=linux $(MAKE) build-examples
@@ -306,7 +309,8 @@ build-platform-binaries: build-agent \
 	build-examples \
 	build-tracegen \
 	build-anonymizer \
-	build-esmapping-generator
+	build-esmapping-generator \
+	build-es-index-cleaner
 
 .PHONY: build-all-platforms
 build-all-platforms: build-binaries-linux build-binaries-windows build-binaries-darwin build-binaries-s390x build-binaries-arm64 build-binaries-ppc64le
@@ -316,10 +320,13 @@ docker-images-cassandra:
 	docker build -t $(DOCKER_NAMESPACE)/jaeger-cassandra-schema:${DOCKER_TAG} plugin/storage/cassandra/
 	@echo "Finished building jaeger-cassandra-schema =============="
 
+docker-images-elastic: TARGET = release
+
 .PHONY: docker-images-elastic
-docker-images-elastic: 
+docker-images-elastic: create-baseimg
 	GOOS=linux GOARCH=$(GOARCH) $(MAKE) build-esmapping-generator
-	docker build -t $(DOCKER_NAMESPACE)/jaeger-es-index-cleaner:${DOCKER_TAG} plugin/storage/es
+	GOOS=linux GOARCH=$(GOARCH) $(MAKE) build-es-index-cleaner
+	docker build --target $(TARGET) -t $(DOCKER_NAMESPACE)/jaeger-es-index-cleaner:${DOCKER_TAG} --build-arg base_image=$(BASE_IMAGE)  --build-arg TARGETARCH=$(GOARCH) cmd/es-index-cleaner
 	docker build -t $(DOCKER_NAMESPACE)/jaeger-es-rollover:${DOCKER_TAG} plugin/storage/es -f plugin/storage/es/Dockerfile.rollover --build-arg TARGETARCH=$(GOARCH)
 	@echo "Finished building jaeger-es-indices-clean =============="
 

cmd/es-index-cleaner/Dockerfile

@@ -0,0 +1,6 @@
+ARG base_image
+
+FROM $base_image AS release
+ARG TARGETARCH
+COPY es-index-cleaner-linux-$TARGETARCH /go/bin/es-index-cleaner-linux
+ENTRYPOINT ["/go/bin/es-index-cleaner-linux"]

cmd/es-index-cleaner/app/flags.go

@@ -45,7 +45,7 @@ type Config struct {
 // AddFlags adds flags for TLS to the FlagSet.
 func (c *Config) AddFlags(flags *flag.FlagSet) {
 	flags.String(indexPrefix, "", "Index prefix")
-	flags.Bool(archive, false, "Whether to remove archive indices")
+	flags.Bool(archive, false, "Whether to remove archive indices. It works only for rollover")
 	flags.Bool(rollover, false, "Whether to remove indices created by rollover")
 	flags.Int(timeout, 120, "Number of seconds to wait for master node response")
 	flags.String(indexDateSeparator, "-", "Index date separator")

cmd/es-index-cleaner/app/index_filter.go

@@ -42,18 +42,13 @@ func (i *IndexFilter) Filter(indices []Index) []Index {
 
 func (i *IndexFilter) filter(indices []Index) []Index {
 	var reg *regexp.Regexp
-	if !i.Rollover && !i.Archive {
-		// daily indices
-		reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-(span|service|dependencies)-\\d{4}%s\\d{2}%s\\d{2}", i.IndexPrefix, i.IndexDateSeparator, i.IndexDateSeparator))
-	} else if !i.Rollover && i.Archive {
-		// daily archive
-		reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-span-archive", i.IndexPrefix))
-	} else if i.Rollover && !i.Archive {
-		// rollover
+	if i.Archive {
+		// archive works only for rollover
+		reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-span-archive-\\d{6}", i.IndexPrefix))
+	} else if i.Rollover {
 		reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-(span|service)-\\d{6}", i.IndexPrefix))
 	} else {
-		// rollover archive
-		reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-span-archive-\\d{6}", i.IndexPrefix))
+		reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-(span|service|dependencies)-\\d{4}%s\\d{2}%s\\d{2}", i.IndexPrefix, i.IndexDateSeparator, i.IndexDateSeparator))
 	}
 
 	var filtered []Index

cmd/es-index-cleaner/app/index_filter_test.go

@@ -229,19 +229,21 @@ func testIndexFilter(t *testing.T, prefix string) {
 			},
 		},
 		{
-			name: "archive indices, remove older 2 days",
+			name: "archive indices, remove older 1 days - archive works only for rollover",
 			filter: &IndexFilter{
 				IndexPrefix:          prefix,
 				IndexDateSeparator:   "-",
 				Archive:              true,
 				Rollover:             false,
-				DeleteBeforeThisDate: time20200807.Add(-time.Hour * 24 * time.Duration(2)),
+				DeleteBeforeThisDate: time20200807.Add(-time.Hour * 24 * time.Duration(1)),
 			},
 			expected: []Index{
 				{
-					Index:        prefix + "jaeger-span-archive",
-					CreationTime: time.Date(2020, time.August, 0, 15, 0, 0, 0, time.UTC),
-					Aliases:      map[string]bool{},
+					Index:        prefix + "jaeger-span-archive-000001",
+					CreationTime: time.Date(2020, time.August, 5, 15, 0, 0, 0, time.UTC),
+					Aliases: map[string]bool{
+						prefix + "jaeger-span-archive-read": true,
+					},
 				},
 			},
 		},

docker/Makefile

@@ -3,9 +3,9 @@ ROOT_IMAGE ?= alpine:3.13
 CERT_IMAGE := $(ROOT_IMAGE)
 GOLANG_IMAGE := golang:1.16-alpine
 
-BASE_IMAGE := localhost:5000/baseimg_alpine:latest
-DEBUG_IMAGE := localhost:5000/debugimg_alpine:latest
-PLATFORMS := linux/amd64,linux/s390x,linux/ppc64le,linux/arm64
+BASE_IMAGE ?= localhost:5000/baseimg_alpine:latest
+DEBUG_IMAGE ?= localhost:5000/debugimg_alpine:latest
+PLATFORMS ?= linux/amd64,linux/s390x,linux/ppc64le,linux/arm64
 
 create-baseimg-debugimg: create-baseimg create-debugimg
 

plugin/storage/es/Dockerfile

@@ -114,10 +114,10 @@ func TestIndexCleaner(t *testing.T) {
 		},
 	}
 	for _, test := range tests {
-		t.Run(fmt.Sprintf("%s_no_prefix", test.name), func(t *testing.T) {
+		t.Run(fmt.Sprintf("%s_no_prefix, %s", test.name, test.envVars), func(t *testing.T) {
 			runIndexCleanerTest(t, client, "", test.expectedIndices, test.envVars)
 		})
-		t.Run(fmt.Sprintf("%s_prefix", test.name), func(t *testing.T) {
+		t.Run(fmt.Sprintf("%s_prefix, %s", test.name, test.envVars), func(t *testing.T) {
 			runIndexCleanerTest(t, client, indexPrefix, test.expectedIndices, append(test.envVars, "INDEX_PREFIX="+indexPrefix))
 		})
 	}