ZMS Syncer implementation for GCP/GCS

.gitignore

@@ -136,6 +136,7 @@ libs/nodejs/auth_core/package-lock.json
 .clover/
 athenz-docker-build.log
 syncers/auth_history_syncer/dynamodb-local-metadata.json
+libs/java/syncer_common/src/test/resources/run_state.json
 
 # Logs
 logs

libs/java/server_aws_common/pom.xml

@@ -83,11 +83,21 @@
       <artifactId>rds</artifactId>
       <version>${aws2.version}</version>
     </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>apache-client</artifactId>
+      <version>${aws2.version}</version>
+    </dependency>
     <dependency>
       <groupId>com.yahoo.athenz</groupId>
       <artifactId>athenz-auth-core</artifactId>
       <version>${project.parent.version}</version>
     </dependency>
+    <dependency>
+      <groupId>com.yahoo.athenz</groupId>
+      <artifactId>athenz-syncer-common</artifactId>
+      <version>${project.parent.version}</version>
+    </dependency>
     <dependency>
       <groupId>com.yahoo.athenz</groupId>
       <artifactId>athenz-server-common</artifactId>

libs/java/server_aws_common/src/main/java/io/athenz/server/aws/common/utils/RetryDynamoDBCommand.java

@@ -1,3 +1,18 @@
+/*
+ * Copyright The Athenz Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package io.athenz.server.aws.common.utils;
 
 import org.slf4j.Logger;

syncers/zms_aws_domain_syncer/src/main/java/com/yahoo/athenz/zms_aws_domain_syncer/AwsSyncer.java → libs/java/server_aws_common/src/main/java/io/athenz/syncer/aws/common/impl/AwsDomainStore.java

@@ -1,23 +1,23 @@
 /*
+ * Copyright The Athenz Authors
  *
- *  * Copyright The Athenz Authors
- *  *
- *  * Licensed under the Apache License, Version 2.0 (the "License");
- *  * you may not use this file except in compliance with the License.
- *  * You may obtain a copy of the License at
- *  *
- *  *     http://www.apache.org/licenses/LICENSE-2.0
- *  *
- *  * Unless required by applicable law or agreed to in writing, software
- *  * distributed under the License is distributed on an "AS IS" BASIS,
- *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  * See the License for the specific language governing permissions and
- *  * limitations under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
-package com.yahoo.athenz.zms_aws_domain_syncer;
+package io.athenz.syncer.aws.common.impl;
 
+import io.athenz.syncer.common.zms.CloudDomainStore;
+import io.athenz.syncer.common.zms.Config;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.slf4j.Logger;
@@ -28,22 +28,25 @@
 import software.amazon.awssdk.services.s3.model.PutObjectRequest;
 import software.amazon.awssdk.services.s3.model.ServerSideEncryption;
 
-public class AwsSyncer {
+import java.lang.invoke.MethodHandles;
+
+public class AwsDomainStore implements CloudDomainStore {
 
-    private static final Logger LOGGER = LoggerFactory.getLogger(AwsSyncer.class);
+    private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
     private static final int MAX_RETRY_COUNT = 3;
 
     private final S3Client s3Client;
 
-    public AwsSyncer() throws Exception {
+    public AwsDomainStore() throws Exception {
         this.s3Client = S3ClientFactory.getS3Client();
     }
 
-    public AwsSyncer(S3Client s3Client) {
+    public AwsDomainStore(S3Client s3Client) {
         this.s3Client = s3Client;
     }
 
+    @Override
     public void uploadDomain(final String domainName, final String domJson) {
 
         final String sseAlgorithm = Config.getInstance().getConfigParam(Config.SYNC_CFG_PARAM_AWS_SSE_ALGORITHM);
@@ -101,6 +104,7 @@ public void uploadDomain(final String domainName, final String domJson) {
         }
     }
 
+    @Override
     public void deleteDomain(final String domainName) {
 
         try {

libs/java/server_aws_common/src/main/java/io/athenz/syncer/aws/common/impl/AwsDomainStoreFactory.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright The Athenz Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.athenz.syncer.aws.common.impl;
+
+import io.athenz.syncer.common.zms.CloudDomainStore;
+import io.athenz.syncer.common.zms.CloudDomainStoreFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.invoke.MethodHandles;
+
+public class AwsDomainStoreFactory implements CloudDomainStoreFactory {
+    private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+    @Override
+    public CloudDomainStore create() {
+        try {
+            return new AwsDomainStore();
+        } catch (Exception e) {
+            LOG.error("Failed to create AwsDomainStore", e);
+            throw new RuntimeException(e);
+        }
+    }
+}

syncers/zms_aws_domain_syncer/src/main/java/com/yahoo/athenz/zms_aws_domain_syncer/StateFileBuilder.java → libs/java/server_aws_common/src/main/java/io/athenz/syncer/aws/common/impl/AwsStateFileBuilder.java

@@ -1,27 +1,24 @@
 /*
+ * Copyright The Athenz Authors
  *
- *  * Copyright The Athenz Authors
- *  *
- *  * Licensed under the Apache License, Version 2.0 (the "License");
- *  * you may not use this file except in compliance with the License.
- *  * You may obtain a copy of the License at
- *  *
- *  *     http://www.apache.org/licenses/LICENSE-2.0
- *  *
- *  * Unless required by applicable law or agreed to in writing, software
- *  * distributed under the License is distributed on an "AS IS" BASIS,
- *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  * See the License for the specific language governing permissions and
- *  * limitations under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
-
-package com.yahoo.athenz.zms_aws_domain_syncer;
+package io.athenz.syncer.aws.common.impl;
 
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.yahoo.athenz.zms.DomainData;
 import com.yahoo.athenz.zms.JWSDomain;
+import io.athenz.syncer.common.zms.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import software.amazon.awssdk.core.ResponseInputStream;
@@ -35,9 +32,8 @@
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
-public class StateFileBuilder {
-
-    private static final Logger LOGGER = LoggerFactory.getLogger(StateFileBuilder.class);
+public class AwsStateFileBuilder implements StateFileBuilder {
+    private static final Logger LOGGER = LoggerFactory.getLogger(AwsStateFileBuilder.class);
 
     private final static String BUCKET_NAME = Config.getInstance().getConfigParam(Config.SYNC_CFG_PARAM_AWS_BUCKET);
     private final static String THREADS_NUMBER = Config.getInstance().getConfigParam(Config.SYNC_CFG_PARAM_STATE_BUILDER_THREADS);
@@ -49,11 +45,13 @@ public class StateFileBuilder {
     private final ObjectMapper jsonMapper = new ObjectMapper();
     private final DomainValidator domainValidator;
 
-    public StateFileBuilder() throws Exception {
+    public AwsStateFileBuilder() throws Exception {
         this(S3ClientFactory.getS3Client(), new DomainValidator());
     }
 
-    public StateFileBuilder(S3Client s3client, DomainValidator domainValidator) {
+    public AwsStateFileBuilder(S3Client s3client, DomainValidator domainValidator) {
+
+        LOGGER.error("XXXXXX creating AwsStateFileBuilder");
 
         this.s3client = s3client;
         this.domainValidator = domainValidator;
@@ -66,7 +64,6 @@ public StateFileBuilder(S3Client s3client, DomainValidator domainValidator) {
     }
 
     public Map<String, DomainState> buildStateMap() {
-
         LOGGER.info("building state map from S3...");
 
         List<String> domains = listObjects(s3client);
@@ -99,25 +96,14 @@ public Map<String, DomainState> buildStateMap() {
                 .filter(entry -> domainValidator.validateJWSDomain(entry.getValue().getJwsDomain()))
                 .collect(Collectors.toMap(
                         Map.Entry::getKey,
-                        value -> getDomainState(domainValidator.getDomainData(value.getValue().getJwsDomain()),
+                        value -> DomainState.getDomainState(domainValidator.getDomainData(value.getValue().getJwsDomain()),
                                 value.getValue().getFetchTime())
                 ));
 
         LOGGER.info("validated signatures of {} domain object from S3 bucket {}", stateMap.size(), BUCKET_NAME);
         return stateMap;
     }
 
-    DomainState getDomainState(final DomainData domData, long fetchTime) {
-        final String domName = domData.getName();
-        final String domMod = domData.getModified().toString();
-
-        DomainState domState = new DomainState();
-        domState.setDomain(domName);
-        domState.setModified(domMod);
-        domState.setFetchTime(fetchTime);
-        return domState;
-    }
-
     /**
      * list the objects in the bucket.
      * @param s3Client aws s3 object
@@ -173,21 +159,6 @@ List<String> listObjects(S3Client s3Client) {
         return domains;
     }
 
-    static class JWSDomainData {
-        JWSDomain jwsDomain;
-        long fetchTime;
-
-        public JWSDomainData(JWSDomain jwsDomain, long fetchTime) {
-            this.jwsDomain = jwsDomain;
-            this.fetchTime = fetchTime;
-        }
-        JWSDomain getJwsDomain() {
-            return jwsDomain;
-        }
-        long getFetchTime() {
-            return fetchTime;
-        }
-    }
 
     class ObjectS3Thread implements Runnable {
 

libs/java/server_aws_common/src/main/java/io/athenz/syncer/aws/common/impl/AwsStateFileBuilderFactory.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright The Athenz Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.athenz.syncer.aws.common.impl;
+
+import io.athenz.syncer.common.zms.StateFileBuilderFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.invoke.MethodHandles;
+
+public class AwsStateFileBuilderFactory implements StateFileBuilderFactory {
+    private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+    @Override
+    public AwsStateFileBuilder create() {
+        try {
+            return new AwsStateFileBuilder();
+        } catch (Exception e) {
+            LOG.error("Failed to create AwsStateFileBuilder", e);
+            throw new RuntimeException(e);
+        }
+    }
+}

syncers/zms_aws_domain_syncer/src/main/java/com/yahoo/athenz/zms_aws_domain_syncer/S3ClientFactory.java → libs/java/server_aws_common/src/main/java/io/athenz/syncer/aws/common/impl/S3ClientFactory.java

@@ -1,23 +1,22 @@
 /*
+ * Copyright The Athenz Authors
  *
- *  * Copyright The Athenz Authors
- *  *
- *  * Licensed under the Apache License, Version 2.0 (the "License");
- *  * you may not use this file except in compliance with the License.
- *  * You may obtain a copy of the License at
- *  *
- *  *     http://www.apache.org/licenses/LICENSE-2.0
- *  *
- *  * Unless required by applicable law or agreed to in writing, software
- *  * distributed under the License is distributed on an "AS IS" BASIS,
- *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  * See the License for the specific language governing permissions and
- *  * limitations under the License.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
-package com.yahoo.athenz.zms_aws_domain_syncer;
+package io.athenz.syncer.aws.common.impl;
 
+import io.athenz.syncer.common.zms.Config;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;