add support for Google Cloud Storage

README.md

@@ -152,6 +152,7 @@ Hash command specific arguments:
 
  * `-l/--limit` - limit the number of repositories to be processed. All repositories will be processed by default
  * `-f/--format` - format of the stored repositories. Supported input data formats that repositories could be stored in are `siva`, `bare` or `standard`, default `siva`
+ * `--gcs-keyfile` - path to [JSON keyfile](https://cloud.google.com/storage/docs/authentication) for authentication in Google Cloud Storage
 
 ## Development
 

build.sbt

@@ -22,6 +22,7 @@ libraryDependencies ++= Seq(
   scalapbGrpc % Compile,
   engine % Compile,
   jgit % Compile,
+  gcs % Compile,
   fixNetty,
   cassandraDriverMetrics % Provided, //needed for using Driver \wo Spark from SparkConnector
   cassandraSparkConnector % Compile,

project/Dependencies.scala

@@ -24,10 +24,11 @@ object Dependencies {
   lazy val hadoopCommon = ("org.apache.hadoop" % "hadoop-common" % "2.6.5")
     .exclude("com.sun.jersey", "jersey-server")
     .exclude("commons-beanutils", "commons-beanutils-core")
-  lazy val scalapb = "com.thesamet.scalapb" %% "scalapb-runtime" % "0.7.1"
-  lazy val scalapbGrpc = "com.thesamet.scalapb" %% "scalapb-runtime-grpc" % "0.7.1"
+  lazy val scalapb = "com.thesamet.scalapb" %% "scalapb-runtime" % "0.8.4"
+  lazy val scalapbGrpc = "com.thesamet.scalapb" %% "scalapb-runtime-grpc" % "0.8.4"
   lazy val ioGrpc = "io.grpc" % "grpc-netty" % "1.10.0"
   lazy val commonsMath = "org.apache.commons" % "commons-math3" % "3.6.1"
   lazy val bblfshClient = "org.bblfsh" % "bblfsh-client" % "1.8.2"
   lazy val scalaJsonParser = "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.6.7"
+  lazy val gcs = "com.google.cloud.bigdataoss" % "gcs-connector" % "hadoop2-1.9.11"
 }

src/main/scala/tech/sourced/gemini/cmd/HashSparkApp.scala

@@ -24,7 +24,8 @@ case class HashAppConfig(
   sparkParallelism: Int = 8,
   docFreqFile: String = "",
   verbose: Boolean = false,
-  mode: String = Gemini.fileSimilarityMode
+  mode: String = Gemini.fileSimilarityMode,
+  gcsKeyFile: String = ""
 )
 
 /**
@@ -94,6 +95,9 @@ object HashSparkApp extends App with Logging {
     opt[String]("doc-freq-file")
       .action((x, c) => c.copy(docFreqFile = x))
       .text("path to file with feature frequencies")
+    opt[String]("gcs-keyfile")
+      .action((x, c) => c.copy(gcsKeyFile = x))
+      .text("path to JSON keyfile for authentication in Google Cloud Storage")
     arg[String]("<path-to-git-repos>")
       .required()
       .action((x, c) => c.copy(reposPath = x))
@@ -120,6 +124,10 @@ object HashSparkApp extends App with Logging {
         .config("spark.tech.sourced.featurext.grpc.port", config.fePort)
         .getOrCreate()
 
+      if (config.gcsKeyFile.nonEmpty) {
+        spark.sparkContext.hadoopConfiguration.set("google.cloud.auth.service.account.json.keyfile", config.gcsKeyFile)
+      }
+
       val reposPath = config.reposPath
       val repos = listRepositories(reposPath, config.format, spark, config.limit)
       printRepositories(reposPath, repos)