feat: generate showcase without post-processing (#1935)

* feat: add

* generate gapic and proto folder

* refactor utilities

* add an action to verify

* checkout googleapis-gen

* setup repo name

* add commit hash of googleapis-gen

* change secret

* change token

* change to git clone

* change user name

* add input list

* include resources folder in main

* remove grpc version in `*ServiceGrpc.java`

* change destination path

* compare generation result with googleapis-gen

* fix type in diff command

* checkout repo using checkout action

* checkout repos as nested repo

* sparse checkout googleapis

* Revert "sparse checkout googleapis"

This reverts commit 3d612f8.

* change library

* change step name

* add a readme

* make grpc version optional

* make protobuf version optional

* checkout master branch, rather than a commit hash

* allow snapshot version of generator

* download snapshot of generator parent pom

* update README

* chore: add test script for showcase

this produces various differences in the output. Mainly additional
protos not being detected, a mismatch between
proto-gapic-showcase-v1beta1/src/proto (arbitrary path)

* chore: fix additional protos logic

* download generator and grpc using mvn

* change error message

* add maven central mirror

* add comments in utilities

* add comments

* add an integration test

* chore: modify showcase source proto folder structure

* fail fast if no file is found

* chore: add showcase integration test

* chore: add action to test on showase

* chore: add diff check for showcase IT

* do not delete google/

* chore: change file mod in protos

* chore: fix diff comparison

* chore: remove unwanted binary

* get protobuf version from WORKSPACE

* add instructions on download `google/` from googleapis

* add comments

* chore: final adjustments for showcase testing

* chore: allow workflow dispatch on workflow

* chore: correct yaml definition

* chore: correct yaml definition ii

* chore: fix util import

* update description of `destination_path`

* chore: correct version parsing

* chore: correct showcase version

* chore: try java 11

* update comments

* chore: infer grpc_version from parent pom

* download dependencies using `curl`

* increase download time

* remove comment

* generate components for showcase

* use script in maven profile

* correct compilation components

* explicit output folder path check

* use poms to resolve versions

* migrate showcase scripts to showcase folder

* remove bazel-based update script

* finalize migration for generation

* fix generation

* change verification profile for gapic-showcase

* remove unused functions in verify.sh

* remove unused CI component

* remove showcase from readme

* remove unused logic

* script comments and cleanup

* improve comment in generate_component

* remove unused bazel files

* remove .gitignore in favor of cleanup

* bash format

* transfer profiles to showcase parent

* fix parent pom script call

* fix wildcard for source replacement

* fix proto golden test, add util script

* install xmllint

* sudo for installing xmlint

* correct xmllint package

* Update library_generation/utilities.sh

Co-authored-by: Joe Wang <[email protected]>

* restore gh action

* move convenience functions to utilities file

* remove duplicate file\

* fix proto path

* use `sed` and `cut` to extract showcase version

* add unit tests for new functions

* add test for versions

---------

Co-authored-by: JoeWang1127 <[email protected]>
Co-authored-by: Joe Wang <[email protected]>

Author: diegomarquezp

library_generation/README.md

@@ -125,15 +125,3 @@ library_generation/generate_library.sh \
 --include_samples true
 ```
 
-## An example to generate showcase client
-```
-library_generation/generate_library.sh \
--p schema/google/showcase/v1beta1 \ # google/ should be in library_generation/.
--d output \
---gapic_generator_version 2.24.0 \
---protobuf_version 23.2 \
---grpc_version 1.55.1 \
---transport grpc+rest \
---rest_numeric_enums false \
---include_samples false
-```

library_generation/generate_library.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 set -eo pipefail
+set -x
 
 # parse input parameters
 while [[ $# -gt 0 ]]; do
@@ -157,3 +158,4 @@ done
 #####################################################
 cd "${destination_path}"
 rm -rf java_gapic_srcjar java_gapic_srcjar_raw.srcjar.zip java_grpc.jar java_proto.jar temp-codegen.srcjar
+set +x

library_generation/test/generate_library_integration_test.sh

@@ -42,34 +42,13 @@ esac
 shift # past argument or value
 done
 
-get_version_from_WORKSPACE() {
-  local version_key_word=$1
-  local workspace=$2
-  local delimiter=$3
-  local version
-  version="$(grep -m 1 "${version_key_word}"  "${workspace}" | sed 's/\"\(.*\)\".*/\1/' | cut -d "${delimiter}" -f2 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
-  echo "${version}"
-}
-
-sparse_clone() {
-  local repo_url=$1
-  local paths=$2
-  local clone_dir
-  clone_dir=$(basename "${repo_url%.*}")
-  rm -rf "${clone_dir}"
-  git clone -n --depth=1 --filter=tree:0 "${repo_url}"
-  cd "${clone_dir}"
-  git sparse-checkout set --no-cone ${paths}
-  git checkout
-  cd ..
-}
-
 script_dir=$(dirname "$(readlink -f "$0")")
+source "${script_dir}/../utilities.sh"
 library_generation_dir="${script_dir}"/..
 cd "${library_generation_dir}"
 # checkout the master branch of googleapis/google (proto files) and WORKSPACE
 echo "Checking out googlapis repository..."
-sparse_clone https://github.com/googleapis/googleapis.git "${proto_path} WORKSPACE google/api google/rpc google/cloud/common_resources.proto"
+sparse_clone https://github.com/googleapis/googleapis.git "${proto_path} WORKSPACE google/api google/rpc google/cloud/common_resources.proto google/iam/v1 google/type google/longrunning"
 cd googleapis
 # parse version of gapic-generator-java, protobuf and grpc from WORKSPACE
 gapic_generator_version=$(get_version_from_WORKSPACE "_gapic_generator_java_version" WORKSPACE "=")
@@ -79,18 +58,25 @@ echo "The version of protobuf is ${protobuf_version}"
 grpc_version=$(get_version_from_WORKSPACE "_grpc_version" WORKSPACE "=")
 echo "The version of protoc-gen-grpc-java plugin is ${gapic_generator_version}."
 # parse GAPIC options from proto_path/BUILD.bazel
-transport="grpc"
-if grep -A 15 "java_gapic_library(" "${proto_path}/BUILD.bazel" | grep -q "grpc+rest"; then
-  transport="grpc+rest"
-fi
-rest_numeric_enums="true"
-if grep -A 15 "java_gapic_library(" "${proto_path}/BUILD.bazel" | grep -q "rest_numeric_enums = False"; then
-  rest_numeric_enums="false"
-fi
-include_samples="false"
-if grep -A 15 "java_gapic_assembly_gradle_pkg(" "${proto_path}/BUILD.bazel" | grep -q "include_samples = True"; then
-  include_samples="true"
-fi
+proto_build_file_path="${proto_path}/BUILD.bazel"
+transport=$(get_config_from_BUILD \
+  "${proto_build_file_path}" \
+  "java_gapic_library(" \
+  "grpc+rest" \
+  "grpc"
+)
+rest_numeric_enums=$(get_config_from_BUILD \
+  "${proto_build_file_path}" \
+  "java_gapic_library(" \
+  "rest_numeric_enums = False" \
+  "true"
+)
+include_samples=$(get_config_from_BUILD \
+  "${proto_build_file_path}" \
+  "java_gapic_assembly_gradle_pkg(" \
+  "include_samples = True" \
+  "false"
+)
 echo "GAPIC options are transport=${transport}, rest_numeric_enums=${rest_numeric_enums}, include_samples=${include_samples}."
 os_architecture="linux-x86_64"
 if [[ "$os_type" == *"macos"* ]]; then

library_generation/test/generate_library_unit_tests.sh

@@ -219,6 +219,48 @@ download_grpc_plugin_failed_with_invalid_arch_test() {
   __assertEquals 1 $((res))
 }
 
+get_config_from_valid_BUILD_test() {
+  build_file="${script_dir}/resources/misc/TESTBUILD.bazel"
+  rule="java_gapic_library("
+  # the pattern we expect to find in the BUILD file
+  pattern_should_match="name"
+  # the pattern that we should not find in the BUILD file
+  pattern_should_not_match="should-not-match"
+  # default value if the pattern was not found
+  if_matched_return="got-a-match"
+  if_not_matched_return="no-match"
+  pattern_matched_result=$(get_config_from_BUILD \
+    "${build_file}" \
+    "${rule}" \
+    "${pattern_should_match}" \
+    "${if_not_matched_return}" \
+    "${if_matched_return}"
+  )
+  __assertEquals "${pattern_matched_result}" "${if_matched_return}"
+  pattern_not_matched_result=$(get_config_from_BUILD \
+    "${build_file}" \
+    "${rule}" \
+    "${pattern_should_not_match}" \
+    "${if_not_matched_return}" \
+    "${if_matched_return}"
+  )
+  __assertEquals "${pattern_not_matched_result}" "${if_not_matched_return}"
+}
+
+get_version_from_valid_WORKSPACE_test() {
+  workspace_file="${script_dir}/resources/misc/TESTWORKSPACE"
+  obtained_ggj_version=$(get_version_from_WORKSPACE "_gapic_generator_java_version" "${workspace_file}")
+  __assertEquals '2.25.1-SNAPSHOT' "${obtained_ggj_version}"
+}
+
+get_version_from_valid_versions_txt_test() {
+  versions_file="${script_dir}/resources/misc/testversions.txt"
+  obtained_ggj_version=$(get_version_from_versions_txt "${versions_file}" "gapic-generator-java")
+  __assertEquals '2.25.1-SNAPSHOT' "${obtained_ggj_version}"
+  obtained_gax_version=$(get_version_from_versions_txt "${versions_file}" "gax")
+  __assertEquals '2.33.1-SNAPSHOT' "${obtained_gax_version}"
+}
+
 # Execute tests.
 # One line per test.
 test_list=(
@@ -244,6 +286,9 @@ test_list=(
   download_grpc_plugin_succeed_with_valid_version_macos_test
   download_grpc_plugin_failed_with_invalid_version_linux_test
   download_grpc_plugin_failed_with_invalid_arch_test
+  get_config_from_valid_BUILD_test
+  get_version_from_valid_WORKSPACE_test
+  get_version_from_valid_versions_txt_test
 )
 
 for ut in "${test_list[@]}"; do

showcase/BUILD.bazel renamed to library_generation/test/resources/misc/TESTBUILD.bazel

@@ -1,5 +1,4 @@
-# Note: The commented section is the framework for gapic-showcase-extended
-# See gapic-showcase-extended folder for more info
+# Test BUILD file, copied from sdk-platform-java/showcase/BUILD.bazel
 
 load(
     "//rules_java_gapic:java_gapic.bzl",

library_generation/test/resources/misc/TESTWORKSPACE

@@ -0,0 +1,133 @@
+# test workspace file obtained from sdk-platform-java
+
+workspace(name = "gapic_generator_java")
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+# gax-java and its transitive dependencies must be imported before
+# gapic-generator-java dependencies to match the order in googleapis repository,
+# which in its turn, prioritizes actual generated clients runtime dependencies
+# over the generator dependencies.
+local_repository(
+   name = "com_google_api_gax_java",
+   path = "gax-java",
+)
+
+load("@com_google_api_gax_java//:repository_rules.bzl", "com_google_api_gax_java_properties")
+
+com_google_api_gax_java_properties(
+    name = "com_google_api_gax_java_properties",
+    file = "@com_google_api_gax_java//:dependencies.properties",
+)
+
+load("@com_google_api_gax_java//:repositories.bzl", "com_google_api_gax_java_repositories")
+
+com_google_api_gax_java_repositories()
+
+_googleapis_commit = "7438480b2a1bc6371d748e974f7a3647f90c4e8d"
+
+http_archive(
+    name = "com_google_googleapis",
+    strip_prefix = "googleapis-%s" % _googleapis_commit,
+    urls = [
+        "https://github.com/googleapis/googleapis/archive/%s.zip" % _googleapis_commit,
+    ],
+)
+
+# protobuf
+RULES_JVM_EXTERNAL_TAG = "4.5"
+
+RULES_JVM_EXTERNAL_SHA = "b17d7388feb9bfa7f2fa09031b32707df529f26c91ab9e5d909eb1676badd9a6"
+
+http_archive(
+    name = "rules_jvm_external",
+    sha256 = RULES_JVM_EXTERNAL_SHA,
+    strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG,
+    url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG,
+)
+
+load("@rules_jvm_external//:repositories.bzl", "rules_jvm_external_deps")
+
+rules_jvm_external_deps()
+
+load("@rules_jvm_external//:setup.bzl", "rules_jvm_external_setup")
+
+rules_jvm_external_setup()
+
+load("@com_google_protobuf//:protobuf_deps.bzl", "PROTOBUF_MAVEN_ARTIFACTS", "protobuf_deps")
+load("@rules_jvm_external//:defs.bzl", "maven_install")
+
+maven_install(
+    artifacts = PROTOBUF_MAVEN_ARTIFACTS,
+    repositories = ["https://repo.maven.apache.org/maven2/"],
+)
+
+_gapic_generator_java_version = "2.25.1-SNAPSHOT"  # {x-version-update:gapic-generator-java:current}
+
+maven_install(
+    artifacts = [
+        "com.google.api:gapic-generator-java:" + _gapic_generator_java_version,
+    ],
+    fail_on_missing_checksum = False,
+    repositories = [
+        "m2Local",
+        "https://repo.maven.apache.org/maven2/",
+    ],
+)
+
+protobuf_deps()
+
+# Bazel rules.
+_rules_gapic_version = "0.5.5"
+
+http_archive(
+    name = "rules_gapic",
+    strip_prefix = "rules_gapic-%s" % _rules_gapic_version,
+    urls = ["https://github.com/googleapis/rules_gapic/archive/v%s.tar.gz" % _rules_gapic_version],
+)
+
+# Java dependencies.
+load("@com_google_googleapis//:repository_rules.bzl", "switched_rules_by_language")
+
+switched_rules_by_language(
+    name = "com_google_googleapis_imports",
+    gapic = True,
+    grpc = True,
+    java = True,
+)
+
+load("@io_grpc_grpc_java//:repositories.bzl", "grpc_java_repositories")
+
+grpc_java_repositories()
+
+_disco_to_proto3_converter_commit = "ce8d8732120cdfb5bf4847c3238b5be8acde87e3"
+
+http_archive(
+    name = "com_google_disco_to_proto3_converter",
+    strip_prefix = "disco-to-proto3-converter-%s" % _disco_to_proto3_converter_commit,
+    urls = ["https://github.com/googleapis/disco-to-proto3-converter/archive/%s.zip" % _disco_to_proto3_converter_commit],
+)
+
+# Showcase
+_showcase_version = "0.28.2"
+
+http_archive(
+    name = "com_google_gapic_showcase",
+    strip_prefix = "gapic-showcase-%s" % _showcase_version,
+    urls = [
+        "https://github.com/googleapis/gapic-showcase/archive/refs/tags/v%s.zip" % _showcase_version,
+    ],
+)
+
+http_archive(
+    name = "rules_pkg",
+    sha256 = "8a298e832762eda1830597d64fe7db58178aa84cd5926d76d5b744d6558941c2",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz",
+        "https://github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz",
+    ],
+)
+
+load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
+
+rules_pkg_dependencies()

library_generation/test/resources/misc/testversions.txt

@@ -0,0 +1,9 @@
+# test versions.txt obtained from sdk-platform-java
+
+gapic-generator-java:2.25.0:2.25.1-SNAPSHOT
+api-common:2.16.0:2.16.1-SNAPSHOT
+gax:2.33.0:2.33.1-SNAPSHOT
+gax-grpc:2.34.0:2.33.1-SNAPSHOT
+gax-httpjson:0.118.0:0.118.1-SNAPSHOT
+proto-google-common-protos:2.24.0:2.24.1-SNAPSHOT
+grpc-google-common-protos:2.24.0:2.24.1-SNAPSHOT