feat: generate a GAPIC library from api definition (#3208)

In this PR:
- Generate a GAPIC library from api definition (proto, service yaml),
rather than from googleapis repository.

To generate a GAPIC library with hermetic build image, the user has to
prepare the api definition to a directory and set
`--api-definition-path` to this directory.

This feature allows the user to generate a GAPIC library from any
protos, rather than checking in protos in googleapis repository.

Internal ticket: b/362705386

Author: JoeWang1127

.github/scripts/hermetic_library_generation.sh

@@ -84,16 +84,29 @@ git show "${target_branch}":"${generation_config}" > "${baseline_generation_conf
 # get .m2 folder so it's mapped into the docker container
 m2_folder=$(dirname "$(mvn help:evaluate -Dexpression=settings.localRepository -q -DforceStdout)")
 
+# download api definitions from googleapis repository
+googleapis_commitish=$(grep googleapis_commitish "${generation_config}" | cut -d ":" -f 2 | xargs)
+api_def_dir=$(mktemp -d)
+git clone https://github.com/googleapis/googleapis.git "${api_def_dir}"
+pushd "${api_def_dir}"
+git checkout "${googleapis_commitish}"
+popd
+
 # run hermetic code generation docker image.
 docker run \
   --rm \
   -u "$(id -u):$(id -g)" \
   -v "$(pwd):${workspace_name}" \
   -v "${m2_folder}":/home/.m2 \
+  -v "${api_def_dir}:${workspace_name}/googleapis" \
   -e GENERATOR_VERSION="${image_tag}" \
   gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \
   --baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \
-  --current-generation-config-path="${workspace_name}/${generation_config}"
+  --current-generation-config-path="${workspace_name}/${generation_config}" \
+  --api-definitions-path="${workspace_name}/googleapis"
+
+# remove api definitions after generation
+rm -rf "${api_def_dir}"
 
 # commit the change to the pull request.
 rm -rdf output googleapis "${baseline_generation_config}"

library_generation/DEVELOPMENT.md

@@ -101,8 +101,14 @@ shell session.
 
 ## Running the script
 The entrypoint script (`library_generation/cli/entry_point.py`) allows you to
-update the target repository with the latest changes starting from the
-googleapis committish declared in `generation_config.yaml`.
+generate a GAPIC repository with a given api definition (proto, service yaml).
+
+### Download the api definition
+For example, googleapis
+```
+git clone https://github.com/googleapis/googleapis
+export api_definitions_path="$(pwd)/googleapis"
+```
 
 ### Download the repo
 For example, google-cloud-java
@@ -118,7 +124,9 @@ python -m pip install .
 
 ### Run the script
 ```
-python cli/entry_point.py generate --repository-path="${path_to_repo}"
+python cli/entry_point.py generate \
+   --repository-path="${path_to_repo}" \
+   --api-definitions-path="${api_definitions_path}"
 ```
 
 
@@ -144,16 +152,25 @@ repo to this folder).
 
 To run the docker container on the google-cloud-java repo, you must run:
 ```bash
-docker run -u "$(id -u)":"$(id -g)"  -v/path/to/google-cloud-java:/workspace $(cat image-id)
+docker run \
+  -u "$(id -u)":"$(id -g)"  \
+  -v /path/to/google-cloud-java:/workspace \
+  -v /path/to/api-definition:/workspace/apis \
+  $(cat image-id) \
+  --api-definitions-path=/workspace/apis
 ```
 
  * `-u "$(id -u)":"$(id -g)"` makes docker run the container impersonating
    yourself. This avoids folder ownership changes since it runs as root by
    default.
- * `-v/path/to/google-cloud-java:/workspace` maps the host machine's
-   google-cloud-java folder to the /workspace folder. The image is configured to
-   perform changes in this directory
- * `$(cat image-id)` obtains the image ID created in the build step
+ * `-v /path/to/google-cloud-java:/workspace` maps the host machine's
+   google-cloud-java folder to the /workspace folder. 
+   The image is configured to perform changes in this directory.
+ * `-v /path/to/api-definition:/workspace/apis` maps the host machine's
+   api-definition folder to /workspace/apis folder.
+ * `$(cat image-id)` obtains the image ID created in the build step.
+ * `--api-definitions-path=/workspace/apis` set the API definition path to
+   `/workspace/apis`.
 
 ## Debug the created containers
 If you are working on changing the way the containers are created, you may want
@@ -173,5 +190,10 @@ We add `less` and `vim` as text tools for further inspection.
 You can also run a shell in a new container by running:
 
 ```bash
-docker run --rm -it -u=$(id -u):$(id -g)  -v/path/to/google-cloud-java:/workspace --entrypoint="bash" $(cat image-id)
+docker run \
+  --rm -it \
+  -u $(id -u):$(id -g) \
+  -v /path/to/google-cloud-java:/workspace \
+  --entrypoint="bash" \
+  $(cat image-id)
 ```

library_generation/README.md

@@ -1,7 +1,7 @@
 # Generate a repository containing GAPIC Client Libraries
 
 The script, `entry_point.py`, allows you to generate a repository containing
-GAPIC client libraries with googleapis commit history (a monorepo, for example,
+GAPIC client libraries with change history (a monorepo, for example, 
 google-cloud-java) from a configuration file.
 
 ## Environment
@@ -48,6 +48,22 @@ right version for each library.
 Please refer [here](go/java-client-releasing#versionstxt-manifest) for more info
 of versions.txt.
 
+### Api definitions path (`api_definitions_path`), optional
+
+The path to where the api definition (proto, service yaml) resides.
+
+The default value is the current working directory when running the script.
+
+Note that you need not only the protos defined the service, but also the transitive
+dependencies of those protos.
+Any missing dependencies will cause `File not found` error.
+
+For example, if your service is defined in `example_service.proto` and it imports
+`google/api/annotations.proto`, you need the `annotations.proto` resides in a
+folder that has the exact structure of the import statement (`google/api` in this
+case), and set `api_definitions_path` to the path contains the root folder (`google`
+in this case).
+
 ## Output of `entry_point.py`
 
 ### GAPIC libraries
@@ -74,11 +90,13 @@ will be created/modified:
 | pom.xml (repo root dir)             | Always generated from inputs                                             |
 | versions.txt                        | New entries will be added if they don’t exist                            |
 
-### googleapis commit history
+### Change history
 
 If both `baseline_generation_config` and `current_generation_config` are
-specified, and they contain different googleapis commit, the commit history will
-be generated into `pr_description.txt` in the `repository_path`.
+specified and the contents are different, the changed contents will be generated
+into `pr_description.txt` in the `repository_path`.
+In addition, if the `googleapis_commitish` is different, the googleapis commit
+history will be generated.
 
 ## Configuration to generate a repository
 
@@ -96,7 +114,7 @@ They are shared by library level parameters.
 | gapic_generator_version |    No    | set through env variable if not specified    |
 | protoc_version          |    No    | inferred from the generator if not specified |
 | grpc_version            |    No    | inferred from the generator if not specified |
-| googleapis-commitish    |   Yes    |                                              |
+| googleapis_commitish    |   Yes    |                                              |
 | libraries_bom_version   |    No    | empty string if not specified                |
 
 ### Library level parameters
@@ -183,22 +201,25 @@ The virtual environment can be installed to any folder, usually it is recommende
 2. Assuming the virtual environment is installed under `sdk-platform-java`. 
 Run the following command under the root folder of `sdk-platform-java` to install the dependencies of `library_generation`
 
-```bash
-python -m pip install -r library_generation/requirements.txt
-```
+   ```bash
+   python -m pip install -r library_generation/requirements.txt
+   ```
 
 3. Run the following command to install `library_generation` as a module, which allows the `library_generation` module to be imported from anywhere
-```bash
-python -m pip install library_generation/
-```
+    ```bash
+    python -m pip install library_generation/
+    ```
+
+4. Download api definition to a local directory
 
 ## An example to generate a repository using `entry_point.py`
 
 ```bash
 python library_generation/entry_point.py generate \
 --baseline-generation-config-path=/path/to/baseline_config_file \
 --current-generation-config-path=/path/to/current_config_file \
---repository-path=path/to/repository
+--repository-path=path/to/repository \
+--api-definitions-path=path/to/api_definition
 ```
 If you run `entry_point.py` with the example [configuration](#an-example-of-generation-configuration)
 shown above, the repository structure is:

library_generation/cli/entry_point.py

@@ -63,10 +63,22 @@ def main(ctx):
     directory.
     """,
 )
+@click.option(
+    "--api-definitions-path",
+    type=str,
+    default=".",
+    show_default=True,
+    help="""
+    The path to which the api definition (proto and service yaml) and its
+    dependencies resides.
+    If not specified, the path is the current working directory.
+    """,
+)
 def generate(
     baseline_generation_config_path: str,
     current_generation_config_path: str,
     repository_path: str,
+    api_definitions_path: str,
 ):
     """
     Compare baseline generation config and current generation config and
@@ -90,14 +102,18 @@ def generate(
     repository_path/pr_description.txt.
     """
     __generate_repo_and_pr_description_impl(
-        baseline_generation_config_path, current_generation_config_path, repository_path
+        baseline_generation_config_path=baseline_generation_config_path,
+        current_generation_config_path=current_generation_config_path,
+        repository_path=repository_path,
+        api_definitions_path=api_definitions_path,
     )
 
 
 def __generate_repo_and_pr_description_impl(
     baseline_generation_config_path: str,
     current_generation_config_path: str,
     repository_path: str,
+    api_definitions_path: str,
 ):
     """
     Implementation method for generate().
@@ -129,13 +145,15 @@ def __generate_repo_and_pr_description_impl(
 
     current_generation_config_path = os.path.abspath(current_generation_config_path)
     repository_path = os.path.abspath(repository_path)
+    api_definitions_path = os.path.abspath(api_definitions_path)
     if not baseline_generation_config_path:
         # Execute full generation based on current_generation_config if
         # baseline_generation_config is not specified.
         # Do not generate pull request description.
         generate_from_yaml(
             config=from_yaml(current_generation_config_path),
             repository_path=repository_path,
+            api_definitions_path=api_definitions_path,
         )
         return
 
@@ -155,6 +173,7 @@ def __generate_repo_and_pr_description_impl(
     generate_from_yaml(
         config=config_change.current_config,
         repository_path=repository_path,
+        api_definitions_path=api_definitions_path,
         target_library_names=target_library_names,
     )
     generate_pr_descriptions(

library_generation/generate_composed_library.py

@@ -59,10 +59,6 @@ def generate_composed_library(
     :return None
     """
     output_folder = repo_config.output_folder
-    util.pull_api_definition(
-        config=config, library=library, output_folder=output_folder
-    )
-
     base_arguments = __construct_tooling_arg(config=config)
     owlbot_cli_source_folder = util.sh_util("mktemp -d")
     os.makedirs(f"{library_path}", exist_ok=True)
@@ -73,7 +69,7 @@ def generate_composed_library(
         # generate postprocessing prerequisite files (.repo-metadata.json, .OwlBot-hermetic.yaml,
         # owlbot.py) here because transport is parsed from BUILD.bazel,
         # which lives in a versioned proto_path. The value of transport will be
-        # overriden by the config object if specified. Note that this override
+        # overridden by the config object if specified. Note that this override
         # does not affect library generation but instead used only for
         # generating postprocessing files such as README.
         util.generate_postprocessing_prerequisite_files(

library_generation/generate_repo.py

@@ -12,6 +12,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import os
+import shutil
+
 import library_generation.utils.utilities as util
 from library_generation.generate_composed_library import generate_composed_library
 from library_generation.model.generation_config import GenerationConfig
@@ -22,6 +25,7 @@
 def generate_from_yaml(
     config: GenerationConfig,
     repository_path: str,
+    api_definitions_path: str,
     target_library_names: list[str] = None,
 ) -> None:
     """
@@ -31,6 +35,7 @@ def generate_from_yaml(
     :param config: a GenerationConfig object.
     :param repository_path: The repository path to which the generated files
     will be sent.
+    :param api_definitions_path: The path to where the api definition resides.
     :param target_library_names: a list of libraries to be generated.
     If specified, only the library whose library_name is in target_library_names
     will be generated.
@@ -43,6 +48,8 @@ def generate_from_yaml(
     repo_config = util.prepare_repo(
         gen_config=config, library_config=target_libraries, repo_path=repository_path
     )
+    # copy api definition to output folder.
+    shutil.copytree(api_definitions_path, repo_config.output_folder, dirs_exist_ok=True)
 
     for library_path, library in repo_config.get_libraries().items():
         print(f"generating library {library.get_library_name()}")

library_generation/test/cli/entry_point_unit_tests.py

@@ -105,9 +105,13 @@ def test_generate_non_monorepo_without_changes_triggers_full_generation(
             baseline_generation_config_path=config_path,
             current_generation_config_path=config_path,
             repository_path=".",
+            api_definitions_path=".",
         )
         generate_from_yaml.assert_called_with(
-            config=ANY, repository_path=ANY, target_library_names=None
+            config=ANY,
+            repository_path=ANY,
+            api_definitions_path=ANY,
+            target_library_names=None,
         )
 
     @patch("library_generation.cli.entry_point.generate_from_yaml")
@@ -134,9 +138,13 @@ def test_generate_non_monorepo_with_changes_triggers_full_generation(
             baseline_generation_config_path=baseline_config_path,
             current_generation_config_path=current_config_path,
             repository_path=".",
+            api_definitions_path=".",
         )
         generate_from_yaml.assert_called_with(
-            config=ANY, repository_path=ANY, target_library_names=None
+            config=ANY,
+            repository_path=ANY,
+            api_definitions_path=ANY,
+            target_library_names=None,
         )
 
     @patch("library_generation.cli.entry_point.generate_from_yaml")
@@ -160,9 +168,13 @@ def test_generate_monorepo_with_common_protos_triggers_full_generation(
             baseline_generation_config_path=config_path,
             current_generation_config_path=config_path,
             repository_path=".",
+            api_definitions_path=".",
         )
         generate_from_yaml.assert_called_with(
-            config=ANY, repository_path=ANY, target_library_names=None
+            config=ANY,
+            repository_path=ANY,
+            api_definitions_path=ANY,
+            target_library_names=None,
         )
 
     @patch("library_generation.cli.entry_point.generate_from_yaml")
@@ -187,7 +199,11 @@ def test_generate_monorepo_without_common_protos_does_not_trigger_full_generatio
             baseline_generation_config_path=config_path,
             current_generation_config_path=config_path,
             repository_path=".",
+            api_definitions_path=".",
         )
         generate_from_yaml.assert_called_with(
-            config=ANY, repository_path=ANY, target_library_names=[]
+            config=ANY,
+            repository_path=ANY,
+            api_definitions_path=ANY,
+            target_library_names=[],
         )