feat: Move release note generation to a sub module (#3299)

In this PR:
- Separate projects to three modules: common, library_generation and
release_note_generation
- Add `generate_release_note.py` to separate PR description generation
from `entrypoint.py`.
- Only install common and library_generation module in image.
- Remove PR description comparison in integration test.

Example generation workflow in downstream library:
https://github.com/googleapis/java-bigtable/actions/runs/11442581326/job/31833887512

Note that `release_note_generation` module still depends on
`library_generation` module because we didn't separate config change
functions into `common` module. This will be in a follow up PR.

Author: JoeWang1127

.cloudbuild/library_generation/library_generation.Dockerfile

@@ -45,19 +45,20 @@ RUN apt-get update && apt-get install -y \
 	&& apt-get clean
 
 # copy source code
-COPY library_generation /src
+COPY hermetic_build/common /src/common
+COPY hermetic_build/library_generation /src/library_generation
 
 # install protoc
 WORKDIR /protoc
-RUN source /src/utils/utilities.sh \
+RUN source /src/library_generation/utils/utilities.sh \
 	&& download_protoc "${PROTOC_VERSION}" "${OS_ARCHITECTURE}"
 # we indicate protoc is available in the container via env vars
 ENV DOCKER_PROTOC_LOCATION=/protoc
 ENV DOCKER_PROTOC_VERSION="${PROTOC_VERSION}"
 
 # install grpc
 WORKDIR /grpc
-RUN source /src/utils/utilities.sh \
+RUN source /src/library_generation/utils/utilities.sh \
 	&& download_grpc_plugin "${GRPC_VERSION}" "${OS_ARCHITECTURE}"
 # similar to protoc, we indicate grpc is available in the container via env vars
 ENV DOCKER_GRPC_LOCATION="/grpc/protoc-gen-grpc-java-${GRPC_VERSION}-${OS_ARCHITECTURE}.exe"
@@ -71,16 +72,18 @@ ENV DOCKER_GRPC_VERSION="${GRPC_VERSION}"
 COPY --from=ggj-build "/sdk-platform-java/gapic-generator-java.jar" "${HOME}/.library_generation/gapic-generator-java.jar"
 RUN chmod 755 "${HOME}/.library_generation/gapic-generator-java.jar"
 
-#  use python 3.11 (the base image has several python versions; here we define the default one)
+#  use python 3.12 (the base image has several python versions; here we define the default one)
 RUN rm $(which python3)
-RUN ln -s $(which python3.11) /usr/local/bin/python
-RUN ln -s $(which python3.11) /usr/local/bin/python3
+RUN ln -s $(which python3.12) /usr/local/bin/python
+RUN ln -s $(which python3.12) /usr/local/bin/python3
 RUN python -m pip install --upgrade pip
 
 # install main scripts as a python package
-WORKDIR /src
-RUN python -m pip install -r requirements.txt
-RUN python -m pip install .
+WORKDIR /
+RUN python -m pip install --require-hashes -r src/common/requirements.txt
+RUN python -m pip install src/common
+RUN python -m pip install --require-hashes -r src/library_generation/requirements.txt
+RUN python -m pip install src/library_generation
 
 # Install nvm with node and npm
 ENV NODE_VERSION 20.12.0
@@ -120,4 +123,4 @@ RUN chmod -R a+rw /home
 RUN chmod -R a+rx /home/.nvm
 
 WORKDIR /workspace
-ENTRYPOINT [ "python", "/src/cli/entry_point.py", "generate" ]
+ENTRYPOINT [ "python", "/src/library_generation/cli/entry_point.py", "generate" ]

.github/scripts/action.yaml

@@ -37,11 +37,33 @@ inputs:
 runs:
   using: "composite"
   steps:
+  - uses: actions/setup-python@v5
+    with:
+      python-version: 3.12
+  - name: Copy python script
+    shell: bash
+    run: |
+      set -x
+      # repository root
+      cd ${{ github.action_path }}/../../
+      rsync -rv \
+        --exclude=tests \
+        hermetic_build "${GITHUB_WORKSPACE}"
   - name: Copy shell script
     shell: bash
     run: |
       cd ${{ github.action_path }}
-      cp hermetic_library_generation.sh $GITHUB_WORKSPACE
+      cp hermetic_library_generation.sh "${GITHUB_WORKSPACE}"
+  - name: Install python packages
+    shell: bash
+    run: |
+      cd "${GITHUB_WORKSPACE}"
+      pip install --require-hashes -r hermetic_build/common/requirements.txt
+      pip install hermetic_build/common
+      pip install --require-hashes -r hermetic_build/library_generation/requirements.txt
+      pip install hermetic_build/library_generation
+      pip install --require-hashes -r hermetic_build/release_note_generation/requirements.txt
+      pip install hermetic_build/release_note_generation
   - name: Generate changed libraries
     shell: bash
     run: |

.github/scripts/hermetic_library_generation.sh

@@ -106,12 +106,16 @@ docker run \
   --current-generation-config-path="${workspace_name}/${generation_config}" \
   --api-definitions-path="${workspace_name}/googleapis"
 
+python hermetic_build/release_note_generation/cli/generate_release_note.py generate \
+  --baseline-generation-config-path="${baseline_generation_config}" \
+  --current-generation-config-path="${generation_config}"
+
 # remove api definitions after generation
 rm -rf "${api_def_dir}"
 
 # commit the change to the pull request.
 rm -rdf output googleapis "${baseline_generation_config}"
-git add --all -- ':!pr_description.txt' ':!hermetic_library_generation.sh'
+git add --all -- ':!pr_description.txt' ':!hermetic_library_generation.sh' ':!hermetic_build'
 changed_files=$(git diff --cached --name-only)
 if [[ "${changed_files}" != "" ]]; then
     echo "Commit changes..."

.github/workflows/verify_library_generation.yaml

@@ -27,7 +27,7 @@ jobs:
           git checkout -b "${head_ref}" fork/${head_ref}
           changed_directories="$(git diff --name-only "fork/${head_ref}" "origin/${base_ref}")"
         fi
-        if [[ ${changed_directories} =~ "library_generation/" ]]; then
+        if [[ ${changed_directories} =~ "hermetic_build/" ]]; then
           echo "should_run=true" >> $GITHUB_OUTPUT
         else
           echo "should_run=false" >> $GITHUB_OUTPUT
@@ -46,32 +46,21 @@ jobs:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
-        python-version: 3.11
-    - name: install pyenv
+        python-version: 3.12
+    - name: install python modules and dependencies
       shell: bash
       run: |
         set -ex
-        curl https://pyenv.run | bash
-        # setup environment
-        export PYENV_ROOT="$HOME/.pyenv"
-        export PATH="$PYENV_ROOT/bin:$PATH"
-        echo "PYENV_ROOT=${PYENV_ROOT}" >> $GITHUB_ENV
-        echo "PATH=${PATH}" >> $GITHUB_ENV
-
-        set +ex
-    - name: install python dependencies
-      shell: bash
-      run: |
-        set -ex
-        pushd library_generation
-        pip install -r requirements.txt
-        pip install .
-        popd
+        pip install --upgrade pip
+        pip install --require-hashes -r hermetic_build/common/requirements.txt
+        pip install hermetic_build/common
+        pip install --require-hashes -r hermetic_build/library_generation/requirements.txt
+        pip install hermetic_build/library_generation
     - name: Run integration tests
       shell: bash
       run: |
         set -x
-        python -m unittest library_generation/test/integration_tests.py
+        python -m unittest hermetic_build/library_generation/tests/integration_tests.py
   library-generation-unit-tests:
     runs-on: ubuntu-22.04
     needs: should-run-library-generation-tests
@@ -80,23 +69,26 @@ jobs:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
-        python-version: 3.11
-    - name: install python dependencies
+        python-version: 3.12
+    - name: install python modules and dependencies
       shell: bash
       run: |
         set -ex
-        pushd library_generation
-        pip install -r requirements.txt
-        pip install .
-        popd
+        pip install --upgrade pip
+        pip install --require-hashes -r hermetic_build/common/requirements.txt
+        pip install hermetic_build/common
+        pip install --require-hashes -r hermetic_build/library_generation/requirements.txt
+        pip install hermetic_build/library_generation
+        pip install --require-hashes -r hermetic_build/release_note_generation/requirements.txt
+        pip install hermetic_build/release_note_generation
     - name: Run shell unit tests
       run: |
         set -x
-        library_generation/test/generate_library_unit_tests.sh
+        hermetic_build/library_generation/tests/generate_library_unit_tests.sh
     - name: Run python unit tests
       run: |
         set -x
-        python -m unittest discover -s library_generation/test/ -p "*unit_tests.py"
+        python -m unittest discover -s hermetic_build -p "*unit_tests.py"
   library-generation-lint-shell:
     runs-on: ubuntu-22.04
     needs: should-run-library-generation-tests
@@ -106,7 +98,7 @@ jobs:
     - name: Run ShellCheck
       uses: ludeeus/[email protected]
       with:
-        scandir: 'library_generation'
+        scandir: 'hermetic_build'
         format: tty
         severity: error
   library-generation-lint-python:
@@ -115,16 +107,23 @@ jobs:
     if: needs.should-run-library-generation-tests.outputs.should_run == 'true'
     steps:
     - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: 3.12
     - name: install python dependencies
       shell: bash
       run: |
         set -ex
-        pushd library_generation
-        pip install -r requirements.txt
-        popd
+        pip install --upgrade pip
+        pip install --require-hashes -r hermetic_build/common/requirements.txt
+        pip install hermetic_build/common
+        pip install --require-hashes -r hermetic_build/library_generation/requirements.txt
+        pip install hermetic_build/library_generation
+        pip install --require-hashes -r hermetic_build/release_note_generation/requirements.txt
+        pip install hermetic_build/release_note_generation
     - name: Lint
       shell: bash
       run: |
         # exclude generated golden files
         # exclude owlbot until further refaction
-        black --check library_generation --exclude "(library_generation/test/resources/goldens)"
+        black --check hermetic_build --exclude "(library_generation/tests/resources/goldens)"

library_generation/__init__.py renamed to hermetic_build/common/__init__.py

@@ -16,8 +16,8 @@
 
 import yaml
 from typing import Optional
-from library_generation.model.library_config import LibraryConfig
-from library_generation.model.gapic_config import GapicConfig
+from common.model.library_config import LibraryConfig
+from common.model.gapic_config import GapicConfig
 
 REPO_LEVEL_PARAMETER = "Repo level parameter"
 LIBRARY_LEVEL_PARAMETER = "Library level parameter"

library_generation/model/gapic_config.py renamed to hermetic_build/common/model/gapic_config.py

@@ -14,8 +14,8 @@
 # limitations under the License.
 from hashlib import sha256
 from typing import Optional
-from library_generation.model.gapic_config import GapicConfig
-from library_generation.model.gapic_inputs import GapicInputs
+from common.model.gapic_config import GapicConfig
+from common.model.gapic_inputs import GapicInputs
 
 
 MAVEN_COORDINATE_SEPARATOR = ":"

library_generation/model/gapic_inputs.py renamed to hermetic_build/common/model/gapic_inputs.py

@@ -0,0 +1,3 @@
+black==24.8.0
+parameterized==0.9.0
+PyYAML==6.0.2