Skip to content

Commit fda19d9

Browse files
Amar3ttoAmar3tto
authored
Merge branch 'apache:master' into master
2 parents fd35cc9 + fb80584 commit fda19d9

File tree

64 files changed

+1363
-251
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

64 files changed

+1363
-251
lines changed

‎.github/actions/setup-default-test-properties/test-properties.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,6 @@
1818
"SPARK_VERSIONS": ["2", "3"]
1919
},
2020
"GoTestProperties": {
21-
"SUPPORTED_VERSIONS": ["1.23"]
21+
"SUPPORTED_VERSIONS": ["1.25"]
2222
}
2323
}

‎.github/actions/setup-environment-action/action.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,5 +83,5 @@ runs:
8383
if: ${{ inputs.go-version != '' }}
8484
uses: actions/setup-go@v5
8585
with:
86-
go-version: ${{ inputs.go-version == 'default' && '1.24' || inputs.go-version }} # never set patch, to get latest patch releases.
86+
go-version: ${{ inputs.go-version == 'default' && '1.25' || inputs.go-version }} # never set patch, to get latest patch releases.
8787
cache-dependency-path: $${{ inputs.disable-cache && '' || 'sdks/go.sum' }}

‎.github/gh-actions-self-hosted-runners/arc/images/Dockerfile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ RUN curl -OL https://nodejs.org/dist/v22.14.0/node-v22.14.0-linux-x64.tar.xz &&
3131
mv /usr/local/node-v22.14.0-linux-x64 /usr/local/node
3232
ENV PATH="${PATH}:/usr/local/node/bin"
3333
#Install Go
34-
ARG go_version=1.24.0
34+
ARG go_version=1.25.2
3535
RUN curl -OL https://go.dev/dl/go${go_version}.linux-amd64.tar.gz && \
3636
tar -C /usr/local -xzf go${go_version}.linux-amd64.tar.gz && \
3737
rm go${go_version}.linux-amd64.tar.gz
@@ -69,7 +69,7 @@ RUN curl -OL https://dl.k8s.io/release/v1.28.1/bin/linux/amd64/kubectl && \
6969
chmod +x ./kubectl && \
7070
mv ./kubectl /usr/local/bin/kubectl
7171
#Install Apache Maven
72-
RUN curl -OL https://dlcdn.apache.org/maven/maven-3/3.9.4/binaries/apache-maven-3.9.4-bin.tar.gz && \
72+
RUN curl -OL https://archive.apache.org/dist/maven/maven-3/3.9.4/binaries/apache-maven-3.9.4-bin.tar.gz && \
7373
tar -xvf apache-maven-3.9.4-bin.tar.gz && \
7474
rm apache-maven-3.9.4-bin.tar.gz && \
7575
mv apache-maven-3.9.4 /usr/local/maven

‎.github/trigger_files/beam_PostCommit_Java_PVR_Spark3_Streaming.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
{
22
"comment": "Modify this file in a trivial way to cause this test suite to run",
3-
"modification": 4
3+
"modification": 5
44
}

‎.github/workflows/beam_LoadTests_Python_CoGBK_Flink_Batch.yml‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -89,9 +89,9 @@ jobs:
8989
test-type: load
9090
test-language: python
9191
argument-file-paths: |
92-
${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Flink_Batch_100b_Single_Key.txt
93-
${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Flink_Batch_100b_Multiple_Keys.txt
94-
${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Flink_Batch_10kB.txt
92+
${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Single_Key.txt
93+
${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Multiple_Keys.txt
94+
${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_10kB.txt
9595
- name: Start Flink with parallelism 5
9696
env:
9797
FLINK_NUM_WORKERS: 5
@@ -108,28 +108,31 @@ jobs:
108108
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
109109
arguments: |
110110
--info \
111+
-PpythonVersion=3.9 \
111112
-PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \
112-
-Prunner=FlinkRunner \
113+
-Prunner=PortableRunner \
113114
'-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Flink_Batch_test_arguments_1 }} --job_name=load-tests-python-flink-batch-cogbk-1-${{ steps.datetime.outputs.datetime }}' \
114115
- name: run CoGBK 2GB of 100B records with multiple keys
115116
uses: ./.github/actions/gradle-command-self-hosted-action
116117
with:
117118
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
118119
arguments: |
119120
--info \
121+
-PpythonVersion=3.9 \
120122
-PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \
121-
-Prunner=FlinkRunner \
123+
-Prunner=PortableRunner \
122124
'-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Flink_Batch_test_arguments_2 }} --job_name=load-tests-python-flink-batch-cogbk-2-${{ steps.datetime.outputs.datetime }}' \
123125
- name: run CoGBK reiterate 4 times 10kB values
124126
uses: ./.github/actions/gradle-command-self-hosted-action
125127
with:
126128
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
127129
arguments: |
128130
--info \
131+
-PpythonVersion=3.9 \
129132
-PloadTest.mainClass=apache_beam.testing.load_tests.co_group_by_key_test \
130-
-Prunner=FlinkRunner \
133+
-Prunner=PortableRunner \
131134
'-PloadTest.args=${{ env.beam_LoadTests_Python_CoGBK_Flink_Batch_test_arguments_2 }} --job_name=load-tests-python-flink-batch-cogbk-3-${{ steps.datetime.outputs.datetime }}' \
132135
- name: Teardown Flink
133136
if: always()
134137
run: |
135-
${{ github.workspace }}/.test-infra/dataproc/flink_cluster.sh delete
138+
${{ github.workspace }}/.test-infra/dataproc/flink_cluster.sh delete

‎.github/workflows/beam_Publish_Docker_Snapshots.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ jobs:
8383
arguments: |
8484
-Pdocker-repository-root=gcr.io/apache-beam-testing/beam_portability \
8585
-Pdocker-tag-list=${{ github.sha }}${LATEST_TAG}
86-
- name: run Publish Docker Snapshots script for Flink
86+
- name: run Publish Docker Snapshots script for Flink 1.17
8787
uses: ./.github/actions/gradle-command-self-hosted-action
8888
with:
8989
gradle-command: :runners:flink:1.17:job-server-container:dockerPush

‎.github/workflows/build_release_candidate.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -451,7 +451,7 @@ jobs:
451451
fi
452452
- uses: actions/setup-go@v6
453453
with:
454-
go-version: '1.24'
454+
go-version: '1.25'
455455
- name: Import GPG key
456456
id: import_gpg
457457
uses: crazy-max/ghaction-import-gpg@e89d40939c28e39f97cf32126055eeae86ba74ec

‎.github/workflows/code_completion_plugin_tests.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ jobs:
7070

7171
# Validate wrapper
7272
- name: Gradle Wrapper Validation
73-
uses: gradle/wrapper-validation-action@v1.0.6
73+
uses: gradle/wrapper-validation-action@v3.5.0
7474

7575
# Setup Java environment for the next steps
7676
- name: Setup Java

.github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Flink_Batch_100b_Multiple_Keys.txt renamed to .github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Multiple_Keys.txt

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,15 @@
1414
# See the License for the specific language governing permissions and
1515
# limitations under the License.
1616

17-
--temp_location=gs://temp-storage-for-perf-tests/loadtests
1817
--publish_to_big_query=true
1918
--metrics_dataset=load_test
2019
--metrics_table=python_flink_batch_cogbk_2
2120
--influx_measurement=python_batch_cogbk_2
22-
--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":5,\\"hot_key_fraction\\":1}''
23-
--co_input_options=''{\\"num_records\\":2000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":5,\\"hot_key_fraction\\":1}''
21+
--input_options=''{\\"num_records\\":200000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":5,\\"hot_key_fraction\\":1}''
22+
--co_input_options=''{\\"num_records\\":20000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":5,\\"hot_key_fraction\\":1}''
2423
--iterations=1
2524
--parallelism=5
26-
--endpoint=localhost:8099
25+
--runner=PortableRunner
26+
--job_endpoint=localhost:8099
2727
--environment_type=DOCKER
28-
--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest
28+
--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest

.github/workflows/load-tests-pipeline-options/python_CoGBK_Dataflow_Flink_Batch_100b_Single_Key.txt renamed to .github/workflows/load-tests-pipeline-options/python_CoGBK_Flink_Batch_100b_Single_Key.txt

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,15 @@
1414
# See the License for the specific language governing permissions and
1515
# limitations under the License.
1616

17-
--temp_location=gs://temp-storage-for-perf-tests/loadtests
1817
--publish_to_big_query=true
1918
--metrics_dataset=load_test
2019
--metrics_table=python_flink_batch_cogbk_1
2120
--influx_measurement=python_batch_cogbk_1
22-
--input_options=''{\\"num_records\\":20000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":1,\\"hot_key_fraction\\":1}''
23-
--co_input_options=''{\\"num_records\\":2000000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":1000,\\"hot_key_fraction\\":1}''
21+
--input_options=''{\\"num_records\\":200000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":1,\\"hot_key_fraction\\":1}''
22+
--co_input_options=''{\\"num_records\\":20000,\\"key_size\\":10,\\"value_size\\":90,\\"num_hot_keys\\":100,\\"hot_key_fraction\\":1}''
2423
--iterations=1
2524
--parallelism=5
26-
--endpoint=localhost:8099
25+
--runner=PortableRunner
26+
--job_endpoint=localhost:8099
2727
--environment_type=DOCKER
28-
--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_go_sdk:latest
28+
--environment_config=gcr.io/apache-beam-testing/beam-sdk/beam_python3.9_sdk:latest

0 commit comments

Comments
 (0)