From 33319ecc1625c69db6fe3d3daecdf7d435afd6b7 Mon Sep 17 00:00:00 2001 From: Ali Ebrahim Date: Mon, 17 Nov 2025 22:33:21 -0800 Subject: [PATCH 1/2] Support newer versions of pyarrow. --- sdks/python/setup.py | 2 +- sdks/python/tox.ini | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index b415c9bb47d1..96b10b136ead 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -145,7 +145,7 @@ def cythonize(*args, **kwargs): pyarrow_dependency = [''] else: pyarrow_dependency = [ - 'pyarrow>=3.0.0,<19.0.0', + 'pyarrow>=3.0.0,<23.0.0', # NOTE(https://github.com/apache/beam/issues/29392): We can remove this # once Beam increases the pyarrow lower bound to a version that fixes CVE. # (lower bound >= 14.0.1) diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index afb9b6520fb6..2ff18af310e5 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -343,7 +343,7 @@ extras = test commands = bash {toxinidir}/scripts/pytest_validates_runner.sh {envname} {toxinidir}/apache_beam/runners/portability/prism_runner_test.py {posargs} -[testenv:py{310,311}-pyarrow-{3,9,10,11,12,13,14,15,16,17,18}] +[testenv:py{310,311}-pyarrow-{3,9,10,11,12,13,14,15,16,17,18,19,20,21,22}] deps = # As a courtesy to users, test against the oldest allowed version of Pyarrow. # We'd have to increase the pyarrow lower bound when Python 3.9 is deprecated. @@ -363,6 +363,10 @@ deps = 16: pyarrow>=16,<17 17: pyarrow>=17,<18 18: pyarrow>=18,<19 + 19: pyarrow>=19,<20 + 20: pyarrow>=20,<21 + 21: pyarrow>=21,<22 + 22: pyarrow>=22,<23 numpy==1.26.4 commands = # Log pyarrow and numpy version for debugging From fc3b21776fb860b0c7cdc9fef8b579b48f8777aa Mon Sep 17 00:00:00 2001 From: Ali Ebrahim Date: Fri, 5 Dec 2025 09:18:25 -0800 Subject: [PATCH 2/2] PR comments. --- .../python/test-suites/tox/py310/build.gradle | 40 ++++++++----------- sdks/python/tox.ini | 5 --- 2 files changed, 16 insertions(+), 29 deletions(-) diff --git a/sdks/python/test-suites/tox/py310/build.gradle b/sdks/python/test-suites/tox/py310/build.gradle index 751faa682ae3..85a113edc1a4 100644 --- a/sdks/python/test-suites/tox/py310/build.gradle +++ b/sdks/python/test-suites/tox/py310/build.gradle @@ -49,30 +49,6 @@ project.tasks.register("postCommitPyDep") {} // For versions that we would like to prioritize for testing, // for example versions released in a timeframe of last 1-2 years. -toxTask "testPy310pyarrow-9", "py310-pyarrow-9", "${posargs}" -test.dependsOn "testPy310pyarrow-9" -postCommitPyDep.dependsOn "testPy310pyarrow-9" - -toxTask "testPy310pyarrow-10", "py310-pyarrow-10", "${posargs}" -test.dependsOn "testPy310pyarrow-10" -postCommitPyDep.dependsOn "testPy310pyarrow-10" - -toxTask "testPy310pyarrow-11", "py310-pyarrow-11", "${posargs}" -test.dependsOn "testPy310pyarrow-11" -postCommitPyDep.dependsOn "testPy310pyarrow-11" - -toxTask "testPy310pyarrow-12", "py310-pyarrow-12", "${posargs}" -test.dependsOn "testPy310pyarrow-12" -postCommitPyDep.dependsOn "testPy310pyarrow-12" - -toxTask "testPy310pyarrow-13", "py310-pyarrow-13", "${posargs}" -test.dependsOn "testPy310pyarrow-13" -postCommitPyDep.dependsOn "testPy310pyarrow-13" - -toxTask "testPy310pyarrow-14", "py310-pyarrow-14", "${posargs}" -test.dependsOn "testPy310pyarrow-14" -postCommitPyDep.dependsOn "testPy310pyarrow-14" - toxTask "testPy310pyarrow-15", "py310-pyarrow-15", "${posargs}" test.dependsOn "testPy310pyarrow-15" postCommitPyDep.dependsOn "testPy310pyarrow-15" @@ -89,6 +65,22 @@ toxTask "testPy310pyarrow-18", "py310-pyarrow-18", "${posargs}" test.dependsOn "testPy310pyarrow-18" postCommitPyDep.dependsOn "testPy310pyarrow-18" +toxTask "testPy310pyarrow-19", "py310-pyarrow-19", "${posargs}" +test.dependsOn "testPy310pyarrow-19" +postCommitPyDep.dependsOn "testPy310pyarrow-19" + +toxTask "testPy310pyarrow-20", "py310-pyarrow-20", "${posargs}" +test.dependsOn "testPy310pyarrow-20" +postCommitPyDep.dependsOn "testPy310pyarrow-20" + +toxTask "testPy310pyarrow-21", "py310-pyarrow-21", "${posargs}" +test.dependsOn "testPy310pyarrow-21" +postCommitPyDep.dependsOn "testPy310pyarrow-21" + +toxTask "testPy310pyarrow-22", "py310-pyarrow-22", "${posargs}" +test.dependsOn "testPy310pyarrow-22" +postCommitPyDep.dependsOn "testPy310pyarrow-22" + // Create a test task for each supported minor version of pandas toxTask "testPy310pandas-14", "py310-pandas-14", "${posargs}" test.dependsOn "testPy310pandas-14" diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index 2ff18af310e5..528602f1b2e5 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -354,11 +354,6 @@ deps = # Test against versions of pyarrow released in last ~2 years. 9: pyarrow>=9,<10 9: pandas==2.1.4 - 10: pyarrow>=10,<11 - 11: pyarrow>=11,<12 - 12: pyarrow>=12,<13 - 13: pyarrow>=13,<14 - 14: pyarrow>=14,<15 15: pyarrow>=15,<16 16: pyarrow>=16,<17 17: pyarrow>=17,<18