From 108749dda54b944ee15983be2222061d79125106 Mon Sep 17 00:00:00 2001 From: Stephen Rosen Date: Mon, 4 May 2026 11:15:10 -0500 Subject: [PATCH 1/2] Make tox factor usage more extensible Previously, the test environments were binary flagged between `mindeps` and `!mindeps`. This is unfortunately bad for adding additional factors, as discovered when testing `orjson` as a factor. Because `tox` does not allow negative factors to be layered with "OR" semantics, it becomes difficult to deselect the locked requirements in `test.txt` in order to replace them. The base testenv is simplified to have unconditional dependencies, and factor-based dependency selection is done in descendant envs which can therefore fully replace the deps declared by the base env. The change in configuration also provides an opportunity to refactor the `depends` declaration into the list form which makes it inheritable. --- tox.ini | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tox.ini b/tox.ini index 5f74c0ae1..c842d66ee 100644 --- a/tox.ini +++ b/tox.ini @@ -18,31 +18,36 @@ labels = # build a wheel, not a tarball, and use a common env to do it (so that the wheel is shared) package = wheel wheel_build_env = build_wheel +deps = -r requirements/py{py_dot_ver}/test.txt +commands = coverage run -m pytest {posargs} +depends = coverage_clean,lint +[testenv:py{3.14,3.13,3.12,3.11,3.10,3.9}-mindeps] +deps = -r requirements/py{py_dot_ver}/test-mindeps.txt + +[testenv:py{3.14,3.13,3.12,3.11,3.10,3.9}-sphinxext] deps = - !mindeps: -r requirements/py{py_dot_ver}/test.txt - mindeps: -r requirements/py{py_dot_ver}/test-mindeps.txt - sphinxext: -r requirements/py{py_dot_ver}/docs.txt -commands = coverage run -m pytest {posargs} -depends = - py{3.14,3.13,3.12,3.11,3.10,3.9}{-mindeps,-sphinxext,}: coverage_clean, lint - coverage_report: py{3.14,3.13,3.12,3.11,3.10,3.9}{-mindeps,-sphinxext,} + -r requirements/py{py_dot_ver}/test.txt + -r requirements/py{py_dot_ver}/docs.txt [testenv:coverage_clean] dependency_groups = coverage skip_install = true commands = coverage erase +depends = [testenv:coverage_report] dependency_groups = coverage skip_install = true commands_pre = -coverage combine commands = coverage report --skip-covered +depends = py{3.14,3.13,3.12,3.11,3.10,3.9}{-mindeps,-sphinxext,} [testenv:lint] deps = pre-commit skip_install = true commands = pre-commit run --all-files +depends = [testenv:mypy,mypy-{py3.9,py3.14}] deps = -r requirements/py{py_dot_ver}/typing.txt From 42e39991836b40cd78bf6023549b5d2b5c7007ff Mon Sep 17 00:00:00 2001 From: Stephen Rosen Date: Fri, 1 May 2026 18:07:29 -0500 Subject: [PATCH 2/2] Support `orjson` as an alternative encoder/decoder Add support for user selection of `orjson` as a faster and more correct JSON encoder/decoder. Request encoding is done by sending encoded bytes through `requests` (which is supported), and request decoding is done via a helper which reads `requests.Response.content` (bytes). This is primarily applied in - the JSON request encoder - the Response class - the APIError base class Additionally, GlobusApp and Transfer retry policies decode response contents, and are here updated to load from bytes. `mypy` runs are expanded to include `orjson` as a dependency, and test runs include it via a factor. New tests can directly target the encoder class, but as there is no central point of control for decoding, there is no clearly testable element for that purpose. tox configuration and testing requirements are updated to pull in `orjson` when the `-orjson` factor is used. GitHub Actions testing is now expanded to include `-orjson` tests. --- .github/workflows/test.yaml | 4 ++ ...1_170758_sirosen_define_orjson_encoder.rst | 8 +++ pyproject.toml | 3 + requirements/py3.10/test-orjson.txt | 55 +++++++++++++++++ requirements/py3.10/typing.txt | 34 ++++++----- requirements/py3.11/test-orjson.txt | 47 +++++++++++++++ requirements/py3.11/typing.txt | 38 ++++++------ requirements/py3.12/test-orjson.txt | 47 +++++++++++++++ requirements/py3.12/typing.txt | 38 ++++++------ requirements/py3.13/test-orjson.txt | 47 +++++++++++++++ requirements/py3.13/typing.txt | 38 ++++++------ requirements/py3.14/test-orjson.txt | 47 +++++++++++++++ requirements/py3.14/typing.txt | 38 ++++++------ requirements/py3.9/test-orjson.txt | 59 +++++++++++++++++++ requirements/py3.9/typing.txt | 34 ++++++----- scripts/ensure_min_python_is_tested.py | 2 +- src/globus_sdk/_internal/orjson_compat.py | 45 ++++++++++++++ src/globus_sdk/config/__init__.py | 8 ++- src/globus_sdk/config/env_vars.py | 15 +++++ src/globus_sdk/exc/api.py | 6 +- .../experimental/transfer_v2/transport.py | 5 +- src/globus_sdk/globus_app/app.py | 6 +- src/globus_sdk/response.py | 6 +- src/globus_sdk/services/transfer/transport.py | 5 +- src/globus_sdk/testing/registry.py | 1 + src/globus_sdk/transport/encoders.py | 33 ++++++++--- tests/unit/test_config.py | 16 +++++ .../unit/transport/test_transport_encoders.py | 34 +++++++++++ tox.ini | 7 +++ 29 files changed, 612 insertions(+), 114 deletions(-) create mode 100644 changelog.d/20260501_170758_sirosen_define_orjson_encoder.rst create mode 100644 requirements/py3.10/test-orjson.txt create mode 100644 requirements/py3.11/test-orjson.txt create mode 100644 requirements/py3.12/test-orjson.txt create mode 100644 requirements/py3.13/test-orjson.txt create mode 100644 requirements/py3.14/test-orjson.txt create mode 100644 requirements/py3.9/test-orjson.txt create mode 100644 src/globus_sdk/_internal/orjson_compat.py diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 88779cf26..591ebbf08 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -44,6 +44,8 @@ jobs: - "3.14" tox-post-environments: - "py3.9-mindeps" + - "py3.9-orjson" + - "py3.14-orjson" - "py3.11-sphinxext" - "coverage_report" @@ -53,6 +55,7 @@ jobs: - "3.11" tox-post-environments: - "py3.11-sphinxext" + - "py3.14-orjson" - "coverage_report" - name: "Windows" @@ -62,6 +65,7 @@ jobs: - "3.11" tox-post-environments: - "py3.9-mindeps" + - "py3.14-orjson" - "py3.11-sphinxext" - "coverage_report" diff --git a/changelog.d/20260501_170758_sirosen_define_orjson_encoder.rst b/changelog.d/20260501_170758_sirosen_define_orjson_encoder.rst new file mode 100644 index 000000000..82ad0184f --- /dev/null +++ b/changelog.d/20260501_170758_sirosen_define_orjson_encoder.rst @@ -0,0 +1,8 @@ +Added +----- + +- The SDK now supports use of ``orjson`` as an alternative JSON encoder and decoder. + When ``GLOBUS_SDK_PREFER_ORJSON=1`` is set, request sending and response decoding + will prefer to use ``orjson`` if it is installed and available, gracefully failing + over to the standard library ``json`` module if it is not. This setting will become + the default behavior in a future major version of the SDK. (:pr:`NUMBER`) diff --git a/pyproject.toml b/pyproject.toml index 3b970ce38..204975b85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ docs = [ "scriv", ] coverage = ["coverage[toml]"] +orjson = ["orjson>=3"] test = [ {include-group = "coverage"}, "pytest", "pytest-xdist", "pytest-randomly", "flaky", @@ -83,6 +84,8 @@ typing = [ "responses", # similarly, sphinx is needed to type-check our sphinx extension "sphinx", + # include any optional test deps + {include-group = "orjson"}, ] typing-mindeps = [ {include-group = "typing"}, diff --git a/requirements/py3.10/test-orjson.txt b/requirements/py3.10/test-orjson.txt new file mode 100644 index 000000000..e87fc1734 --- /dev/null +++ b/requirements/py3.10/test-orjson.txt @@ -0,0 +1,55 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# tox p -m freezedeps +# +certifi==2026.4.22 + # via requests +charset-normalizer==3.4.7 + # via requests +coverage==7.13.5 + # via -r .test.in +exceptiongroup==1.3.1 + # via pytest +execnet==2.1.2 + # via pytest-xdist +flaky==3.8.1 + # via -r .test.in +idna==3.13 + # via requests +iniconfig==2.3.0 + # via pytest +orjson==3.11.8 + # via -r .orjson.in +packaging==26.2 + # via pytest +pluggy==1.6.0 + # via pytest +pygments==2.20.0 + # via pytest +pytest==9.0.3 + # via + # -r .test.in + # pytest-randomly + # pytest-xdist +pytest-randomly==4.1.0 + # via -r .test.in +pytest-xdist==3.8.0 + # via -r .test.in +pyyaml==6.0.3 + # via responses +requests==2.33.1 + # via responses +responses==0.26.0 + # via -r .test.in +tomli==2.4.1 + # via + # coverage + # pytest +typing-extensions==4.15.0 + # via exceptiongroup +urllib3==2.6.3 + # via + # requests + # responses diff --git a/requirements/py3.10/typing.txt b/requirements/py3.10/typing.txt index fb57ddade..51b712ae4 100644 --- a/requirements/py3.10/typing.txt +++ b/requirements/py3.10/typing.txt @@ -6,39 +6,43 @@ # alabaster==1.0.0 # via sphinx -babel==2.17.0 +babel==2.18.0 # via sphinx -certifi==2025.11.12 +certifi==2026.4.22 # via requests -charset-normalizer==3.4.4 +charset-normalizer==3.4.7 # via requests docutils==0.21.2 # via sphinx -idna==3.11 +idna==3.13 # via requests -imagesize==1.4.1 +imagesize==2.0.0 # via sphinx jinja2==3.1.6 # via sphinx +librt==0.9.0 + # via mypy markupsafe==3.0.3 # via jinja2 -mypy==1.18.2 +mypy==1.20.2 # via -r .typing.in mypy-extensions==1.1.0 # via mypy -packaging==25.0 +orjson==3.11.8 + # via -r .typing.in +packaging==26.2 # via sphinx -pathspec==0.12.1 +pathspec==1.1.1 # via mypy -pygments==2.19.2 +pygments==2.20.0 # via sphinx pyyaml==6.0.3 # via responses -requests==2.32.5 +requests==2.33.1 # via # responses # sphinx -responses==0.25.8 +responses==0.26.0 # via -r .typing.in snowballstemmer==3.0.1 # via sphinx @@ -56,23 +60,23 @@ sphinxcontrib-qthelp==2.0.0 # via sphinx sphinxcontrib-serializinghtml==2.0.0 # via sphinx -tomli==2.3.0 +tomli==2.4.1 # via # mypy # sphinx types-cryptography==3.3.23.2 # via types-jwt -types-docutils==0.22.3.20251115 +types-docutils==0.22.3.20260408 # via -r .typing.in types-jwt==0.1.3 # via -r .typing.in -types-requests==2.32.4.20250913 +types-requests==2.33.0.20260408 # via -r .typing.in typing-extensions==4.15.0 # via # -r .typing.in # mypy -urllib3==2.5.0 +urllib3==2.6.3 # via # requests # responses diff --git a/requirements/py3.11/test-orjson.txt b/requirements/py3.11/test-orjson.txt new file mode 100644 index 000000000..9f33bfdf8 --- /dev/null +++ b/requirements/py3.11/test-orjson.txt @@ -0,0 +1,47 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# tox p -m freezedeps +# +certifi==2026.4.22 + # via requests +charset-normalizer==3.4.7 + # via requests +coverage==7.13.5 + # via -r .test.in +execnet==2.1.2 + # via pytest-xdist +flaky==3.8.1 + # via -r .test.in +idna==3.13 + # via requests +iniconfig==2.3.0 + # via pytest +orjson==3.11.8 + # via -r .orjson.in +packaging==26.2 + # via pytest +pluggy==1.6.0 + # via pytest +pygments==2.20.0 + # via pytest +pytest==9.0.3 + # via + # -r .test.in + # pytest-randomly + # pytest-xdist +pytest-randomly==4.1.0 + # via -r .test.in +pytest-xdist==3.8.0 + # via -r .test.in +pyyaml==6.0.3 + # via responses +requests==2.33.1 + # via responses +responses==0.26.0 + # via -r .test.in +urllib3==2.6.3 + # via + # requests + # responses diff --git a/requirements/py3.11/typing.txt b/requirements/py3.11/typing.txt index 45bbb8070..6b278553b 100644 --- a/requirements/py3.11/typing.txt +++ b/requirements/py3.11/typing.txt @@ -6,45 +6,49 @@ # alabaster==1.0.0 # via sphinx -babel==2.17.0 +babel==2.18.0 # via sphinx -certifi==2025.11.12 +certifi==2026.4.22 # via requests -charset-normalizer==3.4.4 +charset-normalizer==3.4.7 # via requests -docutils==0.21.2 +docutils==0.22.4 # via sphinx -idna==3.11 +idna==3.13 # via requests -imagesize==1.4.1 +imagesize==2.0.0 # via sphinx jinja2==3.1.6 # via sphinx +librt==0.9.0 + # via mypy markupsafe==3.0.3 # via jinja2 -mypy==1.18.2 +mypy==1.20.2 # via -r .typing.in mypy-extensions==1.1.0 # via mypy -packaging==25.0 +orjson==3.11.8 + # via -r .typing.in +packaging==26.2 # via sphinx -pathspec==0.12.1 +pathspec==1.1.1 # via mypy -pygments==2.19.2 +pygments==2.20.0 # via sphinx pyyaml==6.0.3 # via responses -requests==2.32.5 +requests==2.33.1 # via # responses # sphinx -responses==0.25.8 +responses==0.26.0 # via -r .typing.in -roman-numerals-py==3.1.0 +roman-numerals==4.1.0 # via sphinx snowballstemmer==3.0.1 # via sphinx -sphinx==8.2.3 +sphinx==9.0.4 # via -r .typing.in sphinxcontrib-applehelp==2.0.0 # via sphinx @@ -60,17 +64,17 @@ sphinxcontrib-serializinghtml==2.0.0 # via sphinx types-cryptography==3.3.23.2 # via types-jwt -types-docutils==0.22.3.20251115 +types-docutils==0.22.3.20260408 # via -r .typing.in types-jwt==0.1.3 # via -r .typing.in -types-requests==2.32.4.20250913 +types-requests==2.33.0.20260408 # via -r .typing.in typing-extensions==4.15.0 # via # -r .typing.in # mypy -urllib3==2.5.0 +urllib3==2.6.3 # via # requests # responses diff --git a/requirements/py3.12/test-orjson.txt b/requirements/py3.12/test-orjson.txt new file mode 100644 index 000000000..6e172c587 --- /dev/null +++ b/requirements/py3.12/test-orjson.txt @@ -0,0 +1,47 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# tox p -m freezedeps +# +certifi==2026.4.22 + # via requests +charset-normalizer==3.4.7 + # via requests +coverage==7.13.5 + # via -r .test.in +execnet==2.1.2 + # via pytest-xdist +flaky==3.8.1 + # via -r .test.in +idna==3.13 + # via requests +iniconfig==2.3.0 + # via pytest +orjson==3.11.8 + # via -r .orjson.in +packaging==26.2 + # via pytest +pluggy==1.6.0 + # via pytest +pygments==2.20.0 + # via pytest +pytest==9.0.3 + # via + # -r .test.in + # pytest-randomly + # pytest-xdist +pytest-randomly==4.1.0 + # via -r .test.in +pytest-xdist==3.8.0 + # via -r .test.in +pyyaml==6.0.3 + # via responses +requests==2.33.1 + # via responses +responses==0.26.0 + # via -r .test.in +urllib3==2.6.3 + # via + # requests + # responses diff --git a/requirements/py3.12/typing.txt b/requirements/py3.12/typing.txt index 89ab34661..cb35ef7c6 100644 --- a/requirements/py3.12/typing.txt +++ b/requirements/py3.12/typing.txt @@ -6,45 +6,49 @@ # alabaster==1.0.0 # via sphinx -babel==2.17.0 +babel==2.18.0 # via sphinx -certifi==2025.11.12 +certifi==2026.4.22 # via requests -charset-normalizer==3.4.4 +charset-normalizer==3.4.7 # via requests -docutils==0.21.2 +docutils==0.22.4 # via sphinx -idna==3.11 +idna==3.13 # via requests -imagesize==1.4.1 +imagesize==2.0.0 # via sphinx jinja2==3.1.6 # via sphinx +librt==0.9.0 + # via mypy markupsafe==3.0.3 # via jinja2 -mypy==1.18.2 +mypy==1.20.2 # via -r .typing.in mypy-extensions==1.1.0 # via mypy -packaging==25.0 +orjson==3.11.8 + # via -r .typing.in +packaging==26.2 # via sphinx -pathspec==0.12.1 +pathspec==1.1.1 # via mypy -pygments==2.19.2 +pygments==2.20.0 # via sphinx pyyaml==6.0.3 # via responses -requests==2.32.5 +requests==2.33.1 # via # responses # sphinx -responses==0.25.8 +responses==0.26.0 # via -r .typing.in -roman-numerals-py==3.1.0 +roman-numerals==4.1.0 # via sphinx snowballstemmer==3.0.1 # via sphinx -sphinx==8.2.3 +sphinx==9.1.0 # via -r .typing.in sphinxcontrib-applehelp==2.0.0 # via sphinx @@ -60,17 +64,17 @@ sphinxcontrib-serializinghtml==2.0.0 # via sphinx types-cryptography==3.3.23.2 # via types-jwt -types-docutils==0.22.3.20251115 +types-docutils==0.22.3.20260408 # via -r .typing.in types-jwt==0.1.3 # via -r .typing.in -types-requests==2.32.4.20250913 +types-requests==2.33.0.20260408 # via -r .typing.in typing-extensions==4.15.0 # via # -r .typing.in # mypy -urllib3==2.5.0 +urllib3==2.6.3 # via # requests # responses diff --git a/requirements/py3.13/test-orjson.txt b/requirements/py3.13/test-orjson.txt new file mode 100644 index 000000000..69e5c8168 --- /dev/null +++ b/requirements/py3.13/test-orjson.txt @@ -0,0 +1,47 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# tox p -m freezedeps +# +certifi==2026.4.22 + # via requests +charset-normalizer==3.4.7 + # via requests +coverage==7.13.5 + # via -r .test.in +execnet==2.1.2 + # via pytest-xdist +flaky==3.8.1 + # via -r .test.in +idna==3.13 + # via requests +iniconfig==2.3.0 + # via pytest +orjson==3.11.8 + # via -r .orjson.in +packaging==26.2 + # via pytest +pluggy==1.6.0 + # via pytest +pygments==2.20.0 + # via pytest +pytest==9.0.3 + # via + # -r .test.in + # pytest-randomly + # pytest-xdist +pytest-randomly==4.1.0 + # via -r .test.in +pytest-xdist==3.8.0 + # via -r .test.in +pyyaml==6.0.3 + # via responses +requests==2.33.1 + # via responses +responses==0.26.0 + # via -r .test.in +urllib3==2.6.3 + # via + # requests + # responses diff --git a/requirements/py3.13/typing.txt b/requirements/py3.13/typing.txt index 06ba2a2c3..ddb345485 100644 --- a/requirements/py3.13/typing.txt +++ b/requirements/py3.13/typing.txt @@ -6,45 +6,49 @@ # alabaster==1.0.0 # via sphinx -babel==2.17.0 +babel==2.18.0 # via sphinx -certifi==2025.11.12 +certifi==2026.4.22 # via requests -charset-normalizer==3.4.4 +charset-normalizer==3.4.7 # via requests -docutils==0.21.2 +docutils==0.22.4 # via sphinx -idna==3.11 +idna==3.13 # via requests -imagesize==1.4.1 +imagesize==2.0.0 # via sphinx jinja2==3.1.6 # via sphinx +librt==0.9.0 + # via mypy markupsafe==3.0.3 # via jinja2 -mypy==1.18.2 +mypy==1.20.2 # via -r .typing.in mypy-extensions==1.1.0 # via mypy -packaging==25.0 +orjson==3.11.8 + # via -r .typing.in +packaging==26.2 # via sphinx -pathspec==0.12.1 +pathspec==1.1.1 # via mypy -pygments==2.19.2 +pygments==2.20.0 # via sphinx pyyaml==6.0.3 # via responses -requests==2.32.5 +requests==2.33.1 # via # responses # sphinx -responses==0.25.8 +responses==0.26.0 # via -r .typing.in -roman-numerals-py==3.1.0 +roman-numerals==4.1.0 # via sphinx snowballstemmer==3.0.1 # via sphinx -sphinx==8.2.3 +sphinx==9.1.0 # via -r .typing.in sphinxcontrib-applehelp==2.0.0 # via sphinx @@ -60,17 +64,17 @@ sphinxcontrib-serializinghtml==2.0.0 # via sphinx types-cryptography==3.3.23.2 # via types-jwt -types-docutils==0.22.3.20251115 +types-docutils==0.22.3.20260408 # via -r .typing.in types-jwt==0.1.3 # via -r .typing.in -types-requests==2.32.4.20250913 +types-requests==2.33.0.20260408 # via -r .typing.in typing-extensions==4.15.0 # via # -r .typing.in # mypy -urllib3==2.5.0 +urllib3==2.6.3 # via # requests # responses diff --git a/requirements/py3.14/test-orjson.txt b/requirements/py3.14/test-orjson.txt new file mode 100644 index 000000000..1ae899e61 --- /dev/null +++ b/requirements/py3.14/test-orjson.txt @@ -0,0 +1,47 @@ +# +# This file is autogenerated by pip-compile with Python 3.14 +# by the following command: +# +# tox p -m freezedeps +# +certifi==2026.4.22 + # via requests +charset-normalizer==3.4.7 + # via requests +coverage==7.13.5 + # via -r .test.in +execnet==2.1.2 + # via pytest-xdist +flaky==3.8.1 + # via -r .test.in +idna==3.13 + # via requests +iniconfig==2.3.0 + # via pytest +orjson==3.11.8 + # via -r .orjson.in +packaging==26.2 + # via pytest +pluggy==1.6.0 + # via pytest +pygments==2.20.0 + # via pytest +pytest==9.0.3 + # via + # -r .test.in + # pytest-randomly + # pytest-xdist +pytest-randomly==4.1.0 + # via -r .test.in +pytest-xdist==3.8.0 + # via -r .test.in +pyyaml==6.0.3 + # via responses +requests==2.33.1 + # via responses +responses==0.26.0 + # via -r .test.in +urllib3==2.6.3 + # via + # requests + # responses diff --git a/requirements/py3.14/typing.txt b/requirements/py3.14/typing.txt index ec731f1f7..b371ae76b 100644 --- a/requirements/py3.14/typing.txt +++ b/requirements/py3.14/typing.txt @@ -6,45 +6,49 @@ # alabaster==1.0.0 # via sphinx -babel==2.17.0 +babel==2.18.0 # via sphinx -certifi==2025.11.12 +certifi==2026.4.22 # via requests -charset-normalizer==3.4.4 +charset-normalizer==3.4.7 # via requests -docutils==0.21.2 +docutils==0.22.4 # via sphinx -idna==3.11 +idna==3.13 # via requests -imagesize==1.4.1 +imagesize==2.0.0 # via sphinx jinja2==3.1.6 # via sphinx +librt==0.9.0 + # via mypy markupsafe==3.0.3 # via jinja2 -mypy==1.18.2 +mypy==1.20.2 # via -r .typing.in mypy-extensions==1.1.0 # via mypy -packaging==25.0 +orjson==3.11.8 + # via -r .typing.in +packaging==26.2 # via sphinx -pathspec==0.12.1 +pathspec==1.1.1 # via mypy -pygments==2.19.2 +pygments==2.20.0 # via sphinx pyyaml==6.0.3 # via responses -requests==2.32.5 +requests==2.33.1 # via # responses # sphinx -responses==0.25.8 +responses==0.26.0 # via -r .typing.in -roman-numerals-py==3.1.0 +roman-numerals==4.1.0 # via sphinx snowballstemmer==3.0.1 # via sphinx -sphinx==8.2.3 +sphinx==9.1.0 # via -r .typing.in sphinxcontrib-applehelp==2.0.0 # via sphinx @@ -60,17 +64,17 @@ sphinxcontrib-serializinghtml==2.0.0 # via sphinx types-cryptography==3.3.23.2 # via types-jwt -types-docutils==0.22.3.20251115 +types-docutils==0.22.3.20260408 # via -r .typing.in types-jwt==0.1.3 # via -r .typing.in -types-requests==2.32.4.20250913 +types-requests==2.33.0.20260408 # via -r .typing.in typing-extensions==4.15.0 # via # -r .typing.in # mypy -urllib3==2.5.0 +urllib3==2.6.3 # via # requests # responses diff --git a/requirements/py3.9/test-orjson.txt b/requirements/py3.9/test-orjson.txt new file mode 100644 index 000000000..a1c033c68 --- /dev/null +++ b/requirements/py3.9/test-orjson.txt @@ -0,0 +1,59 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# tox p -m freezedeps +# +certifi==2026.4.22 + # via requests +charset-normalizer==3.4.7 + # via requests +coverage==7.10.7 + # via -r .test.in +exceptiongroup==1.3.1 + # via pytest +execnet==2.1.2 + # via pytest-xdist +flaky==3.8.1 + # via -r .test.in +idna==3.13 + # via requests +importlib-metadata==8.7.1 + # via pytest-randomly +iniconfig==2.1.0 + # via pytest +orjson==3.11.5 + # via -r .orjson.in +packaging==26.2 + # via pytest +pluggy==1.6.0 + # via pytest +pygments==2.20.0 + # via pytest +pytest==8.4.2 + # via + # -r .test.in + # pytest-randomly + # pytest-xdist +pytest-randomly==4.0.1 + # via -r .test.in +pytest-xdist==3.8.0 + # via -r .test.in +pyyaml==6.0.3 + # via responses +requests==2.32.5 + # via responses +responses==0.26.0 + # via -r .test.in +tomli==2.4.1 + # via + # coverage + # pytest +typing-extensions==4.15.0 + # via exceptiongroup +urllib3==2.6.3 + # via + # requests + # responses +zipp==3.23.1 + # via importlib-metadata diff --git a/requirements/py3.9/typing.txt b/requirements/py3.9/typing.txt index 1dc4046b1..6ed3d898a 100644 --- a/requirements/py3.9/typing.txt +++ b/requirements/py3.9/typing.txt @@ -6,33 +6,37 @@ # alabaster==0.7.16 # via sphinx -babel==2.17.0 +babel==2.18.0 # via sphinx -certifi==2025.11.12 +certifi==2026.4.22 # via requests -charset-normalizer==3.4.4 +charset-normalizer==3.4.7 # via requests docutils==0.21.2 # via sphinx -idna==3.11 +idna==3.13 # via requests -imagesize==1.4.1 +imagesize==1.5.0 # via sphinx -importlib-metadata==8.7.0 +importlib-metadata==8.7.1 # via sphinx jinja2==3.1.6 # via sphinx +librt==0.9.0 + # via mypy markupsafe==3.0.3 # via jinja2 -mypy==1.18.2 +mypy==1.19.1 # via -r .typing.in mypy-extensions==1.1.0 # via mypy -packaging==25.0 +orjson==3.11.5 + # via -r .typing.in +packaging==26.2 # via sphinx -pathspec==0.12.1 +pathspec==1.1.1 # via mypy -pygments==2.19.2 +pygments==2.20.0 # via sphinx pyyaml==6.0.3 # via responses @@ -40,7 +44,7 @@ requests==2.32.5 # via # responses # sphinx -responses==0.25.8 +responses==0.26.0 # via -r .typing.in snowballstemmer==3.0.1 # via sphinx @@ -58,7 +62,7 @@ sphinxcontrib-qthelp==2.0.0 # via sphinx sphinxcontrib-serializinghtml==2.0.0 # via sphinx -tomli==2.3.0 +tomli==2.4.1 # via # mypy # sphinx @@ -68,16 +72,16 @@ types-docutils==0.22.3.20251115 # via -r .typing.in types-jwt==0.1.3 # via -r .typing.in -types-requests==2.32.4.20250913 +types-requests==2.32.4.20260107 # via -r .typing.in typing-extensions==4.15.0 # via # -r .typing.in # mypy -urllib3==2.5.0 +urllib3==2.6.3 # via # requests # responses # types-requests -zipp==3.23.0 +zipp==3.23.1 # via importlib-metadata diff --git a/scripts/ensure_min_python_is_tested.py b/scripts/ensure_min_python_is_tested.py index f785fed1e..a3f237a33 100644 --- a/scripts/ensure_min_python_is_tested.py +++ b/scripts/ensure_min_python_is_tested.py @@ -24,7 +24,7 @@ else: raise ValueError("Could not find 'Linux' in the test matrix.") - for environment in include["tox-post-environments"]: + for environment in include["tox-environments"]: if environment.endswith("-mindeps"): break else: diff --git a/src/globus_sdk/_internal/orjson_compat.py b/src/globus_sdk/_internal/orjson_compat.py new file mode 100644 index 000000000..6d7500a2b --- /dev/null +++ b/src/globus_sdk/_internal/orjson_compat.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import json +import typing as t + +from globus_sdk import config + +if t.TYPE_CHECKING: + import requests + +dumps: t.Callable[[t.Any], bytes] +loads: t.Callable[[str | bytes], t.Any] +try: + import orjson + + ORJSON_AVAILABLE: bool = True +except ImportError: + ORJSON_AVAILABLE = False + + def dumps(obj: t.Any) -> bytes: + return json.dumps(obj).encode() + + def loads(data: str | bytes) -> t.Any: + return json.loads(data) + +else: + dumps = orjson.dumps + loads = orjson.loads + + +def get_response_loader() -> t.Callable[[requests.Response], t.Any]: + # IMPORTANT: getting the config setting can error, so this function + # must be called outside of any error handling context for reading the response data + # which would catch ValueErrors + if ORJSON_AVAILABLE and config.get_prefer_orjson(): + + def loader(r: requests.Response) -> t.Any: + return loads(r.content) + + else: + + def loader(r: requests.Response) -> t.Any: + return r.json() + + return loader diff --git a/src/globus_sdk/config/__init__.py b/src/globus_sdk/config/__init__.py index da5268112..fafb8b95e 100644 --- a/src/globus_sdk/config/__init__.py +++ b/src/globus_sdk/config/__init__.py @@ -1,4 +1,9 @@ -from .env_vars import get_environment_name, get_http_timeout, get_ssl_verify +from .env_vars import ( + get_environment_name, + get_http_timeout, + get_prefer_orjson, + get_ssl_verify, +) from .environments import EnvConfig, get_service_url, get_webapp_url __all__ = ( @@ -6,6 +11,7 @@ "get_environment_name", "get_ssl_verify", "get_http_timeout", + "get_prefer_orjson", "get_service_url", "get_webapp_url", ) diff --git a/src/globus_sdk/config/env_vars.py b/src/globus_sdk/config/env_vars.py index 8e34958fa..2a9c5b6ba 100644 --- a/src/globus_sdk/config/env_vars.py +++ b/src/globus_sdk/config/env_vars.py @@ -17,6 +17,7 @@ ENVNAME_VAR = "GLOBUS_SDK_ENVIRONMENT" HTTP_TIMEOUT_VAR = "GLOBUS_SDK_HTTP_TIMEOUT" SSL_VERIFY_VAR = "GLOBUS_SDK_VERIFY_SSL" +PREFER_ORJSON_VAR = "GLOBUS_SDK_PREFER_ORJSON" def get_environment_name(inputenv: str | None = None) -> str: @@ -56,6 +57,11 @@ def get_http_timeout(value: float | None = None) -> float | None: return result +def get_prefer_orjson() -> bool: + var = os.getenv(PREFER_ORJSON_VAR, "false") + return _bool_cast(var) + + def _ssl_verify_cast(value: t.Any) -> bool | str: if isinstance(value, bool): return value @@ -83,3 +89,12 @@ def _float_cast(value: str) -> float: except ValueError as e: log.error(f'Value "{value}" can\'t cast to float') raise ValueError(f"Invalid config float: {value}") from e + + +def _bool_cast(value: str) -> bool: + if isinstance(value, str): + if value.lower() in {"y", "yes", "t", "true", "on", "1"}: + return True + if value.lower() in {"n", "no", "f", "false", "off", "0"}: + return False + raise ValueError(f"Invalid config bool: {value}") diff --git a/src/globus_sdk/exc/api.py b/src/globus_sdk/exc/api.py index 393293dc6..3c91d370a 100644 --- a/src/globus_sdk/exc/api.py +++ b/src/globus_sdk/exc/api.py @@ -6,7 +6,7 @@ import textwrap import typing as t -from globus_sdk._internal import guards +from globus_sdk._internal import guards, orjson_compat from .base import GlobusError from .err_info import ErrorInfoContainer @@ -132,11 +132,13 @@ def raw_json(self) -> dict[str, t.Any] | None: if self._cached_raw_json == _CACHE_SENTINEL: self._cached_raw_json = None if self._json_mimetype(): + response_loader = orjson_compat.get_response_loader() + try: # technically, this could be a non-dict JSON type, like a list or # string but in those cases the user can just cast -- the "normal" # case is a dict - self._cached_raw_json = self._underlying_response.json() + self._cached_raw_json = response_loader(self._underlying_response) except ValueError: log.error( "Error body could not be JSON decoded! " diff --git a/src/globus_sdk/experimental/transfer_v2/transport.py b/src/globus_sdk/experimental/transfer_v2/transport.py index c18609978..8f845dded 100644 --- a/src/globus_sdk/experimental/transfer_v2/transport.py +++ b/src/globus_sdk/experimental/transfer_v2/transport.py @@ -5,6 +5,7 @@ from __future__ import annotations +from globus_sdk._internal import orjson_compat from globus_sdk.transport import RetryCheck, RetryCheckResult, RetryContext from globus_sdk.transport.default_retry_checks import ( DEFAULT_RETRY_CHECKS, @@ -28,10 +29,12 @@ def check_transfer_v2_transient_error(ctx: RetryContext) -> RetryCheckResult: if ctx.response is not None and ( ctx.response.status_code in retry_config.transient_error_status_codes ): + response_loader = orjson_compat.get_response_loader() + try: # if any of the error objects have a `code` of ExternalError or # EndpointError then the error likely isn't transient - errors = ctx.response.json()["errors"] + errors = response_loader(ctx.response)["errors"] for error in errors: if error["code"] in ("ExternalError", "EndpointError"): return RetryCheckResult.no_decision diff --git a/src/globus_sdk/globus_app/app.py b/src/globus_sdk/globus_app/app.py index fc7348041..6e8fff29a 100644 --- a/src/globus_sdk/globus_app/app.py +++ b/src/globus_sdk/globus_app/app.py @@ -18,6 +18,7 @@ GlobusSDKUsageError, IDTokenDecoder, ) +from globus_sdk._internal import orjson_compat from globus_sdk._internal.type_definitions import Closable from globus_sdk.authorizers import GlobusAuthorizer from globus_sdk.gare import GARE, GlobusAuthorizationParameters, to_gare @@ -594,8 +595,11 @@ def _load_response_gare(response: Response | None) -> GARE | None: """Return a parsed GARE from a 403 response or None if not possible.""" if response is None or response.status_code != 403: return None + + response_loader = orjson_compat.get_response_loader() + try: - decoded_body = response.json() + decoded_body = response_loader(response) except JSONDecodeError: return None else: diff --git a/src/globus_sdk/response.py b/src/globus_sdk/response.py index 01d47f59b..d373a262e 100644 --- a/src/globus_sdk/response.py +++ b/src/globus_sdk/response.py @@ -6,7 +6,7 @@ import typing as t from functools import cached_property -from globus_sdk._internal import guards +from globus_sdk._internal import guards, orjson_compat log = logging.getLogger(__name__) @@ -78,8 +78,10 @@ def _parsed_json(self) -> t.Any: return self._wrapped._parsed_json if self._response is not None: + response_loader = orjson_compat.get_response_loader() + try: - return self._response.json() + return response_loader(self._response) except ValueError: log.warning("response data did not parse as JSON, data=None") return None diff --git a/src/globus_sdk/services/transfer/transport.py b/src/globus_sdk/services/transfer/transport.py index 4ab2c6888..0f9ddcc7f 100644 --- a/src/globus_sdk/services/transfer/transport.py +++ b/src/globus_sdk/services/transfer/transport.py @@ -5,6 +5,7 @@ from __future__ import annotations +from globus_sdk._internal import orjson_compat from globus_sdk.transport import RetryCheck, RetryCheckResult, RetryContext from globus_sdk.transport.default_retry_checks import ( DEFAULT_RETRY_CHECKS, @@ -25,8 +26,10 @@ def check_transfer_transient_error(ctx: RetryContext) -> RetryCheckResult: if ctx.response is not None and ( ctx.response.status_code in retry_config.transient_error_status_codes ): + response_loader = orjson_compat.get_response_loader() + try: - code = ctx.response.json()["code"] + code = response_loader(ctx.response)["code"] except (ValueError, KeyError): code = "" diff --git a/src/globus_sdk/testing/registry.py b/src/globus_sdk/testing/registry.py index 54bd47af9..b6fca8c5a 100644 --- a/src/globus_sdk/testing/registry.py +++ b/src/globus_sdk/testing/registry.py @@ -7,6 +7,7 @@ import responses import globus_sdk +import globus_sdk.experimental from .models import RegisteredResponse, ResponseList, ResponseSet diff --git a/src/globus_sdk/transport/encoders.py b/src/globus_sdk/transport/encoders.py index 5052f47e9..03c4876a8 100644 --- a/src/globus_sdk/transport/encoders.py +++ b/src/globus_sdk/transport/encoders.py @@ -6,6 +6,8 @@ import requests +from globus_sdk import config +from globus_sdk._internal import orjson_compat from globus_sdk._missing import MISSING, filter_missing @@ -105,6 +107,13 @@ class JSONRequestEncoder(RequestEncoder): that APIs requiring a content-type of "application/json" are able to read the data. """ + def __init__(self, *, use_orjson: bool | None = None) -> None: + self.use_orjson = ( + use_orjson + if use_orjson is not None + else (orjson_compat.ORJSON_AVAILABLE and config.get_prefer_orjson()) + ) + def encode( self, method: str, @@ -115,13 +124,23 @@ def encode( ) -> requests.Request: if data is not None: headers = {"Content-Type": "application/json", **headers} - return requests.Request( - method, - url, - json=self._prepare_data(data), - params=self._prepare_params(params), - headers=self._prepare_headers(headers), - ) + + if self.use_orjson: + return requests.Request( + method, + url, + data=orjson_compat.dumps(self._prepare_data(data)), + params=self._prepare_params(params), + headers=self._prepare_headers(headers), + ) + else: + return requests.Request( + method, + url, + json=self._prepare_data(data), + params=self._prepare_params(params), + headers=self._prepare_headers(headers), + ) class FormRequestEncoder(RequestEncoder): diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 06b506c90..0f62b05b5 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -211,3 +211,19 @@ def test_service_url_from_env_var(): globus_sdk.config.get_webapp_url() == f"https://app.{env}.globuscs.info/" ) + + +@pytest.mark.parametrize( + "value, expected_result", + [(x, True) for x in ["1", "YES", "true", "t", "True", "ON"]] + + [(x, False) for x in ["0", "NO", "false", "f", "False", "OFF"]] + + [("invalid", ValueError), ("1.0", ValueError)], +) +def test_get_prefer_orjson(value, expected_result, monkeypatch): + monkeypatch.setenv("GLOBUS_SDK_PREFER_ORJSON", value) + if expected_result is not ValueError: + assert globus_sdk.config.get_prefer_orjson() == expected_result + + else: + with pytest.raises(expected_result): + globus_sdk.config.get_prefer_orjson() diff --git a/tests/unit/transport/test_transport_encoders.py b/tests/unit/transport/test_transport_encoders.py index 559227b9b..f9b565acb 100644 --- a/tests/unit/transport/test_transport_encoders.py +++ b/tests/unit/transport/test_transport_encoders.py @@ -6,6 +6,13 @@ from globus_sdk._payload import GlobusPayload from globus_sdk.transport import FormRequestEncoder, JSONRequestEncoder, RequestEncoder +try: + import orjson # noqa: F401 + + has_orjson = True +except ImportError: + has_orjson = False + @pytest.mark.parametrize("data", ("foo", b"bar")) def test_text_request_encoder_accepts_string_data(data): @@ -159,3 +166,30 @@ def test_form_encoder_payload_preparation( headers={}, ) assert request.data == expected_data + + +@pytest.mark.skipif(has_orjson, reason="test requires that orjson is not installed") +@pytest.mark.parametrize("env_var_is_set", (True, False)) +def test_json_encoder_never_prefers_orjson_if_not_installed( + monkeypatch, env_var_is_set +): + if env_var_is_set: + monkeypatch.setenv("GLOBUS_SDK_PREFER_ORJSON", "true") + assert JSONRequestEncoder().use_orjson is False + + +@pytest.mark.skipif(not has_orjson, reason="test requires that orjson is installed") +@pytest.mark.parametrize("env_var_is_set", (True, False)) +def test_json_encoder_prefers_orjson_based_on_env_var(monkeypatch, env_var_is_set): + if env_var_is_set: + monkeypatch.setenv("GLOBUS_SDK_PREFER_ORJSON", "true") + + assert JSONRequestEncoder().use_orjson is env_var_is_set + + +@pytest.mark.parametrize("env_var_is_set", (True, False)) +@pytest.mark.parametrize("initarg", (True, False)) +def test_json_encoder_can_force_orjson_usage(monkeypatch, env_var_is_set, initarg): + if env_var_is_set: + monkeypatch.setenv("GLOBUS_SDK_PREFER_ORJSON", "true") + assert JSONRequestEncoder(use_orjson=initarg).use_orjson is initarg diff --git a/tox.ini b/tox.ini index c842d66ee..3ec770253 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,7 @@ envlist = test-lazy-imports coverage_clean py{3.14,3.13,3.12,3.11,3.10,3.9} + py{3.14,3.9}-orjson py3.9-mindeps py3.11-sphinxext coverage_report @@ -25,6 +26,9 @@ depends = coverage_clean,lint [testenv:py{3.14,3.13,3.12,3.11,3.10,3.9}-mindeps] deps = -r requirements/py{py_dot_ver}/test-mindeps.txt +[testenv:py{3.14,3.13,3.12,3.11,3.10,3.9}-orjson] +deps = -r requirements/py{py_dot_ver}/test-orjson.txt + [testenv:py{3.14,3.13,3.12,3.11,3.10,3.9}-sphinxext] deps = -r requirements/py{py_dot_ver}/test.txt @@ -112,6 +116,7 @@ deps = dependency-groups>=1,<2 commands = python -m dependency_groups test -o requirements/.test.in python -m dependency_groups typing -o requirements/.typing.in + python -m dependency_groups orjson -o requirements/.orjson.in python -m dependency_groups test-mindeps -o requirements/.test-mindeps.in python -m dependency_groups docs -o requirements/.docs.in [testenv:freezedeps-py{3.14,3.13,3.12,3.11,3.10,3.9}] @@ -127,6 +132,8 @@ commands = # Minimum dependencies are only tested against the lowest supported Python version. py3.9: pip-compile --strip-extras -q -U --resolver=backtracking .test-mindeps.in -o py{py_dot_ver}/test-mindeps.txt + # orjson testing happens on every version + pip-compile --strip-extras -q -U --resolver=backtracking -r .test.in -r .orjson.in -o py{py_dot_ver}/test-orjson.txt # The docs requirements are only generated for Python 3.11. py3.11: pip-compile --strip-extras -q -U --resolver=backtracking .docs.in -o py{py_dot_ver}/docs.txt