From 3a31e524af5e36ef06a086e1cdecf4f0970c4cc2 Mon Sep 17 00:00:00 2001 From: Steven Bal Date: Fri, 21 Mar 2025 11:05:08 +0100 Subject: [PATCH 1/4] :heavy_plus_sign: Replace pytest-benchmark with codspeed --- requirements/ci.txt | 9 +++++---- requirements/dev.txt | 15 ++++++++------- requirements/test-tools.in | 2 +- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/requirements/ci.txt b/requirements/ci.txt index 187a78ee..9f7e3480 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -84,6 +84,7 @@ cffi==1.16.0 # -c requirements/base.txt # -r requirements/base.txt # cryptography + # pytest-codspeed charset-normalizer==3.3.2 # via # -c requirements/base.txt @@ -379,6 +380,8 @@ factory-boy==3.2.0 # via -r requirements/test-tools.in faker==8.1.0 # via factory-boy +filelock==3.18.0 + # via pytest-codspeed flake8==7.1.2 # via -r requirements/test-tools.in flower==2.0.1 @@ -534,8 +537,6 @@ psycopg2==2.9.9 # -c requirements/base.txt # -r requirements/base.txt # open-api-framework -py-cpuinfo==9.0.0 - # via pytest-benchmark pycodestyle==2.12.1 # via flake8 pycparser==2.20 @@ -588,8 +589,8 @@ pyrsistent==0.17.3 pytest==8.3.3 # via # -r requirements/test-tools.in - # pytest-benchmark -pytest-benchmark==5.1.0 + # pytest-codspeed +pytest-codspeed==2.2.1 # via -r requirements/test-tools.in python-dateutil==2.9.0.post0 # via diff --git a/requirements/dev.txt b/requirements/dev.txt index fd701307..5bd2e6fe 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -107,6 +107,7 @@ cffi==1.16.0 # -c requirements/ci.txt # -r requirements/ci.txt # cryptography + # pytest-codspeed charset-normalizer==3.3.2 # via # -c requirements/ci.txt @@ -430,6 +431,11 @@ faker==8.1.0 # -c requirements/ci.txt # -r requirements/ci.txt # factory-boy +filelock==3.18.0 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt + # pytest-codspeed flake8==7.1.2 # via # -c requirements/ci.txt @@ -640,11 +646,6 @@ psycopg2==2.9.9 # -c requirements/ci.txt # -r requirements/ci.txt # open-api-framework -py-cpuinfo==9.0.0 - # via - # -c requirements/ci.txt - # -r requirements/ci.txt - # pytest-benchmark pycodestyle==2.12.1 # via # -c requirements/ci.txt @@ -716,8 +717,8 @@ pytest==8.3.3 # via # -c requirements/ci.txt # -r requirements/ci.txt - # pytest-benchmark -pytest-benchmark==5.1.0 + # pytest-codspeed +pytest-codspeed==2.2.1 # via # -c requirements/ci.txt # -r requirements/ci.txt diff --git a/requirements/test-tools.in b/requirements/test-tools.in index 120cdd00..4f31402d 100644 --- a/requirements/test-tools.in +++ b/requirements/test-tools.in @@ -4,7 +4,7 @@ # Dependencies only relevant for (unit) testing codecov pytest -pytest-benchmark +pytest-codspeed coverage < 5.0 django-webtest factory-boy From c95eddbdc340bf003025ffa8a34bb127a2ad6c11 Mon Sep 17 00:00:00 2001 From: Steven Bal Date: Fri, 21 Mar 2025 11:05:38 +0100 Subject: [PATCH 2/4] :white_check_mark: Modify performance tests for codspeed --- performance_test/conftest.py | 13 ------------- performance_test/test_objects_list.py | 6 +----- 2 files changed, 1 insertion(+), 18 deletions(-) delete mode 100644 performance_test/conftest.py diff --git a/performance_test/conftest.py b/performance_test/conftest.py deleted file mode 100644 index 02cff041..00000000 --- a/performance_test/conftest.py +++ /dev/null @@ -1,13 +0,0 @@ -import pytest - - -@pytest.fixture -def benchmark_assertions(benchmark): - def wrapper(**kwargs): - stats = benchmark.stats["stats"] - for name, value in kwargs.items(): - assert ( - getattr(stats, name) < value - ), f"{name} {getattr(stats, name)}s exceeded {value}s" - - return wrapper diff --git a/performance_test/test_objects_list.py b/performance_test/test_objects_list.py index 8cfc0676..c367341f 100644 --- a/performance_test/test_objects_list.py +++ b/performance_test/test_objects_list.py @@ -1,4 +1,3 @@ -import pytest import requests from furl import furl @@ -6,8 +5,7 @@ AUTH_HEADERS = {"Authorization": "Token secret"} -@pytest.mark.benchmark(max_time=60, min_rounds=5) -def test_objects_api_list(benchmark, benchmark_assertions): +def test_objects_api_list(benchmark): """ Regression test for maykinmedia/objects-api#538 """ @@ -25,5 +23,3 @@ def make_request(): assert result.status_code == 200 assert result.json()["count"] == 5000 - - benchmark_assertions(mean=1, max=1) From 4a5f36f9d0e85a6e1aa339eaf5e25bbe4727facc Mon Sep 17 00:00:00 2001 From: Steven Bal Date: Fri, 21 Mar 2025 11:10:39 +0100 Subject: [PATCH 3/4] :construction_worker: Add codspeed to actions --- .github/workflows/ci.yml | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 371664e6..71f5b525 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,13 +65,33 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} - performance-tests: - name: Run the performance test suite + # performance-tests: + # name: Run the performance test suite + # runs-on: ubuntu-latest + + # steps: + # - uses: actions/checkout@v4 + + # - name: Bring up docker compose and load data + # run: | + # docker compose up -d --build || ( docker compose logs >&2 && exit 1; ) + # until docker compose logs web | grep -q "spawned uWSGI worker"; do + # echo "uWSGI not running yet, waiting..." + # sleep 3 + # done + # docker compose exec --user root web pip install factory-boy + # cat performance_test/create_data.py | docker compose exec -T web src/manage.py shell + + # - name: Run tests + # run: | + # pip install -r requirements/ci.txt + # pytest performance_test/ --benchmark-json output.json + + benchmarks: + name: Run benchmarks runs-on: ubuntu-latest - steps: - uses: actions/checkout@v4 - - name: Bring up docker compose and load data run: | docker compose up -d --build || ( docker compose logs >&2 && exit 1; ) @@ -82,10 +102,15 @@ jobs: docker compose exec --user root web pip install factory-boy cat performance_test/create_data.py | docker compose exec -T web src/manage.py shell - - name: Run tests + - name: Install dependencies run: | pip install -r requirements/ci.txt - pytest performance_test/ --benchmark-json output.json + + - name: Run benchmarks + uses: CodSpeedHQ/action@v3 + with: + token: ${{ secrets.CODSPEED_TOKEN }} + run: "pytest performance_test/ --codspeed" docs: runs-on: ubuntu-latest From ed9d44295a6f7411df63fa0a8ce22a306065fd79 Mon Sep 17 00:00:00 2001 From: Steven Bal Date: Fri, 21 Mar 2025 12:50:00 +0100 Subject: [PATCH 4/4] :construction_worker: --- .github/workflows/ci.yml | 47 +++++++++++++++++++-------- performance_test/conftest.py | 24 ++++++++++++++ performance_test/test_objects_list.py | 13 ++++---- requirements/ci.txt | 3 ++ requirements/dev.txt | 5 +++ requirements/test-tools.in | 1 + setup.cfg | 4 +++ 7 files changed, 77 insertions(+), 20 deletions(-) create mode 100644 performance_test/conftest.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71f5b525..29cd12b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,27 +90,46 @@ jobs: benchmarks: name: Run benchmarks runs-on: ubuntu-latest + + services: + postgres: + image: postgis/postgis:17-3.5 + env: + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + # needed because the postgres container does not provide a healthcheck + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + steps: - uses: actions/checkout@v4 - - name: Bring up docker compose and load data - run: | - docker compose up -d --build || ( docker compose logs >&2 && exit 1; ) - until docker compose logs web | grep -q "spawned uWSGI worker"; do - echo "uWSGI not running yet, waiting..." - sleep 3 - done - docker compose exec --user root web pip install factory-boy - cat performance_test/create_data.py | docker compose exec -T web src/manage.py shell - - - name: Install dependencies - run: | - pip install -r requirements/ci.txt + # - name: Bring up docker compose and load data + # run: | + # docker compose up -d --build || ( docker compose logs >&2 && exit 1; ) + # until docker compose logs web | grep -q "spawned uWSGI worker"; do + # echo "uWSGI not running yet, waiting..." + # sleep 3 + # done + # docker compose exec --user root web pip install factory-boy + # cat performance_test/create_data.py | docker compose exec -T web src/manage.py shell + + - name: Set up backend environment + uses: maykinmedia/setup-django-backend@v1.3 + with: + apt-packages: 'libgdal-dev gdal-bin' + python-version: '3.11' + setup-node: true - name: Run benchmarks uses: CodSpeedHQ/action@v3 with: token: ${{ secrets.CODSPEED_TOKEN }} - run: "pytest performance_test/ --codspeed" + run: "pytest performance_test/ --codspeed --codspeed-mode=walltime" + env: + DJANGO_SETTINGS_MODULE: objects.conf.ci + SECRET_KEY: dummy + DB_USER: postgres + DB_PASSWORD: '' docs: runs-on: ubuntu-latest diff --git a/performance_test/conftest.py b/performance_test/conftest.py new file mode 100644 index 00000000..d43cccb0 --- /dev/null +++ b/performance_test/conftest.py @@ -0,0 +1,24 @@ +import pytest + +from objects.core.tests.factories import ObjectRecordFactory, ObjectTypeFactory +from objects.token.tests.factories import TokenAuthFactory + + +@pytest.fixture(scope="session") +def setup(django_db_setup, django_db_blocker): + print("setup") + with django_db_blocker.unblock(): + object_type = ObjectTypeFactory.create( + service__api_root="http://localhost:8001/api/v2/", + uuid="f1220670-8ab7-44f1-a318-bd0782e97662", + ) + + TokenAuthFactory(token="secret", is_superuser=True) + + ObjectRecordFactory.create_batch( + 5000, + object__object_type=object_type, + start_at="2020-01-01", + version=1, + data={"kiemjaar": "1234"}, + ) diff --git a/performance_test/test_objects_list.py b/performance_test/test_objects_list.py index c367341f..7c76aaac 100644 --- a/performance_test/test_objects_list.py +++ b/performance_test/test_objects_list.py @@ -1,11 +1,12 @@ -import requests +import pytest from furl import furl BASE_URL = furl("http://localhost:8000/api/v2/") AUTH_HEADERS = {"Authorization": "Token secret"} -def test_objects_api_list(benchmark): +@pytest.mark.django_db +def test_objects_api_list(benchmark, client, setup): """ Regression test for maykinmedia/objects-api#538 """ @@ -17,9 +18,9 @@ def test_objects_api_list(benchmark): } def make_request(): - return requests.get((BASE_URL / "objects").set(params), headers=AUTH_HEADERS) + return client.get((BASE_URL / "objects").set(params), headers=AUTH_HEADERS) - result = benchmark(make_request) + response = benchmark(make_request) - assert result.status_code == 200 - assert result.json()["count"] == 5000 + assert response.status_code == 200 + assert response.json()["count"] == 5000 diff --git a/requirements/ci.txt b/requirements/ci.txt index 9f7e3480..603a0786 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -590,8 +590,11 @@ pytest==8.3.3 # via # -r requirements/test-tools.in # pytest-codspeed + # pytest-django pytest-codspeed==2.2.1 # via -r requirements/test-tools.in +pytest-django==4.10.0 + # via -r requirements/test-tools.in python-dateutil==2.9.0.post0 # via # -c requirements/base.txt diff --git a/requirements/dev.txt b/requirements/dev.txt index 5bd2e6fe..379e10ea 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -718,10 +718,15 @@ pytest==8.3.3 # -c requirements/ci.txt # -r requirements/ci.txt # pytest-codspeed + # pytest-django pytest-codspeed==2.2.1 # via # -c requirements/ci.txt # -r requirements/ci.txt +pytest-django==4.10.0 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt python-dateutil==2.9.0.post0 # via # -c requirements/ci.txt diff --git a/requirements/test-tools.in b/requirements/test-tools.in index 4f31402d..773080e5 100644 --- a/requirements/test-tools.in +++ b/requirements/test-tools.in @@ -4,6 +4,7 @@ # Dependencies only relevant for (unit) testing codecov pytest +pytest-django pytest-codspeed coverage < 5.0 django-webtest diff --git a/setup.cfg b/setup.cfg index 91adacab..345b9afb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,3 +14,7 @@ skip_glob = **/migrations/** known_django=django known_first_party=objects sections=FUTURE,STDLIB,DJANGO,THIRDPARTY,FIRSTPARTY,LOCALFOLDER + +[tool:pytest] +DJANGO_SETTINGS_MODULE = objects.conf.ci +pythonpath = . src