Feature/codspeed

.github/workflows/ci.yml

@@ -65,27 +65,71 @@ jobs:
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
-  performance-tests:
-    name: Run the performance test suite
+  # performance-tests:
+  #   name: Run the performance test suite
+  #   runs-on: ubuntu-latest
+
+  #   steps:
+  #     - uses: actions/checkout@v4
+
+  #     - name: Bring up docker compose and load data
+  #       run: |
+  #         docker compose up -d --build || ( docker compose logs >&2 && exit 1; )
+  #         until docker compose logs web | grep -q "spawned uWSGI worker"; do
+  #           echo "uWSGI not running yet, waiting..."
+  #           sleep 3
+  #         done
+  #         docker compose exec --user root web pip install factory-boy
+  #         cat performance_test/create_data.py | docker compose exec -T web src/manage.py shell
+
+  #     - name: Run tests
+  #       run: |
+  #         pip install -r requirements/ci.txt
+  #         pytest performance_test/ --benchmark-json output.json
+
+  benchmarks:
+    name: Run benchmarks
     runs-on: ubuntu-latest
 
+    services:
+      postgres:
+        image: postgis/postgis:17-3.5
+        env:
+          POSTGRES_HOST_AUTH_METHOD: trust
+        ports:
+          - 5432:5432
+        # needed because the postgres container does not provide a healthcheck
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+
     steps:
       - uses: actions/checkout@v4
+      # - name: Bring up docker compose and load data
+      #   run: |
+      #     docker compose up -d --build || ( docker compose logs >&2 && exit 1; )
+      #     until docker compose logs web | grep -q "spawned uWSGI worker"; do
+      #       echo "uWSGI not running yet, waiting..."
+      #       sleep 3
+      #     done
+      #     docker compose exec --user root web pip install factory-boy
+      #     cat performance_test/create_data.py | docker compose exec -T web src/manage.py shell
 
-      - name: Bring up docker compose and load data
-        run: |
-          docker compose up -d --build || ( docker compose logs >&2 && exit 1; )
-          until docker compose logs web | grep -q "spawned uWSGI worker"; do
-            echo "uWSGI not running yet, waiting..."
-            sleep 3
-          done
-          docker compose exec --user root web pip install factory-boy
-          cat performance_test/create_data.py | docker compose exec -T web src/manage.py shell
+      - name: Set up backend environment
+        uses: maykinmedia/setup-django-backend@v1.3
+        with:
+          apt-packages: 'libgdal-dev gdal-bin'
+          python-version: '3.11'
+          setup-node: true
 
-      - name: Run tests
-        run: |
-          pip install -r requirements/ci.txt
-          pytest performance_test/ --benchmark-json output.json
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@v3
+        with:
+          token: ${{ secrets.CODSPEED_TOKEN }}
+          run: "pytest performance_test/ --codspeed --codspeed-mode=walltime"
+        env:
+          DJANGO_SETTINGS_MODULE: objects.conf.ci
+          SECRET_KEY: dummy
+          DB_USER: postgres
+          DB_PASSWORD: ''
 
   docs:
     runs-on: ubuntu-latest

performance_test/conftest.py

@@ -1,13 +1,24 @@
 import pytest
 
+from objects.core.tests.factories import ObjectRecordFactory, ObjectTypeFactory
+from objects.token.tests.factories import TokenAuthFactory
 
-@pytest.fixture
-def benchmark_assertions(benchmark):
-    def wrapper(**kwargs):
-        stats = benchmark.stats["stats"]
-        for name, value in kwargs.items():
-            assert (
-                getattr(stats, name) < value
-            ), f"{name} {getattr(stats, name)}s exceeded {value}s"
 
-    return wrapper
+@pytest.fixture(scope="session")
+def setup(django_db_setup, django_db_blocker):
+    print("setup")
+    with django_db_blocker.unblock():
+        object_type = ObjectTypeFactory.create(
+            service__api_root="http://localhost:8001/api/v2/",
+            uuid="f1220670-8ab7-44f1-a318-bd0782e97662",
+        )
+
+        TokenAuthFactory(token="secret", is_superuser=True)
+
+        ObjectRecordFactory.create_batch(
+            5000,
+            object__object_type=object_type,
+            start_at="2020-01-01",
+            version=1,
+            data={"kiemjaar": "1234"},
+        )

performance_test/test_objects_list.py

@@ -1,13 +1,12 @@
 import pytest
-import requests
 from furl import furl
 
 BASE_URL = furl("http://localhost:8000/api/v2/")
 AUTH_HEADERS = {"Authorization": "Token secret"}
 
 
-@pytest.mark.benchmark(max_time=60, min_rounds=5)
-def test_objects_api_list(benchmark, benchmark_assertions):
+@pytest.mark.django_db
+def test_objects_api_list(benchmark, client, setup):
     """
     Regression test for maykinmedia/objects-api#538
     """
@@ -19,11 +18,9 @@ def test_objects_api_list(benchmark, benchmark_assertions):
     }
 
     def make_request():
-        return requests.get((BASE_URL / "objects").set(params), headers=AUTH_HEADERS)
+        return client.get((BASE_URL / "objects").set(params), headers=AUTH_HEADERS)
 
-    result = benchmark(make_request)
+    response = benchmark(make_request)
 
-    assert result.status_code == 200
-    assert result.json()["count"] == 5000
-
-    benchmark_assertions(mean=1, max=1)
+    assert response.status_code == 200
+    assert response.json()["count"] == 5000

requirements/ci.txt

@@ -84,6 +84,7 @@ cffi==1.16.0
     #   -c requirements/base.txt
     #   -r requirements/base.txt
     #   cryptography
+    #   pytest-codspeed
 charset-normalizer==3.3.2
     # via
     #   -c requirements/base.txt
@@ -379,6 +380,8 @@ factory-boy==3.2.0
     # via -r requirements/test-tools.in
 faker==8.1.0
     # via factory-boy
+filelock==3.18.0
+    # via pytest-codspeed
 flake8==7.1.2
     # via -r requirements/test-tools.in
 flower==2.0.1
@@ -534,8 +537,6 @@ psycopg2==2.9.9
     #   -c requirements/base.txt
     #   -r requirements/base.txt
     #   open-api-framework
-py-cpuinfo==9.0.0
-    # via pytest-benchmark
 pycodestyle==2.12.1
     # via flake8
 pycparser==2.20
@@ -588,8 +589,11 @@ pyrsistent==0.17.3
 pytest==8.3.3
     # via
     #   -r requirements/test-tools.in
-    #   pytest-benchmark
-pytest-benchmark==5.1.0
+    #   pytest-codspeed
+    #   pytest-django
+pytest-codspeed==2.2.1
+    # via -r requirements/test-tools.in
+pytest-django==4.10.0
     # via -r requirements/test-tools.in
 python-dateutil==2.9.0.post0
     # via

requirements/dev.txt

@@ -107,6 +107,7 @@ cffi==1.16.0
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
     #   cryptography
+    #   pytest-codspeed
 charset-normalizer==3.3.2
     # via
     #   -c requirements/ci.txt
@@ -430,6 +431,11 @@ faker==8.1.0
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
     #   factory-boy
+filelock==3.18.0
+    # via
+    #   -c requirements/ci.txt
+    #   -r requirements/ci.txt
+    #   pytest-codspeed
 flake8==7.1.2
     # via
     #   -c requirements/ci.txt
@@ -640,11 +646,6 @@ psycopg2==2.9.9
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
     #   open-api-framework
-py-cpuinfo==9.0.0
-    # via
-    #   -c requirements/ci.txt
-    #   -r requirements/ci.txt
-    #   pytest-benchmark
 pycodestyle==2.12.1
     # via
     #   -c requirements/ci.txt
@@ -716,8 +717,13 @@ pytest==8.3.3
     # via
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
-    #   pytest-benchmark
-pytest-benchmark==5.1.0
+    #   pytest-codspeed
+    #   pytest-django
+pytest-codspeed==2.2.1
+    # via
+    #   -c requirements/ci.txt
+    #   -r requirements/ci.txt
+pytest-django==4.10.0
     # via
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt

requirements/test-tools.in

@@ -4,7 +4,8 @@
 # Dependencies only relevant for (unit) testing
 codecov
 pytest
-pytest-benchmark
+pytest-django
+pytest-codspeed
 coverage < 5.0
 django-webtest
 factory-boy

setup.cfg

@@ -14,3 +14,7 @@ skip_glob = **/migrations/**
 known_django=django
 known_first_party=objects
 sections=FUTURE,STDLIB,DJANGO,THIRDPARTY,FIRSTPARTY,LOCALFOLDER
+
+[tool:pytest]
+DJANGO_SETTINGS_MODULE = objects.conf.ci
+pythonpath = . src