diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 371664e6..29cd12b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,27 +65,71 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} - performance-tests: - name: Run the performance test suite + # performance-tests: + # name: Run the performance test suite + # runs-on: ubuntu-latest + + # steps: + # - uses: actions/checkout@v4 + + # - name: Bring up docker compose and load data + # run: | + # docker compose up -d --build || ( docker compose logs >&2 && exit 1; ) + # until docker compose logs web | grep -q "spawned uWSGI worker"; do + # echo "uWSGI not running yet, waiting..." + # sleep 3 + # done + # docker compose exec --user root web pip install factory-boy + # cat performance_test/create_data.py | docker compose exec -T web src/manage.py shell + + # - name: Run tests + # run: | + # pip install -r requirements/ci.txt + # pytest performance_test/ --benchmark-json output.json + + benchmarks: + name: Run benchmarks runs-on: ubuntu-latest + services: + postgres: + image: postgis/postgis:17-3.5 + env: + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + # needed because the postgres container does not provide a healthcheck + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + steps: - uses: actions/checkout@v4 + # - name: Bring up docker compose and load data + # run: | + # docker compose up -d --build || ( docker compose logs >&2 && exit 1; ) + # until docker compose logs web | grep -q "spawned uWSGI worker"; do + # echo "uWSGI not running yet, waiting..." + # sleep 3 + # done + # docker compose exec --user root web pip install factory-boy + # cat performance_test/create_data.py | docker compose exec -T web src/manage.py shell - - name: Bring up docker compose and load data - run: | - docker compose up -d --build || ( docker compose logs >&2 && exit 1; ) - until docker compose logs web | grep -q "spawned uWSGI worker"; do - echo "uWSGI not running yet, waiting..." - sleep 3 - done - docker compose exec --user root web pip install factory-boy - cat performance_test/create_data.py | docker compose exec -T web src/manage.py shell + - name: Set up backend environment + uses: maykinmedia/setup-django-backend@v1.3 + with: + apt-packages: 'libgdal-dev gdal-bin' + python-version: '3.11' + setup-node: true - - name: Run tests - run: | - pip install -r requirements/ci.txt - pytest performance_test/ --benchmark-json output.json + - name: Run benchmarks + uses: CodSpeedHQ/action@v3 + with: + token: ${{ secrets.CODSPEED_TOKEN }} + run: "pytest performance_test/ --codspeed --codspeed-mode=walltime" + env: + DJANGO_SETTINGS_MODULE: objects.conf.ci + SECRET_KEY: dummy + DB_USER: postgres + DB_PASSWORD: '' docs: runs-on: ubuntu-latest diff --git a/performance_test/conftest.py b/performance_test/conftest.py index 02cff041..d43cccb0 100644 --- a/performance_test/conftest.py +++ b/performance_test/conftest.py @@ -1,13 +1,24 @@ import pytest +from objects.core.tests.factories import ObjectRecordFactory, ObjectTypeFactory +from objects.token.tests.factories import TokenAuthFactory -@pytest.fixture -def benchmark_assertions(benchmark): - def wrapper(**kwargs): - stats = benchmark.stats["stats"] - for name, value in kwargs.items(): - assert ( - getattr(stats, name) < value - ), f"{name} {getattr(stats, name)}s exceeded {value}s" - return wrapper +@pytest.fixture(scope="session") +def setup(django_db_setup, django_db_blocker): + print("setup") + with django_db_blocker.unblock(): + object_type = ObjectTypeFactory.create( + service__api_root="http://localhost:8001/api/v2/", + uuid="f1220670-8ab7-44f1-a318-bd0782e97662", + ) + + TokenAuthFactory(token="secret", is_superuser=True) + + ObjectRecordFactory.create_batch( + 5000, + object__object_type=object_type, + start_at="2020-01-01", + version=1, + data={"kiemjaar": "1234"}, + ) diff --git a/performance_test/test_objects_list.py b/performance_test/test_objects_list.py index 8cfc0676..7c76aaac 100644 --- a/performance_test/test_objects_list.py +++ b/performance_test/test_objects_list.py @@ -1,13 +1,12 @@ import pytest -import requests from furl import furl BASE_URL = furl("http://localhost:8000/api/v2/") AUTH_HEADERS = {"Authorization": "Token secret"} -@pytest.mark.benchmark(max_time=60, min_rounds=5) -def test_objects_api_list(benchmark, benchmark_assertions): +@pytest.mark.django_db +def test_objects_api_list(benchmark, client, setup): """ Regression test for maykinmedia/objects-api#538 """ @@ -19,11 +18,9 @@ def test_objects_api_list(benchmark, benchmark_assertions): } def make_request(): - return requests.get((BASE_URL / "objects").set(params), headers=AUTH_HEADERS) + return client.get((BASE_URL / "objects").set(params), headers=AUTH_HEADERS) - result = benchmark(make_request) + response = benchmark(make_request) - assert result.status_code == 200 - assert result.json()["count"] == 5000 - - benchmark_assertions(mean=1, max=1) + assert response.status_code == 200 + assert response.json()["count"] == 5000 diff --git a/requirements/ci.txt b/requirements/ci.txt index 187a78ee..603a0786 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -84,6 +84,7 @@ cffi==1.16.0 # -c requirements/base.txt # -r requirements/base.txt # cryptography + # pytest-codspeed charset-normalizer==3.3.2 # via # -c requirements/base.txt @@ -379,6 +380,8 @@ factory-boy==3.2.0 # via -r requirements/test-tools.in faker==8.1.0 # via factory-boy +filelock==3.18.0 + # via pytest-codspeed flake8==7.1.2 # via -r requirements/test-tools.in flower==2.0.1 @@ -534,8 +537,6 @@ psycopg2==2.9.9 # -c requirements/base.txt # -r requirements/base.txt # open-api-framework -py-cpuinfo==9.0.0 - # via pytest-benchmark pycodestyle==2.12.1 # via flake8 pycparser==2.20 @@ -588,8 +589,11 @@ pyrsistent==0.17.3 pytest==8.3.3 # via # -r requirements/test-tools.in - # pytest-benchmark -pytest-benchmark==5.1.0 + # pytest-codspeed + # pytest-django +pytest-codspeed==2.2.1 + # via -r requirements/test-tools.in +pytest-django==4.10.0 # via -r requirements/test-tools.in python-dateutil==2.9.0.post0 # via diff --git a/requirements/dev.txt b/requirements/dev.txt index fd701307..379e10ea 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -107,6 +107,7 @@ cffi==1.16.0 # -c requirements/ci.txt # -r requirements/ci.txt # cryptography + # pytest-codspeed charset-normalizer==3.3.2 # via # -c requirements/ci.txt @@ -430,6 +431,11 @@ faker==8.1.0 # -c requirements/ci.txt # -r requirements/ci.txt # factory-boy +filelock==3.18.0 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt + # pytest-codspeed flake8==7.1.2 # via # -c requirements/ci.txt @@ -640,11 +646,6 @@ psycopg2==2.9.9 # -c requirements/ci.txt # -r requirements/ci.txt # open-api-framework -py-cpuinfo==9.0.0 - # via - # -c requirements/ci.txt - # -r requirements/ci.txt - # pytest-benchmark pycodestyle==2.12.1 # via # -c requirements/ci.txt @@ -716,8 +717,13 @@ pytest==8.3.3 # via # -c requirements/ci.txt # -r requirements/ci.txt - # pytest-benchmark -pytest-benchmark==5.1.0 + # pytest-codspeed + # pytest-django +pytest-codspeed==2.2.1 + # via + # -c requirements/ci.txt + # -r requirements/ci.txt +pytest-django==4.10.0 # via # -c requirements/ci.txt # -r requirements/ci.txt diff --git a/requirements/test-tools.in b/requirements/test-tools.in index 120cdd00..773080e5 100644 --- a/requirements/test-tools.in +++ b/requirements/test-tools.in @@ -4,7 +4,8 @@ # Dependencies only relevant for (unit) testing codecov pytest -pytest-benchmark +pytest-django +pytest-codspeed coverage < 5.0 django-webtest factory-boy diff --git a/setup.cfg b/setup.cfg index 91adacab..345b9afb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,3 +14,7 @@ skip_glob = **/migrations/** known_django=django known_first_party=objects sections=FUTURE,STDLIB,DJANGO,THIRDPARTY,FIRSTPARTY,LOCALFOLDER + +[tool:pytest] +DJANGO_SETTINGS_MODULE = objects.conf.ci +pythonpath = . src