From 0ed8d4d1f71754f193323025b225c9cfc121e6f4 Mon Sep 17 00:00:00 2001 From: Sam Avis Date: Fri, 6 Jun 2025 18:07:36 +0100 Subject: [PATCH 1/4] Added pytest-profiling dep. Created unit test folder --- pyproject.toml | 1 + tests/{ => unit}/conftest.py | 0 tests/{ => unit}/test_grid.py | 0 tests/{ => unit}/test_initialization.py | 0 tests/{ => unit}/test_placeholder.py | 0 5 files changed, 1 insertion(+) rename tests/{ => unit}/conftest.py (100%) rename tests/{ => unit}/test_grid.py (100%) rename tests/{ => unit}/test_initialization.py (100%) rename tests/{ => unit}/test_placeholder.py (100%) diff --git a/pyproject.toml b/pyproject.toml index 3c8646a..e90a848 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ dev = [ "pre-commit>=4.2.0", "ruff>=0.11.4", "pytest>=7.0", + "pytest-profiling", ] [tool.pytest.ini_options] diff --git a/tests/conftest.py b/tests/unit/conftest.py similarity index 100% rename from tests/conftest.py rename to tests/unit/conftest.py diff --git a/tests/test_grid.py b/tests/unit/test_grid.py similarity index 100% rename from tests/test_grid.py rename to tests/unit/test_grid.py diff --git a/tests/test_initialization.py b/tests/unit/test_initialization.py similarity index 100% rename from tests/test_initialization.py rename to tests/unit/test_initialization.py diff --git a/tests/test_placeholder.py b/tests/unit/test_placeholder.py similarity index 100% rename from tests/test_placeholder.py rename to tests/unit/test_placeholder.py From 14b5af5724a852a7abe0f1fd64cdf8e91860580e Mon Sep 17 00:00:00 2001 From: Sam Avis Date: Mon, 9 Jun 2025 13:53:42 +0100 Subject: [PATCH 2/4] Add profiling test --- pyproject.toml | 3 ++- tests/{unit => }/conftest.py | 0 tests/profiling/test_profiling.py | 18 ++++++++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) rename tests/{unit => }/conftest.py (100%) create mode 100644 tests/profiling/test_profiling.py diff --git a/pyproject.toml b/pyproject.toml index e90a848..85bd93d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,9 +15,10 @@ dev = [ "ruff>=0.11.4", "pytest>=7.0", "pytest-profiling", + "flameprof", ] [tool.pytest.ini_options] testpaths = ["tests"] python_files = "test_*.py" -addopts = "-v" +addopts = "-v -m 'not profiling'" diff --git a/tests/unit/conftest.py b/tests/conftest.py similarity index 100% rename from tests/unit/conftest.py rename to tests/conftest.py diff --git a/tests/profiling/test_profiling.py b/tests/profiling/test_profiling.py new file mode 100644 index 0000000..5a00973 --- /dev/null +++ b/tests/profiling/test_profiling.py @@ -0,0 +1,18 @@ +"""Test(s) to use with pytest-profiling to identify bottlenecks in the code.""" +import pytest + +@pytest.mark.profiling +def test_profiling(monkeypatch): + + # Define the command line arguments + import sys + argv = ['dementpy.py', 'grassland', 'output', '20250402', 'scrubland'] + monkeypatch.setattr(sys, 'argv', argv) + + # Move to subfolder so input and output folders will be correct + import os + os.chdir('src') + + # Run dementpy + import dementpy + dementpy.main() From 10731c005e8ca846008ee7a7833a51c85139724c Mon Sep 17 00:00:00 2001 From: Sam Avis Date: Mon, 9 Jun 2025 14:13:29 +0100 Subject: [PATCH 3/4] Prevent dementpy from running automatically on import --- src/dementpy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/dementpy.py b/src/dementpy.py index c04f8b6..daa0148 100644 --- a/src/dementpy.py +++ b/src/dementpy.py @@ -103,4 +103,5 @@ def main(): os.chdir('../'+output_folder) export(Output_init, site, outname) -main() \ No newline at end of file +if __name__ == '__main__': + main() From 1499480dc636480da1b7fac72667fc9356216eb5 Mon Sep 17 00:00:00 2001 From: "M. A. Kowalski" Date: Tue, 24 Jun 2025 14:18:34 +0100 Subject: [PATCH 4/4] fix: implicit conversion to float64 There was an implicit conversion to float64 taking place due to a cast to the Python's list (and hence Python's float). This resulted in a serious (factor ~10) degradation in performance, which should now be fixed. The performance degradation was a result of temporary allocation performed by pandas when a dtype of a frame was implicitly changed in updates of the form e.g.: ```python import pandas as pd import numpy as np df = pd.DataFrame({"f32": np.ones(2, dtype="float32")}) df.iloc[1:2] = np.float64(1/3) ``` since pandas 2.1, such operations raise a FutureWarning. All occurences of that warning in DEMENTpy are resolved in this commit. --- src/grid.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/grid.py b/src/grid.py index f86065a..ec1cb4c 100644 --- a/src/grid.py +++ b/src/grid.py @@ -420,8 +420,8 @@ def metabolism(self,day): # Update Substrates pools with dead enzymes DeadEnz_df = pd.concat( [Enzyme_Loss, - Enzyme_Loss.mul(self.Enz_Attrib['N_cost'].tolist()*self.gridsize,axis=0), - Enzyme_Loss.mul(self.Enz_Attrib['P_cost'].tolist()*self.gridsize,axis=0)], + Enzyme_Loss.mul(np.repeat(self.Enz_Attrib['N_cost'].values, self.gridsize), axis=0), + Enzyme_Loss.mul(np.repeat(self.Enz_Attrib['P_cost'].values, self.gridsize), axis=0)], axis=1 ) DeadEnz_df.index = [np.arange(self.gridsize).repeat(self.n_enzymes), DeadEnz_df.index] # create a multi-index @@ -713,5 +713,5 @@ def reinitialization(self,initialization,microbes_pp,output,mode,pulse,*args): # last: assign microbes to each grid box randomly based on prior densities choose_taxa = np.zeros((self.n_taxa,self.gridsize), dtype='int8') for i in range(self.n_taxa): - choose_taxa[i,:] = np.random.choice([1,0], self.gridsize, replace=True, p=[frequencies[i], 1-frequencies[i]]) + choose_taxa[i,:] = np.random.binomial(1, frequencies[i], self.gridsize) self.Microbes.loc[np.ravel(choose_taxa,order='F')==0] = np.float32(0) # NOTE order='F' \ No newline at end of file