From 5e57ca108cc2290ed366c8dd88149b0acc6f7173 Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Thu, 23 Jan 2025 10:50:56 -0500 Subject: [PATCH 01/24] Add initial project files --- .../.bumpversion.cfg | 29 ++++ formats/pyramid-generator-3d-tool/.gitignore | 162 ++++++++++++++++++ .../pyramid-generator-3d-tool/.python-version | 1 + formats/pyramid-generator-3d-tool/Dockerfile | 25 +++ formats/pyramid-generator-3d-tool/README.md | 12 ++ formats/pyramid-generator-3d-tool/VERSION | 1 + .../pyramid-generator-3d-tool/build-docker.sh | 21 +++ formats/pyramid-generator-3d-tool/plugin.json | 67 ++++++++ .../pyramid-generator-3d-tool/pyproject.toml | 29 ++++ .../pyramid-generator-3d-tool/run-plugin.sh | 20 +++ .../formats/pyramid_generator_3d/__init__.py | 5 + .../formats/pyramid_generator_3d/__main__.py | 57 ++++++ .../pyramid_generator_3d.py | 6 + .../pyramid-generator-3d-tool/template.cwl | 35 ++++ .../tests/__init__.py | 1 + .../tests/conftest.py | 21 +++ .../tests/test_cli.py | 34 ++++ .../tests/test_pyramid_generator_3d.py | 25 +++ 18 files changed, 551 insertions(+) create mode 100644 formats/pyramid-generator-3d-tool/.bumpversion.cfg create mode 100644 formats/pyramid-generator-3d-tool/.gitignore create mode 100644 formats/pyramid-generator-3d-tool/.python-version create mode 100644 formats/pyramid-generator-3d-tool/Dockerfile create mode 100644 formats/pyramid-generator-3d-tool/README.md create mode 100644 formats/pyramid-generator-3d-tool/VERSION create mode 100755 formats/pyramid-generator-3d-tool/build-docker.sh create mode 100644 formats/pyramid-generator-3d-tool/plugin.json create mode 100644 formats/pyramid-generator-3d-tool/pyproject.toml create mode 100755 formats/pyramid-generator-3d-tool/run-plugin.sh create mode 100644 formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__init__.py create mode 100644 formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__main__.py create mode 100644 formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/pyramid_generator_3d.py create mode 100644 formats/pyramid-generator-3d-tool/template.cwl create mode 100644 formats/pyramid-generator-3d-tool/tests/__init__.py create mode 100644 formats/pyramid-generator-3d-tool/tests/conftest.py create mode 100644 formats/pyramid-generator-3d-tool/tests/test_cli.py create mode 100644 formats/pyramid-generator-3d-tool/tests/test_pyramid_generator_3d.py diff --git a/formats/pyramid-generator-3d-tool/.bumpversion.cfg b/formats/pyramid-generator-3d-tool/.bumpversion.cfg new file mode 100644 index 000000000..55930811b --- /dev/null +++ b/formats/pyramid-generator-3d-tool/.bumpversion.cfg @@ -0,0 +1,29 @@ +[bumpversion] +current_version = 0.1.0 +commit = True +tag = False +parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? +serialize = + {major}.{minor}.{patch}-{release}{dev} + {major}.{minor}.{patch} + +[bumpversion:part:release] +optional_value = _ +first_value = dev +values = + dev + _ + +[bumpversion:part:dev] + +[bumpversion:file:pyproject.toml] +search = version = "{current_version}" +replace = version = "{new_version}" + +[bumpversion:file:VERSION] + +[bumpversion:file:README.md] + +[bumpversion:file:plugin.json] + +[bumpversion:file:src/polus/images/formats/pyramid_generator_3d/__init__.py] diff --git a/formats/pyramid-generator-3d-tool/.gitignore b/formats/pyramid-generator-3d-tool/.gitignore new file mode 100644 index 000000000..db9ae5e04 --- /dev/null +++ b/formats/pyramid-generator-3d-tool/.gitignore @@ -0,0 +1,162 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ diff --git a/formats/pyramid-generator-3d-tool/.python-version b/formats/pyramid-generator-3d-tool/.python-version new file mode 100644 index 000000000..bd28b9c5c --- /dev/null +++ b/formats/pyramid-generator-3d-tool/.python-version @@ -0,0 +1 @@ +3.9 diff --git a/formats/pyramid-generator-3d-tool/Dockerfile b/formats/pyramid-generator-3d-tool/Dockerfile new file mode 100644 index 000000000..22846828d --- /dev/null +++ b/formats/pyramid-generator-3d-tool/Dockerfile @@ -0,0 +1,25 @@ +FROM polusai/bfio:2.1.9 + +# environment variables defined in polusai/bfio +ENV EXEC_DIR="/opt/executables" +ENV POLUS_IMG_EXT=".ome.tif" +ENV POLUS_TAB_EXT=".csv" +ENV POLUS_LOG="INFO" + +# Work directory defined in the base container +WORKDIR ${EXEC_DIR} + +# TODO: Change the tool_dir to the tool directory +ENV TOOL_DIR="formats/pyramid-generator-3d" + +# Copy the repository into the container +RUN mkdir image-tools +COPY . ${EXEC_DIR}/image-tools + +# Install the tool +RUN pip3 install "${EXEC_DIR}/image-tools/${TOOL_DIR}" --no-cache-dir + +# Set the entrypoint +# TODO: Change the entrypoint to the tool entrypoint +ENTRYPOINT ["python3", "-m", "polus.images.formats.pyramid_generator_3d"] +CMD ["--help"] diff --git a/formats/pyramid-generator-3d-tool/README.md b/formats/pyramid-generator-3d-tool/README.md new file mode 100644 index 000000000..fd1ee9608 --- /dev/null +++ b/formats/pyramid-generator-3d-tool/README.md @@ -0,0 +1,12 @@ +# pyramid_generator_3d (0.1.0) + +Generate 3D or Volume Image Pyramid + +## Building + +To build the Docker image for the tool, run `./build-docker.sh`. + +## Install WIPP Plugin + +If WIPP is running, navigate to the plugins page and add a new plugin. Paste the +contents of `plugin.json` into the pop-up window and submit. diff --git a/formats/pyramid-generator-3d-tool/VERSION b/formats/pyramid-generator-3d-tool/VERSION new file mode 100644 index 000000000..6e8bf73aa --- /dev/null +++ b/formats/pyramid-generator-3d-tool/VERSION @@ -0,0 +1 @@ +0.1.0 diff --git a/formats/pyramid-generator-3d-tool/build-docker.sh b/formats/pyramid-generator-3d-tool/build-docker.sh new file mode 100755 index 000000000..03dc913ba --- /dev/null +++ b/formats/pyramid-generator-3d-tool/build-docker.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Change the name of the tool here +tool_dir="formats/pyramid-generator-3d" + +# The version is read from the VERSION file +version=$("] +readme = "README.md" +packages = [{include = "polus", from = "src"}] + +[tool.poetry.dependencies] +python = ">=3.9,<3.12" +bfio = {version = ">=2.3.3,<3.0", extras = ["all"]} +filepattern = ">=2.0.4,<3.0" +typer = "^0.7.0" + +[tool.poetry.group.dev.dependencies] +bump2version = "^1.0.1" +pytest = "^8.3" +pytest-sugar = "^1.0" +pytest-xdist = "^3.6" +pre-commit = "^3.8" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.pytest.ini_options] +pythonpath = [ + "." +] diff --git a/formats/pyramid-generator-3d-tool/run-plugin.sh b/formats/pyramid-generator-3d-tool/run-plugin.sh new file mode 100755 index 000000000..b0bfa6621 --- /dev/null +++ b/formats/pyramid-generator-3d-tool/run-plugin.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +version=$( None: + """Add options to pytest.""" + parser.addoption( + "--slow", + action="store_true", + dest="slow", + default=False, + help="Run slow tests", + ) + parser.addoption( + "--downloads", + action="store_true", + dest="downloads", + default=False, + help="Run tests that download large data files", + ) diff --git a/formats/pyramid-generator-3d-tool/tests/test_cli.py b/formats/pyramid-generator-3d-tool/tests/test_cli.py new file mode 100644 index 000000000..6926040b6 --- /dev/null +++ b/formats/pyramid-generator-3d-tool/tests/test_cli.py @@ -0,0 +1,34 @@ +"""Testing the Command Line Tool.""" + +import faulthandler +import json +from pathlib import Path +from typer.testing import CliRunner + +from .conftest import FixtureReturnType + +from polus.images.formats.pyramid_generator_3d.__main__ import app + +faulthandler.enable() + + +def test_cli(): + """Test the command line.""" + # TODO: Set up parameters for the test + inp_dir, out_dir, = Path("tests/data/inp"), Path("tests/data/out") + + runner = CliRunner() + result = runner.invoke( + app, + [ + "--inpDir", + inp_dir, + "--outDir", + out_dir, + ], + ) + + # Test for a successful run + assert result.exit_code == 0 + + # TODO: Add more tests diff --git a/formats/pyramid-generator-3d-tool/tests/test_pyramid_generator_3d.py b/formats/pyramid-generator-3d-tool/tests/test_pyramid_generator_3d.py new file mode 100644 index 000000000..bd7bfac0d --- /dev/null +++ b/formats/pyramid-generator-3d-tool/tests/test_pyramid_generator_3d.py @@ -0,0 +1,25 @@ +"""Tests for pyramid_generator_3d.""" + +import pytest + +from polus.images.formats.pyramid_generator_3d.pyramid_generator_3d import pyramid_generator_3d + + +def test_pyramid_generator_3d(): + """Test pyramid_generator_3d.""" + # TODO: Add tests + pass + + +@pytest.mark.skipif("not config.getoption('slow')") +def test_slow_pyramid_generator_3d(): + """Test that can take a long time to run.""" + # TODO: Add optional tests + pass + + +@pytest.mark.skipif("not config.getoption('downloads')") +def test_download_pyramid_generator_3d(): + """Test thatdownload data from.""" + # TODO: Add optional tests + pass From a0227f95410a1a638c511fae4a933250eb6392ca Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Fri, 24 Jan 2025 14:04:40 -0500 Subject: [PATCH 02/24] Updated cli option parsing --- .../formats/pyramid_generator_3d/__init__.py | 2 +- .../formats/pyramid_generator_3d/__main__.py | 247 ++++++++++++++++-- .../pyramid_generator_3d.py | 50 +++- 3 files changed, 269 insertions(+), 30 deletions(-) diff --git a/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__init__.py b/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__init__.py index d82e79b56..cc23c1b1d 100644 --- a/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__init__.py +++ b/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__init__.py @@ -2,4 +2,4 @@ __version__ = "0.1.0" -from polus.images.formats.pyramid_generator_3d.pyramid_generator_3d import pyramid_generator_3d +# from polus.images.formats.pyramid_generator_3d.pyramid_generator_3d import pyramid_generator_3d diff --git a/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__main__.py b/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__main__.py index fd7b276e8..7a29f7945 100644 --- a/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__main__.py +++ b/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__main__.py @@ -3,10 +3,14 @@ import logging import os import pathlib +import typing import typer - -from polus.images.formats.pyramid_generator_3d.pyramid_generator_3d import pyramid_generator_3d +from polus.images.formats.pyramid_generator_3d.pyramid_generator_3d import ( + SubCommand, + gen_py3d, + gen_volume, +) logging.basicConfig( format="%(asctime)s - %(name)-8s - %(levelname)-8s - %(message)s", @@ -20,38 +24,229 @@ POLUS_TAB_EXT = os.environ.get("POLUS_TAB_EXT", ".csv") app = typer.Typer() + +def sub_cmd_callback(ctx: typer.Context, value: SubCommand) -> SubCommand: + """Parse cmd type and set custom context object. + + set ctx.obj["sub_cmd"] according to the subcommand value, such that we can + check the validity of the parameters for the subcommand. + Args: + ctx (typer.Context): typer context object + value (str): passed in parameter value + """ + ctx.ensure_object(dict) + ctx.obj["cmd"] = "Py3D" if value == SubCommand.Py3D else "Vol" + return value + + +def _camal_case(s: str) -> str: + """Convert string to camel case. + + Args: + s (str): input string + + Returns: + str: camel case string + """ + s_ = s.split("_") + return s_[0] + "".join(word.capitalize() for word in s_[1:]) + + +def vol_option_callback( + ctx: typer.Context, param: typer.CallbackParam, value: typing.Any +): + """Determine validity of Vol options if using Vol subcommand. + + Args: + ctx (typer.Context): typer context object + value (str): passed in parameter value + """ + if ctx.obj["cmd"] == "Vol" or ctx.obj["cmd"] == "Vol_Py3D": + if not value: + raise typer.BadParameter( + f"--{_camal_case(param.name)} are required for volume generation." + ) + return value + + +def inp_dir_callback(ctx: typer.Context, value: typing.Union[pathlib.Path, None]): + """Determine validity of input directory if using Vol subcommand. + + Args: + ctx (typer.Context): typer context object + value (pathlib.Path): passed in parameter value + """ + if ctx.obj["cmd"] == "Vol" or ctx.obj["cmd"] == "Vol_Py3D": + if not value: + raise typer.BadParameter( + "Input directory is required for volume generation." + ) + if not value.exists(): + raise typer.BadParameter("Input directory does not exist.") + if not value.is_dir(): + raise typer.BadParameter("Input directory is not a directory.") + if not os.access(value, os.R_OK): + raise typer.BadParameter("Input directory is not readable.") + + return value.resolve() if value else value + + +def out_dir_callback(ctx: typer.Context, value: typing.Union[pathlib.Path, None]): + """Determine validity of output directory if using Vol subcommand. + + Args: + ctx (typer.Context): typer context object + value (pathlib.Path): passed in parameter value + """ + if ctx.obj["cmd"] == "Vol" or ctx.obj["cmd"] == "Vol_Py3D": + if not value: + raise typer.BadParameter( + "Output directory is required for volume generation." + ) + if not value.exists(): + raise typer.BadParameter("Output directory does not exist.") + if not value.is_dir(): + raise typer.BadParameter("Output directory is not a directory.") + if not os.access(value, os.W_OK): + raise typer.BadParameter("Output directory is not writable.") + + return value.resolve() if value else value + + +def py3d_option_callback( + ctx: typer.Context, param: typer.CallbackParam, value: typing.Any +): + """Determine validity of parameter if using Py3D subcommand. + + Args: + ctx (typer.Context): typer context object + value (int): passed in parameter value + """ + if ctx.obj["cmd"] == "Py3D": + if not value: + raise typer.BadParameter( + f"--{_camal_case(param.name)} is required for 3D pyramid generation." + ) + return value + + +def zarr_dir_callback(ctx: typer.Context, value: typing.Union[pathlib.Path, None]): + """Determine validity of zarr directory if using Py3D subcommand. + + Args: + ctx (typer.Context): typer context object + value (pathlib.Path): passed in parameter value + """ + if ctx.obj["cmd"] == "Py3D": + if value: # use zarr directory + if not value.exists(): + raise typer.BadParameter("Zarr directory does not exist.") + if not value.is_dir(): + raise typer.BadParameter("Zarr directory is not a directory.") + if not os.access(value, os.R_OK): + raise typer.BadParameter("Zarr directory is not readable.") + else: # None value, use inpDir instead + # change context label + ctx.obj["cmd"] = "Vol_Py3D" + logger.info( + "Zarr dir not provided, using inpDir to perform first " + "volume generation, and then 3D pyramid generation." + ) + + # fully resolve path + return value.resolve() if value else value + + @app.command() def main( - fileExtension: str = typer.Option( + ctx: typer.Context, + sub_cmd: SubCommand = typer.Option( ..., - help="None", + "--subCmd", + help="Subcommand to run. Choose 'Py3D' for 3D pyramid generation or 'Vol' for volume generation.", + callback=sub_cmd_callback, ), - filePattern: str = typer.Option( - , - help="None", + zarr_dir: pathlib.Path = typer.Option( + None, + "--zarrDir", + help=( + "Directory containing the input zarr files for 3D pyramid generation. " + "If not provided, inpDir needs to be provided." + ), + callback=zarr_dir_callback, ), - inpDir: pathlib.Path = typer.Option( - ..., - help="None", - exists=True, - file_okay=False, - dir_okay=True, - resolve_path=True, - readable=True, - ), - outDir: pathlib.Path = typer.Option( - ..., - help="None", - exists=True, - file_okay=False, - dir_okay=True, - resolve_path=True, - readable=True, + inp_dir: pathlib.Path = typer.Option( + None, + "--inpDir", + help="Directory containing the input images for 3D pyramid generation.", + callback=inp_dir_callback, + ), + file_pattern: str = typer.Option( + None, + "--filePattern", + help="File pattern for selecting images for 3D pyramid generation.", + callback=vol_option_callback, + ), + group_by: str = typer.Option( + None, + "--groupBy", + help="Image dimension, e.g., 'z', to group images for 3D pyramid generation.", + callback=vol_option_callback, + ), + out_dir: pathlib.Path = typer.Option( + None, + "--outDir", + help="Output directory for 3D pyramid generation.", + callback=out_dir_callback, + ), + out_img_name: str = typer.Option( + None, + "--outImgName", + help="Name of the output image name for 3D pyramid generation.", + callback=vol_option_callback, + ), + base_scale_key: int = typer.Option( + 0, + "--baseScaleKey", + help="Base scale key for volume generation.", + ), + num_levels: int = typer.Option( + None, + "--numLevels", + help="Number of levels for volume generation.", + callback=py3d_option_callback, ), ): """CLI for the pyramid_generator_3d tool.""" - pass + # for some reason after the callback the sub_cmd only receives None value. + # need to determine type based on stored context custom object. nevertheless + # other param checks are correct based on the sub_cmd value + # seems to be problem specific to using typer callback and enum + sub_cmd = ctx.obj["cmd"] + logger.info("Starting pyramid_generator_3d...") + logger.info("subCmd: %s", sub_cmd) + logger.info("zarrDir: %s", zarr_dir) + logger.info("inpDir: %s", inp_dir) + logger.info("groupBy: %s", group_by) + logger.info("filePattern: %s", file_pattern) + logger.info("outDir: %s", out_dir) + logger.info("outImgName: %s", out_img_name) + logger.info("baseScaleKey: %d", base_scale_key) + logger.info("numLevels: %s", num_levels) + + # call argolid + if sub_cmd.startswith("Vol"): + logger.info("Starting Volume Generation...") + gen_volume(inp_dir, group_by, file_pattern, out_dir, out_img_name) + logger.info("Volume generation completed.") + + if sub_cmd.endswith("Py3D"): + logger.info("Starting 3D Pyramid Generation...") + # use out_dir if zarr_dir is None. volume generation outputs zarr to out_dir + zarr_dir = zarr_dir if zarr_dir else out_dir + gen_py3d(zarr_dir, base_scale_key, num_levels) + logger.info("3D pyramid generation completed.") -if __name__ == '__main__': +if __name__ == "__main__": app() diff --git a/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/pyramid_generator_3d.py b/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/pyramid_generator_3d.py index af37d5db9..dff8c50f9 100644 --- a/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/pyramid_generator_3d.py +++ b/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/pyramid_generator_3d.py @@ -1,6 +1,50 @@ """pyramid_generator_3d.""" +from enum import Enum +from pathlib import Path +from argolid import PyramidGenerator3D, VolumeGenerator -def pyramid_generator_3d(): - """pyramid_generator_3d.""" - pass \ No newline at end of file + +class SubCommand(str, Enum): + """SubCommand.""" + + Py3D = "Py3D" # only perform 3D pyramid generation + Vol = "Vol" # only perform volume generation + + +def gen_volume( + inp_dir: Path, + group_by: str, + file_pattern: str, + out_dir: Path, + out_img_name: str, +): + """Generate volume.using argolid. + + Args: + inp_dir (Path): input directory + group_by (str): image dimension to group by + file_pattern (str): file pattern to search for images + out_dir (Path): output directory + out_img_name (str): output image name + """ + volume_gen = VolumeGenerator( + str(inp_dir), group_by, file_pattern, str(out_dir), out_img_name + ) + volume_gen.generate_volume() + + +def gen_py3d( + zarr_dir: Path, + base_scale_key: int, + num_levels: int, +): + """Generate 3d pyramid using argolid. + + Args: + zarr_dir (Path): path to zarr arrays + base_scale_key (int): base scale key + num_levels (int): number of levels for pyramid + """ + pyramid_gen = PyramidGenerator3D(zarr_dir, base_scale_key) + pyramid_gen.generate_pyramid(num_levels) From 9fc7ba3a3ecd8c84f06d3d8dfe59dec6459ac197 Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Fri, 24 Jan 2025 14:08:50 -0500 Subject: [PATCH 03/24] Created CLI and option parsing tests --- .../tests/test_tool.py | 247 ++++++++++++++++++ 1 file changed, 247 insertions(+) create mode 100644 formats/pyramid-generator-3d-tool/tests/test_tool.py diff --git a/formats/pyramid-generator-3d-tool/tests/test_tool.py b/formats/pyramid-generator-3d-tool/tests/test_tool.py new file mode 100644 index 000000000..7f9acb7cf --- /dev/null +++ b/formats/pyramid-generator-3d-tool/tests/test_tool.py @@ -0,0 +1,247 @@ +"""Testing the Command Line Tool.""" + +import faulthandler +import logging +import shutil +import typing +from pathlib import Path + +import pytest +import requests +from polus.images.formats.pyramid_generator_3d.__main__ import app +from typer.testing import CliRunner + +faulthandler.enable() +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + +_OPTION_NAMES_VOL = [ + "--subCmd", + "--inpDir", + "--filePattern", + "--groupBy", + "--outDir", + "--outImgName", +] +_OPTION_NAMES_PY3D = ["--subCmd", "--zarrDir", "--baseScaleKey", "--numLevels"] +_OPTION_NAMES_VOL_PY3D = _OPTION_NAMES_VOL + _OPTION_NAMES_PY3D[2:] + +OPTION_NAMES_VOL = tuple(_OPTION_NAMES_VOL) +OPTION_NAMES_PY3D = tuple(_OPTION_NAMES_PY3D) +OPTION_NAMES_VOL_PY3D = tuple(_OPTION_NAMES_VOL_PY3D) +BAD_PARAM_TEST_COUNTER = 0 + + +def _get_real_img(path2save: Path, filename=None) -> Path: + """Download a real image from the internet. + + Args: + path2save (Path): path to save the image + + Returns: + Path : path to the downloaded image + """ + # Download the data if it doesn't exist + URL = "https://github.com/usnistgov/WIPP/raw/master/data/PyramidBuilding/inputCollection/" + filename = "img_r001_c001.ome.tif" if filename is None else filename + if not (path2save / filename).exists(): + content = requests.get(URL + filename, timeout=10.0).content + (path2save / filename).open("wb").write(content) + + return path2save / filename + + +@pytest.fixture +def gen_data_path() -> typing.Generator[Path, None, None]: + """Generate a temporary path holding test data.""" + data_path = Path("data") + data_path.mkdir(parents=True, exist_ok=True) + + yield data_path + + # delete the temporary path + shutil.rmtree(data_path) + + +@pytest.fixture +def gen_image_collection_path( + gen_data_path, +) -> typing.Generator[typing.Tuple[Path, Path], None, None]: + """Generate input and output path for image collection test.""" + data_path = gen_data_path + inp_dir = data_path / "input/image_collection" + inp_dir.mkdir(parents=True, exist_ok=True) + + out_dir = data_path / "output/image_collection" + out_dir.mkdir(parents=True, exist_ok=True) + + yield inp_dir, out_dir + + # delete the input and output path + shutil.rmtree(inp_dir) + shutil.rmtree(out_dir) + + +@pytest.fixture +def gen_image_collection( + gen_image_collection_path, +) -> typing.Generator[typing.Tuple[Path, Path, str, str], None, None]: + """Create an image collection.""" + inp_dir, out_dir = gen_image_collection_path + + img_paths = [] + for r in range(1, 5): # r = 1,2,3,4 + for c in range(1, 5): # c = 1,2,3,4 + img_path = _get_real_img(inp_dir, f"img_r{r:03d}_c{c:03d}.ome.tif") + img_paths.append(img_path) + + file_pattern = "img_r001_c{c:ddd}.ome.tif" + out_img_name = "output_img" + yield inp_dir, out_dir, file_pattern, out_img_name + + # delete the image + for img_path in img_paths: + img_path.unlink() + + +@pytest.fixture +def default_params() -> typing.Generator[typing.Tuple[str, int, int], None, None]: + """Return default params for group_by, base_scale_key, num_levels.""" + yield "c", 0, 2 + + +def test_cli(gen_image_collection, default_params): + """Test the command line.""" + inp_dir, out_dir, file_pattern, out_img_name = gen_image_collection + group_by, base_scale_key, num_levels = default_params + runner = CliRunner() + + # do test with Vol + param_list = [ + "--subCmd", + "Vol", + "--inpDir", + inp_dir, + "--filePattern", + file_pattern, + "--groupBy", + group_by, + "--outDir", + out_dir, + "--outImgName", + out_img_name, + ] + result = runner.invoke(app, param_list) + assert result.exit_code == 0 + + # do test with Py3D, using previous output as input + param_list = [ + "--subCmd", + "Py3D", + "--zarrDir", + out_dir, + "--baseScaleKey", + base_scale_key, + "--numLevels", + num_levels, + ] + result = runner.invoke(app, param_list) + + # remove all file and folders in out_dir + for item in out_dir.iterdir(): + if item.is_file(): + item.unlink() + else: + shutil.rmtree(item) + + # test for Py3D with no zarrDir but inpDir provided + param_list = [ + "--subCmd", + "Py3D", + "--inpDir", + inp_dir, + "--filePattern", + file_pattern, + "--groupBy", + group_by, + "--outDir", + out_dir, + "--outImgName", + out_img_name, + "--baseScaleKey", + base_scale_key, + "--numLevels", + num_levels, + ] + assert result.exit_code == 0 + + +@pytest.fixture() +def complete_param(gen_image_collection, default_params): + """Generate complete params.""" + inp_dir, out_dir, file_pattern, out_img_name = gen_image_collection + group_by, base_scale_key, num_levels = default_params + OPTIONS = [ + "--zarrDir", + "--inpDir", + "--filePattern", + "--groupBy", + "--outDir", + "--outImgName", + "--baseScaleKey", + "--numLevels", + ] + option_values = [ + out_dir, + inp_dir, + file_pattern, + group_by, + out_dir, + out_img_name, + base_scale_key, + num_levels, + ] + return dict(zip(OPTIONS, option_values)) + + +def gen_bad_params(sub_cmd, option_names): + """Generate bad params for Vol subcommand.""" + lst_tmp = [] + lst = [] + key_lst = option_names + missing_names_lst = [] + for i, key in enumerate(key_lst): + lst.append((lst_tmp, sub_cmd)) + # create a string that formats the current time + missing_names_lst.append(f"subCmd={sub_cmd}, missing {key}.") + if i == len(key_lst) - 1: # skip the last key + break + lst_tmp.append(key) + return lst, missing_names_lst + + +def combine_bad_params(): + """Combine bad params into a single set of tests.""" + v1, n1 = gen_bad_params("Vol", OPTION_NAMES_VOL) + v2, n2 = gen_bad_params("Py3D", OPTION_NAMES_PY3D) + v3, n3 = gen_bad_params("Py3D", OPTION_NAMES_VOL_PY3D) + return v1 + v2 + v3, n1 + n2 + n3 + + +@pytest.mark.parametrize( + "bad_params, sub_cmd", + argvalues=combine_bad_params()[0], + ids=combine_bad_params()[1], +) +def test_bad_params(bad_params, sub_cmd, complete_param): + """Test the command line with bad params for Vol subcommand.""" + runner = CliRunner() + param_name_list = bad_params + param_list = [] + for param_name in param_name_list: + if param_name == "--subCmd": + param_list += [param_name, sub_cmd] + else: + param_list += [param_name, complete_param[param_name]] + result = runner.invoke(app, param_list) + assert result.exit_code != 0 From 824f4732adec7c4819b00c461f104189ece0e56f Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Fri, 24 Jan 2025 14:09:20 -0500 Subject: [PATCH 04/24] Updated dependencies --- formats/pyramid-generator-3d-tool/pyproject.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/formats/pyramid-generator-3d-tool/pyproject.toml b/formats/pyramid-generator-3d-tool/pyproject.toml index d4769a384..5d9c61983 100644 --- a/formats/pyramid-generator-3d-tool/pyproject.toml +++ b/formats/pyramid-generator-3d-tool/pyproject.toml @@ -10,7 +10,8 @@ packages = [{include = "polus", from = "src"}] python = ">=3.9,<3.12" bfio = {version = ">=2.3.3,<3.0", extras = ["all"]} filepattern = ">=2.0.4,<3.0" -typer = "^0.7.0" +typer = "0.7.0" +argolid = "^0.0.6" [tool.poetry.group.dev.dependencies] bump2version = "^1.0.1" @@ -18,6 +19,7 @@ pytest = "^8.3" pytest-sugar = "^1.0" pytest-xdist = "^3.6" pre-commit = "^3.8" +requests = "^2.32.3" [build-system] requires = ["poetry-core"] From 4d31f7f44ab7649dd7d3572570674fbcaa8121dc Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Fri, 24 Jan 2025 14:10:04 -0500 Subject: [PATCH 05/24] Removed unwanted tests scripts --- .../tests/conftest.py | 21 ------------ .../tests/test_cli.py | 34 ------------------- .../tests/test_pyramid_generator_3d.py | 25 -------------- 3 files changed, 80 deletions(-) delete mode 100644 formats/pyramid-generator-3d-tool/tests/conftest.py delete mode 100644 formats/pyramid-generator-3d-tool/tests/test_cli.py delete mode 100644 formats/pyramid-generator-3d-tool/tests/test_pyramid_generator_3d.py diff --git a/formats/pyramid-generator-3d-tool/tests/conftest.py b/formats/pyramid-generator-3d-tool/tests/conftest.py deleted file mode 100644 index dd33c6ea5..000000000 --- a/formats/pyramid-generator-3d-tool/tests/conftest.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Test fixtures. -""" - -import pytest - -def pytest_addoption(parser: pytest.Parser) -> None: - """Add options to pytest.""" - parser.addoption( - "--slow", - action="store_true", - dest="slow", - default=False, - help="Run slow tests", - ) - parser.addoption( - "--downloads", - action="store_true", - dest="downloads", - default=False, - help="Run tests that download large data files", - ) diff --git a/formats/pyramid-generator-3d-tool/tests/test_cli.py b/formats/pyramid-generator-3d-tool/tests/test_cli.py deleted file mode 100644 index 6926040b6..000000000 --- a/formats/pyramid-generator-3d-tool/tests/test_cli.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Testing the Command Line Tool.""" - -import faulthandler -import json -from pathlib import Path -from typer.testing import CliRunner - -from .conftest import FixtureReturnType - -from polus.images.formats.pyramid_generator_3d.__main__ import app - -faulthandler.enable() - - -def test_cli(): - """Test the command line.""" - # TODO: Set up parameters for the test - inp_dir, out_dir, = Path("tests/data/inp"), Path("tests/data/out") - - runner = CliRunner() - result = runner.invoke( - app, - [ - "--inpDir", - inp_dir, - "--outDir", - out_dir, - ], - ) - - # Test for a successful run - assert result.exit_code == 0 - - # TODO: Add more tests diff --git a/formats/pyramid-generator-3d-tool/tests/test_pyramid_generator_3d.py b/formats/pyramid-generator-3d-tool/tests/test_pyramid_generator_3d.py deleted file mode 100644 index bd7bfac0d..000000000 --- a/formats/pyramid-generator-3d-tool/tests/test_pyramid_generator_3d.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Tests for pyramid_generator_3d.""" - -import pytest - -from polus.images.formats.pyramid_generator_3d.pyramid_generator_3d import pyramid_generator_3d - - -def test_pyramid_generator_3d(): - """Test pyramid_generator_3d.""" - # TODO: Add tests - pass - - -@pytest.mark.skipif("not config.getoption('slow')") -def test_slow_pyramid_generator_3d(): - """Test that can take a long time to run.""" - # TODO: Add optional tests - pass - - -@pytest.mark.skipif("not config.getoption('downloads')") -def test_download_pyramid_generator_3d(): - """Test thatdownload data from.""" - # TODO: Add optional tests - pass From c084f5ab790bb664158df7cb11c248bee971b96a Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Fri, 24 Jan 2025 14:35:44 -0500 Subject: [PATCH 06/24] Changed group_by to enum --- .../formats/pyramid_generator_3d/__main__.py | 28 ++++++++++--------- .../pyramid_generator_3d.py | 12 ++++++-- .../tests/test_tool.py | 2 +- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__main__.py b/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__main__.py index 7a29f7945..699892fad 100644 --- a/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__main__.py +++ b/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__main__.py @@ -7,6 +7,7 @@ import typer from polus.images.formats.pyramid_generator_3d.pyramid_generator_3d import ( + GroupBy, SubCommand, gen_py3d, gen_volume, @@ -20,8 +21,6 @@ logger = logging.getLogger("polus.images.formats.pyramid_generator_3d") logger.setLevel(POLUS_LOG) -POLUS_IMG_EXT = os.environ.get("POLUS_IMG_EXT", ".ome.tif") -POLUS_TAB_EXT = os.environ.get("POLUS_TAB_EXT", ".csv") app = typer.Typer() @@ -35,7 +34,7 @@ def sub_cmd_callback(ctx: typer.Context, value: SubCommand) -> SubCommand: value (str): passed in parameter value """ ctx.ensure_object(dict) - ctx.obj["cmd"] = "Py3D" if value == SubCommand.Py3D else "Vol" + ctx.obj["sub_cmd"] = "Py3D" if value == SubCommand.Py3D else "Vol" return value @@ -61,11 +60,13 @@ def vol_option_callback( ctx (typer.Context): typer context object value (str): passed in parameter value """ - if ctx.obj["cmd"] == "Vol" or ctx.obj["cmd"] == "Vol_Py3D": + if ctx.obj["sub_cmd"].startswith("Vol"): if not value: raise typer.BadParameter( f"--{_camal_case(param.name)} are required for volume generation." ) + if param.name == "group_by": + ctx.obj["group_by"] = value return value @@ -76,7 +77,7 @@ def inp_dir_callback(ctx: typer.Context, value: typing.Union[pathlib.Path, None] ctx (typer.Context): typer context object value (pathlib.Path): passed in parameter value """ - if ctx.obj["cmd"] == "Vol" or ctx.obj["cmd"] == "Vol_Py3D": + if ctx.obj["sub_cmd"].startswith("Vol"): if not value: raise typer.BadParameter( "Input directory is required for volume generation." @@ -98,7 +99,7 @@ def out_dir_callback(ctx: typer.Context, value: typing.Union[pathlib.Path, None] ctx (typer.Context): typer context object value (pathlib.Path): passed in parameter value """ - if ctx.obj["cmd"] == "Vol" or ctx.obj["cmd"] == "Vol_Py3D": + if ctx.obj["sub_cmd"].startswith("Vol"): if not value: raise typer.BadParameter( "Output directory is required for volume generation." @@ -122,7 +123,7 @@ def py3d_option_callback( ctx (typer.Context): typer context object value (int): passed in parameter value """ - if ctx.obj["cmd"] == "Py3D": + if ctx.obj["sub_cmd"] == "Py3D": if not value: raise typer.BadParameter( f"--{_camal_case(param.name)} is required for 3D pyramid generation." @@ -137,7 +138,7 @@ def zarr_dir_callback(ctx: typer.Context, value: typing.Union[pathlib.Path, None ctx (typer.Context): typer context object value (pathlib.Path): passed in parameter value """ - if ctx.obj["cmd"] == "Py3D": + if ctx.obj["sub_cmd"] == "Py3D": if value: # use zarr directory if not value.exists(): raise typer.BadParameter("Zarr directory does not exist.") @@ -147,10 +148,10 @@ def zarr_dir_callback(ctx: typer.Context, value: typing.Union[pathlib.Path, None raise typer.BadParameter("Zarr directory is not readable.") else: # None value, use inpDir instead # change context label - ctx.obj["cmd"] = "Vol_Py3D" + ctx.obj["sub_cmd"] = "Vol_Py3D" logger.info( - "Zarr dir not provided, using inpDir to perform first " - "volume generation, and then 3D pyramid generation." + "Zarr dir not provided, using inpDir to perform volume " + "generation first, and then 3D pyramid generation." ) # fully resolve path @@ -187,7 +188,7 @@ def main( help="File pattern for selecting images for 3D pyramid generation.", callback=vol_option_callback, ), - group_by: str = typer.Option( + group_by: GroupBy = typer.Option( None, "--groupBy", help="Image dimension, e.g., 'z', to group images for 3D pyramid generation.", @@ -222,7 +223,8 @@ def main( # need to determine type based on stored context custom object. nevertheless # other param checks are correct based on the sub_cmd value # seems to be problem specific to using typer callback and enum - sub_cmd = ctx.obj["cmd"] + sub_cmd = ctx.obj["sub_cmd"] + group_by = ctx.obj.get("group_by", None) logger.info("Starting pyramid_generator_3d...") logger.info("subCmd: %s", sub_cmd) logger.info("zarrDir: %s", zarr_dir) diff --git a/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/pyramid_generator_3d.py b/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/pyramid_generator_3d.py index dff8c50f9..0ffef7252 100644 --- a/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/pyramid_generator_3d.py +++ b/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/pyramid_generator_3d.py @@ -8,8 +8,16 @@ class SubCommand(str, Enum): """SubCommand.""" - Py3D = "Py3D" # only perform 3D pyramid generation - Vol = "Vol" # only perform volume generation + Py3D = "Py3D" # Perform 3D pyramid generation from zarr arrays or from image collection + Vol = "Vol" # Perform volume generation from image collection + + +class GroupBy(str, Enum): + """GroupBy.""" + + t = "t" + z = "z" + c = "c" def gen_volume( diff --git a/formats/pyramid-generator-3d-tool/tests/test_tool.py b/formats/pyramid-generator-3d-tool/tests/test_tool.py index 7f9acb7cf..f710c36a8 100644 --- a/formats/pyramid-generator-3d-tool/tests/test_tool.py +++ b/formats/pyramid-generator-3d-tool/tests/test_tool.py @@ -213,7 +213,7 @@ def gen_bad_params(sub_cmd, option_names): for i, key in enumerate(key_lst): lst.append((lst_tmp, sub_cmd)) # create a string that formats the current time - missing_names_lst.append(f"subCmd={sub_cmd}, missing {key}.") + missing_names_lst.append(f"subCmd={sub_cmd}, missing {key} ") if i == len(key_lst) - 1: # skip the last key break lst_tmp.append(key) From a8f7d79e05fbff05c58383948d34593816e75d7b Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Fri, 24 Jan 2025 16:07:57 -0500 Subject: [PATCH 07/24] Updated README --- formats/pyramid-generator-3d-tool/README.md | 42 ++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/formats/pyramid-generator-3d-tool/README.md b/formats/pyramid-generator-3d-tool/README.md index fd1ee9608..32ef085a7 100644 --- a/formats/pyramid-generator-3d-tool/README.md +++ b/formats/pyramid-generator-3d-tool/README.md @@ -1,6 +1,46 @@ # pyramid_generator_3d (0.1.0) -Generate 3D or Volume Image Pyramid +Generate 3D Image Pyramid from an image collection or Zarr directory. This plugin is a wrapper for argolid. +This tool offers 2 subcommands: `Vol` and `Py3D`, for volume generation and 3D pyramid generation respectively. See [Usage](##usage) section for details. + +## Options +| Name | Description | I/O | Type | +|-------------|-----------------------------------------------------------------------------|-----|--------| +|`--subCmd` | Subcommand to invoke. Options are `Vol` and `Py3D`. |Input|string | +|`--zarrDir` | Directory to Zarr arrays for generating 3D pyramid. |Input|collection| +|`--inpDir` | Directory to input image collection. Required if `--zarrDir` is unspecified.|Input|collection| +|`--filePattern` | File pattern for discovering images in `--inpDir`. |Input|collection| +|`--groupBy` | Grouping variable for images. Options are `t`, `z`, `c`. |Input|string| +|`--outDir` | Output directory. |Output|collection| +|`--outImgName` | Output name for Zarr arrays when using volume generation. |Input|string| +|`--baseScaleKey`| Base scale key for 3D pyramid generation. Default to 0. |Input|integer| +|`--numLevels` | Number of levels for 3D pyramid. |Input|integer| + +## Usage +### Volume Generation +Use `Vol` subcommand to generate Zarr arrays from image stacks. It reads images from the input directory, groups them by specific dimension, and writes Zarr array into the output directory. +The ***required*** options for `Vol` subcommand are `--inpDir`, `--filePattern`, `--groupBy`, `outDir`, `--outImgName` +Example usage: +``` +python3 -m polus.images.formats.pyramid_generator_3d --subCmd Vol --inpDir /path/to/input/images --filePattern img_r{r:ddd}_c{c:ddd}.ome.tif --groupBy c --outDir /path/to/output --outImgName output_image +``` + +### 3D Pyramid +Use `Py3D` subcommand to generate 3D pyramid from either (1) a directory with Zarr array or (2) a directory of images. +#### From Zarr directory +When generating from a Zarr directory, the ***required*** options are `--zarrDir` and `--numLevels`. `--baseScaleKey` defaults to 0. +Example usage: +``` +python -m polus.images.formats.pyramid_generator_3d --subCmd Py3D --zarrDir /path/to/zarr/array --baseScaleKey 0 --numLevels 2 +``` + +#### From image collection +When generating directly from an image collection, the current tool firsts calls the volume generation routine first to generate Zarr array, from which 3D pyramid is subsequently generated. Thus, all options required for `Vol` subcommand are required in addition to the required options of `Py3D` (excluding `--zarrDir`). +Together, the ***required*** options are `--inpDir`, `--filePattern`, `--groupBy`, `outDir`, `--outImgName`, `--numLevels`. `--baseScaleKey` defaults to 0. +Example usage: +``` +python -m polus.images.formats.pyramid_generator_3d --subCmd Py3D --inpDir /path/to/input/images --filePattern img_r{r:ddd}_c{c:ddd}.ome.tif --groupBy c --outDir /path/to/output --outImgName test_output --baseScaleKey 0 --numLevels 2 +``` ## Building From 528759b4c12c8983245705e4a415f931548b32e5 Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Fri, 24 Jan 2025 16:47:19 -0500 Subject: [PATCH 08/24] Updated README --- formats/pyramid-generator-3d-tool/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/formats/pyramid-generator-3d-tool/README.md b/formats/pyramid-generator-3d-tool/README.md index 32ef085a7..55703cff2 100644 --- a/formats/pyramid-generator-3d-tool/README.md +++ b/formats/pyramid-generator-3d-tool/README.md @@ -28,15 +28,15 @@ python3 -m polus.images.formats.pyramid_generator_3d --subCmd Vol --inpDir /path ### 3D Pyramid Use `Py3D` subcommand to generate 3D pyramid from either (1) a directory with Zarr array or (2) a directory of images. #### From Zarr directory -When generating from a Zarr directory, the ***required*** options are `--zarrDir` and `--numLevels`. `--baseScaleKey` defaults to 0. +When generating from a Zarr directory, the ***required*** options are `--zarrDir`, `--outDir`, and `--numLevels`. `--baseScaleKey` defaults to 0. Since the output will be written into the Zarr directory, use the same directory for `--zarrDir` and `--outDir`. Example usage: ``` -python -m polus.images.formats.pyramid_generator_3d --subCmd Py3D --zarrDir /path/to/zarr/array --baseScaleKey 0 --numLevels 2 +python -m polus.images.formats.pyramid_generator_3d --subCmd Py3D --zarrDir /path/to/zarr/array --outDir /path/to/zarr/array --baseScaleKey 0 --numLevels 2 ``` #### From image collection When generating directly from an image collection, the current tool firsts calls the volume generation routine first to generate Zarr array, from which 3D pyramid is subsequently generated. Thus, all options required for `Vol` subcommand are required in addition to the required options of `Py3D` (excluding `--zarrDir`). -Together, the ***required*** options are `--inpDir`, `--filePattern`, `--groupBy`, `outDir`, `--outImgName`, `--numLevels`. `--baseScaleKey` defaults to 0. +Together, the ***required*** options are `--inpDir`, `--filePattern`, `--groupBy`, `--outDir`, `--outImgName`, `--numLevels`. `--baseScaleKey` defaults to 0. Example usage: ``` python -m polus.images.formats.pyramid_generator_3d --subCmd Py3D --inpDir /path/to/input/images --filePattern img_r{r:ddd}_c{c:ddd}.ome.tif --groupBy c --outDir /path/to/output --outImgName test_output --baseScaleKey 0 --numLevels 2 From 29cedfeff42803eee56a72e67bce6eefdfb58861 Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Fri, 24 Jan 2025 16:47:57 -0500 Subject: [PATCH 09/24] Made --outDir required option --- .../formats/pyramid_generator_3d/__main__.py | 31 ++++--------------- .../tests/test_tool.py | 12 +++++-- 2 files changed, 16 insertions(+), 27 deletions(-) diff --git a/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__main__.py b/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__main__.py index 699892fad..a22f96fc0 100644 --- a/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__main__.py +++ b/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__main__.py @@ -92,28 +92,6 @@ def inp_dir_callback(ctx: typer.Context, value: typing.Union[pathlib.Path, None] return value.resolve() if value else value -def out_dir_callback(ctx: typer.Context, value: typing.Union[pathlib.Path, None]): - """Determine validity of output directory if using Vol subcommand. - - Args: - ctx (typer.Context): typer context object - value (pathlib.Path): passed in parameter value - """ - if ctx.obj["sub_cmd"].startswith("Vol"): - if not value: - raise typer.BadParameter( - "Output directory is required for volume generation." - ) - if not value.exists(): - raise typer.BadParameter("Output directory does not exist.") - if not value.is_dir(): - raise typer.BadParameter("Output directory is not a directory.") - if not os.access(value, os.W_OK): - raise typer.BadParameter("Output directory is not writable.") - - return value.resolve() if value else value - - def py3d_option_callback( ctx: typer.Context, param: typer.CallbackParam, value: typing.Any ): @@ -195,10 +173,13 @@ def main( callback=vol_option_callback, ), out_dir: pathlib.Path = typer.Option( - None, + ..., "--outDir", - help="Output directory for 3D pyramid generation.", - callback=out_dir_callback, + help="Output directory.", + exists=True, + file_okay=False, + writable=True, + resolve_path=True, ), out_img_name: str = typer.Option( None, diff --git a/formats/pyramid-generator-3d-tool/tests/test_tool.py b/formats/pyramid-generator-3d-tool/tests/test_tool.py index f710c36a8..55840a228 100644 --- a/formats/pyramid-generator-3d-tool/tests/test_tool.py +++ b/formats/pyramid-generator-3d-tool/tests/test_tool.py @@ -23,8 +23,14 @@ "--outDir", "--outImgName", ] -_OPTION_NAMES_PY3D = ["--subCmd", "--zarrDir", "--baseScaleKey", "--numLevels"] -_OPTION_NAMES_VOL_PY3D = _OPTION_NAMES_VOL + _OPTION_NAMES_PY3D[2:] +_OPTION_NAMES_PY3D = [ + "--subCmd", + "--zarrDir", + "--outDir", + "--baseScaleKey", + "--numLevels", +] +_OPTION_NAMES_VOL_PY3D = _OPTION_NAMES_VOL + _OPTION_NAMES_PY3D[-2:] OPTION_NAMES_VOL = tuple(_OPTION_NAMES_VOL) OPTION_NAMES_PY3D = tuple(_OPTION_NAMES_PY3D) @@ -140,6 +146,8 @@ def test_cli(gen_image_collection, default_params): "Py3D", "--zarrDir", out_dir, + "--outDir", + out_dir, "--baseScaleKey", base_scale_key, "--numLevels", From 28d0560ff40dbe9432ec239968a2f0c6985af7cc Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Fri, 24 Jan 2025 17:25:31 -0500 Subject: [PATCH 10/24] Updated README --- formats/pyramid-generator-3d-tool/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/formats/pyramid-generator-3d-tool/README.md b/formats/pyramid-generator-3d-tool/README.md index 55703cff2..524149245 100644 --- a/formats/pyramid-generator-3d-tool/README.md +++ b/formats/pyramid-generator-3d-tool/README.md @@ -11,7 +11,7 @@ This tool offers 2 subcommands: `Vol` and `Py3D`, for volume generation and 3D p |`--inpDir` | Directory to input image collection. Required if `--zarrDir` is unspecified.|Input|collection| |`--filePattern` | File pattern for discovering images in `--inpDir`. |Input|collection| |`--groupBy` | Grouping variable for images. Options are `t`, `z`, `c`. |Input|string| -|`--outDir` | Output directory. |Output|collection| +|`--outDir` | Path of output directory. |Output|collection| |`--outImgName` | Output name for Zarr arrays when using volume generation. |Input|string| |`--baseScaleKey`| Base scale key for 3D pyramid generation. Default to 0. |Input|integer| |`--numLevels` | Number of levels for 3D pyramid. |Input|integer| From f6ff3a3555b00fa303154261fadaa6dc4d79c2ce Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Fri, 24 Jan 2025 17:25:58 -0500 Subject: [PATCH 11/24] Updated plugin.json --- formats/pyramid-generator-3d-tool/plugin.json | 129 ++++++++++++++---- 1 file changed, 101 insertions(+), 28 deletions(-) diff --git a/formats/pyramid-generator-3d-tool/plugin.json b/formats/pyramid-generator-3d-tool/plugin.json index 98580c8de..2f54cbcc5 100644 --- a/formats/pyramid-generator-3d-tool/plugin.json +++ b/formats/pyramid-generator-3d-tool/plugin.json @@ -15,25 +15,73 @@ "polus.images.formats.pyramid_generator_3d" ], "inputs": { + "subCmd": { + "type": "enum", + "options": { + "values": [ + "Vol", + "Py3D" + ] + }, + "title": "Subcommand", + "description": "Subcommand to invoke. Options are Vol and Py3D.", + "required": "True" + }, + "zarrDir": { + "type": "collection", + "title": "Zarr array directory", + "description": "Directory to Zarr arrays for generating 3D pyramid.", + "required": "False" + }, "inpDir": { "type": "collection", "title": "Input collection", "description": "Input image collection to be processed by this tool.", - "required": "True" + "required": "False" }, "filePattern": { "type": "string", "title": "Filename pattern", - "description": "Filename pattern used to separate data.", - "required": "False", - "default": ".*" + "description": "Filename pattern used to select images.", + "required": "False" }, - "preview": { - "type": "boolean", - "title": "Preview", - "description": "Generate an output preview.", + "groupBy": { + "type": "enum", + "options": { + "values": [ + "t", + "z", + "c" + ] + }, + "title": "Group by", + "description": "Grouping variable for images. Options are 't', 'z', 'c'.", + "required": "False" + }, + "outDir": { + "type": "collection", + "title": "Output directory", + "description": "Path of output directory.", + "required": "True" + }, + "outImgName": { + "type": "string", + "title": "Output image name", + "description": "Output name for Zarr arrays when using volume generation.", + "required": "False" + }, + "baseScaleKey":{ + "type": "number", + "title": "Base scale key", + "description": "Base scale key for 3D pyramid generation. Default to 0.", "required": "False", - "default": "False" + "default": "0" + }, + "numLevels":{ + "type": "number", + "title": "Number of levels", + "description": "Number of levels for 3D pyramid.", + "required": "False" } }, "outputs": { @@ -42,26 +90,51 @@ "description": "Output collection." } }, - "ui": { - "inpDir": { - "type": "collection", - "title": "Input collection", - "description": "Input image collection to be processed by this tool.", - "required": "True" + "ui": [ + { + "key":"inputs.subCmd", + "title": "Subcommand", + "description": "Subcommand to invoke. Options are Vol and Py3D." }, - "filePattern": { - "type": "string", - "title": "Filename pattern", - "description": "Filename pattern used to separate data.", - "required": "False", - "default": "False" + { + "key": "inputs.zarrDir", + "title": "Zarr array directory", + "description": "Directory to Zarr arrays for generating 3D pyramid." }, - "preview": { - "type": "boolean", + { + "key": "inputs.inpDir", + "title": "Image directory", + "description": "Path to directory containing images or path to a single image." + }, + { + "key": "inputs.filePattern", "title": "Filename pattern", - "description": "Generate an output preview.", - "required": "False", - "default": "False" + "description": "Filename pattern used to find images." + }, + { + "key": "inputs.groupBy", + "title": "Group by", + "description": "Grouping variable for images. Options are 't', 'z', 'c'." + }, + { + "key": "inputs.outDir", + "title": "Output directory", + "description": "Path of output directory." + }, + { + "key": "inputs.outImgName", + "title": "Output image name", + "description": "Output name for Zarr arrays when using volume generation." + }, + { + "key": "inputs.baseScaleKey", + "title": "Base scale key", + "description": "Base scale key for 3D pyramid generation. Default to 0." + }, + { + "key": "inputs.numLevels", + "title": "Number of levels", + "description": "Number of levels for 3D pyramid." } - } -} \ No newline at end of file + ] +} From eee1b654824d0031e4e7cc2b2e66e63f2b68caa7 Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Fri, 24 Jan 2025 17:26:29 -0500 Subject: [PATCH 12/24] Updated cwl file --- .../pyramidgenerator3d.cwl | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 formats/pyramid-generator-3d-tool/pyramidgenerator3d.cwl diff --git a/formats/pyramid-generator-3d-tool/pyramidgenerator3d.cwl b/formats/pyramid-generator-3d-tool/pyramidgenerator3d.cwl new file mode 100644 index 000000000..f05420445 --- /dev/null +++ b/formats/pyramid-generator-3d-tool/pyramidgenerator3d.cwl @@ -0,0 +1,55 @@ +class: CommandLineTool +cwlVersion: v1.2 + +inputs: + subCmd: + inputBinding: + prefix: --subCmd + type: string + zarrDir: + inputBinding: + prefix: --zarrDir + type: Directory? + inpDir: + inputBinding: + prefix: --inpDir + type: Directory + filePattern: + inputBinding: + prefix: --filePattern + type: string? + groupBy: + inputBinding: + prefix: --groupBy + type: string? + outDir: + inputBinding: + prefix: --outDir + type: Directory + outImgName: + inputBinding: + prefix: --outImgName + type: string? + baseScaleKey: + inputBinding: + prefix: --baseScaleKey + type: int? + numLevels: + inputBinding: + prefix: --numLevels + type: int? + +outputs: + outDir: + outputBinding: + glob: $(inputs.outDir.basename) + type: Directory + +requirements: + DockerRequirement: + dockerPull: polusai/pyramid-generator-3d-tool:0.1.0 + InitialWorkDirRequirement: + listing: + - entry: $(inputs.outDir) + writable: true + InlineJavascriptRequirement: {} From c62ee74af3e45554bc689050eed87c73defb61e3 Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Fri, 24 Jan 2025 17:26:55 -0500 Subject: [PATCH 13/24] Removed template cwl file --- .../pyramid-generator-3d-tool/template.cwl | 35 ------------------- 1 file changed, 35 deletions(-) delete mode 100644 formats/pyramid-generator-3d-tool/template.cwl diff --git a/formats/pyramid-generator-3d-tool/template.cwl b/formats/pyramid-generator-3d-tool/template.cwl deleted file mode 100644 index b6f8f296a..000000000 --- a/formats/pyramid-generator-3d-tool/template.cwl +++ /dev/null @@ -1,35 +0,0 @@ -class: CommandLineTool -cwlVersion: v1.2 - -inputs: - fileExtension: - inputBinding: - prefix: --fileExtension - type: string - filePattern: - inputBinding: - prefix: --filePattern - type: string? - inpDir: - inputBinding: - prefix: --inpDir - type: Directory - outDir: - inputBinding: - prefix: --outDir - type: Directory - -outputs: - outDir: - outputBinding: - glob: $(inputs.outDir.basename) - type: Directory - -requirements: - DockerRequirement: - dockerPull: polusai/ome-converter-tool:0.1.0 - InitialWorkDirRequirement: - listing: - - entry: $(inputs.outDir) - writable: true - InlineJavascriptRequirement: {} \ No newline at end of file From 9a0dbe053e00d8afff0fbd4a02454880984c4d34 Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Mon, 27 Jan 2025 09:47:38 -0500 Subject: [PATCH 14/24] Updated bfio base image version --- formats/pyramid-generator-3d-tool/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/formats/pyramid-generator-3d-tool/Dockerfile b/formats/pyramid-generator-3d-tool/Dockerfile index 22846828d..fc2cf372f 100644 --- a/formats/pyramid-generator-3d-tool/Dockerfile +++ b/formats/pyramid-generator-3d-tool/Dockerfile @@ -1,4 +1,4 @@ -FROM polusai/bfio:2.1.9 +FROM polusai/bfio:2.4.7 # environment variables defined in polusai/bfio ENV EXEC_DIR="/opt/executables" @@ -10,7 +10,7 @@ ENV POLUS_LOG="INFO" WORKDIR ${EXEC_DIR} # TODO: Change the tool_dir to the tool directory -ENV TOOL_DIR="formats/pyramid-generator-3d" +ENV TOOL_DIR="formats/pyramid-generator-3d-tool" # Copy the repository into the container RUN mkdir image-tools From 8ecb08afe42859607a8718aefe07212fda0967a0 Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Mon, 27 Jan 2025 09:48:12 -0500 Subject: [PATCH 15/24] Updated tool name and path --- formats/pyramid-generator-3d-tool/build-docker.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/formats/pyramid-generator-3d-tool/build-docker.sh b/formats/pyramid-generator-3d-tool/build-docker.sh index 03dc913ba..d38722be1 100755 --- a/formats/pyramid-generator-3d-tool/build-docker.sh +++ b/formats/pyramid-generator-3d-tool/build-docker.sh @@ -1,20 +1,22 @@ #!/bin/bash # Change the name of the tool here -tool_dir="formats/pyramid-generator-3d" +tool_dir="formats/" +tool_name="pyramid-generator-3d-tool" # The version is read from the VERSION file version=$( Date: Mon, 27 Jan 2025 09:48:29 -0500 Subject: [PATCH 16/24] Added new dependency --- formats/pyramid-generator-3d-tool/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/formats/pyramid-generator-3d-tool/pyproject.toml b/formats/pyramid-generator-3d-tool/pyproject.toml index 5d9c61983..f7d789e7d 100644 --- a/formats/pyramid-generator-3d-tool/pyproject.toml +++ b/formats/pyramid-generator-3d-tool/pyproject.toml @@ -12,6 +12,7 @@ bfio = {version = ">=2.3.3,<3.0", extras = ["all"]} filepattern = ">=2.0.4,<3.0" typer = "0.7.0" argolid = "^0.0.6" +lxml = "^5.3.0" [tool.poetry.group.dev.dependencies] bump2version = "^1.0.1" From 3b1c77f224961a444bb4333b215e757bdac36920 Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Mon, 27 Jan 2025 09:49:45 -0500 Subject: [PATCH 17/24] Added 3 example usages --- .../pyramid-generator-3d-tool/run-plugin.sh | 54 +++++++++++++++++-- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/formats/pyramid-generator-3d-tool/run-plugin.sh b/formats/pyramid-generator-3d-tool/run-plugin.sh index b0bfa6621..5676c3d02 100755 --- a/formats/pyramid-generator-3d-tool/run-plugin.sh +++ b/formats/pyramid-generator-3d-tool/run-plugin.sh @@ -2,19 +2,63 @@ version=$( Date: Mon, 27 Jan 2025 09:50:25 -0500 Subject: [PATCH 18/24] Fixed output path --- .../src/polus/images/formats/pyramid_generator_3d/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__main__.py b/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__main__.py index a22f96fc0..cd36904e7 100644 --- a/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__main__.py +++ b/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__main__.py @@ -226,7 +226,7 @@ def main( if sub_cmd.endswith("Py3D"): logger.info("Starting 3D Pyramid Generation...") # use out_dir if zarr_dir is None. volume generation outputs zarr to out_dir - zarr_dir = zarr_dir if zarr_dir else out_dir + zarr_dir = zarr_dir if zarr_dir else out_dir / out_img_name gen_py3d(zarr_dir, base_scale_key, num_levels) logger.info("3D pyramid generation completed.") From caf848f5d8856e1eb6ac222c03fb31ef305bfb61 Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Mon, 27 Jan 2025 09:55:48 -0500 Subject: [PATCH 19/24] =?UTF-8?q?Bump=20version:=200.1.0=20=E2=86=92=200.1?= =?UTF-8?q?.1-dev0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- formats/pyramid-generator-3d-tool/.bumpversion.cfg | 4 +++- formats/pyramid-generator-3d-tool/README.md | 2 +- formats/pyramid-generator-3d-tool/VERSION | 2 +- formats/pyramid-generator-3d-tool/plugin.json | 4 ++-- formats/pyramid-generator-3d-tool/pyproject.toml | 2 +- formats/pyramid-generator-3d-tool/pyramidgenerator3d.cwl | 2 +- .../src/polus/images/formats/pyramid_generator_3d/__init__.py | 2 +- 7 files changed, 10 insertions(+), 8 deletions(-) diff --git a/formats/pyramid-generator-3d-tool/.bumpversion.cfg b/formats/pyramid-generator-3d-tool/.bumpversion.cfg index 55930811b..42da417f2 100644 --- a/formats/pyramid-generator-3d-tool/.bumpversion.cfg +++ b/formats/pyramid-generator-3d-tool/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.1.0 +current_version = 0.1.1-dev0 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? @@ -27,3 +27,5 @@ replace = version = "{new_version}" [bumpversion:file:plugin.json] [bumpversion:file:src/polus/images/formats/pyramid_generator_3d/__init__.py] + +[bumpversion:file:pyramidgenerator3d.cwl] diff --git a/formats/pyramid-generator-3d-tool/README.md b/formats/pyramid-generator-3d-tool/README.md index 524149245..99dc2fad3 100644 --- a/formats/pyramid-generator-3d-tool/README.md +++ b/formats/pyramid-generator-3d-tool/README.md @@ -1,4 +1,4 @@ -# pyramid_generator_3d (0.1.0) +# pyramid_generator_3d (0.1.1-dev0) Generate 3D Image Pyramid from an image collection or Zarr directory. This plugin is a wrapper for argolid. This tool offers 2 subcommands: `Vol` and `Py3D`, for volume generation and 3D pyramid generation respectively. See [Usage](##usage) section for details. diff --git a/formats/pyramid-generator-3d-tool/VERSION b/formats/pyramid-generator-3d-tool/VERSION index 6e8bf73aa..44bf4db83 100644 --- a/formats/pyramid-generator-3d-tool/VERSION +++ b/formats/pyramid-generator-3d-tool/VERSION @@ -1 +1 @@ -0.1.0 +0.1.1-dev0 diff --git a/formats/pyramid-generator-3d-tool/plugin.json b/formats/pyramid-generator-3d-tool/plugin.json index 2f54cbcc5..123532f6e 100644 --- a/formats/pyramid-generator-3d-tool/plugin.json +++ b/formats/pyramid-generator-3d-tool/plugin.json @@ -1,6 +1,6 @@ { "name": "pyramid_generator_3d", - "version": "0.1.0", + "version": "0.1.1-dev0", "title": "pyramid_generator_3d", "description": "Generate 3D or Volume Image Pyramid", "author": "Ji Liu (ji.liu@axleinfo.com)", @@ -8,7 +8,7 @@ "repository": "https://github.com/polusAI/tabular-tools", "website": "https://ncats.nih.gov/preclinical/core/informatics", "citation": "", - "containerId": "polusai/pyramid-generator-3d-tool:0.1.0", + "containerId": "polusai/pyramid-generator-3d-tool:0.1.1-dev0", "baseCommand": [ "python3", "-m", diff --git a/formats/pyramid-generator-3d-tool/pyproject.toml b/formats/pyramid-generator-3d-tool/pyproject.toml index f7d789e7d..905bdf68f 100644 --- a/formats/pyramid-generator-3d-tool/pyproject.toml +++ b/formats/pyramid-generator-3d-tool/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "polus-images-formats-pyramid-generator-3d" -version = "0.1.0" +version = "0.1.1-dev0" description = "Generate 3D or Volume Image Pyramid" authors = ["Ji Liu "] readme = "README.md" diff --git a/formats/pyramid-generator-3d-tool/pyramidgenerator3d.cwl b/formats/pyramid-generator-3d-tool/pyramidgenerator3d.cwl index f05420445..8c924cdbf 100644 --- a/formats/pyramid-generator-3d-tool/pyramidgenerator3d.cwl +++ b/formats/pyramid-generator-3d-tool/pyramidgenerator3d.cwl @@ -47,7 +47,7 @@ outputs: requirements: DockerRequirement: - dockerPull: polusai/pyramid-generator-3d-tool:0.1.0 + dockerPull: polusai/pyramid-generator-3d-tool:0.1.1-dev0 InitialWorkDirRequirement: listing: - entry: $(inputs.outDir) diff --git a/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__init__.py b/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__init__.py index cc23c1b1d..5c6780f35 100644 --- a/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__init__.py +++ b/formats/pyramid-generator-3d-tool/src/polus/images/formats/pyramid_generator_3d/__init__.py @@ -1,5 +1,5 @@ """pyramid_generator_3d.""" -__version__ = "0.1.0" +__version__ = "0.1.1-dev0" # from polus.images.formats.pyramid_generator_3d.pyramid_generator_3d import pyramid_generator_3d From 4da6b0195e0dc3d3ac6115a0d9708a6cf016e330 Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Mon, 27 Jan 2025 16:18:45 -0500 Subject: [PATCH 20/24] Added description of testing with real data. --- formats/pyramid-generator-3d-tool/README.md | 23 +++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/formats/pyramid-generator-3d-tool/README.md b/formats/pyramid-generator-3d-tool/README.md index 99dc2fad3..224399b4e 100644 --- a/formats/pyramid-generator-3d-tool/README.md +++ b/formats/pyramid-generator-3d-tool/README.md @@ -42,6 +42,29 @@ Example usage: python -m polus.images.formats.pyramid_generator_3d --subCmd Py3D --inpDir /path/to/input/images --filePattern img_r{r:ddd}_c{c:ddd}.ome.tif --groupBy c --outDir /path/to/output --outImgName test_output --baseScaleKey 0 --numLevels 2 ``` +## Testing with real data + +1. install ome-zarr +``` +pip install ome-zarr +``` + +2. download an example dataset +``` +cd +ome_zarr download https://uk1s3.embassy.ebi.ac.uk/idr/zarr/v0.1/1884807.zarr +``` + +More dataset can be found [here](https://www.openmicroscopy.org/2020/11/04/zarr-data.html). To download other dataset, replace the link in the command above with other links copied from the "S3-entrypoint" column from the website. + +3. modify .zarray file under `/1884807.zarr/0`, i.e., delete the first element in "chunks" and "shape", which correspond to the T dimension. + +4. run the following command: +``` +python -m polus.images.formats.pyramid_generator_3d --subCmd Py3D --zarrDir /1884807.zarr --outDir /1884807.zarr --baseScaleKey 0 --numLevels 2 +``` +Pyramid data will be written into the same folder. + ## Building To build the Docker image for the tool, run `./build-docker.sh`. From 7d87cef8b4b5ed6fb3cf6805fa6bbb2d51cdba0a Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Tue, 28 Jan 2025 09:58:04 -0500 Subject: [PATCH 21/24] Fixed path bug and added output file assertion --- formats/pyramid-generator-3d-tool/tests/test_tool.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/formats/pyramid-generator-3d-tool/tests/test_tool.py b/formats/pyramid-generator-3d-tool/tests/test_tool.py index 55840a228..370ca1d28 100644 --- a/formats/pyramid-generator-3d-tool/tests/test_tool.py +++ b/formats/pyramid-generator-3d-tool/tests/test_tool.py @@ -139,21 +139,28 @@ def test_cli(gen_image_collection, default_params): ] result = runner.invoke(app, param_list) assert result.exit_code == 0 + # check presence of .zarray file + assert Path(out_dir / out_img_name / "0" / ".zarray").exists() # do test with Py3D, using previous output as input param_list = [ "--subCmd", "Py3D", "--zarrDir", - out_dir, + out_dir / out_img_name, "--outDir", - out_dir, + out_dir / out_img_name, "--baseScaleKey", base_scale_key, "--numLevels", num_levels, ] result = runner.invoke(app, param_list) + assert result.exit_code == 0 + + # check presence of .zarray file + for level in range(1, num_levels + 1): + assert Path(out_dir / out_img_name / f"{level}" / ".zarray").exists() # remove all file and folders in out_dir for item in out_dir.iterdir(): From 9ba6bf30adc38a57ff056a36518d26a02bc5f01a Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Tue, 28 Jan 2025 14:42:20 -0500 Subject: [PATCH 22/24] fix: add missing ict.yml file --- formats/pyramid-generator-3d-tool/ict.yml | 116 ++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 formats/pyramid-generator-3d-tool/ict.yml diff --git a/formats/pyramid-generator-3d-tool/ict.yml b/formats/pyramid-generator-3d-tool/ict.yml new file mode 100644 index 000000000..0668ff9d5 --- /dev/null +++ b/formats/pyramid-generator-3d-tool/ict.yml @@ -0,0 +1,116 @@ +author: +- Ji Liu +contact: ji.liu@axleinfo.com +container: polusai/pyramid-generator-3d-tool:0.1.1-dev0 +description: "This plugin generates 3D image pyramid." +entrypoint: '[python3, -m, polus.images.formats.pyramid_generator_3d]' +inputs: +- description: Subcommand to invoke. Options are Vol and Py3D. + format: + - enum + name: subCmd + required: true + type: string +- description: Directory to Zarr arrays for generating 3D pyramid. + format: + - collection + name: zarrDir + required: false + type: path +- description: Input image collection to be processed by this tool. + format: + - collection + name: inpDir + required: false + type: path +- description: Filename pattern used to select images from inpDir. + format: + - string + name: filePattern + required: false + type: string +- description: Grouping variable for images. Options are 't', 'z', 'c'. + format: + - enum + name: groupBy + required: false + type: string +- description: Path of output directory. + format: + - collection + name: outDir + required: true + type: path +- description: Output name for Zarr arrays when using volume generation. + format: + - string + name: outImgName + required: false + type: string +- description: Base scale key for 3D pyramid generation. Default to 0. + format: + - integer + name: baseScaleKey + required: false + type: integer +- description: Number of levels for 3D pyramid. + format: + - integer + name: numLevels + required: false + type: integer +outputs: +- description: Path of output directory. + format: + - collection + name: outDir + required: true + type: path +repository: https://github.com/PolusAI/image-tools +specVersion: 1.0.0 +title: 3D Pyramid Generator Plugin +ui: +- description: Subcommand to invoke. Options are Vol and Py3D. + fields: + - Vol + - Py3D + key: inputs.subCmd + title: Subcommand + type: select +- description: Directory to Zarr arrays for generating 3D pyramid. + key: inputs.zarrDir + title: Directory to zarr arrays + type: path +- description: Input image collection to be processed by this tool. + key: inputs.inpDir + title: Input directory + type: path +- description: Filename pattern used to select images from inpDir. + key: inputs.filePattern + title: File pattern + type: string +- description: Grouping variable for images. Options are 't', 'z', 'c'. + fields: + - t + - z + - c + key: inputs.groupBy + title: Group by + type: select +- description: Path of output directory. + key: inputs.outDir + title: Output directory + type: path +- description: Output name for Zarr arrays when using volume generation. + key: inputs.outImgName + title: Output image name + type: string +- description: Base scale key for 3D pyramid generation. Default to 0. + key: inputs.baseScaleKey + title: Base scale key + type: integer +- description: Number of levels for 3D pyramid. + key: inputs.numLevels + title: Number of levels + type: integer +version: 0.1.1-dev0 From 6d88bd5194432fd422125b9514c965f68ebafce4 Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Tue, 28 Jan 2025 14:43:47 -0500 Subject: [PATCH 23/24] fix: add ict.yml to bumpversion track list --- formats/pyramid-generator-3d-tool/.bumpversion.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/formats/pyramid-generator-3d-tool/.bumpversion.cfg b/formats/pyramid-generator-3d-tool/.bumpversion.cfg index 42da417f2..005c971a8 100644 --- a/formats/pyramid-generator-3d-tool/.bumpversion.cfg +++ b/formats/pyramid-generator-3d-tool/.bumpversion.cfg @@ -29,3 +29,5 @@ replace = version = "{new_version}" [bumpversion:file:src/polus/images/formats/pyramid_generator_3d/__init__.py] [bumpversion:file:pyramidgenerator3d.cwl] + +[bumpversion:ict.yml] From 1d29ef84e50e7dba7b6a890dbf40c7697976f418 Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Tue, 28 Jan 2025 14:45:16 -0500 Subject: [PATCH 24/24] fix: change description and correct repo link --- formats/pyramid-generator-3d-tool/plugin.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/formats/pyramid-generator-3d-tool/plugin.json b/formats/pyramid-generator-3d-tool/plugin.json index 123532f6e..d2fc7f21b 100644 --- a/formats/pyramid-generator-3d-tool/plugin.json +++ b/formats/pyramid-generator-3d-tool/plugin.json @@ -2,10 +2,10 @@ "name": "pyramid_generator_3d", "version": "0.1.1-dev0", "title": "pyramid_generator_3d", - "description": "Generate 3D or Volume Image Pyramid", + "description": "Generate 3D Image Pyramid", "author": "Ji Liu (ji.liu@axleinfo.com)", "institution": "National Center for Advancing Translational Sciences, National Institutes of Health", - "repository": "https://github.com/polusAI/tabular-tools", + "repository": "https://github.com/PolusAI/image-tools", "website": "https://ncats.nih.gov/preclinical/core/informatics", "citation": "", "containerId": "polusai/pyramid-generator-3d-tool:0.1.1-dev0",