From 82d8e7f26c422b219672c736afe48d8dd762994e Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Wed, 8 Jan 2025 15:48:27 -0500 Subject: [PATCH 01/14] added new input flag to indicate if keep orig img dtype --- .../images/apply_flatfield/apply_flatfield.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/apply_flatfield.py b/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/apply_flatfield.py index d81ef71da..bc8335252 100644 --- a/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/apply_flatfield.py +++ b/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/apply_flatfield.py @@ -4,6 +4,7 @@ import operator import pathlib import typing +import numpy as np import bfio import numpy @@ -135,6 +136,7 @@ def _unshade_batch( out_dir: pathlib.Path, ff_image: numpy.ndarray, df_image: typing.Optional[numpy.ndarray] = None, + keep_orig_dtype : typing.Optional[bool] = True, ) -> None: """Apply flatfield correction to a batch of images. @@ -168,6 +170,15 @@ def _unshade_batch( img_stack /= ff_image + 1e-8 + if keep_orig_dtype: + orig_dtype = images[0].dtype + max_val = numpy.iinfo(orig_dtype).max + + # clamp values to the original dtype range + img_stack[img_stack < 0] = 0 + img_stack[img_stack > max_val] = max_val + img_stack = np.round(img_stack).astype(orig_dtype) + # Save outputs with preadator.ProcessManager( name="unshade_batch::save", From d1707fe1c5d148cd0c66230ba2e381833b6caf10 Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Wed, 8 Jan 2025 15:56:45 -0500 Subject: [PATCH 02/14] added the same flag to other associated functions --- .../images/apply_flatfield/apply_flatfield.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/apply_flatfield.py b/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/apply_flatfield.py index bc8335252..3b3b181a4 100644 --- a/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/apply_flatfield.py +++ b/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/apply_flatfield.py @@ -27,6 +27,7 @@ def apply( # noqa: PLR0913 df_pattern: typing.Optional[str], out_dir: pathlib.Path, preview: bool = False, + keep_orig_dtype: typing.Optional[bool] = True, ) -> list[pathlib.Path]: """Run batch-wise flatfield correction on the image collection. @@ -42,6 +43,9 @@ def apply( # noqa: PLR0913 saved. preview: if True, return the paths to the images that would be saved without actually performing any other computation. + keep_orig_dtype: if True, the output images will be saved with the same + dtype as the input images. If False, the output images will be saved as + float32. """ img_fp = FilePattern(str(img_dir), img_pattern) img_variables = img_fp.get_variables() @@ -83,7 +87,8 @@ def apply( # noqa: PLR0913 if preview: out_files.extend(img_paths) else: - _unshade_images(img_paths, out_dir, ff_path, df_path) + _unshade_images(img_paths, out_dir, ff_path, df_path, + keep_orig_dtype) return out_files @@ -93,6 +98,7 @@ def _unshade_images( out_dir: pathlib.Path, ff_path: pathlib.Path, df_path: typing.Optional[pathlib.Path], + keep_orig_dtype: typing.Optional[bool] = True, ) -> None: """Remove the given flatfield components from all images and save outputs. @@ -101,6 +107,9 @@ def _unshade_images( out_dir: directory to save the corrected images ff_path: path to the flatfield image df_path: path to the darkfield image + keep_orig_dtype: if True, the output images will be saved with the same + dtype as the input images. If False, the output images will be saved as + float32. """ logger.info(f"Applying flatfield correction to {len(img_paths)} images ...") logger.info(f"{ff_path.name = } ...") @@ -128,6 +137,7 @@ def _unshade_images( out_dir, ff_image, df_image, + keep_orig_dtype ) @@ -145,6 +155,9 @@ def _unshade_batch( out_dir: directory to save the corrected images ff_image: component to be used for flatfield correction df_image: component to be used for flatfield correction + keep_orig_dtype: if True, the output images will be saved with the same + dtype as the input images. If False, the output images will be saved as + float32. """ # Load images with preadator.ProcessManager( From 652f64b0459ef02bb419c8633b3ccc73090761ca Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Wed, 8 Jan 2025 16:03:31 -0500 Subject: [PATCH 03/14] added cmd line input for keep orig dtype --- .../images/transforms/images/apply_flatfield/__main__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/__main__.py b/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/__main__.py index 87b57e2c4..b50bb8f75 100644 --- a/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/__main__.py +++ b/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/__main__.py @@ -69,6 +69,11 @@ def main( # noqa: PLR0913 "--preview", help="Preview the output without saving.", ), + keep_orig_dtype: typing.Optional[bool] = typer.Option( + True, + "--keepOrigDtype", + help="Keep the original dtype of the input images.", + ) ) -> None: """CLI for the Apply Flatfield plugin. @@ -87,6 +92,7 @@ def main( # noqa: PLR0913 logger.info(f"dfPattern = {df_pattern}") logger.info(f"outDir = {out_dir}") logger.info(f"preview = {preview}") + logger.info(f"keepOrigDtype = {keep_orig_dtype}") out_files = apply( img_dir=img_dir, @@ -96,6 +102,7 @@ def main( # noqa: PLR0913 df_pattern=df_pattern, out_dir=out_dir, preview=preview, + keep_orig_dtype=keep_orig_dtype, ) if preview: From c58fab16ca48fb613f38a19eaf173f52cc17c068 Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Wed, 8 Jan 2025 16:03:47 -0500 Subject: [PATCH 04/14] version modification --- .../polus/images/transforms/images/apply_flatfield/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/__init__.py b/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/__init__.py index ab03b2528..203e0a3ea 100644 --- a/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/__init__.py +++ b/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/__init__.py @@ -3,4 +3,4 @@ from . import utils from .apply_flatfield import apply -__version__ = "2.0.1" +__version__ = "2.0.1-dev" From 39c0add55478d381acc244f14e1c45d21387d52f Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Wed, 8 Jan 2025 17:29:34 -0500 Subject: [PATCH 05/14] version number update --- transforms/images/apply-flatfield-tool/.bumpversion.cfg | 2 +- transforms/images/apply-flatfield-tool/VERSION | 2 +- transforms/images/apply-flatfield-tool/pyproject.toml | 2 +- .../polus/images/transforms/images/apply_flatfield/__init__.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/transforms/images/apply-flatfield-tool/.bumpversion.cfg b/transforms/images/apply-flatfield-tool/.bumpversion.cfg index a2f2fd1cf..4e0208558 100644 --- a/transforms/images/apply-flatfield-tool/.bumpversion.cfg +++ b/transforms/images/apply-flatfield-tool/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.0.1 +current_version = 2.0.1-dev1 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? diff --git a/transforms/images/apply-flatfield-tool/VERSION b/transforms/images/apply-flatfield-tool/VERSION index 38f77a65b..6fb571b65 100644 --- a/transforms/images/apply-flatfield-tool/VERSION +++ b/transforms/images/apply-flatfield-tool/VERSION @@ -1 +1 @@ -2.0.1 +2.0.1-dev1 diff --git a/transforms/images/apply-flatfield-tool/pyproject.toml b/transforms/images/apply-flatfield-tool/pyproject.toml index cf82fd99d..6899dec1f 100644 --- a/transforms/images/apply-flatfield-tool/pyproject.toml +++ b/transforms/images/apply-flatfield-tool/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "polus-images-transforms-images-apply-flatfield" -version = "2.0.1" +version = "2.0.1-dev1" description = "" authors = [ "Nick Schaub ", diff --git a/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/__init__.py b/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/__init__.py index 203e0a3ea..2fdc0b8ad 100644 --- a/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/__init__.py +++ b/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/__init__.py @@ -3,4 +3,4 @@ from . import utils from .apply_flatfield import apply -__version__ = "2.0.1-dev" +__version__ = "2.0.1-dev1" From 4ebbc38c95122230fb6e72478a7232788f39c02a Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Wed, 8 Jan 2025 17:29:49 -0500 Subject: [PATCH 06/14] added new flag description --- transforms/images/apply-flatfield-tool/applyflatfield.cwl | 4 ++++ transforms/images/apply-flatfield-tool/ict.yaml | 6 ++++++ transforms/images/apply-flatfield-tool/plugin.json | 8 +++++++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/transforms/images/apply-flatfield-tool/applyflatfield.cwl b/transforms/images/apply-flatfield-tool/applyflatfield.cwl index 949e8e114..d41b44710 100644 --- a/transforms/images/apply-flatfield-tool/applyflatfield.cwl +++ b/transforms/images/apply-flatfield-tool/applyflatfield.cwl @@ -29,6 +29,10 @@ inputs: inputBinding: prefix: --preview type: boolean? + keepOrigDtype: + inputBinding: + prefix: --keepOrigDtype + type: boolean? outputs: outDir: outputBinding: diff --git a/transforms/images/apply-flatfield-tool/ict.yaml b/transforms/images/apply-flatfield-tool/ict.yaml index 032a24751..d85312a50 100644 --- a/transforms/images/apply-flatfield-tool/ict.yaml +++ b/transforms/images/apply-flatfield-tool/ict.yaml @@ -42,6 +42,12 @@ inputs: name: preview required: false type: boolean +- description: Keep original image data type if true, otherwise convert to float + format: + - boolean + name: keepOrigDtype + required: false + type: boolean name: polusai/ApplyFlatfield outputs: - description: Output collection diff --git a/transforms/images/apply-flatfield-tool/plugin.json b/transforms/images/apply-flatfield-tool/plugin.json index 60a1bafda..f2db88eb1 100644 --- a/transforms/images/apply-flatfield-tool/plugin.json +++ b/transforms/images/apply-flatfield-tool/plugin.json @@ -50,7 +50,13 @@ "type": "boolean", "description": "Preview the output images' names without actually running computation", "required": false - } + }, + { + "name": "keepOrigDtype", + "type": "boolean", + "description": "Keep original image data type if true, otherwise convert to float", + "required": false + } ], "outputs": [ { From 33791fe820916a572c4342725e0810ef59fbb16a Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Fri, 10 Jan 2025 14:10:18 -0500 Subject: [PATCH 07/14] updated dtype value calculation --- .../images/apply_flatfield/apply_flatfield.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/apply_flatfield.py b/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/apply_flatfield.py index 3b3b181a4..893a390a2 100644 --- a/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/apply_flatfield.py +++ b/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/apply_flatfield.py @@ -185,12 +185,16 @@ def _unshade_batch( if keep_orig_dtype: orig_dtype = images[0].dtype - max_val = numpy.iinfo(orig_dtype).max - - # clamp values to the original dtype range - img_stack[img_stack < 0] = 0 - img_stack[img_stack > max_val] = max_val - img_stack = np.round(img_stack).astype(orig_dtype) + # if integer type + if np.issubdtype(orig_dtype, np.integer): + min_val = numpy.iinfo(orig_dtype).min + max_val = numpy.iinfo(orig_dtype).max + # clamp values to the original dtype range + img_stack[img_stack < min_val] = min_val + img_stack[img_stack > max_val] = max_val + img_stack = np.round(img_stack).astype(orig_dtype) + elif np.issubdtype(orig_dtype, np.floating): + img_stack = img_stack.astype(orig_dtype) # Save outputs with preadator.ProcessManager( From 9fadb2e8aaf5a0d64f071c6aaf61a9e119a977e9 Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Fri, 10 Jan 2025 14:15:49 -0500 Subject: [PATCH 08/14] added source img and output img dtype condition to the tests --- .../apply-flatfield-tool/tests/test_plugin.py | 49 ++++++++++++------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/transforms/images/apply-flatfield-tool/tests/test_plugin.py b/transforms/images/apply-flatfield-tool/tests/test_plugin.py index d0faea18c..a65d30bdb 100644 --- a/transforms/images/apply-flatfield-tool/tests/test_plugin.py +++ b/transforms/images/apply-flatfield-tool/tests/test_plugin.py @@ -21,19 +21,24 @@ def _make_random_image( path: pathlib.Path, rng: numpy.random.Generator, size: int, + dtype: numpy.dtype = numpy.float32, ) -> None: with bfio.BioWriter(path) as writer: writer.X = size writer.Y = size - writer.dtype = numpy.float32 - - writer[:] = rng.random(size=(size, size), dtype=writer.dtype) + writer.dtype = dtype + if dtype == numpy.float32: + writer[:] = rng.random(size=(size, size), dtype=writer.dtype) + else: + writer[:] = rng.integers(low=0, high=numpy.iinfo(writer.dtype).max, + size=(size, size), dtype=writer.dtype) FixtureReturnType = tuple[pathlib.Path, str, pathlib.Path, str] -def gen_once(num_groups: int, img_size: int) -> FixtureReturnType: +def gen_once(num_groups: int, img_size: int, + dtype : numpy.dtype=numpy.float32) -> FixtureReturnType: """Generate a set of random images for testing.""" img_pattern = "img_x{x}_c{c}.ome.tif" @@ -45,15 +50,19 @@ def gen_once(num_groups: int, img_size: int) -> FixtureReturnType: rng = numpy.random.default_rng(42) for i in range(num_groups): - ff_path = ff_dir.joinpath(f"{ff_pattern.format(c=i + 1)}_flatfield.ome.tif") - _make_random_image(ff_path, rng, img_size) + ff_path = ff_dir.joinpath( + f"{ff_pattern.format(c=i + 1)}_flatfield.ome.tif" + ) + _make_random_image(ff_path, rng, img_size, dtype) - df_path = ff_dir.joinpath(f"{ff_pattern.format(c=i + 1)}_darkfield.ome.tif") - _make_random_image(df_path, rng, img_size) + df_path = ff_dir.joinpath( + f"{ff_pattern.format(c=i + 1)}_darkfield.ome.tif" + ) + _make_random_image(df_path, rng, img_size, dtype) for j in range(10): # 10 images in each group img_path = img_dir.joinpath(img_pattern.format(x=j + 1, c=i + 1)) - _make_random_image(img_path, rng, img_size) + _make_random_image(img_path, rng, img_size, dtype) image_names = list(sorted(p.name for p in img_dir.iterdir())) logger.debug(f"Generated {image_names} images in {img_dir}") @@ -68,8 +77,12 @@ def gen_once(num_groups: int, img_size: int) -> FixtureReturnType: NUM_GROUPS = [1, 4] IMG_SIZES = [1024, 4096] -PARAMS = list(itertools.product(NUM_GROUPS, IMG_SIZES)) -IDS = [f"{num_groups}_{img_size}" for num_groups, img_size in PARAMS] +IMG_DTYPE = [numpy.float32, numpy.uint16] +KEEP_ORIG_DTYPE = [True, False] +PARAMS = list(itertools.product(NUM_GROUPS, IMG_SIZES, IMG_DTYPE, + KEEP_ORIG_DTYPE)) +IDS = [f"{num_groups}_{img_size}_{dtype}_{keep_orig}" \ + for num_groups, img_size, dtype, keep_orig in PARAMS] @pytest.fixture(params=PARAMS, ids=IDS) @@ -77,10 +90,11 @@ def gen_images(request: pytest.FixtureRequest) -> FixtureReturnType: """Generate a set of random images for testing.""" num_groups: int img_size: int - num_groups, img_size = request.param - img_dir, img_pattern, ff_dir, ff_pattern = gen_once(num_groups, img_size) + num_groups, img_size, dtype, keep_orig = request.param + img_dir, img_pattern, ff_dir, ff_pattern = gen_once(num_groups, img_size, + dtype) - yield img_dir, img_pattern, ff_dir, ff_pattern + yield img_dir, img_pattern, ff_dir, ff_pattern, keep_orig # Cleanup shutil.rmtree(img_dir) @@ -90,7 +104,7 @@ def gen_images(request: pytest.FixtureRequest) -> FixtureReturnType: def test_estimate(gen_images: FixtureReturnType) -> None: """Test the `estimate` function.""" - img_dir, img_pattern, ff_dir, ff_pattern = gen_images + img_dir, img_pattern, ff_dir, ff_pattern, keep_orig = gen_images out_dir = pathlib.Path(tempfile.mkdtemp(suffix="out_dir")) apply( @@ -100,10 +114,11 @@ def test_estimate(gen_images: FixtureReturnType) -> None: ff_pattern=f"{ff_pattern}_flatfield.ome.tif", df_pattern=f"{ff_pattern}_darkfield.ome.tif", out_dir=out_dir, + keep_orig_dtype=keep_orig ) - img_names = [p.name for p in img_dir.iterdir()] - out_names = [p.name for p in out_dir.iterdir()] + img_names = sorted([p.name for p in img_dir.iterdir()]) + out_names = sorted([p.name for p in out_dir.iterdir()]) for name in img_names: assert name in out_names, f"{name} not in {out_names}" From f9328aff81078b3ed28a16b42c19fbdf922991b7 Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Sun, 12 Jan 2025 20:57:09 -0500 Subject: [PATCH 09/14] updated readme --- transforms/images/apply-flatfield-tool/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/transforms/images/apply-flatfield-tool/README.md b/transforms/images/apply-flatfield-tool/README.md index c2ce5d8b9..25b29be3c 100644 --- a/transforms/images/apply-flatfield-tool/README.md +++ b/transforms/images/apply-flatfield-tool/README.md @@ -55,3 +55,4 @@ Command line options: | `--dfPattern` | Filename pattern used to match darkfield files to image files | Input | string | | `--outDir` | Output collection | Output | collection | | `--preview` | Preview the output images' names without actually running computation | Input | boolean | +| `--keepOrigData` | Keep the original image data type if true | Input | boolean | \ No newline at end of file From b0e7774b784efc5f433edf2dd388b58bae338c0d Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Sun, 12 Jan 2025 21:03:57 -0500 Subject: [PATCH 10/14] added changelog --- .../images/apply-flatfield-tool/CHANGELOG.md | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 transforms/images/apply-flatfield-tool/CHANGELOG.md diff --git a/transforms/images/apply-flatfield-tool/CHANGELOG.md b/transforms/images/apply-flatfield-tool/CHANGELOG.md new file mode 100644 index 000000000..0f3afcc9b --- /dev/null +++ b/transforms/images/apply-flatfield-tool/CHANGELOG.md @@ -0,0 +1,29 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog], +and this project adheres to [Semantic Versioning]. + + +## [2.0.1-dev1] - 2025-01-12 + +### Added + +- "keepOrigDtype" option is added to the tool, so that the end user can choose to keep the original data type after the correction is applied, e.g., uint16 -> uint16. Otherwise, floating point output is kept. + +### Changed + +### Deprecated + +### Removed + +### Fixed + +### Security + + + +[keep a changelog]: https://keepachangelog.com/en/1.0.0/ +[semantic versioning]: https://semver.org/spec/v2.0.0.html + From 63a50eccec24723ff10a59b948314cbf879db2d2 Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Sun, 12 Jan 2025 21:04:04 -0500 Subject: [PATCH 11/14] updated readme --- transforms/images/apply-flatfield-tool/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transforms/images/apply-flatfield-tool/README.md b/transforms/images/apply-flatfield-tool/README.md index 25b29be3c..700da0695 100644 --- a/transforms/images/apply-flatfield-tool/README.md +++ b/transforms/images/apply-flatfield-tool/README.md @@ -55,4 +55,4 @@ Command line options: | `--dfPattern` | Filename pattern used to match darkfield files to image files | Input | string | | `--outDir` | Output collection | Output | collection | | `--preview` | Preview the output images' names without actually running computation | Input | boolean | -| `--keepOrigData` | Keep the original image data type if true | Input | boolean | \ No newline at end of file +| `--keepOrigDtype`| Keep the original image data type if true | Input | boolean | \ No newline at end of file From e5287c31ea91e41e241468b04a7d521c879e34f7 Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Mon, 13 Jan 2025 14:50:12 -0500 Subject: [PATCH 12/14] Reformatted files using pre-commit --- .../images/apply_flatfield/__main__.py | 5 ++--- .../images/apply_flatfield/apply_flatfield.py | 19 +++++++------------ .../images/apply_flatfield/utils.py | 4 ++-- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/__main__.py b/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/__main__.py index b50bb8f75..63054a101 100644 --- a/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/__main__.py +++ b/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/__main__.py @@ -6,8 +6,7 @@ import typing import typer -from polus.images.transforms.images.apply_flatfield import apply -from polus.images.transforms.images.apply_flatfield import utils +from polus.images.transforms.images.apply_flatfield import apply, utils # Initialize the logger logging.basicConfig( @@ -73,7 +72,7 @@ def main( # noqa: PLR0913 True, "--keepOrigDtype", help="Keep the original dtype of the input images.", - ) + ), ) -> None: """CLI for the Apply Flatfield plugin. diff --git a/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/apply_flatfield.py b/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/apply_flatfield.py index 893a390a2..b57f7078c 100644 --- a/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/apply_flatfield.py +++ b/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/apply_flatfield.py @@ -4,10 +4,10 @@ import operator import pathlib import typing -import numpy as np import bfio import numpy +import numpy as np import preadator import tqdm from filepattern import FilePattern @@ -28,7 +28,7 @@ def apply( # noqa: PLR0913 out_dir: pathlib.Path, preview: bool = False, keep_orig_dtype: typing.Optional[bool] = True, -) -> list[pathlib.Path]: +) -> typing.List[pathlib.Path]: """Run batch-wise flatfield correction on the image collection. Args: @@ -87,14 +87,13 @@ def apply( # noqa: PLR0913 if preview: out_files.extend(img_paths) else: - _unshade_images(img_paths, out_dir, ff_path, df_path, - keep_orig_dtype) + _unshade_images(img_paths, out_dir, ff_path, df_path, keep_orig_dtype) return out_files def _unshade_images( - img_paths: list[pathlib.Path], + img_paths: typing.List[pathlib.Path], out_dir: pathlib.Path, ff_path: pathlib.Path, df_path: typing.Optional[pathlib.Path], @@ -133,20 +132,16 @@ def _unshade_images( total=len(batch_indices) - 1, ): _unshade_batch( - img_paths[i_start:i_end], - out_dir, - ff_image, - df_image, - keep_orig_dtype + img_paths[i_start:i_end], out_dir, ff_image, df_image, keep_orig_dtype ) def _unshade_batch( - batch_paths: list[pathlib.Path], + batch_paths: typing.List[pathlib.Path], out_dir: pathlib.Path, ff_image: numpy.ndarray, df_image: typing.Optional[numpy.ndarray] = None, - keep_orig_dtype : typing.Optional[bool] = True, + keep_orig_dtype: typing.Optional[bool] = True, ) -> None: """Apply flatfield correction to a batch of images. diff --git a/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/utils.py b/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/utils.py index d32e577a1..85146bdd1 100644 --- a/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/utils.py +++ b/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/utils.py @@ -1,9 +1,9 @@ """Utilities for the apply flatfield plugin.""" - import logging import multiprocessing import os import pathlib +import typing import bfio import numpy @@ -13,7 +13,7 @@ MAX_WORKERS = max(1, multiprocessing.cpu_count() // 2) -def load_img(path: pathlib.Path, i: int) -> tuple[int, numpy.ndarray]: +def load_img(path: pathlib.Path, i: int) -> typing.Tuple[int, numpy.ndarray]: """Load image from path. This method is intended to be used in a thread. The index is used to From 1664ed26be8dba057fec799df32789ff93774b3f Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Mon, 13 Jan 2025 14:53:39 -0500 Subject: [PATCH 13/14] ran pre-commit checks --- .../apply-flatfield-tool/tests/test_plugin.py | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/transforms/images/apply-flatfield-tool/tests/test_plugin.py b/transforms/images/apply-flatfield-tool/tests/test_plugin.py index a65d30bdb..374a5a36a 100644 --- a/transforms/images/apply-flatfield-tool/tests/test_plugin.py +++ b/transforms/images/apply-flatfield-tool/tests/test_plugin.py @@ -1,10 +1,10 @@ """Tests for the plugin.""" - import itertools import logging import pathlib import shutil import tempfile +import typing import bfio import numpy @@ -30,17 +30,21 @@ def _make_random_image( if dtype == numpy.float32: writer[:] = rng.random(size=(size, size), dtype=writer.dtype) else: - writer[:] = rng.integers(low=0, high=numpy.iinfo(writer.dtype).max, - size=(size, size), dtype=writer.dtype) + writer[:] = rng.integers( + low=0, + high=numpy.iinfo(writer.dtype).max, + size=(size, size), + dtype=writer.dtype, + ) -FixtureReturnType = tuple[pathlib.Path, str, pathlib.Path, str] +FixtureReturnType = typing.Tuple[pathlib.Path, str, pathlib.Path, str, bool] -def gen_once(num_groups: int, img_size: int, - dtype : numpy.dtype=numpy.float32) -> FixtureReturnType: +def gen_once( + num_groups: int, img_size: int, dtype: numpy.dtype = numpy.float32 +) -> FixtureReturnType: """Generate a set of random images for testing.""" - img_pattern = "img_x{x}_c{c}.ome.tif" ff_pattern = "img_x(1-10)_c{c}" @@ -50,14 +54,10 @@ def gen_once(num_groups: int, img_size: int, rng = numpy.random.default_rng(42) for i in range(num_groups): - ff_path = ff_dir.joinpath( - f"{ff_pattern.format(c=i + 1)}_flatfield.ome.tif" - ) + ff_path = ff_dir.joinpath(f"{ff_pattern.format(c=i + 1)}_flatfield.ome.tif") _make_random_image(ff_path, rng, img_size, dtype) - df_path = ff_dir.joinpath( - f"{ff_pattern.format(c=i + 1)}_darkfield.ome.tif" - ) + df_path = ff_dir.joinpath(f"{ff_pattern.format(c=i + 1)}_darkfield.ome.tif") _make_random_image(df_path, rng, img_size, dtype) for j in range(10): # 10 images in each group @@ -72,27 +72,29 @@ def gen_once(num_groups: int, img_size: int, img_pattern = "img_x{x:d+}_c{c:d}.ome.tif" ff_pattern = "img_x\\(1-10\\)_c{c:d}" - return img_dir, img_pattern, ff_dir, ff_pattern + return img_dir, img_pattern, ff_dir, ff_pattern, True NUM_GROUPS = [1, 4] IMG_SIZES = [1024, 4096] IMG_DTYPE = [numpy.float32, numpy.uint16] KEEP_ORIG_DTYPE = [True, False] -PARAMS = list(itertools.product(NUM_GROUPS, IMG_SIZES, IMG_DTYPE, - KEEP_ORIG_DTYPE)) -IDS = [f"{num_groups}_{img_size}_{dtype}_{keep_orig}" \ - for num_groups, img_size, dtype, keep_orig in PARAMS] +PARAMS = list(itertools.product(NUM_GROUPS, IMG_SIZES, IMG_DTYPE, KEEP_ORIG_DTYPE)) +IDS = [ + f"{num_groups}_{img_size}_{dtype}_{keep_orig}" + for num_groups, img_size, dtype, keep_orig in PARAMS +] @pytest.fixture(params=PARAMS, ids=IDS) -def gen_images(request: pytest.FixtureRequest) -> FixtureReturnType: +def gen_images( + request: pytest.FixtureRequest, +) -> typing.Generator[FixtureReturnType, None, None]: """Generate a set of random images for testing.""" num_groups: int img_size: int num_groups, img_size, dtype, keep_orig = request.param - img_dir, img_pattern, ff_dir, ff_pattern = gen_once(num_groups, img_size, - dtype) + img_dir, img_pattern, ff_dir, ff_pattern, _ = gen_once(num_groups, img_size, dtype) yield img_dir, img_pattern, ff_dir, ff_pattern, keep_orig @@ -103,7 +105,6 @@ def gen_images(request: pytest.FixtureRequest) -> FixtureReturnType: def test_estimate(gen_images: FixtureReturnType) -> None: """Test the `estimate` function.""" - img_dir, img_pattern, ff_dir, ff_pattern, keep_orig = gen_images out_dir = pathlib.Path(tempfile.mkdtemp(suffix="out_dir")) @@ -114,7 +115,7 @@ def test_estimate(gen_images: FixtureReturnType) -> None: ff_pattern=f"{ff_pattern}_flatfield.ome.tif", df_pattern=f"{ff_pattern}_darkfield.ome.tif", out_dir=out_dir, - keep_orig_dtype=keep_orig + keep_orig_dtype=keep_orig, ) img_names = sorted([p.name for p in img_dir.iterdir()]) @@ -128,8 +129,7 @@ def test_estimate(gen_images: FixtureReturnType) -> None: def test_cli() -> None: """Test the CLI.""" - - img_dir, img_pattern, ff_dir, ff_pattern = gen_once(2, 2_048) + img_dir, img_pattern, ff_dir, ff_pattern, _ = gen_once(2, 2_048) out_dir = pathlib.Path(tempfile.mkdtemp(suffix="out_dir")) runner = typer.testing.CliRunner() @@ -154,9 +154,9 @@ def test_cli() -> None: assert result.exit_code == 0, result.stdout - img_paths = set(p.name for p in img_dir.iterdir() if p.name.endswith(".ome.tif")) + img_paths = {p.name for p in img_dir.iterdir() if p.name.endswith(".ome.tif")} - out_names = set(p.name for p in out_dir.iterdir() if p.name.endswith(".ome.tif")) + out_names = {p.name for p in out_dir.iterdir() if p.name.endswith(".ome.tif")} assert img_paths == out_names, f"{(img_paths)} != {out_names}" From cb2ea3bf05fa5f9b6e1e864811c30cbcd3195d29 Mon Sep 17 00:00:00 2001 From: Ji Liu Date: Mon, 13 Jan 2025 15:47:39 -0500 Subject: [PATCH 14/14] Updated img scaling logic --- .../images/apply_flatfield/apply_flatfield.py | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/apply_flatfield.py b/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/apply_flatfield.py index b57f7078c..0a882c0d1 100644 --- a/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/apply_flatfield.py +++ b/transforms/images/apply-flatfield-tool/src/polus/images/transforms/images/apply_flatfield/apply_flatfield.py @@ -172,23 +172,42 @@ def _unshade_batch( images = [img for _, img in sorted(images, key=operator.itemgetter(0))] img_stack = numpy.stack(images, axis=0).astype(numpy.float32) + # find min and max values of original images (across last 2 axes) + def get_min_max(img_stack): + min_val = img_stack.min(axis=(-1, -2), keepdims=True) + max_val = img_stack.max(axis=(-1, -2), keepdims=True) + return min_val, max_val + + min_orig, max_orig = get_min_max(img_stack) # dim: n_images x 1 x 1 + # Apply flatfield correction if df_image is not None: img_stack -= df_image img_stack /= ff_image + 1e-8 + # calculate min and max values of corrected images + min_new, max_new = get_min_max(img_stack) + + # scale corrected images to original range + img_stack = (img_stack - min_new) / (max_new - min_new) * ( + max_orig - min_orig + ) + min_orig + if keep_orig_dtype: orig_dtype = images[0].dtype # if integer type if np.issubdtype(orig_dtype, np.integer): - min_val = numpy.iinfo(orig_dtype).min - max_val = numpy.iinfo(orig_dtype).max - # clamp values to the original dtype range - img_stack[img_stack < min_val] = min_val - img_stack[img_stack > max_val] = max_val + # clip out of range values for orig dtype + dtype_info = np.iinfo(orig_dtype) + img_stack = np.clip(img_stack, dtype_info.min, dtype_info.max) + + # round and cast to original dtype img_stack = np.round(img_stack).astype(orig_dtype) + elif np.issubdtype(orig_dtype, np.floating): + # floating point image, clamp to [0,1] + img_stack = np.clip(img_stack, 0.0, 1.0) img_stack = img_stack.astype(orig_dtype) # Save outputs