From b98b521f265139380c7a081facfca8d20d2a56bc Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Wed, 14 Jul 2021 23:00:28 +0100 Subject: [PATCH 01/23] Start setting up some checks for existing nexus files --- src/nexgen/nxs_check/__init__.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 src/nexgen/nxs_check/__init__.py diff --git a/src/nexgen/nxs_check/__init__.py b/src/nexgen/nxs_check/__init__.py new file mode 100644 index 00000000..2488e977 --- /dev/null +++ b/src/nexgen/nxs_check/__init__.py @@ -0,0 +1,3 @@ +""" +Utilities for checking that NeXus format files are correct +""" From a735715c1c2d2d859c1fe7a721a50abdf06ad924 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Thu, 29 Jul 2021 17:18:31 +0100 Subject: [PATCH 02/23] Try using pathlib --- src/nexgen/nxs_copy/CopyTristanNexus.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/nexgen/nxs_copy/CopyTristanNexus.py b/src/nexgen/nxs_copy/CopyTristanNexus.py index 20eddb67..e2b054a8 100644 --- a/src/nexgen/nxs_copy/CopyTristanNexus.py +++ b/src/nexgen/nxs_copy/CopyTristanNexus.py @@ -6,6 +6,8 @@ import h5py import numpy as np +from pathlib import Path + from . import get_nexus_tree, identify_scan_axis, convert_scan_axis from .. import create_attributes @@ -92,7 +94,10 @@ def multiple_images_nexus( Returns: The name of the output NeXus file. """ - nxs_filename = os.path.splitext(data_file)[0] + ".nxs" + # nxs_filename = os.path.splitext(data_file)[0] + ".nxs" + if type(data_file) is str: + data_file = Path(data_file).expanduser().resolve() + nxs_filename = data_file.parent / f"{data_file.stem}.nxs" with h5py.File(tristan_nexus, "r") as nxs_in, h5py.File( nxs_filename, write_mode ) as nxs_out: From 6d85ad2ce12f458b7f576b541512f49956061ab2 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Thu, 29 Jul 2021 17:57:55 +0100 Subject: [PATCH 03/23] Use pathlib --- src/nexgen/nxs_copy/CopyTristanNexus.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/nexgen/nxs_copy/CopyTristanNexus.py b/src/nexgen/nxs_copy/CopyTristanNexus.py index e2b054a8..42744490 100644 --- a/src/nexgen/nxs_copy/CopyTristanNexus.py +++ b/src/nexgen/nxs_copy/CopyTristanNexus.py @@ -2,7 +2,6 @@ Tools for copying the metadata from Tristan NeXus files. """ -import os import h5py import numpy as np @@ -29,7 +28,9 @@ def single_image_nexus(data_file, tristan_nexus, write_mode="x"): Returns: The name of the output NeXus file. """ - nxs_filename = os.path.splitext(data_file)[0] + ".nxs" + if type(data_file) is str: + data_file = Path(data_file).expanduser().resolve() + nxs_filename = data_file.parent / f"{data_file.stem}.nxs" with h5py.File(tristan_nexus, "r") as nxs_in, h5py.File( nxs_filename, write_mode ) as nxs_out: @@ -38,7 +39,7 @@ def single_image_nexus(data_file, tristan_nexus, write_mode="x"): # Create nxdata group nxdata = nxentry.create_group("data") # Add link to data - nxdata["data"] = h5py.ExternalLink(os.path.basename(data_file), "data") + nxdata["data"] = h5py.ExternalLink(data_file.name, "data") # Compute and write axis information ax, ax_attr = identify_scan_axis(nxs_in) create_attributes( @@ -106,7 +107,7 @@ def multiple_images_nexus( # Create nxdata group nxdata = nxentry.create_group("data") # Add link to data - nxdata["data"] = h5py.ExternalLink(os.path.basename(data_file), "data") + nxdata["data"] = h5py.ExternalLink(data_file.name, "data") # Compute and write axis information ax, ax_attr = identify_scan_axis(nxs_in) create_attributes( @@ -179,7 +180,9 @@ def pump_probe_nexus(data_file, tristan_nexus, write_mode="x", mode="static"): "['static', 'powder_diffraction', 'rotation']" ) - nxs_filename = os.path.splitext(data_file)[0] + ".nxs" + if type(data_file) is str: + data_file = Path(data_file).expanduser().resolve() + nxs_filename = data_file.parent / f"{data_file.stem}.nxs" with h5py.File(tristan_nexus, "r") as nxs_in, h5py.File( nxs_filename, write_mode ) as nxs_out: @@ -188,7 +191,7 @@ def pump_probe_nexus(data_file, tristan_nexus, write_mode="x", mode="static"): # Create nxdata group nxdata = nxentry.create_group("data") # Add link to data - nxdata["data"] = h5py.ExternalLink(os.path.basename(data_file), "data") + nxdata["data"] = h5py.ExternalLink(data_file.name, "data") # Compute and write axis information ax, ax_attr = identify_scan_axis(nxs_in) create_attributes( From 851112a317f6dbab6518d945587dbdb1ffc1f817 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 30 Jul 2021 11:40:51 +0100 Subject: [PATCH 04/23] Add definition check --- src/nexgen/nxs_check/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nexgen/nxs_check/__init__.py b/src/nexgen/nxs_check/__init__.py index 2488e977..a1437491 100644 --- a/src/nexgen/nxs_check/__init__.py +++ b/src/nexgen/nxs_check/__init__.py @@ -1,3 +1,3 @@ """ -Utilities for checking that NeXus format files are correct +General utilities for checking that NeXus format files are correct """ From d3cfe440e7b442eec776427007bef58161cd8876 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 30 Jul 2021 15:30:11 +0100 Subject: [PATCH 05/23] Check and fix two_theta and det_z --- src/nexgen/nxs_check/CheckTristanNexus.py | 89 +++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 src/nexgen/nxs_check/CheckTristanNexus.py diff --git a/src/nexgen/nxs_check/CheckTristanNexus.py b/src/nexgen/nxs_check/CheckTristanNexus.py new file mode 100644 index 00000000..35bc9fde --- /dev/null +++ b/src/nexgen/nxs_check/CheckTristanNexus.py @@ -0,0 +1,89 @@ +""" +Tools to check and eventually fix NeXus files for Tristan LATRD detector. +""" + +import h5py +import logging + +# Define logger +logger = logging.getLogger("TristanNXSChecks") + + +def check_definition(nxsfile: h5py.File): + """ + Checks and eventually fixes that the definition is set to "NXmx" + """ + definition = nxsfile["entry/definition"][()] + logger.info(f"Application definition: {definition}") + if definition != "NXmx": + del nxsfile["entry/definition"] + nxsfile["entry"].create_dataset("definition", data="NXmx") + logger.info("Fixing definition to NXmx") + logger.info("") + + +def check_detector_transformations(NXtransf: h5py.Group): + """ + Checks and eventually fixes the values of detector_z and two_theta fields and their attributes. + """ + logger.info( + "On I19-2 the Tristan detector does not sit on two_theta arm, which must be set to 0." + ) + try: + ds_name = "two_theta" + two_theta = NXtransf["two_theta"] + except KeyError: + # For older versions of Tristan nexus file + ds_name = "twotheta" + two_theta = NXtransf["twotheta"] + if two_theta[ds_name][()] != 0: + logger.info("Correcting the value of two_theta arm ...") + d = {} + for k in two_theta[ds_name].attrs.keys(): + d[k] = two_theta[ds_name].attrs[k] + del two_theta[ds_name] + tt = two_theta.create_dataset(ds_name, data=[(0.0)]) + for k, v in d.items(): + tt.attrs[k] = v + del d + + logger.info("Additionally, the detector_z vector should be [0,0,-1]") + det_z = NXtransf["detector_z/det_z"] + if det_z.attrs["vector"] != [0, 0, -1]: + logger.info("Overwriting det_z vector ...") + det_z.attrs["vector"] = [0, 0, -1] + + logger.info("Checking dependency tree for typos ...") + + +def run_checks(tristan_nexus_file): + """ + Instigates the functions to check nexus files generated after binning of Tristan data. + """ + wdir = tristan_nexus_file.parent + logfile = wdir / "NeXusChecks.log" # widr is a PosixPath + logging.basicConfig( + filename=logfile, + format="%(message)s", + level="DEBUG", + ) + logger.info(f"Running checks on {tristan_nexus_file} ...") + + with h5py.File(tristan_nexus_file, "r+") as nxsfile: + logger.info("Check application definition") + check_definition(nxsfile) + logger.info("-" * 10) + logger.info( + "Check that detector_z and two_theta fields are correctly set to I19-2 configuration for Tristan." + ) + try: + # According to NXmx Gold Standard + check_detector_transformations( + nxsfile["entry/instrument/detector/transformations"] + ) + except KeyError: + # For earlier versions of the Tristan nexus file (before June 2021): + # NXtransformations group was placed under NXinstrument group + check_detector_transformations(nxsfile["entry/instrument/transformations"]) + logger.info("-" * 10) + logger.info("EOF") From cd8bb768674f8366bb332ddc646c83e6f4628972 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 30 Jul 2021 16:46:41 +0100 Subject: [PATCH 06/23] Temporary fix to dependency tree errors - a bit hard coded. WIP. --- src/nexgen/nxs_check/CheckTristanNexus.py | 38 +++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/src/nexgen/nxs_check/CheckTristanNexus.py b/src/nexgen/nxs_check/CheckTristanNexus.py index 35bc9fde..12596ad1 100644 --- a/src/nexgen/nxs_check/CheckTristanNexus.py +++ b/src/nexgen/nxs_check/CheckTristanNexus.py @@ -1,10 +1,12 @@ """ -Tools to check and eventually fix NeXus files for Tristan LATRD detector. +Tools to check and eventually fix NeXus files for Tristan LATRD detector on I19-2 beamline at DLS. """ import h5py import logging +import numpy as np + # Define logger logger = logging.getLogger("TristanNXSChecks") @@ -53,7 +55,37 @@ def check_detector_transformations(NXtransf: h5py.Group): logger.info("Overwriting det_z vector ...") det_z.attrs["vector"] = [0, 0, -1] - logger.info("Checking dependency tree for typos ...") + # TBD + # logger.info("Checking dependency tree for typos ...") + + +def check_I19_dependency_tree(NXtransf: h5py.Group): + """ + Check and fix that the dependency tree in "entry/sample/transformations" is consistent with I19-2. + """ + # FIXME Quick hard coded way, works for now but needs to be generalized. + if NXtransf["omega"].attrs["depends_on"] != b".": + NXtransf["omega"].attrs["depends_on"] = np.string_(".") + if NXtransf["kappa"].attrs["depends_on"] != b"/entry/sample/transformations/omega": + NXtransf["kappa"].attrs["depends_on"] = np.string_( + "/entry/sample/transformations/omega" + ) + if NXtransf["phi"].attrs["depends_on"] != b"/entry/sample/transformations/kappa": + NXtransf["phi"].attrs["depends_on"] = np.string_( + "/entry/sample/transformations/kappa" + ) + if NXtransf["sam_x"].attrs["depends_on"] != b"/entry/sample/transformations/sam_y": + NXtransf["sam_x"].attrs["depends_on"] = np.string_( + "/entry/sample/transformations/sam_y" + ) + if NXtransf["sam_y"].attrs["depends_on"] != b"/entry/sample/transformations/sam_z": + NXtransf["sam_y"].attrs["depends_on"] = np.string_( + "/entry/sample/transformations/sam_z" + ) + if NXtransf["sam_z"].attrs["depends_on"] != b"/entry/sample/transformations/phi": + NXtransf["sam_Z"].attrs["depends_on"] = np.string_( + "/entry/sample/transformations/phi" + ) def run_checks(tristan_nexus_file): @@ -86,4 +118,6 @@ def run_checks(tristan_nexus_file): # NXtransformations group was placed under NXinstrument group check_detector_transformations(nxsfile["entry/instrument/transformations"]) logger.info("-" * 10) + logger.info("Check goniometer dependency tree") + check_I19_dependency_tree(nxsfile["entry/sample/transformations"]) logger.info("EOF") From 532feb923e3087d5f38bb15a4735a5f1c03517f6 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 30 Jul 2021 17:16:10 +0100 Subject: [PATCH 07/23] Update logger --- src/nexgen/nxs_check/CheckTristanNexus.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nexgen/nxs_check/CheckTristanNexus.py b/src/nexgen/nxs_check/CheckTristanNexus.py index 12596ad1..54997354 100644 --- a/src/nexgen/nxs_check/CheckTristanNexus.py +++ b/src/nexgen/nxs_check/CheckTristanNexus.py @@ -64,6 +64,8 @@ def check_I19_dependency_tree(NXtransf: h5py.Group): Check and fix that the dependency tree in "entry/sample/transformations" is consistent with I19-2. """ # FIXME Quick hard coded way, works for now but needs to be generalized. + logger.info("The dependency tree on I19-2 should follow this order:") + logger.info("x - y - z - phi - kappa - omega") if NXtransf["omega"].attrs["depends_on"] != b".": NXtransf["omega"].attrs["depends_on"] = np.string_(".") if NXtransf["kappa"].attrs["depends_on"] != b"/entry/sample/transformations/omega": @@ -95,9 +97,7 @@ def run_checks(tristan_nexus_file): wdir = tristan_nexus_file.parent logfile = wdir / "NeXusChecks.log" # widr is a PosixPath logging.basicConfig( - filename=logfile, - format="%(message)s", - level="DEBUG", + filename=logfile, format="%(asctime)s %(message)s", level="DEBUG", filemode="w" ) logger.info(f"Running checks on {tristan_nexus_file} ...") From e3a0d64d5f339280f5ca9ec059353344a73e3585 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 30 Jul 2021 17:28:21 +0100 Subject: [PATCH 08/23] Fix detector_z vector comparison --- src/nexgen/nxs_check/CheckTristanNexus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nexgen/nxs_check/CheckTristanNexus.py b/src/nexgen/nxs_check/CheckTristanNexus.py index 54997354..79afd3e8 100644 --- a/src/nexgen/nxs_check/CheckTristanNexus.py +++ b/src/nexgen/nxs_check/CheckTristanNexus.py @@ -51,7 +51,7 @@ def check_detector_transformations(NXtransf: h5py.Group): logger.info("Additionally, the detector_z vector should be [0,0,-1]") det_z = NXtransf["detector_z/det_z"] - if det_z.attrs["vector"] != [0, 0, -1]: + if np.any(det_z.attrs["vector"] != [0, 0, -1]): logger.info("Overwriting det_z vector ...") det_z.attrs["vector"] = [0, 0, -1] From 419738e0b29ea4d9c25821974485e56c7401de6f Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 30 Jul 2021 17:30:16 +0100 Subject: [PATCH 09/23] Fix typo --- src/nexgen/nxs_check/CheckTristanNexus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nexgen/nxs_check/CheckTristanNexus.py b/src/nexgen/nxs_check/CheckTristanNexus.py index 79afd3e8..68f25667 100644 --- a/src/nexgen/nxs_check/CheckTristanNexus.py +++ b/src/nexgen/nxs_check/CheckTristanNexus.py @@ -85,7 +85,7 @@ def check_I19_dependency_tree(NXtransf: h5py.Group): "/entry/sample/transformations/sam_z" ) if NXtransf["sam_z"].attrs["depends_on"] != b"/entry/sample/transformations/phi": - NXtransf["sam_Z"].attrs["depends_on"] = np.string_( + NXtransf["sam_z"].attrs["depends_on"] = np.string_( "/entry/sample/transformations/phi" ) From e025b626f8993ff3b5c85b15a130e661b3ecba0a Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 30 Jul 2021 17:43:53 +0100 Subject: [PATCH 10/23] Add check for sample depends_on field --- src/nexgen/nxs_check/CheckTristanNexus.py | 59 ++++++++++++++++------- 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/src/nexgen/nxs_check/CheckTristanNexus.py b/src/nexgen/nxs_check/CheckTristanNexus.py index 68f25667..64f68135 100644 --- a/src/nexgen/nxs_check/CheckTristanNexus.py +++ b/src/nexgen/nxs_check/CheckTristanNexus.py @@ -24,7 +24,7 @@ def check_definition(nxsfile: h5py.File): logger.info("") -def check_detector_transformations(NXtransf: h5py.Group): +def check_detector_transformations(nxtransf: h5py.Group): """ Checks and eventually fixes the values of detector_z and two_theta fields and their attributes. """ @@ -33,11 +33,11 @@ def check_detector_transformations(NXtransf: h5py.Group): ) try: ds_name = "two_theta" - two_theta = NXtransf["two_theta"] + two_theta = nxtransf["two_theta"] except KeyError: # For older versions of Tristan nexus file ds_name = "twotheta" - two_theta = NXtransf["twotheta"] + two_theta = nxtransf["twotheta"] if two_theta[ds_name][()] != 0: logger.info("Correcting the value of two_theta arm ...") d = {} @@ -50,7 +50,7 @@ def check_detector_transformations(NXtransf: h5py.Group): del d logger.info("Additionally, the detector_z vector should be [0,0,-1]") - det_z = NXtransf["detector_z/det_z"] + det_z = nxtransf["detector_z/det_z"] if np.any(det_z.attrs["vector"] != [0, 0, -1]): logger.info("Overwriting det_z vector ...") det_z.attrs["vector"] = [0, 0, -1] @@ -59,33 +59,53 @@ def check_detector_transformations(NXtransf: h5py.Group): # logger.info("Checking dependency tree for typos ...") -def check_I19_dependency_tree(NXtransf: h5py.Group): +def check_sample_depends_on(nxsample: h5py.Group): + """ + Check that the sample depends_on field exists and is correct. + For I19-2 Tristan it should be "/entry/sample/transformations/phi" + """ + try: + dep = nxsample["depends_on"][()] + if dep != b"/entry/sample/transformations/phi": + logger.info("Fixing sample depends_on field ...") + del nxsample["depends_on"] + nxsample.create_dataset( + "depends_on", data=np.string_("/entry/sample/transformations/phi") + ) + except KeyError: + logger.info("Sample depends_on field did not exist, creating now ...") + nxsample.create_dataset( + "depends_on", data=np.string_("/entry/sample/transformations/phi") + ) + + +def check_I19_dependency_tree(nxtransf: h5py.Group): """ Check and fix that the dependency tree in "entry/sample/transformations" is consistent with I19-2. """ # FIXME Quick hard coded way, works for now but needs to be generalized. logger.info("The dependency tree on I19-2 should follow this order:") logger.info("x - y - z - phi - kappa - omega") - if NXtransf["omega"].attrs["depends_on"] != b".": - NXtransf["omega"].attrs["depends_on"] = np.string_(".") - if NXtransf["kappa"].attrs["depends_on"] != b"/entry/sample/transformations/omega": - NXtransf["kappa"].attrs["depends_on"] = np.string_( + if nxtransf["omega"].attrs["depends_on"] != b".": + nxtransf["omega"].attrs["depends_on"] = np.string_(".") + if nxtransf["kappa"].attrs["depends_on"] != b"/entry/sample/transformations/omega": + nxtransf["kappa"].attrs["depends_on"] = np.string_( "/entry/sample/transformations/omega" ) - if NXtransf["phi"].attrs["depends_on"] != b"/entry/sample/transformations/kappa": - NXtransf["phi"].attrs["depends_on"] = np.string_( + if nxtransf["phi"].attrs["depends_on"] != b"/entry/sample/transformations/kappa": + nxtransf["phi"].attrs["depends_on"] = np.string_( "/entry/sample/transformations/kappa" ) - if NXtransf["sam_x"].attrs["depends_on"] != b"/entry/sample/transformations/sam_y": - NXtransf["sam_x"].attrs["depends_on"] = np.string_( + if nxtransf["sam_x"].attrs["depends_on"] != b"/entry/sample/transformations/sam_y": + nxtransf["sam_x"].attrs["depends_on"] = np.string_( "/entry/sample/transformations/sam_y" ) - if NXtransf["sam_y"].attrs["depends_on"] != b"/entry/sample/transformations/sam_z": - NXtransf["sam_y"].attrs["depends_on"] = np.string_( + if nxtransf["sam_y"].attrs["depends_on"] != b"/entry/sample/transformations/sam_z": + nxtransf["sam_y"].attrs["depends_on"] = np.string_( "/entry/sample/transformations/sam_z" ) - if NXtransf["sam_z"].attrs["depends_on"] != b"/entry/sample/transformations/phi": - NXtransf["sam_z"].attrs["depends_on"] = np.string_( + if nxtransf["sam_z"].attrs["depends_on"] != b"/entry/sample/transformations/phi": + nxtransf["sam_z"].attrs["depends_on"] = np.string_( "/entry/sample/transformations/phi" ) @@ -97,7 +117,7 @@ def run_checks(tristan_nexus_file): wdir = tristan_nexus_file.parent logfile = wdir / "NeXusChecks.log" # widr is a PosixPath logging.basicConfig( - filename=logfile, format="%(asctime)s %(message)s", level="DEBUG", filemode="w" + filename=logfile, format="%(message)s", level="DEBUG", filemode="w" ) logger.info(f"Running checks on {tristan_nexus_file} ...") @@ -118,6 +138,9 @@ def run_checks(tristan_nexus_file): # NXtransformations group was placed under NXinstrument group check_detector_transformations(nxsfile["entry/instrument/transformations"]) logger.info("-" * 10) + logger.info("Check sample depends on") + check_sample_depends_on(nxsfile["entry/sample"]) + logger.info("-" * 10) logger.info("Check goniometer dependency tree") check_I19_dependency_tree(nxsfile["entry/sample/transformations"]) logger.info("EOF") From 32d0ec48e5f5a4c38829984532ec51803f5772ce Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 30 Jul 2021 17:54:04 +0100 Subject: [PATCH 11/23] Check and fix typos in dependency chain for detector transformations --- src/nexgen/nxs_check/CheckTristanNexus.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/nexgen/nxs_check/CheckTristanNexus.py b/src/nexgen/nxs_check/CheckTristanNexus.py index 64f68135..14dcb900 100644 --- a/src/nexgen/nxs_check/CheckTristanNexus.py +++ b/src/nexgen/nxs_check/CheckTristanNexus.py @@ -55,8 +55,18 @@ def check_detector_transformations(nxtransf: h5py.Group): logger.info("Overwriting det_z vector ...") det_z.attrs["vector"] = [0, 0, -1] - # TBD - # logger.info("Checking dependency tree for typos ...") + logger.info("Checking dependency tree of detector for typos ...") + if two_theta[ds_name].attrs["depends_on"] != b".": + logger.info("Setting two_theta as base ...") + two_theta[ds_name].attrs["depends_on"] = np.string_(".") + if ( + det_z.attrs["depends_on"] + != b"/entry/instrument/detector/transformations/two_theta/two_theta" + ): + logger.info("Fixing typo in det_z dependency ...") + det_z.attrs["depends_on"] == np.string_( + "/entry/instrument/detector/transformations/two_theta/two_theta" + ) def check_sample_depends_on(nxsample: h5py.Group): From e9d529e24ebd74390944e217cd19200c61744487 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 30 Jul 2021 18:01:12 +0100 Subject: [PATCH 12/23] Fix another typo --- src/nexgen/nxs_check/CheckTristanNexus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nexgen/nxs_check/CheckTristanNexus.py b/src/nexgen/nxs_check/CheckTristanNexus.py index 14dcb900..d7c61c29 100644 --- a/src/nexgen/nxs_check/CheckTristanNexus.py +++ b/src/nexgen/nxs_check/CheckTristanNexus.py @@ -64,7 +64,7 @@ def check_detector_transformations(nxtransf: h5py.Group): != b"/entry/instrument/detector/transformations/two_theta/two_theta" ): logger.info("Fixing typo in det_z dependency ...") - det_z.attrs["depends_on"] == np.string_( + det_z.attrs["depends_on"] = np.string_( "/entry/instrument/detector/transformations/two_theta/two_theta" ) From 4839289c589691c78d1e9455b7134ce55f84267d Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 30 Jul 2021 18:04:46 +0100 Subject: [PATCH 13/23] Add some logging messages --- src/nexgen/nxs_check/CheckTristanNexus.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/nexgen/nxs_check/CheckTristanNexus.py b/src/nexgen/nxs_check/CheckTristanNexus.py index d7c61c29..81cc86e1 100644 --- a/src/nexgen/nxs_check/CheckTristanNexus.py +++ b/src/nexgen/nxs_check/CheckTristanNexus.py @@ -97,24 +97,30 @@ def check_I19_dependency_tree(nxtransf: h5py.Group): logger.info("The dependency tree on I19-2 should follow this order:") logger.info("x - y - z - phi - kappa - omega") if nxtransf["omega"].attrs["depends_on"] != b".": + logger.info("Fixing omega ...") nxtransf["omega"].attrs["depends_on"] = np.string_(".") if nxtransf["kappa"].attrs["depends_on"] != b"/entry/sample/transformations/omega": + logger.info("Fixing kappa ...") nxtransf["kappa"].attrs["depends_on"] = np.string_( "/entry/sample/transformations/omega" ) if nxtransf["phi"].attrs["depends_on"] != b"/entry/sample/transformations/kappa": + logger.info("Fixing phi ...") nxtransf["phi"].attrs["depends_on"] = np.string_( "/entry/sample/transformations/kappa" ) if nxtransf["sam_x"].attrs["depends_on"] != b"/entry/sample/transformations/sam_y": + logger.info("Fixing sam_x ...") nxtransf["sam_x"].attrs["depends_on"] = np.string_( "/entry/sample/transformations/sam_y" ) if nxtransf["sam_y"].attrs["depends_on"] != b"/entry/sample/transformations/sam_z": + logger.info("Fixing sam_y ...") nxtransf["sam_y"].attrs["depends_on"] = np.string_( "/entry/sample/transformations/sam_z" ) if nxtransf["sam_z"].attrs["depends_on"] != b"/entry/sample/transformations/phi": + logger.info("Fixing sam_z ...") nxtransf["sam_z"].attrs["depends_on"] = np.string_( "/entry/sample/transformations/phi" ) From cca3ceef01f11606c05f93f8c17be9009a9e2876 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 30 Jul 2021 19:45:11 +0100 Subject: [PATCH 14/23] Make sure file is a path object --- src/nexgen/nxs_check/CheckTristanNexus.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/nexgen/nxs_check/CheckTristanNexus.py b/src/nexgen/nxs_check/CheckTristanNexus.py index 81cc86e1..68712346 100644 --- a/src/nexgen/nxs_check/CheckTristanNexus.py +++ b/src/nexgen/nxs_check/CheckTristanNexus.py @@ -2,11 +2,14 @@ Tools to check and eventually fix NeXus files for Tristan LATRD detector on I19-2 beamline at DLS. """ +import sys import h5py import logging import numpy as np +from pathlib import Path + # Define logger logger = logging.getLogger("TristanNXSChecks") @@ -130,6 +133,8 @@ def run_checks(tristan_nexus_file): """ Instigates the functions to check nexus files generated after binning of Tristan data. """ + if type(tristan_nexus_file) is str: + tristan_nexus_file = Path(tristan_nexus_file).expanduser().resolve() wdir = tristan_nexus_file.parent logfile = wdir / "NeXusChecks.log" # widr is a PosixPath logging.basicConfig( @@ -160,3 +165,6 @@ def run_checks(tristan_nexus_file): logger.info("Check goniometer dependency tree") check_I19_dependency_tree(nxsfile["entry/sample/transformations"]) logger.info("EOF") + + +run_checks(sys.argv[1]) From e8c91a837553324d43e00d1456e46cf1e60436fb Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 3 Sep 2021 14:58:11 +0100 Subject: [PATCH 15/23] Add check on sample_x/y/z vectors --- src/nexgen/nxs_check/CheckTristanNexus.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/nexgen/nxs_check/CheckTristanNexus.py b/src/nexgen/nxs_check/CheckTristanNexus.py index 68712346..06163fec 100644 --- a/src/nexgen/nxs_check/CheckTristanNexus.py +++ b/src/nexgen/nxs_check/CheckTristanNexus.py @@ -113,20 +113,29 @@ def check_I19_dependency_tree(nxtransf: h5py.Group): "/entry/sample/transformations/kappa" ) if nxtransf["sam_x"].attrs["depends_on"] != b"/entry/sample/transformations/sam_y": - logger.info("Fixing sam_x ...") + logger.info("Fixing sam_x dependency...") nxtransf["sam_x"].attrs["depends_on"] = np.string_( "/entry/sample/transformations/sam_y" ) + if nxtransf["sam_x"].attrs["vector"] != [-1, 0, 0]: + logger.info("Fixing sam_x vector...") + nxtransf["sam_x"].attrs["vector"] = [-1, 0, 0] if nxtransf["sam_y"].attrs["depends_on"] != b"/entry/sample/transformations/sam_z": logger.info("Fixing sam_y ...") nxtransf["sam_y"].attrs["depends_on"] = np.string_( "/entry/sample/transformations/sam_z" ) + if nxtransf["sam_y"].attrs["vector"] != [0, -1, 0]: + logger.info("Fixing sam_y vector...") + nxtransf["sam_y"].attrs["vector"] = [0, -1, 0] if nxtransf["sam_z"].attrs["depends_on"] != b"/entry/sample/transformations/phi": logger.info("Fixing sam_z ...") nxtransf["sam_z"].attrs["depends_on"] = np.string_( "/entry/sample/transformations/phi" ) + if nxtransf["sam_z"].attrs["vector"] != [0, 0, -1]: + logger.info("Fixing sam_z vector...") + nxtransf["sam_z"].attrs["vector"] = [0, 0, -1] def run_checks(tristan_nexus_file): From f7f946cafcfb893f8659b6792e24fc7fd67e5762 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Fri, 3 Sep 2021 15:23:06 +0100 Subject: [PATCH 16/23] Fix ValueError --- src/nexgen/nxs_check/CheckTristanNexus.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nexgen/nxs_check/CheckTristanNexus.py b/src/nexgen/nxs_check/CheckTristanNexus.py index 06163fec..965811f7 100644 --- a/src/nexgen/nxs_check/CheckTristanNexus.py +++ b/src/nexgen/nxs_check/CheckTristanNexus.py @@ -117,7 +117,7 @@ def check_I19_dependency_tree(nxtransf: h5py.Group): nxtransf["sam_x"].attrs["depends_on"] = np.string_( "/entry/sample/transformations/sam_y" ) - if nxtransf["sam_x"].attrs["vector"] != [-1, 0, 0]: + if np.any(nxtransf["sam_x"].attrs["vector"] != [-1, 0, 0]): logger.info("Fixing sam_x vector...") nxtransf["sam_x"].attrs["vector"] = [-1, 0, 0] if nxtransf["sam_y"].attrs["depends_on"] != b"/entry/sample/transformations/sam_z": @@ -125,7 +125,7 @@ def check_I19_dependency_tree(nxtransf: h5py.Group): nxtransf["sam_y"].attrs["depends_on"] = np.string_( "/entry/sample/transformations/sam_z" ) - if nxtransf["sam_y"].attrs["vector"] != [0, -1, 0]: + if np.any(nxtransf["sam_y"].attrs["vector"] != [0, -1, 0]): logger.info("Fixing sam_y vector...") nxtransf["sam_y"].attrs["vector"] = [0, -1, 0] if nxtransf["sam_z"].attrs["depends_on"] != b"/entry/sample/transformations/phi": @@ -133,7 +133,7 @@ def check_I19_dependency_tree(nxtransf: h5py.Group): nxtransf["sam_z"].attrs["depends_on"] = np.string_( "/entry/sample/transformations/phi" ) - if nxtransf["sam_z"].attrs["vector"] != [0, 0, -1]: + if np.any(nxtransf["sam_z"].attrs["vector"] != [0, 0, -1]): logger.info("Fixing sam_z vector...") nxtransf["sam_z"].attrs["vector"] = [0, 0, -1] From 718e37ece1b07445ecb8e630c5a990e343dd93f7 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Mon, 6 Sep 2021 17:05:49 +0100 Subject: [PATCH 17/23] Fix number values --- src/nexgen/nxs_check/CheckTristanNexus.py | 41 ++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/src/nexgen/nxs_check/CheckTristanNexus.py b/src/nexgen/nxs_check/CheckTristanNexus.py index 965811f7..848e5bef 100644 --- a/src/nexgen/nxs_check/CheckTristanNexus.py +++ b/src/nexgen/nxs_check/CheckTristanNexus.py @@ -138,6 +138,42 @@ def check_I19_dependency_tree(nxtransf: h5py.Group): nxtransf["sam_z"].attrs["vector"] = [0, 0, -1] +def check_values(nxentry: h5py.Group): + """ + Checks tht all dataset values that are supposed to be floats/ints aren't saved as strings. + """ + instr = nxentry["instrument"] + if type(instr["beam/incident_wavelength"][()]) is str: + logger.info("Fixing incident wavelength value ...") + d = {} + for k, v in instr["beam/incident_wavelength"].attrs.items(): + d[k] = v + val = float(instr["beam/incident_wavelength"][()]) + del instr["beam/incident_wavelength"] + wl = instr["beam"].create_dataset("incident_wavelength", data=val) + for k, v in d.items(): + wl.attrs[k] = v + del d, val + if type(instr["attenuator/attenuator_transmission"][()]) is str: + logger.info("Fixing attenuator transmission value...") + val = float(instr["attenuator/attenuator_transmission"][()]) + del instr["attenuator/attenuator_traansmission"] + instr["attenuator"].create_dataset("attenuator_transmission", data=val) + del val + det = instr["detector"] + if type(det["sensor_thickness"]) is str: + logger.info("Fixing sensor thickness value ...") + d = {} + for k, v in det["sensor_thickness"].attrs.items(): + d[k] = v + val = float(det["sensor_thickness"][()]) + del det["sensor_thickness"] + th = det.create_dataset("sensor_thickness", data=val) + for k, v in d.items(): + th.attrs[k] = v + del d, val + + def run_checks(tristan_nexus_file): """ Instigates the functions to check nexus files generated after binning of Tristan data. @@ -147,7 +183,7 @@ def run_checks(tristan_nexus_file): wdir = tristan_nexus_file.parent logfile = wdir / "NeXusChecks.log" # widr is a PosixPath logging.basicConfig( - filename=logfile, format="%(message)s", level="DEBUG", filemode="w" + filename=logfile, format="%(message)s", level="DEBUG", filemode="a" ) logger.info(f"Running checks on {tristan_nexus_file} ...") @@ -173,6 +209,9 @@ def run_checks(tristan_nexus_file): logger.info("-" * 10) logger.info("Check goniometer dependency tree") check_I19_dependency_tree(nxsfile["entry/sample/transformations"]) + logger.info("-" * 10) + logger.info("Check that number values are saved as floats and not strings") + check_values(nxsfile["entry"]) logger.info("EOF") From c1cc8bcb6382a0654645e800579605d83fdc9714 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Mon, 6 Sep 2021 17:23:01 +0100 Subject: [PATCH 18/23] Check against correct type --- src/nexgen/nxs_check/CheckTristanNexus.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nexgen/nxs_check/CheckTristanNexus.py b/src/nexgen/nxs_check/CheckTristanNexus.py index 848e5bef..2a517961 100644 --- a/src/nexgen/nxs_check/CheckTristanNexus.py +++ b/src/nexgen/nxs_check/CheckTristanNexus.py @@ -143,7 +143,7 @@ def check_values(nxentry: h5py.Group): Checks tht all dataset values that are supposed to be floats/ints aren't saved as strings. """ instr = nxentry["instrument"] - if type(instr["beam/incident_wavelength"][()]) is str: + if type(instr["beam/incident_wavelength"][()]) is bytes: logger.info("Fixing incident wavelength value ...") d = {} for k, v in instr["beam/incident_wavelength"].attrs.items(): @@ -154,14 +154,14 @@ def check_values(nxentry: h5py.Group): for k, v in d.items(): wl.attrs[k] = v del d, val - if type(instr["attenuator/attenuator_transmission"][()]) is str: + if type(instr["attenuator/attenuator_transmission"][()]) is np.bytes_: logger.info("Fixing attenuator transmission value...") val = float(instr["attenuator/attenuator_transmission"][()]) del instr["attenuator/attenuator_traansmission"] instr["attenuator"].create_dataset("attenuator_transmission", data=val) del val det = instr["detector"] - if type(det["sensor_thickness"]) is str: + if type(det["sensor_thickness"]) is np.bytes_: logger.info("Fixing sensor thickness value ...") d = {} for k, v in det["sensor_thickness"].attrs.items(): From 6f24ac8bebd28227186c98159fc42cf9fd396596 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Mon, 6 Sep 2021 17:24:22 +0100 Subject: [PATCH 19/23] Check against correct type, part 2 --- src/nexgen/nxs_check/CheckTristanNexus.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nexgen/nxs_check/CheckTristanNexus.py b/src/nexgen/nxs_check/CheckTristanNexus.py index 2a517961..8ef96970 100644 --- a/src/nexgen/nxs_check/CheckTristanNexus.py +++ b/src/nexgen/nxs_check/CheckTristanNexus.py @@ -154,14 +154,14 @@ def check_values(nxentry: h5py.Group): for k, v in d.items(): wl.attrs[k] = v del d, val - if type(instr["attenuator/attenuator_transmission"][()]) is np.bytes_: + if type(instr["attenuator/attenuator_transmission"][()]) is bytes: logger.info("Fixing attenuator transmission value...") val = float(instr["attenuator/attenuator_transmission"][()]) del instr["attenuator/attenuator_traansmission"] instr["attenuator"].create_dataset("attenuator_transmission", data=val) del val det = instr["detector"] - if type(det["sensor_thickness"]) is np.bytes_: + if type(det["sensor_thickness"]) is bytes: logger.info("Fixing sensor thickness value ...") d = {} for k, v in det["sensor_thickness"].attrs.items(): From c4821bac08aa66287a595842f6536d7f59358b12 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Mon, 6 Sep 2021 17:26:36 +0100 Subject: [PATCH 20/23] Get value --- src/nexgen/nxs_check/CheckTristanNexus.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nexgen/nxs_check/CheckTristanNexus.py b/src/nexgen/nxs_check/CheckTristanNexus.py index 8ef96970..838d9618 100644 --- a/src/nexgen/nxs_check/CheckTristanNexus.py +++ b/src/nexgen/nxs_check/CheckTristanNexus.py @@ -160,8 +160,8 @@ def check_values(nxentry: h5py.Group): del instr["attenuator/attenuator_traansmission"] instr["attenuator"].create_dataset("attenuator_transmission", data=val) del val - det = instr["detector"] - if type(det["sensor_thickness"]) is bytes: + det = nxentry["instrument/detector"] + if type(det["sensor_thickness"][()]) is bytes: logger.info("Fixing sensor thickness value ...") d = {} for k, v in det["sensor_thickness"].attrs.items(): From b1d00854580b62450af2da346267e9bc4b0c01d1 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Mon, 6 Sep 2021 17:28:59 +0100 Subject: [PATCH 21/23] Check against correct type, part 3 --- src/nexgen/nxs_check/CheckTristanNexus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nexgen/nxs_check/CheckTristanNexus.py b/src/nexgen/nxs_check/CheckTristanNexus.py index 838d9618..902f5e16 100644 --- a/src/nexgen/nxs_check/CheckTristanNexus.py +++ b/src/nexgen/nxs_check/CheckTristanNexus.py @@ -161,7 +161,7 @@ def check_values(nxentry: h5py.Group): instr["attenuator"].create_dataset("attenuator_transmission", data=val) del val det = nxentry["instrument/detector"] - if type(det["sensor_thickness"][()]) is bytes: + if type(det["sensor_thickness"][()]) is np.bytes_: logger.info("Fixing sensor thickness value ...") d = {} for k, v in det["sensor_thickness"].attrs.items(): From 8ace170d6bef6f79e9c8dd2fd26f4745ad1d54a9 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Mon, 6 Sep 2021 17:52:46 +0100 Subject: [PATCH 22/23] Flip detector --- src/nexgen/nxs_check/CheckTristanNexus.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nexgen/nxs_check/CheckTristanNexus.py b/src/nexgen/nxs_check/CheckTristanNexus.py index 902f5e16..fdd37325 100644 --- a/src/nexgen/nxs_check/CheckTristanNexus.py +++ b/src/nexgen/nxs_check/CheckTristanNexus.py @@ -54,9 +54,9 @@ def check_detector_transformations(nxtransf: h5py.Group): logger.info("Additionally, the detector_z vector should be [0,0,-1]") det_z = nxtransf["detector_z/det_z"] - if np.any(det_z.attrs["vector"] != [0, 0, -1]): + if np.any(det_z.attrs["vector"] != [0, 0, 1]): logger.info("Overwriting det_z vector ...") - det_z.attrs["vector"] = [0, 0, -1] + det_z.attrs["vector"] = [0, 0, 1] # [0,0,-1] logger.info("Checking dependency tree of detector for typos ...") if two_theta[ds_name].attrs["depends_on"] != b".": From 26a5282bb0fcc7e30409caa426165eee0f174310 Mon Sep 17 00:00:00 2001 From: Noemi Frisina Date: Tue, 7 Sep 2021 17:37:42 +0100 Subject: [PATCH 23/23] Allow for omega scan when checking sample depends_on --- src/nexgen/nxs_check/CheckTristanNexus.py | 29 +++++++++++++---------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/nexgen/nxs_check/CheckTristanNexus.py b/src/nexgen/nxs_check/CheckTristanNexus.py index fdd37325..c1bf4583 100644 --- a/src/nexgen/nxs_check/CheckTristanNexus.py +++ b/src/nexgen/nxs_check/CheckTristanNexus.py @@ -72,24 +72,29 @@ def check_detector_transformations(nxtransf: h5py.Group): ) -def check_sample_depends_on(nxsample: h5py.Group): +def check_sample_depends_on(nxsample: h5py.Group, scan_axis: str): """ Check that the sample depends_on field exists and is correct. - For I19-2 Tristan it should be "/entry/sample/transformations/phi" + For I19-2 Tristan it should be "/entry/sample/transformations/phi" for a phi scan + and "/entry/sample/transformations/omega" for on omega scan. """ + logger.info(f"Passed value for scan axis: {scan_axis}") + if scan_axis == "phi": + dep_string = np.string_("/entry/sample/transformations/phi") + elif scan_axis == "omega": + dep_string = np.string_("/entry/sample/transformations/omega") + # else: + # print("Wrong scan_axis, please choose either phi or omega") + # logger.warning("Wrong scan axis inserted, couldn't correct") try: dep = nxsample["depends_on"][()] - if dep != b"/entry/sample/transformations/phi": + if dep != dep_string: logger.info("Fixing sample depends_on field ...") del nxsample["depends_on"] - nxsample.create_dataset( - "depends_on", data=np.string_("/entry/sample/transformations/phi") - ) + nxsample.create_dataset("depends_on", data=dep_string) except KeyError: logger.info("Sample depends_on field did not exist, creating now ...") - nxsample.create_dataset( - "depends_on", data=np.string_("/entry/sample/transformations/phi") - ) + nxsample.create_dataset("depends_on", data=dep_string) def check_I19_dependency_tree(nxtransf: h5py.Group): @@ -174,7 +179,7 @@ def check_values(nxentry: h5py.Group): del d, val -def run_checks(tristan_nexus_file): +def run_checks(tristan_nexus_file, scan_axis): """ Instigates the functions to check nexus files generated after binning of Tristan data. """ @@ -205,7 +210,7 @@ def run_checks(tristan_nexus_file): check_detector_transformations(nxsfile["entry/instrument/transformations"]) logger.info("-" * 10) logger.info("Check sample depends on") - check_sample_depends_on(nxsfile["entry/sample"]) + check_sample_depends_on(nxsfile["entry/sample"], scan_axis) logger.info("-" * 10) logger.info("Check goniometer dependency tree") check_I19_dependency_tree(nxsfile["entry/sample/transformations"]) @@ -215,4 +220,4 @@ def run_checks(tristan_nexus_file): logger.info("EOF") -run_checks(sys.argv[1]) +run_checks(sys.argv[1], sys.argv[2])