From e45e592c4c69de858531aa06069929aa5f668885 Mon Sep 17 00:00:00 2001 From: Jonathan Karlsen Date: Tue, 9 Dec 2025 15:09:40 +0100 Subject: [PATCH 1/6] Fix export to runpath uses scientific notation for design_matrix values This commit fixes the issue where Ert converts large intergers to scientific notation when exporting design_matrix values to parameters.json, parameters.txt, and run templates. --- src/ert/run_models/_create_run_path.py | 13 +-- .../ert/unit_tests/test_run_path_creation.py | 81 ++++++++++++++++++- 2 files changed, 87 insertions(+), 7 deletions(-) diff --git a/src/ert/run_models/_create_run_path.py b/src/ert/run_models/_create_run_path.py index 2da938f10bf..89c86dcbcf0 100644 --- a/src/ert/run_models/_create_run_path.py +++ b/src/ert/run_models/_create_run_path.py @@ -61,7 +61,10 @@ def _value_export_txt( for param, value in param_map.items(): if isinstance(value, (int | float)): if key == DESIGN_MATRIX_GROUP: - print(f"{param} {value:g}", file=f) + if isinstance(value, float): + print(f"{param} {value:g}", file=f) + else: + print(f"{param} {value}", file=f) else: print(f"{key}:{param} {value:g}", file=f) elif key == DESIGN_MATRIX_GROUP: @@ -203,10 +206,10 @@ def _make_param_substituter( param_substituter = deepcopy(substituter) for values in param_data.values(): for param_name, value in values.items(): - if isinstance(value, (int, float)): - formatted_value = f"{value:.6g}" - else: - formatted_value = str(value) + formatted_value = ( + f"{value:.6g}" if isinstance(value, (float)) else str(value) + ) + param_substituter[f"<{param_name}>"] = formatted_value return param_substituter diff --git a/tests/ert/unit_tests/test_run_path_creation.py b/tests/ert/unit_tests/test_run_path_creation.py index fb43226b004..4101f8c2a6e 100644 --- a/tests/ert/unit_tests/test_run_path_creation.py +++ b/tests/ert/unit_tests/test_run_path_creation.py @@ -7,6 +7,7 @@ import numpy as np import orjson +import polars as pl import pytest import xtgeo @@ -20,6 +21,7 @@ from ert.plugins import get_site_plugins from ert.run_arg import create_run_arguments from ert.run_models._create_run_path import _make_param_substituter, create_run_path +from ert.run_models.model_factory import _merge_parameters from ert.runpaths import Runpaths from ert.sample_prior import sample_prior from ert.storage import ( @@ -27,6 +29,7 @@ load_realization_parameters_and_responses, ) from ert.substitutions import Substitutions +from tests.ert.conftest import _create_design_matrix from tests.ert.unit_tests.config.summary_generator import simple_smspec, simple_unsmry @@ -1059,7 +1062,81 @@ def test_that_parameters_as_magic_strings_are_substituted(): { "GROUP1": {"a": 1.01}, "GROUP2": {"b": "value"}, + "GROUP3": {"c": 4213131313}, }, - ).substitute(" and ") - == "1.01 and value" + ).substitute(" and and ") + == "1.01 and value and 4213131313" + ) + + +@pytest.mark.usefixtures("use_tmpdir") +def test_design_matrix_scalars_are_not_exported_with_scientific_notation( + run_args, storage +): + """ + This is a regression test to make sure that the integers will remain full + numbers in parameters.txt, paramaters.json, and template files, when + exporting values from the design matrix. + """ + design_matrix_filename = "dm.xlsx" + some_integer = 4294912121 + design_sheet = pl.DataFrame({"REAL": [0], "my_value": [some_integer]}) + _create_design_matrix(design_matrix_filename, design_sheet) + + run_template_file_name = "my_file.tmpl" + run_template_output_file_name = "my_file.txt" + Path(run_template_file_name).write_text("my_value is ", encoding="utf-8") + + config = ErtConfig.from_file_contents( + "NUM_REALIZATIONS 1\n" + f"DESIGN_MATRIX {design_matrix_filename} DESIGN_SHEET:DesignSheet\n" + f"RUN_TEMPLATE {run_template_file_name} {run_template_output_file_name}" + ) + + _, design_matrix = _merge_parameters( + config.analysis_config.design_matrix, + config.ensemble_config.parameter_configuration, + ) + experiment_id = storage.create_experiment( + parameters=config.parameter_configurations_with_design_matrix, + templates=config.ert_templates, + ) + prior_ensemble = storage.create_ensemble( + experiment_id, name="prior", ensemble_size=1 + ) + sample_prior(prior_ensemble, [0], 123, design_matrix_df=design_matrix.to_polars()) + runargs = run_args(config, prior_ensemble, 1) + runpaths = Runpaths.from_config(config) + create_run_path( + run_args=runargs, + ensemble=prior_ensemble, + user_config_file=config.user_config_file, + forward_model_steps=config.forward_model_steps, + env_vars=config.env_vars, + env_pr_fm_step=config.env_pr_fm_step, + substitutions=config.substitutions, + parameters_file="parameters", + runpaths=runpaths, + ) + + experiment_path = Path("simulations/realization-0/iter-0") + assert experiment_path.exists() + + parameters_text = experiment_path / "parameters.txt" + assert parameters_text.exists() + assert ( + parameters_text.read_text(encoding="utf-8").strip() + == f"my_value {some_integer}" + ) + + parameters_json = experiment_path / "parameters.json" + assert parameters_json.exists() + assert orjson.loads(parameters_json.read_text(encoding="utf-8")) == { + "my_value": {"value": some_integer} + } + + run_template_output = experiment_path / run_template_output_file_name + assert run_template_output.exists() + assert ( + run_template_output.read_text(encoding="utf-8") == f"my_value is {some_integer}" ) From 885ac2eefa3ee650cf8e1009e23aeb490a03e831 Mon Sep 17 00:00:00 2001 From: Jonathan Karlsen Date: Tue, 16 Dec 2025 15:24:46 +0100 Subject: [PATCH 2/6] fixup:should also cover small numbers --- src/ert/run_models/_create_run_path.py | 11 ++------- .../ert/unit_tests/test_run_path_creation.py | 23 +++++++++++-------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/ert/run_models/_create_run_path.py b/src/ert/run_models/_create_run_path.py index 89c86dcbcf0..a0755d8322a 100644 --- a/src/ert/run_models/_create_run_path.py +++ b/src/ert/run_models/_create_run_path.py @@ -61,10 +61,7 @@ def _value_export_txt( for param, value in param_map.items(): if isinstance(value, (int | float)): if key == DESIGN_MATRIX_GROUP: - if isinstance(value, float): - print(f"{param} {value:g}", file=f) - else: - print(f"{param} {value}", file=f) + print(f"{param} {value}", file=f) else: print(f"{key}:{param} {value:g}", file=f) elif key == DESIGN_MATRIX_GROUP: @@ -206,11 +203,7 @@ def _make_param_substituter( param_substituter = deepcopy(substituter) for values in param_data.values(): for param_name, value in values.items(): - formatted_value = ( - f"{value:.6g}" if isinstance(value, (float)) else str(value) - ) - - param_substituter[f"<{param_name}>"] = formatted_value + param_substituter[f"<{param_name}>"] = str(value) return param_substituter diff --git a/tests/ert/unit_tests/test_run_path_creation.py b/tests/ert/unit_tests/test_run_path_creation.py index 4101f8c2a6e..0024e3174e9 100644 --- a/tests/ert/unit_tests/test_run_path_creation.py +++ b/tests/ert/unit_tests/test_run_path_creation.py @@ -1074,18 +1074,22 @@ def test_design_matrix_scalars_are_not_exported_with_scientific_notation( run_args, storage ): """ - This is a regression test to make sure that the integers will remain full + This is a regression test to make sure that the integers and floats will remain full numbers in parameters.txt, paramaters.json, and template files, when exporting values from the design matrix. """ design_matrix_filename = "dm.xlsx" - some_integer = 4294912121 - design_sheet = pl.DataFrame({"REAL": [0], "my_value": [some_integer]}) + + design_sheet = pl.DataFrame( + {"REAL": [0], "my_big_value": [4294912121], "my_small_value": [1.0000000000272]} + ) _create_design_matrix(design_matrix_filename, design_sheet) run_template_file_name = "my_file.tmpl" run_template_output_file_name = "my_file.txt" - Path(run_template_file_name).write_text("my_value is ", encoding="utf-8") + Path(run_template_file_name).write_text( + "big is \nsmall is ", encoding="utf-8" + ) config = ErtConfig.from_file_contents( "NUM_REALIZATIONS 1\n" @@ -1126,17 +1130,18 @@ def test_design_matrix_scalars_are_not_exported_with_scientific_notation( assert parameters_text.exists() assert ( parameters_text.read_text(encoding="utf-8").strip() - == f"my_value {some_integer}" + == "my_big_value 4294912121\nmy_small_value 1.0000000000272" ) parameters_json = experiment_path / "parameters.json" assert parameters_json.exists() - assert orjson.loads(parameters_json.read_text(encoding="utf-8")) == { - "my_value": {"value": some_integer} - } + parameters_json_dict = orjson.loads(parameters_json.read_text(encoding="utf-8")) + assert str(parameters_json_dict["my_big_value"]["value"]) == "4294912121" + assert str(parameters_json_dict["my_small_value"]["value"]) == "1.0000000000272" run_template_output = experiment_path / run_template_output_file_name assert run_template_output.exists() assert ( - run_template_output.read_text(encoding="utf-8") == f"my_value is {some_integer}" + run_template_output.read_text(encoding="utf-8") + == "big is 4294912121\nsmall is 1.0000000000272" ) From 619326d5a3ccf9d57e2e8921f69594a1819c37fa Mon Sep 17 00:00:00 2001 From: Jonathan Karlsen Date: Wed, 17 Dec 2025 07:57:55 +0100 Subject: [PATCH 3/6] limit parameters.txt and run_template scalar parameters to .6f format when exporting --- src/ert/run_models/_create_run_path.py | 12 +++++++++--- tests/ert/unit_tests/test_run_path_creation.py | 4 ++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/ert/run_models/_create_run_path.py b/src/ert/run_models/_create_run_path.py index a0755d8322a..4ae352b75ee 100644 --- a/src/ert/run_models/_create_run_path.py +++ b/src/ert/run_models/_create_run_path.py @@ -59,11 +59,16 @@ def _value_export_txt( with path.open("w") as f: for key, param_map in values.items(): for param, value in param_map.items(): - if isinstance(value, (int | float)): + if isinstance(value, (int)): if key == DESIGN_MATRIX_GROUP: print(f"{param} {value}", file=f) else: - print(f"{key}:{param} {value:g}", file=f) + print(f"{key}:{param} {value}", file=f) + elif isinstance(value, (float)): + if key == DESIGN_MATRIX_GROUP: + print(f"{param} {value:.6f}", file=f) + else: + print(f"{key}:{param} {value:.6f}", file=f) elif key == DESIGN_MATRIX_GROUP: print(f"{param} {value}", file=f) else: @@ -203,7 +208,8 @@ def _make_param_substituter( param_substituter = deepcopy(substituter) for values in param_data.values(): for param_name, value in values.items(): - param_substituter[f"<{param_name}>"] = str(value) + formatted_value = f"{value:.6f}" if isinstance(value, float) else str(value) + param_substituter[f"<{param_name}>"] = formatted_value return param_substituter diff --git a/tests/ert/unit_tests/test_run_path_creation.py b/tests/ert/unit_tests/test_run_path_creation.py index 0024e3174e9..a01d760c6e6 100644 --- a/tests/ert/unit_tests/test_run_path_creation.py +++ b/tests/ert/unit_tests/test_run_path_creation.py @@ -1130,7 +1130,7 @@ def test_design_matrix_scalars_are_not_exported_with_scientific_notation( assert parameters_text.exists() assert ( parameters_text.read_text(encoding="utf-8").strip() - == "my_big_value 4294912121\nmy_small_value 1.0000000000272" + == "my_big_value 4294912121\nmy_small_value 1.000000" ) parameters_json = experiment_path / "parameters.json" @@ -1143,5 +1143,5 @@ def test_design_matrix_scalars_are_not_exported_with_scientific_notation( assert run_template_output.exists() assert ( run_template_output.read_text(encoding="utf-8") - == "big is 4294912121\nsmall is 1.0000000000272" + == "big is 4294912121\nsmall is 1.000000" ) From 9839425c686042717ea0e10b4249d5b557a48878 Mon Sep 17 00:00:00 2001 From: Jonathan Karlsen Date: Wed, 17 Dec 2025 09:03:05 +0100 Subject: [PATCH 4/6] add test for conditional formatting --- src/ert/run_models/_create_run_path.py | 22 ++++++++++++++++--- .../ert/unit_tests/test_run_path_creation.py | 22 ++++++++++++++++++- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/src/ert/run_models/_create_run_path.py b/src/ert/run_models/_create_run_path.py index 4ae352b75ee..7a3e267e5cc 100644 --- a/src/ert/run_models/_create_run_path.py +++ b/src/ert/run_models/_create_run_path.py @@ -37,6 +37,15 @@ logger = logging.getLogger(__name__) +def _conditionally_format_float(num: float) -> str: + str_num = str(num) + if "." not in str_num: + return str_num + float_parts = str_num.split(".") + formatted_str = f"{num:.6f}" if len(float_parts[1]) >= 6 else str(num) + return formatted_str.rstrip("0").rstrip(".") + + def _backup_if_existing(path: Path) -> None: if not path.exists(): return @@ -66,9 +75,12 @@ def _value_export_txt( print(f"{key}:{param} {value}", file=f) elif isinstance(value, (float)): if key == DESIGN_MATRIX_GROUP: - print(f"{param} {value:.6f}", file=f) + print(f"{param} {_conditionally_format_float(value)}", file=f) else: - print(f"{key}:{param} {value:.6f}", file=f) + print( + f"{key}:{param} {_conditionally_format_float(value)}", + file=f, + ) elif key == DESIGN_MATRIX_GROUP: print(f"{param} {value}", file=f) else: @@ -208,7 +220,11 @@ def _make_param_substituter( param_substituter = deepcopy(substituter) for values in param_data.values(): for param_name, value in values.items(): - formatted_value = f"{value:.6f}" if isinstance(value, float) else str(value) + formatted_value = ( + _conditionally_format_float(value) + if isinstance(value, float) + else str(value) + ) param_substituter[f"<{param_name}>"] = formatted_value return param_substituter diff --git a/tests/ert/unit_tests/test_run_path_creation.py b/tests/ert/unit_tests/test_run_path_creation.py index a01d760c6e6..6b442a0cf5b 100644 --- a/tests/ert/unit_tests/test_run_path_creation.py +++ b/tests/ert/unit_tests/test_run_path_creation.py @@ -20,7 +20,11 @@ ) from ert.plugins import get_site_plugins from ert.run_arg import create_run_arguments -from ert.run_models._create_run_path import _make_param_substituter, create_run_path +from ert.run_models._create_run_path import ( + _conditionally_format_float, + _make_param_substituter, + create_run_path, +) from ert.run_models.model_factory import _merge_parameters from ert.runpaths import Runpaths from ert.sample_prior import sample_prior @@ -1145,3 +1149,19 @@ def test_design_matrix_scalars_are_not_exported_with_scientific_notation( run_template_output.read_text(encoding="utf-8") == "big is 4294912121\nsmall is 1.000000" ) + + +@pytest.mark.parametrize( + "input_float, expected_string", + [ + (10, "10"), + (10.0, "10"), + (0.10, "0.1"), + (0.101, "0.101"), + (12.1111114, "12.111111"), + (0.100100100, "0.1001"), + (0.100100900, "0.100101"), + ], +) +def test_run_path_parameters_format_floats_correctly(input_float, expected_string): + assert _conditionally_format_float(input_float) == expected_string From 00839dffc5fcbcf66bf4a26d64a02cd30a6c4f2a Mon Sep 17 00:00:00 2001 From: Jonathan Karlsen Date: Wed, 17 Dec 2025 09:32:14 +0100 Subject: [PATCH 5/6] update snapshot --- .../test_misfit_collector/0/misfit_collector.csv | 10 +++++----- tests/ert/unit_tests/test_run_path_creation.py | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/ert/unit_tests/snapshots/test_libres_facade/test_misfit_collector/0/misfit_collector.csv b/tests/ert/unit_tests/snapshots/test_libres_facade/test_misfit_collector/0/misfit_collector.csv index 04935eac725..c187774c8f7 100644 --- a/tests/ert/unit_tests/snapshots/test_libres_facade/test_misfit_collector/0/misfit_collector.csv +++ b/tests/ert/unit_tests/snapshots/test_libres_facade/test_misfit_collector/0/misfit_collector.csv @@ -1,6 +1,6 @@ Realization,MISFIT:FOPR,MISFIT:WOPR_OP1_108,MISFIT:WOPR_OP1_144,MISFIT:WOPR_OP1_190,MISFIT:WOPR_OP1_36,MISFIT:WOPR_OP1_72,MISFIT:WOPR_OP1_9,MISFIT:WPR_DIFF_1,MISFIT:TOTAL -0,1572.4551,4.6631575,1.2280039,24.150873,0.16579537,16.603199,0.5786172,17.52338,1637.3682 -1,564.73254,4.3687825,32.65306,2.25,7.513238,7.502955,4.0,3.9172122,626.9378 -2,760.21344,0.67585987,0.04954808,0.8789783,0.53148407,10.315068,0.56918764,21.326956,794.5605 -3,762.2885,0.0573729,2.0035706,89.3921,1.0729967,0.2363393,1.9635296,4.454344,861.46875 -4,978.6855,0.60999763,11.165132,2.3617368,0.51387596,41.033993,0.04177265,27.461798,1061.8737 +0,1572.4546,4.6631527,1.2280097,24.150967,0.16579399,16.603184,0.57861996,17.523544,1637.3679 +1,564.73285,4.3688073,32.65306,2.25,7.513266,7.503007,4.0,3.9172914,626.9383 +2,760.21466,0.67585325,0.0495441,0.8790346,0.5314902,10.31503,0.56919664,21.326956,794.56177 +3,762.2892,0.05737214,2.0035608,89.39229,1.0730002,0.23634046,1.9635346,4.45436,861.46954 +4,978.6881,0.6100113,11.165296,2.3617568,0.5138796,41.034237,0.04177362,27.46182,1061.8767 diff --git a/tests/ert/unit_tests/test_run_path_creation.py b/tests/ert/unit_tests/test_run_path_creation.py index 6b442a0cf5b..308de1ad468 100644 --- a/tests/ert/unit_tests/test_run_path_creation.py +++ b/tests/ert/unit_tests/test_run_path_creation.py @@ -1085,7 +1085,7 @@ def test_design_matrix_scalars_are_not_exported_with_scientific_notation( design_matrix_filename = "dm.xlsx" design_sheet = pl.DataFrame( - {"REAL": [0], "my_big_value": [4294912121], "my_small_value": [1.0000000000272]} + {"REAL": [0], "my_big_value": [4294912121], "my_small_value": [1.00000377]} ) _create_design_matrix(design_matrix_filename, design_sheet) @@ -1134,20 +1134,20 @@ def test_design_matrix_scalars_are_not_exported_with_scientific_notation( assert parameters_text.exists() assert ( parameters_text.read_text(encoding="utf-8").strip() - == "my_big_value 4294912121\nmy_small_value 1.000000" + == "my_big_value 4294912121\nmy_small_value 1.000004" ) parameters_json = experiment_path / "parameters.json" assert parameters_json.exists() parameters_json_dict = orjson.loads(parameters_json.read_text(encoding="utf-8")) assert str(parameters_json_dict["my_big_value"]["value"]) == "4294912121" - assert str(parameters_json_dict["my_small_value"]["value"]) == "1.0000000000272" + assert str(parameters_json_dict["my_small_value"]["value"]) == "1.00000377" run_template_output = experiment_path / run_template_output_file_name assert run_template_output.exists() assert ( run_template_output.read_text(encoding="utf-8") - == "big is 4294912121\nsmall is 1.000000" + == "big is 4294912121\nsmall is 1.000004" ) From 81396aa7eea4872ae788187de98721488fac085d Mon Sep 17 00:00:00 2001 From: Jonathan Karlsen Date: Wed, 17 Dec 2025 10:21:07 +0100 Subject: [PATCH 6/6] update test --- .../test_export_misfit/0/csv_data.csv | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/ert/unit_tests/plugins/snapshots/test_export_misfit/test_export_misfit/0/csv_data.csv b/tests/ert/unit_tests/plugins/snapshots/test_export_misfit/test_export_misfit/0/csv_data.csv index 41c2d92ab19..fd1c84b37f8 100644 --- a/tests/ert/unit_tests/plugins/snapshots/test_export_misfit/test_export_misfit/0/csv_data.csv +++ b/tests/ert/unit_tests/plugins/snapshots/test_export_misfit/test_export_misfit/0/csv_data.csv @@ -1,6 +1,6 @@ Realization,FOPR,WOPR_OP1_108,WOPR_OP1_144,WOPR_OP1_190,WOPR_OP1_36,WOPR_OP1_72,WOPR_OP1_9,WPR_DIFF_1 -0,1572.4551,4.6631575,1.2280039,24.150873,0.16579537,16.603199,0.5786172,17.52338 -1,564.73254,4.3687825,32.65306,2.25,7.5132375,7.502955,4.0,3.917212 -2,760.21344,0.67585987,0.04954808,0.8789783,0.53148407,10.315068,0.56918764,21.326956 -3,762.2886,0.057372905,2.0035706,89.3921,1.0729967,0.2363393,1.9635296,4.454344 -4,978.6854,0.60999763,11.165132,2.3617368,0.51387596,41.033993,0.04177265,27.461798 +0,1572.4546,4.6631527,1.2280097,24.150967,0.16579399,16.603184,0.57861996,17.523544 +1,564.73285,4.3688073,32.65306,2.25,7.513266,7.503007,4.0,3.9172916 +2,760.21466,0.6758533,0.0495441,0.8790346,0.5314902,10.31503,0.56919664,21.326956 +3,762.2892,0.05737214,2.0035608,89.392296,1.0730002,0.23634046,1.9635346,4.45436 +4,978.6881,0.6100113,11.165296,2.3617568,0.5138796,41.034237,0.041773625,27.461823