From f98b9c27012724597f912c4b791b320c04e5c508 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Sun, 7 Dec 2025 16:29:31 -0800 Subject: [PATCH 1/4] make datetime fields constants in the DatabaseLogic --- .../stac_fastapi/opensearch/database_logic.py | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 05aac1763..3987f2289 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -154,6 +154,11 @@ def __attrs_post_init__(self): aggregation_mapping: Dict[str, Dict[str, Any]] = AGGREGATION_MAPPING + # Private constant for the datetime property field + PROPERTIES_DATETIME_FIELD = "properties.datetime" + PROPERTIES_START_DATETIME_FIELD = "properties.start_datetime" + PROPERTIES_END_DATETIME_FIELD = "properties.end_datetime" + """CORE LOGIC""" async def get_all_collections( @@ -488,7 +493,7 @@ def apply_datetime_filter( Q( "bool", filter=[ - Q("exists", field="properties.datetime"), + Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD), Q( "term", **{"properties__datetime": datetime_search["eq"]}, @@ -497,10 +502,10 @@ def apply_datetime_filter( ), Q( "bool", - must_not=[Q("exists", field="properties.datetime")], + must_not=[Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD)], filter=[ - Q("exists", field="properties.start_datetime"), - Q("exists", field="properties.end_datetime"), + Q("exists", field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD), + Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD), Q( "range", properties__start_datetime={ @@ -522,7 +527,7 @@ def apply_datetime_filter( Q( "bool", filter=[ - Q("exists", field="properties.datetime"), + Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD), Q( "range", properties__datetime={ @@ -534,21 +539,17 @@ def apply_datetime_filter( ), Q( "bool", - must_not=[Q("exists", field="properties.datetime")], + must_not=[Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD)], filter=[ - Q("exists", field="properties.start_datetime"), - Q("exists", field="properties.end_datetime"), + Q("exists", field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD), + Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD), Q( "range", - properties__start_datetime={ - "lte": datetime_search["lte"] - }, + properties__start_datetime={"lte": datetime_search["lte"]}, ), Q( "range", - properties__end_datetime={ - "gte": datetime_search["gte"] - }, + properties__end_datetime={"gte": datetime_search["gte"]}, ), ], ), @@ -563,8 +564,8 @@ def apply_datetime_filter( filter_query = Q( "bool", filter=[ - Q("exists", field="properties.start_datetime"), - Q("exists", field="properties.end_datetime"), + Q("exists", field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD), + Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD), Q( "range", properties__start_datetime={"lte": datetime_search["eq"]}, @@ -579,8 +580,8 @@ def apply_datetime_filter( filter_query = Q( "bool", filter=[ - Q("exists", field="properties.start_datetime"), - Q("exists", field="properties.end_datetime"), + Q("exists", field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD), + Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD), Q( "range", properties__start_datetime={"lte": datetime_search["lte"]}, From 9268b46f067312105cf24c84353d63485680ebae Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Mon, 8 Dec 2025 18:47:01 -0800 Subject: [PATCH 2/4] update obsolete comment --- .../stac_fastapi/opensearch/database_logic.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 3987f2289..f92385dd5 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -154,10 +154,14 @@ def __attrs_post_init__(self): aggregation_mapping: Dict[str, Dict[str, Any]] = AGGREGATION_MAPPING - # Private constant for the datetime property field + # constants for field names + # they are used in multiple methods + # and could be overwritten in subclasses used with alternate opensearch mappings. PROPERTIES_DATETIME_FIELD = "properties.datetime" PROPERTIES_START_DATETIME_FIELD = "properties.start_datetime" PROPERTIES_END_DATETIME_FIELD = "properties.end_datetime" + COLLECTION_FIELD = "collection" + GEOMETRY_FIELD = "geometry" """CORE LOGIC""" @@ -441,7 +445,7 @@ def apply_ids_filter(search: Search, item_ids: List[str]): @staticmethod def apply_collections_filter(search: Search, collection_ids: List[str]): """Database logic to search a list of STAC collection ids.""" - return search.filter("terms", collection=collection_ids) + return search.filter("terms", **{DatabaseLogic.COLLECTION_FIELD:collection_ids}) @staticmethod def apply_free_text_filter(search: Search, free_text_queries: Optional[List[str]]): @@ -613,7 +617,7 @@ def apply_bbox_filter(search: Search, bbox: List): Q( { "geo_shape": { - "geometry": { + DatabaseLogic.GEOMETRY_FIELD: { "shape": { "type": "polygon", "coordinates": bbox2polygon(*bbox), From 310a25d058ae19a463d34069ba8a1c0a2c8e05e6 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Mon, 8 Dec 2025 20:27:30 -0800 Subject: [PATCH 3/4] update CHANGELOG, lint fix --- CHANGELOG.md | 3 ++ .../stac_fastapi/opensearch/database_logic.py | 50 +++++++++++++++---- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 679f1ba62..546b30c3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,8 +11,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Added catalogs route support to enable federated hierarchical catalog browsing and navigation in the STAC API. [#547](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/547) + ### Changed +- Have opensearch datetime, geometry and collections fields defined as constant strings [#553](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/553) + ### Fixed - Fix unawaited coroutine in `stac_fastapi.core.core`. [#551](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/551) diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 673c14ee1..3f5bf70d0 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -445,7 +445,9 @@ def apply_ids_filter(search: Search, item_ids: List[str]): @staticmethod def apply_collections_filter(search: Search, collection_ids: List[str]): """Database logic to search a list of STAC collection ids.""" - return search.filter("terms", **{DatabaseLogic.COLLECTION_FIELD:collection_ids}) + return search.filter( + "terms", **{DatabaseLogic.COLLECTION_FIELD: collection_ids} + ) @staticmethod def apply_free_text_filter(search: Search, free_text_queries: Optional[List[str]]): @@ -506,10 +508,18 @@ def apply_datetime_filter( ), Q( "bool", - must_not=[Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD)], + must_not=[ + Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD) + ], filter=[ - Q("exists", field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD), - Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD), + Q( + "exists", + field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD, + ), + Q( + "exists", + field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD, + ), Q( "range", properties__start_datetime={ @@ -543,17 +553,29 @@ def apply_datetime_filter( ), Q( "bool", - must_not=[Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD)], + must_not=[ + Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD) + ], filter=[ - Q("exists", field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD), - Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD), + Q( + "exists", + field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD, + ), + Q( + "exists", + field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD, + ), Q( "range", - properties__start_datetime={"lte": datetime_search["lte"]}, + properties__start_datetime={ + "lte": datetime_search["lte"] + }, ), Q( "range", - properties__end_datetime={"gte": datetime_search["gte"]}, + properties__end_datetime={ + "gte": datetime_search["gte"] + }, ), ], ), @@ -568,7 +590,10 @@ def apply_datetime_filter( filter_query = Q( "bool", filter=[ - Q("exists", field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD), + Q( + "exists", + field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD, + ), Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD), Q( "range", @@ -584,7 +609,10 @@ def apply_datetime_filter( filter_query = Q( "bool", filter=[ - Q("exists", field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD), + Q( + "exists", + field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD, + ), Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD), Q( "range", From c84db1607b95f84fe97a4017635a47788d3e7916 Mon Sep 17 00:00:00 2001 From: thomas loubrieu Date: Thu, 11 Dec 2025 19:31:18 -0800 Subject: [PATCH 4/4] fix forgotten nested properties management, extend changes to elasticsearch --- .../elasticsearch/database_logic.py | 137 ++++++++++++++---- .../stac_fastapi/opensearch/database_logic.py | 78 +++++++--- 2 files changed, 168 insertions(+), 47 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 759156073..7576f2870 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -154,6 +154,20 @@ def __attrs_post_init__(self): aggregation_mapping: Dict[str, Dict[str, Any]] = AGGREGATION_MAPPING + # constants for field names + # they are used in multiple methods + # and could be overwritten in subclasses used with alternate opensearch mappings. + PROPERTIES_DATETIME_FIELD = "properties.datetime" + PROPERTIES_START_DATETIME_FIELD = "properties.start_datetime" + PROPERTIES_END_DATETIME_FIELD = "properties.end_datetime" + COLLECTION_FIELD = "collection" + GEOMETRY_FIELD = "geometry" + + @staticmethod + def __nested_field__(field: str): + """Convert opensearch field to nested field format.""" + return field.replace(".", "__") + """CORE LOGIC""" async def get_all_collections( @@ -436,7 +450,10 @@ def apply_ids_filter(search: Search, item_ids: List[str]): @staticmethod def apply_collections_filter(search: Search, collection_ids: List[str]): """Database logic to search a list of STAC collection ids.""" - return search.filter("terms", collection=collection_ids) + collection_nested_field = DatabaseLogic.__nested_field__( + DatabaseLogic.COLLECTION_FIELD + ) + return search.filter("terms", **{collection_nested_field: collection_ids}) @staticmethod def apply_datetime_filter( @@ -461,6 +478,16 @@ def apply_datetime_filter( if not datetime_search: return search, datetime_search + nested_datetime_field = DatabaseLogic.__nested_field__( + DatabaseLogic.PROPERTIES_DATETIME_FIELD + ) + nested_start_datetime_field = DatabaseLogic.__nested_field__( + DatabaseLogic.PROPERTIES_START_DATETIME_FIELD + ) + nested_end_datetime_field = DatabaseLogic.__nested_field__( + DatabaseLogic.PROPERTIES_END_DATETIME_FIELD + ) + if USE_DATETIME: if "eq" in datetime_search: # For exact matches, include: @@ -470,28 +497,42 @@ def apply_datetime_filter( Q( "bool", filter=[ - Q("exists", field="properties.datetime"), + Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD), Q( "term", - **{"properties__datetime": datetime_search["eq"]}, + **{nested_datetime_field: datetime_search["eq"]}, ), ], ), Q( "bool", - must_not=[Q("exists", field="properties.datetime")], + must_not=[ + Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD) + ], filter=[ - Q("exists", field="properties.start_datetime"), - Q("exists", field="properties.end_datetime"), + Q( + "exists", + field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD, + ), + Q( + "exists", + field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD, + ), Q( "range", - properties__start_datetime={ - "lte": datetime_search["eq"] + **{ + nested_start_datetime_field: { + "lte": datetime_search["eq"] + } }, ), Q( "range", - properties__end_datetime={"gte": datetime_search["eq"]}, + **{ + nested_end_datetime_field: { + "gte": datetime_search["eq"] + } + }, ), ], ), @@ -504,32 +545,46 @@ def apply_datetime_filter( Q( "bool", filter=[ - Q("exists", field="properties.datetime"), + Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD), Q( "range", - properties__datetime={ - "gte": datetime_search["gte"], - "lte": datetime_search["lte"], + **{ + nested_datetime_field: { + "gte": datetime_search["gte"], + "lte": datetime_search["lte"], + } }, ), ], ), Q( "bool", - must_not=[Q("exists", field="properties.datetime")], + must_not=[ + Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD) + ], filter=[ - Q("exists", field="properties.start_datetime"), - Q("exists", field="properties.end_datetime"), + Q( + "exists", + field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD, + ), + Q( + "exists", + field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD, + ), Q( "range", - properties__start_datetime={ - "lte": datetime_search["lte"] + **{ + nested_start_datetime_field: { + "lte": datetime_search["lte"] + } }, ), Q( "range", - properties__end_datetime={ - "gte": datetime_search["gte"] + **{ + nested_end_datetime_field: { + "gte": datetime_search["gte"] + } }, ), ], @@ -545,15 +600,26 @@ def apply_datetime_filter( filter_query = Q( "bool", filter=[ - Q("exists", field="properties.start_datetime"), - Q("exists", field="properties.end_datetime"), + Q( + "exists", + field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD, + ), + Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD), Q( "range", - properties__start_datetime={"lte": datetime_search["eq"]}, + **{ + nested_start_datetime_field: { + "lte": datetime_search["eq"] + } + }, ), Q( "range", - properties__end_datetime={"gte": datetime_search["eq"]}, + **{ + nested_end_datetime_field: { + "gte": datetime_search["eq"] + } + }, ), ], ) @@ -561,15 +627,26 @@ def apply_datetime_filter( filter_query = Q( "bool", filter=[ - Q("exists", field="properties.start_datetime"), - Q("exists", field="properties.end_datetime"), + Q( + "exists", + field=DatabaseLogic.PROPERTIES_START_DATETIME_FIELD, + ), + Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD), Q( "range", - properties__start_datetime={"lte": datetime_search["lte"]}, + **{ + nested_start_datetime_field: { + "lte": datetime_search["lte"] + } + }, ), Q( "range", - properties__end_datetime={"gte": datetime_search["gte"]}, + **{ + nested_end_datetime_field: { + "gte": datetime_search["gte"] + } + }, ), ], ) @@ -594,7 +671,7 @@ def apply_bbox_filter(search: Search, bbox: List): Q( { "geo_shape": { - "geometry": { + DatabaseLogic.GEOMETRY_FIELD: { "shape": { "type": "polygon", "coordinates": bbox2polygon(*bbox), @@ -1708,7 +1785,7 @@ def bulk_sync( kwargs = kwargs or {} # Resolve the `refresh` parameter - refresh = kwargs.get("refresh", self.async_settings.database_refresh) + refresh = kwargs.get("refresh", self.sync_settings.database_refresh) refresh = validate_refresh(refresh) # Log the bulk insert attempt diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 3f5bf70d0..1bf6c0c12 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -163,6 +163,11 @@ def __attrs_post_init__(self): COLLECTION_FIELD = "collection" GEOMETRY_FIELD = "geometry" + @staticmethod + def __nested_field__(field: str): + """Convert opensearch field to nested field format.""" + return field.replace(".", "__") + """CORE LOGIC""" async def get_all_collections( @@ -445,9 +450,10 @@ def apply_ids_filter(search: Search, item_ids: List[str]): @staticmethod def apply_collections_filter(search: Search, collection_ids: List[str]): """Database logic to search a list of STAC collection ids.""" - return search.filter( - "terms", **{DatabaseLogic.COLLECTION_FIELD: collection_ids} + collection_nested_field = DatabaseLogic.__nested_field__( + DatabaseLogic.COLLECTION_FIELD ) + return search.filter("terms", **{collection_nested_field: collection_ids}) @staticmethod def apply_free_text_filter(search: Search, free_text_queries: Optional[List[str]]): @@ -490,6 +496,16 @@ def apply_datetime_filter( # False: Always search only by start/end datetime USE_DATETIME = get_bool_env("USE_DATETIME", default=True) + nested_datetime_field = DatabaseLogic.__nested_field__( + DatabaseLogic.PROPERTIES_DATETIME_FIELD + ) + nested_start_datetime_field = DatabaseLogic.__nested_field__( + DatabaseLogic.PROPERTIES_START_DATETIME_FIELD + ) + nested_end_datetime_field = DatabaseLogic.__nested_field__( + DatabaseLogic.PROPERTIES_END_DATETIME_FIELD + ) + if USE_DATETIME: if "eq" in datetime_search: # For exact matches, include: @@ -502,7 +518,7 @@ def apply_datetime_filter( Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD), Q( "term", - **{"properties__datetime": datetime_search["eq"]}, + **{nested_datetime_field: datetime_search["eq"]}, ), ], ), @@ -522,13 +538,19 @@ def apply_datetime_filter( ), Q( "range", - properties__start_datetime={ - "lte": datetime_search["eq"] + **{ + nested_start_datetime_field: { + "lte": datetime_search["eq"] + } }, ), Q( "range", - properties__end_datetime={"gte": datetime_search["eq"]}, + **{ + nested_end_datetime_field: { + "gte": datetime_search["eq"] + } + }, ), ], ), @@ -544,9 +566,11 @@ def apply_datetime_filter( Q("exists", field=DatabaseLogic.PROPERTIES_DATETIME_FIELD), Q( "range", - properties__datetime={ - "gte": datetime_search["gte"], - "lte": datetime_search["lte"], + **{ + nested_datetime_field: { + "gte": datetime_search["gte"], + "lte": datetime_search["lte"], + } }, ), ], @@ -567,14 +591,18 @@ def apply_datetime_filter( ), Q( "range", - properties__start_datetime={ - "lte": datetime_search["lte"] + **{ + nested_start_datetime_field: { + "lte": datetime_search["lte"] + } }, ), Q( "range", - properties__end_datetime={ - "gte": datetime_search["gte"] + **{ + nested_end_datetime_field: { + "gte": datetime_search["gte"] + } }, ), ], @@ -597,11 +625,19 @@ def apply_datetime_filter( Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD), Q( "range", - properties__start_datetime={"lte": datetime_search["eq"]}, + **{ + nested_start_datetime_field: { + "lte": datetime_search["eq"] + } + }, ), Q( "range", - properties__end_datetime={"gte": datetime_search["eq"]}, + **{ + nested_end_datetime_field: { + "gte": datetime_search["eq"] + } + }, ), ], ) @@ -616,11 +652,19 @@ def apply_datetime_filter( Q("exists", field=DatabaseLogic.PROPERTIES_END_DATETIME_FIELD), Q( "range", - properties__start_datetime={"lte": datetime_search["lte"]}, + **{ + nested_start_datetime_field: { + "lte": datetime_search["lte"] + } + }, ), Q( "range", - properties__end_datetime={"gte": datetime_search["gte"]}, + **{ + nested_end_datetime_field: { + "gte": datetime_search["gte"] + } + }, ), ], )