feat: add metadata_fields, sort, show_collections and fq params to "my data" API

doc/release-notes/12009-my-data-api-params.md

@@ -0,0 +1 @@
+The My Data API now supports the `metadata_fields`, `sort` and `order`, `show_collections` and `fq` parameters, which enhances its functionality and brings it in line with the search API.

doc/sphinx-guides/source/api/native-api.rst

@@ -8579,6 +8579,16 @@ Parameters:
 
 ``per_page`` Number of results returned per page.
 
+``metadata_fields`` Includes the requested fields for each dataset in the response. Multiple "metadata_fields" parameters can be used to include several fields. See :doc:`search` for further information on this parameter.
+
+``show_collections`` Whether or not to include a list of parent and linked collections for each dataset search result.
+
+``sort`` The sort field. Supported values include "name", "date" and "relevance".
+
+``order`` The order in which to sort. Can either be "asc" or "desc".
+
+``fq`` A filter query to filter the list returned. Multiple "fq" parameters can be used.
+
 MyData Collection List
 ~~~~~~~~~~~~~~~~~~~~~~
 

doc/sphinx-guides/source/api/search.rst

@@ -27,7 +27,7 @@ Name             Type     Description
 q                string   The search term or terms. Using "title:data" will search only the "title" field. "*" can be used as a wildcard either alone or adjacent to a term (i.e. "bird*"). For example, https://demo.dataverse.org/api/search?q=title:data . For a list of fields to search, please see https://github.com/IQSS/dataverse/issues/2558 (for now).
 type             string   Can be either "dataverse", "dataset", or "file". Multiple "type" parameters can be used to include multiple types (i.e. ``type=dataset&type=file``). If omitted, all types will be returned.  For example, https://demo.dataverse.org/api/search?q=*&type=dataset
 subtree          string   The identifier of the Dataverse collection to which the search should be narrowed. The subtree of this Dataverse collection and all its children will be searched.  Multiple "subtree" parameters can be used to include multiple Dataverse collections. For example, https://demo.dataverse.org/api/search?q=data&subtree=birds&subtree=cats .
-sort             string   The sort field. Supported values include "name" and "date". See example under "order".
+sort             string   The sort field. Supported values include "name", "date" and "relevance". See example under "order".
 order            string   The order in which to sort. Can either be "asc" or "desc".  For example, https://demo.dataverse.org/api/search?q=data&sort=name&order=asc
 per_page         int      The number of results to return per request. The default is 10. The max is 1000. See :ref:`iteration example <iteration-example>`.
 start            int      A cursor for paging through search results. See :ref:`iteration example <iteration-example>`.

src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java

@@ -9,20 +9,14 @@
 import edu.harvard.iq.dataverse.authorization.users.GuestUser;
 import edu.harvard.iq.dataverse.authorization.users.User;
 import edu.harvard.iq.dataverse.engine.command.impl.GetUserPermittedCollectionsCommand;
-import edu.harvard.iq.dataverse.search.SolrQueryResponse;
-import edu.harvard.iq.dataverse.search.SolrSearchResult;
+import edu.harvard.iq.dataverse.search.*;
 import edu.harvard.iq.dataverse.api.AbstractApiBean;
 import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
 import edu.harvard.iq.dataverse.authorization.DataverseRole;
 import edu.harvard.iq.dataverse.authorization.DataverseRolePermissionHelper;
 import edu.harvard.iq.dataverse.authorization.groups.GroupServiceBean;
 import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
 import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
-import edu.harvard.iq.dataverse.search.SearchConstants;
-import edu.harvard.iq.dataverse.search.SearchException;
-import edu.harvard.iq.dataverse.search.SearchFields;
-import edu.harvard.iq.dataverse.search.SearchServiceFactory;
-import edu.harvard.iq.dataverse.search.SortBy;
 
 import java.util.Arrays;
 import java.util.List;
@@ -154,13 +148,18 @@ private void verifyAuth (ContainerRequestContext crc, String userIdentifier) thr
     public String retrieveMyDataAsJsonString(
             @Context ContainerRequestContext crc,
             @QueryParam("dvobject_types") List<DvObject.DType> dvobject_types,
-            @QueryParam("published_states") List<String> published_states, 
+            @QueryParam("published_states") List<String> published_states,
+            @QueryParam("metadata_fields") List<String> metadataFields,
             @QueryParam("selected_page") Integer selectedPage, 
             @QueryParam("mydata_search_term") String searchTerm,             
             @QueryParam("role_ids") List<Long> roleIds, 
             @QueryParam("userIdentifier") String userIdentifier,
             @QueryParam("filter_validities") Boolean filterValidities,
-            @QueryParam("dataset_valid") List<Boolean> datasetValidities) {
+            @QueryParam("dataset_valid") List<Boolean> datasetValidities,
+            @QueryParam("show_collections") boolean showCollections,
+            @QueryParam("sort") String sortField,
+            @QueryParam("order") String sortOrder,
+            @QueryParam("fq") final List<String> filterQueries) {
         boolean otherUser;
 
         String noMsgResultsFound = BundleUtil.getStringFromBundle("dataretrieverAPI.noMsgResultsFound");
@@ -224,23 +223,37 @@ public String retrieveMyDataAsJsonString(
 
         //msg("search with user: " + searchUser.getIdentifier());
 
-        List<String> filterQueries = this.myDataFinder.getSolrFilterQueries();
-        if (filterQueries==null){
+        List<String> defaultFilterQueries = this.myDataFinder.getSolrFilterQueries();
+        if (defaultFilterQueries==null){
             logger.fine("No ids found for this search");
             return this.getJSONErrorString(noMsgResultsFound, null);
         }
+        filterQueries.addAll(defaultFilterQueries);
+
+        SortBy sortBy;
+        try {
+            sortBy = SearchUtil.getSortBy(sortField, sortOrder, SearchFields.RELEASE_OR_CREATE_DATE);
+        } catch (Exception ex) {
+            return this.getJSONErrorString(ex.getLocalizedMessage(), null);
+        }
 
         try {
                 solrQueryResponse = searchService.getDefaultSearchService().search(
                         dataverseRequest,
                         null, // subtree, default it to Dataverse for now
                         filterParams.getSearchTerm(),  //"*", //
                         filterQueries,//filterQueries,
-                        //SearchFields.NAME_SORT, SortBy.ASCENDING,
-                        SearchFields.RELEASE_OR_CREATE_DATE, SortBy.DESCENDING,
+                        sortBy.getField(),
+                        sortBy.getOrder(),
                         solrCardStart, //paginationStart,
                         true, // dataRelatedToMe
-                        SearchConstants.NUM_SOLR_DOCS_TO_RETRIEVE //10 // SearchFields.NUM_SOLR_DOCS_TO_RETRIEVE
+                        SearchConstants.NUM_SOLR_DOCS_TO_RETRIEVE, //10 // SearchFields.NUM_SOLR_DOCS_TO_RETRIEVE
+                        true,
+                        null,
+                        null,
+                        true,
+                        true,
+                        showCollections
                 );
 
             if (this.solrQueryResponse.getNumResultsFound()==0){
@@ -284,7 +297,7 @@ public String retrieveMyDataAsJsonString(
                         Json.createObjectBuilder()
                                 .add("pagination", pager.asJsonObjectBuilderUsingCardTerms())
                                 //.add(SearchConstants.SEARCH_API_ITEMS, this.formatSolrDocs(solrQueryResponse, filterParams, this.myDataFinder))
-                                .add(SearchConstants.SEARCH_API_ITEMS, this.formatSolrDocs(solrQueryResponse, roleTagRetriever))
+                                .add(SearchConstants.SEARCH_API_ITEMS, this.formatSolrDocs(solrQueryResponse, roleTagRetriever, metadataFields))
                                 .add(SearchConstants.SEARCH_API_TOTAL_COUNT, solrQueryResponse.getNumResultsFound())
                                 .add(SearchConstants.SEARCH_API_START, solrQueryResponse.getResultsStart())
                                 .add("search_term",  filterParams.getSearchTerm())
@@ -347,7 +360,7 @@ private JsonObjectBuilder getPublicationStatusCounts(SolrQueryResponse solrRespo
      * @param roleTagRetriever
      * @return 
      */
-    private JsonArrayBuilder formatSolrDocs(SolrQueryResponse solrResponse, RoleTagRetriever roleTagRetriever ){
+    private JsonArrayBuilder formatSolrDocs(SolrQueryResponse solrResponse, RoleTagRetriever roleTagRetriever, List<String> metadataFields){
         if (solrResponse == null){
             throw new NullPointerException("DataRetrieverAPI.formatSolrDocs:  solrResponse should not be null");     
         }
@@ -365,7 +378,7 @@ private JsonArrayBuilder formatSolrDocs(SolrQueryResponse solrResponse, RoleTagR
             // (a) Get core card data from solr
             // -------------------------------------------
 
-            myDataCardInfo = doc.getJsonForMyData(isValid(doc));
+            myDataCardInfo = doc.getJsonForMyData(isValid(doc), metadataFields);
 
             if (doc.getEntity() != null && !doc.getEntity().isInstanceofDataFile()){
                 String parentAlias = dataverseService.getParentAliasString(doc);

src/main/java/edu/harvard/iq/dataverse/search/SearchUtil.java

@@ -68,8 +68,18 @@ public static String getTimestampOrNull(Timestamp timestamp) {
     }
 
     public static SortBy getSortBy(String sortField, String sortOrder) throws Exception {
+        return getSortBy(sortField, sortOrder, SearchFields.RELEVANCE);
+    }
+
+    public static SortBy getSortBy(String sortField, String sortOrder, String defaultSortField) throws Exception {
+        List<String> allowedDefaultSortFieldValues = SortBy.allowedFieldStrings();
+        if (!allowedDefaultSortFieldValues.contains(defaultSortField)) {
+            throw new Exception("The 'defaultSortField' was set to '" + defaultSortField + "' but expected one of " + allowedDefaultSortFieldValues + ".");
+        }
 
         if (StringUtils.isBlank(sortField)) {
+            sortField = defaultSortField;
+        } else if (sortField.equals("relevance")) {
             sortField = SearchFields.RELEVANCE;
         } else if (sortField.equals("name")) {
             // "name" sounds better than "name_sort" so we convert it here so users don't have to pass in "name_sort"

src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java

@@ -438,9 +438,9 @@ public JsonArrayBuilder getRelevance() {
      *
      * @return
      */
-    public JsonObjectBuilder getJsonForMyData(boolean isValid) {
+    public JsonObjectBuilder getJsonForMyData(boolean isValid, List<String> metadataFields) {
 
-        JsonObjectBuilder myDataJson = json(true, true, true);// boolean showRelevance, boolean showEntityIds, boolean showApiUrls)
+        JsonObjectBuilder myDataJson = json(true, true, true, metadataFields);
 
         myDataJson.add("publication_statuses", this.getPublicationStatusesAsJSON())
                 .add("is_draft_state", this.isDraftState()).add("is_in_review_state", this.isInReviewState())

src/main/java/edu/harvard/iq/dataverse/search/SortBy.java

@@ -14,6 +14,11 @@ public static List<String> allowedOrderStrings() {
     }
 
     private final String field;
+
+    public static List<String> allowedFieldStrings() {
+        return Arrays.asList(SearchFields.RELEVANCE, SearchFields.NAME_SORT, SearchFields.RELEASE_OR_CREATE_DATE);
+    }
+
     private final String order;
 
     public SortBy(String field, String order) {

src/test/java/edu/harvard/iq/dataverse/api/DataRetrieverApiIT.java

@@ -7,6 +7,8 @@
 import edu.harvard.iq.dataverse.util.BundleUtil;
 
 import io.restassured.path.json.JsonPath;
+import org.hamcrest.CoreMatchers;
+import org.hamcrest.Matchers;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 
@@ -407,6 +409,108 @@ public void testRetrieveMyDataAsJsonStringSortOrder() {
         assertEquals(OK.getStatusCode(), deleteSuperUserResponse.getStatusCode());
     }
 
+    @Test
+    public void testRetrieveMyDataWithMetadataFields() {
+
+        Response createUser = UtilIT.createRandomUser();
+        createUser.prettyPrint();
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.prettyPrint();
+        String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse);
+
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.prettyPrint();
+
+        Response myDataWithAuthor = UtilIT.retrieveMyDataAsJsonString(apiToken, "", new ArrayList<>(Arrays.asList(6L)), "&metadata_fields=citation:author");
+        myDataWithAuthor.prettyPrint();
+        myDataWithAuthor.then().assertThat()
+                .body("data.items[0].metadataBlocks.citation.displayName", CoreMatchers.equalTo("Citation Metadata"))
+                .body("data.items[0].metadataBlocks.citation.fields[0].typeName", CoreMatchers.equalTo("author"))
+                .body("data.items[0].metadataBlocks.citation.fields[0].value[0].authorName.value", CoreMatchers.equalTo("Finch, Fiona"))
+                .body("data.items[0].metadataBlocks.citation.fields[0].value[0].authorAffiliation.value", CoreMatchers.equalTo("Birds Inc."))
+                .statusCode(OK.getStatusCode());
+
+        Response subFieldsNotSupported = UtilIT.retrieveMyDataAsJsonString(apiToken, "", new ArrayList<>(Arrays.asList(6L)), "&metadata_fields=citation:authorAffiliation");
+        subFieldsNotSupported.prettyPrint();
+        subFieldsNotSupported.then().assertThat()
+                .body("data.items[0].metadataBlocks.citation.displayName", CoreMatchers.equalTo("Citation Metadata"))
+                // No fields returned. authorAffiliation is a subfield of author and not supported.
+                .body("data.items[0].metadataBlocks.citation.fields", Matchers.empty())
+                .statusCode(OK.getStatusCode());
+
+        Response myDataWithAllFieldsFromCitation = UtilIT.retrieveMyDataAsJsonString(apiToken, "", new ArrayList<>(Arrays.asList(6L)), "&metadata_fields=citation:*");
+        // Many more fields printed
+        myDataWithAllFieldsFromCitation.prettyPrint();
+        myDataWithAllFieldsFromCitation.then().assertThat()
+                .body("data.items[0].metadataBlocks.citation.displayName", CoreMatchers.equalTo("Citation Metadata"))
+                // Many fields returned, all of the citation block that has been filled in.
+                .body("data.items[0].metadataBlocks.citation.fields", Matchers.hasSize(5))
+                .statusCode(OK.getStatusCode());
+
+    }
+
+    @Test
+    public void testRetrieveMyDataWithCollections() {
+        Response createUser = UtilIT.createRandomUser();
+        String apiToken = UtilIT.getApiTokenFromResponse(createUser);
+
+        Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken);
+        createDataverseResponse.prettyPrint();
+        createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        JsonPath createdDataverse = JsonPath.from(createDataverseResponse.body().asString());
+        String dataverseName = createdDataverse.getString("data.name");
+        String dataverseAlias = createdDataverse.getString("data.alias");
+        Integer dataverseId = createdDataverse.getInt("data.id");
+
+        UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken).then().assertThat().statusCode(OK.getStatusCode());
+
+        Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken);
+        createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode());
+        JsonPath createdDataset = JsonPath.from(createDatasetResponse.body().asString());
+        int datasetId = createdDataset.getInt("data.id");
+        String datasetPid = createdDataset.getString("data.persistentId");
+
+        UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken).then().assertThat().statusCode(OK.getStatusCode());
+
+        // Test that the Dataverse collection that the dataset was created in is returned
+        Response myDataResponse = UtilIT.retrieveMyDataAsJsonString(apiToken, "", new ArrayList<>(Arrays.asList(6L)), "&show_collections=true");
+        myDataResponse.prettyPrint();
+        myDataResponse.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data.items[0].collections.size()", CoreMatchers.is(1))
+                .body("data.items[0].collections[0].id", CoreMatchers.is(dataverseId))
+                .body("data.items[0].collections[0].name", CoreMatchers.is(dataverseName))
+                .body("data.items[0].collections[0].alias", CoreMatchers.is(dataverseAlias));
+
+        Response createDataverse2Response = UtilIT.createRandomDataverse(apiToken);
+        createDataverse2Response.prettyPrint();
+        createDataverse2Response.then().assertThat().statusCode(CREATED.getStatusCode());
+        JsonPath createDataverse2 = JsonPath.from(createDataverse2Response.body().asString());
+        String dataverse2Name = createDataverse2.getString("data.name");
+        String dataverse2Alias = createDataverse2.getString("data.alias");
+        Integer dataverse2Id = createDataverse2.getInt("data.id");
+
+        UtilIT.publishDataverseViaNativeApi(dataverse2Alias, apiToken).then().assertThat().statusCode(OK.getStatusCode());
+
+        UtilIT.linkDataset(datasetPid, dataverse2Alias, apiToken).then().assertThat().statusCode(OK.getStatusCode());
+
+        UtilIT.sleepForReindex(String.valueOf(datasetId), apiToken, 5);
+
+        // Test that the Dataverse collection that the dataset was linked to is also returned
+        myDataResponse = UtilIT.retrieveMyDataAsJsonString(apiToken, "", new ArrayList<>(Arrays.asList(6L)), "&show_collections=true");
+        myDataResponse.prettyPrint();
+        myDataResponse.then().assertThat()
+                .statusCode(OK.getStatusCode())
+                .body("data.items[0].collections.size()", CoreMatchers.is(2))
+                .body("data.items[0].collections", CoreMatchers.hasItems(
+                        Map.of("id", dataverseId, "name", dataverseName, "alias", dataverseAlias),
+                        Map.of("id", dataverse2Id, "name", dataverse2Name, "alias", dataverse2Alias)
+                ));
+
+    }
+
     private static String prettyPrintError(String resourceBundleKey, List<String> params) {
         final String errorMessage;
         if (params == null || params.isEmpty()) {

src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java

@@ -4126,17 +4126,21 @@ static Response importDatasetViaNativeApi(String apiToken, String dataverseAlias
     }
 
 
-    static Response retrieveMyDataAsJsonString(String apiToken, String userIdentifier, ArrayList<Long> roleIds) {
+    static Response retrieveMyDataAsJsonString(String apiToken, String userIdentifier, ArrayList<Long> roleIds, String parameterString) {
         Response response = given()
                 .header(API_TOKEN_HTTP_HEADER, apiToken)
                 .contentType("application/json; charset=utf-8")
                 .queryParam("role_ids", roleIds)
                 .queryParam("dvobject_types", MyDataFilterParams.defaultDvObjectTypes)
                 .queryParam("published_states", MyDataFilterParams.defaultPublishedStates)
-                .get("/api/mydata/retrieve?userIdentifier=" + userIdentifier);
+                .get("/api/mydata/retrieve?userIdentifier=" + userIdentifier + parameterString);
         return response;
     }
 
+    static Response retrieveMyDataAsJsonString(String apiToken, String userIdentifier, ArrayList<Long> roleIds) {
+        return retrieveMyDataAsJsonString(apiToken, userIdentifier, roleIds, "");
+    }
+
     static Response retrieveMyCollectionList(String apiToken, String userIdentifier) {
         RequestSpecification requestSpecification = given();
         if (apiToken != null) {