From 561aeeb7012e64fff6f2df7f6d0920607cdf9fb6 Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Thu, 19 May 2022 13:04:06 -0500
Subject: [PATCH 01/36] upgrade Puma

---
 Gemfile      | 2 +-
 Gemfile.lock | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/Gemfile b/Gemfile
index 88ac8b9..80c970e 100644
--- a/Gemfile
+++ b/Gemfile
@@ -11,7 +11,7 @@ gem 'rails', '~> 6.0.2'
 # Use sqlite3 as the database for Active Record
 gem 'sqlite3'
 # Use Puma as the app server
-gem 'puma', '~> 3.7'
+gem 'puma', '~> 4.3'
 # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder
 # gem 'jbuilder', '~> 2.5'
 # Use Redis adapter to run Action Cable in production
diff --git a/Gemfile.lock b/Gemfile.lock
index c38a145..d94298a 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -96,7 +96,8 @@ GEM
     nokogiri (1.13.6)
       mini_portile2 (~> 2.8.0)
       racc (~> 1.4)
-    puma (3.12.6)
+    puma (4.3.12)
+      nio4r (~> 2.0)
     racc (1.6.0)
     rack (2.2.3)
     rack-test (1.1.0)
@@ -168,7 +169,7 @@ DEPENDENCIES
   bootsnap
   byebug
   listen (>= 3.0.5, < 3.2)
-  puma (~> 3.7)
+  puma (~> 4.3)
   rails (~> 6.0.2)
   rest-client (>= 2.1.0.rc1, < 2.2)
   spring

From deb704ac4e5273980a620adf52b59fafa5e8bee5 Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Thu, 19 May 2022 13:13:40 -0500
Subject: [PATCH 02/36] add specific version of puma to avoid security warnings

---
 Gemfile      | 2 +-
 Gemfile.lock | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Gemfile b/Gemfile
index 80c970e..6d7da95 100644
--- a/Gemfile
+++ b/Gemfile
@@ -11,7 +11,7 @@ gem 'rails', '~> 6.0.2'
 # Use sqlite3 as the database for Active Record
 gem 'sqlite3'
 # Use Puma as the app server
-gem 'puma', '~> 4.3'
+gem 'puma', '>= 4.3.12'
 # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder
 # gem 'jbuilder', '~> 2.5'
 # Use Redis adapter to run Action Cable in production
diff --git a/Gemfile.lock b/Gemfile.lock
index d94298a..3c04aa9 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -169,7 +169,7 @@ DEPENDENCIES
   bootsnap
   byebug
   listen (>= 3.0.5, < 3.2)
-  puma (~> 4.3)
+  puma (>= 4.3.12)
   rails (~> 6.0.2)
   rest-client (>= 2.1.0.rc1, < 2.2)
   spring

From 065857a7bef10fe1152b769d6ae3e59fc1dcc3c3 Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Wed, 25 May 2022 10:46:36 -0500
Subject: [PATCH 03/36] update .ruby-version

---
 .ruby-version | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ruby-version b/.ruby-version
index 324db8d..4e34c4d 100644
--- a/.ruby-version
+++ b/.ruby-version
@@ -1 +1 @@
-ruby-2.6.8
+ruby-2.7.6

From 38b02b7a32e1a343d5ec714aff18df7cb930f382 Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Wed, 25 May 2022 10:50:33 -0500
Subject: [PATCH 04/36] update to later version of puma

---
 Gemfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gemfile b/Gemfile
index 6d7da95..4170559 100644
--- a/Gemfile
+++ b/Gemfile
@@ -11,7 +11,7 @@ gem 'rails', '~> 6.0.2'
 # Use sqlite3 as the database for Active Record
 gem 'sqlite3'
 # Use Puma as the app server
-gem 'puma', '>= 4.3.12'
+gem 'puma', '>= 5.6'
 # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder
 # gem 'jbuilder', '~> 2.5'
 # Use Redis adapter to run Action Cable in production

From e763bb1f0c9f98873a0cf7a004deff586214440c Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Wed, 25 May 2022 10:51:14 -0500
Subject: [PATCH 05/36] another round of gem updates

---
 Gemfile.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index 3c04aa9..3f1cd9c 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -69,7 +69,7 @@ GEM
     globalid (1.0.0)
       activesupport (>= 5.0)
     http-accept (1.7.0)
-    http-cookie (1.0.4)
+    http-cookie (1.0.5)
       domain_name (~> 0.5)
     i18n (1.10.0)
       concurrent-ruby (~> 1.0)
@@ -96,7 +96,7 @@ GEM
     nokogiri (1.13.6)
       mini_portile2 (~> 2.8.0)
       racc (~> 1.4)
-    puma (4.3.12)
+    puma (5.6.4)
       nio4r (~> 2.0)
     racc (1.6.0)
     rack (2.2.3)
@@ -169,7 +169,7 @@ DEPENDENCIES
   bootsnap
   byebug
   listen (>= 3.0.5, < 3.2)
-  puma (>= 4.3.12)
+  puma (>= 5.6)
   rails (~> 6.0.2)
   rest-client (>= 2.1.0.rc1, < 2.2)
   spring

From 2769319a13dfa6fe3877fcd6cc6c2dddbc7ae3bc Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Tue, 24 May 2022 13:40:03 -0500
Subject: [PATCH 06/36] add facet for matching nested facet

---
 app/services/search_item_req.rb | 39 +++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb
index e9260c0..64c683b 100644
--- a/app/services/search_item_req.rb
+++ b/app/services/search_item_req.rb
@@ -104,6 +104,45 @@ def facets
             "order" => { f_type => dir },
           }
         }
+      elsif f.include?("[")
+        #or nest it inside the next one?
+        #this will be the same
+        facet = f.split("[")
+        path = facet.split(".").first
+        condition = f[/(?<=\[).+?(?=\])/]
+        subject = condition.split(".").first
+        predicate = condition.split(".").last
+        aggs[f] = {
+          "nested" => {
+            "path" => path
+          },
+          "aggs" => {
+            "query" => {
+              "term" => {
+                subject => predicate
+              }
+            },
+            "aggs" => {
+              f => {
+                "terms" => {
+                  "field" => facet,
+                  "order" => { type => dir },
+                  "size" => size
+                },
+                "aggs" => {
+                  "top_matches" => {
+                    "top_hits" => {
+                      "_source" => {
+                        "includes" => [ f ]
+                      },
+                      "size" => 1
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
       # if nested, has extra syntax
       elsif f.include?(".")
         path = f.split(".").first

From 3b33d4afcf38391e9825e3f81b41dbe68837c6d9 Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Tue, 24 May 2022 14:31:49 -0500
Subject: [PATCH 07/36] add filter for matching nested facet

---
 app/services/search_item_req.rb | 37 ++++++++++++++++++++++++++++-----
 1 file changed, 32 insertions(+), 5 deletions(-)

diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb
index 64c683b..c58036d 100644
--- a/app/services/search_item_req.rb
+++ b/app/services/search_item_req.rb
@@ -104,9 +104,8 @@ def facets
             "order" => { f_type => dir },
           }
         }
+      #nested facet, matching on another nested facet
       elsif f.include?("[")
-        #or nest it inside the next one?
-        #this will be the same
         facet = f.split("[")
         path = facet.split(".").first
         condition = f[/(?<=\[).+?(?=\])/]
@@ -143,7 +142,7 @@ def facets
             }
           }
         }
-      # if nested, has extra syntax
+      # ordinary nested facet
       elsif f.include?(".")
         path = f.split(".").first
         aggs[f] = {
@@ -200,8 +199,36 @@ def filters
     # (type 2 will only be used for dates)
     filters = fields.map {|f| f.split(@@filter_separator, 3) }
     filters.each do |filter|
-      # NESTED FIELD FILTER
-      if filter[0].include?(".")
+      # NESTED matching
+      if filter[0].include?("[")
+        facet = f.split("[")
+        path = facet.split(".").first
+        condition = f[/(?<=\[).+?(?=\])/]
+        subject = condition.split(".").first
+        predicate = condition.split(".").last
+        # this is a nested field and must be treated differently
+        nested = {
+          "nested" => {
+      
+            "path" => path,
+            "query" => {
+              "bool" => {
+                "must" => {
+                  "term" => {
+                    # "person.name" => "oliver wendell holmes"
+                    # Remove CR's added by hidden input field values with returns
+                    facet => filter[1].gsub(/\r/, "")
+                    # "person.role" => "judge"
+                    subject => predicate
+                  }
+                }
+              }
+            }
+          }
+        }
+        filter_list << nested
+      #ordinary nested facet
+      elsif filter[0].include?(".")
         path = filter[0].split(".").first
         # this is a nested field and must be treated differently
         nested = {

From fa5c245dfeba7a951d335d95c94948e64566622b Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Tue, 24 May 2022 17:26:08 -0500
Subject: [PATCH 08/36] change split character, add missing comma

---
 app/services/search_item_req.rb | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb
index c58036d..f79894e 100644
--- a/app/services/search_item_req.rb
+++ b/app/services/search_item_req.rb
@@ -109,8 +109,8 @@ def facets
         facet = f.split("[")
         path = facet.split(".").first
         condition = f[/(?<=\[).+?(?=\])/]
-        subject = condition.split(".").first
-        predicate = condition.split(".").last
+        subject = condition.split("|").first
+        predicate = condition.split("|").last
         aggs[f] = {
           "nested" => {
             "path" => path
@@ -204,8 +204,8 @@ def filters
         facet = f.split("[")
         path = facet.split(".").first
         condition = f[/(?<=\[).+?(?=\])/]
-        subject = condition.split(".").first
-        predicate = condition.split(".").last
+        subject = condition.split("|").first
+        predicate = condition.split("|").last
         # this is a nested field and must be treated differently
         nested = {
           "nested" => {
@@ -217,7 +217,7 @@ def filters
                   "term" => {
                     # "person.name" => "oliver wendell holmes"
                     # Remove CR's added by hidden input field values with returns
-                    facet => filter[1].gsub(/\r/, "")
+                    facet => filter[1].gsub(/\r/, ""),
                     # "person.role" => "judge"
                     subject => predicate
                   }

From 21310ef0548c6b224da50c2b0f79f5ddd98747c2 Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Thu, 26 May 2022 11:24:05 -0500
Subject: [PATCH 09/36] parse the array for matching nested fields

---
 app/services/search_item_req.rb | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb
index f79894e..d63b755 100644
--- a/app/services/search_item_req.rb
+++ b/app/services/search_item_req.rb
@@ -84,7 +84,6 @@ def facets
     Array.wrap(@params["facet"]).each do |f|
       # histograms use a different ordering terminology than normal aggs
       f_type = type == "_term" ? "_key" : "_count"
-
       if f.include?("date") || f[/_d$/]
         # NOTE: if nested fields will ever have dates we will
         # need to refactor this to be available to both
@@ -105,13 +104,18 @@ def facets
           }
         }
       #nested facet, matching on another nested facet
+      
       elsif f.include?("[")
-        facet = f.split("[")
-        path = facet.split(".").first
-        condition = f[/(?<=\[).+?(?=\])/]
+        # will be an array including the original, and an alternate aggregation name
+        options = JSON.parse(f)
+        agg_name = options[1]
+        original = options[2]
+        facet = original.split("[")
+        path = original.split(".").first
+        condition = original[/(?<=\[).+?(?=\])/]
         subject = condition.split("|").first
         predicate = condition.split("|").last
-        aggs[f] = {
+        aggs[agg_name] = {
           "nested" => {
             "path" => path
           },
@@ -122,7 +126,7 @@ def facets
               }
             },
             "aggs" => {
-              f => {
+              agg_name => {
                 "terms" => {
                   "field" => facet,
                   "order" => { type => dir },
@@ -132,7 +136,7 @@ def facets
                   "top_matches" => {
                     "top_hits" => {
                       "_source" => {
-                        "includes" => [ f ]
+                        "includes" => [ agg_name ]
                       },
                       "size" => 1
                     }
@@ -201,9 +205,11 @@ def filters
     filters.each do |filter|
       # NESTED matching
       if filter[0].include?("[")
-        facet = f.split("[")
-        path = facet.split(".").first
-        condition = f[/(?<=\[).+?(?=\])/]
+        options = JSON.parse(f)
+        original = options[2]
+        facet = original.split("[")
+        path = original.split(".").first
+        condition = original[/(?<=\[).+?(?=\])/]
         subject = condition.split("|").first
         predicate = condition.split("|").last
         # this is a nested field and must be treated differently

From 4656dc6e104292fbd2982673d62321b738378710 Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Thu, 26 May 2022 16:21:47 -0500
Subject: [PATCH 10/36] change how compound facet name is parsed

---
 app/services/search_item_req.rb | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb
index d63b755..e8231a0 100644
--- a/app/services/search_item_req.rb
+++ b/app/services/search_item_req.rb
@@ -108,10 +108,10 @@ def facets
       elsif f.include?("[")
         # will be an array including the original, and an alternate aggregation name
         options = JSON.parse(f)
+        original = options[0]
         agg_name = options[1]
-        original = options[2]
-        facet = original.split("[")
-        path = original.split(".").first
+        facet = original.split("[")[0]
+        path = facet.split(".").first
         condition = original[/(?<=\[).+?(?=\])/]
         subject = condition.split("|").first
         predicate = condition.split("|").last
@@ -206,9 +206,9 @@ def filters
       # NESTED matching
       if filter[0].include?("[")
         options = JSON.parse(f)
-        original = options[2]
-        facet = original.split("[")
-        path = original.split(".").first
+        original = options[1]
+        facet = original.split("[")[0]
+        path = facet.split(".").first
         condition = original[/(?<=\[).+?(?=\])/]
         subject = condition.split("|").first
         predicate = condition.split("|").last

From 2b3af8e3e5a7b8ce6df764504dea4710e833bf48 Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Thu, 26 May 2022 16:22:44 -0500
Subject: [PATCH 11/36] use facet name as agg name

this is not the most semantically correct, but it doesn't break anything and orchid seems to expect it
---
 app/services/search_item_req.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb
index e8231a0..c30c9ea 100644
--- a/app/services/search_item_req.rb
+++ b/app/services/search_item_req.rb
@@ -115,7 +115,7 @@ def facets
         condition = original[/(?<=\[).+?(?=\])/]
         subject = condition.split("|").first
         predicate = condition.split("|").last
-        aggs[agg_name] = {
+        aggs[f] = {
           "nested" => {
             "path" => path
           },
@@ -126,7 +126,7 @@ def facets
               }
             },
             "aggs" => {
-              agg_name => {
+              f => {
                 "terms" => {
                   "field" => facet,
                   "order" => { type => dir },

From 100ac901e8271799ed4319edfdc0ab498ab391ad Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Thu, 26 May 2022 16:23:20 -0500
Subject: [PATCH 12/36] change query to filter

this is necessary to make this sort of query work
---
 app/services/search_item_req.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb
index c30c9ea..0981253 100644
--- a/app/services/search_item_req.rb
+++ b/app/services/search_item_req.rb
@@ -120,7 +120,7 @@ def facets
             "path" => path
           },
           "aggs" => {
-            "query" => {
+            "filter" => {
               "term" => {
                 subject => predicate
               }

From 3dd66085004562bad1060ed06073d73145354238 Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Fri, 27 May 2022 12:31:31 -0500
Subject: [PATCH 13/36] fix nested filter aggregation so it doesn't cause 400
 error

---
 app/services/search_item_req.rb | 44 +++++++++++++++++----------------
 1 file changed, 23 insertions(+), 21 deletions(-)

diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb
index 0981253..a3aa58d 100644
--- a/app/services/search_item_req.rb
+++ b/app/services/search_item_req.rb
@@ -115,30 +115,32 @@ def facets
         condition = original[/(?<=\[).+?(?=\])/]
         subject = condition.split("|").first
         predicate = condition.split("|").last
-        aggs[f] = {
+        aggs[agg_name] = {
           "nested" => {
             "path" => path
           },
           "aggs" => {
-            "filter" => {
-              "term" => {
-                subject => predicate
-              }
-            },
-            "aggs" => {
-              f => {
-                "terms" => {
-                  "field" => facet,
-                  "order" => { type => dir },
-                  "size" => size
-                },
-                "aggs" => {
-                  "top_matches" => {
-                    "top_hits" => {
-                      "_source" => {
-                        "includes" => [ agg_name ]
-                      },
-                      "size" => 1
+            agg_name => {
+              "filter" => {
+                "term" => {
+                  subject => predicate
+                }
+              },
+              "aggs" => {
+                agg_name => {
+                  "terms" => {
+                    "field" => facet,
+                    "order" => { type => dir },
+                    "size" => size
+                  },
+                  "aggs" => {
+                    "top_matches" => {
+                      "top_hits" => {
+                        "_source" => {
+                          "includes" => [ agg_name ]
+                        },
+                        "size" => 1
+                      }
                     }
                   }
                 }
@@ -220,7 +222,7 @@ def filters
             "query" => {
               "bool" => {
                 "must" => {
-                  "term" => {
+                  "terms" => {
                     # "person.name" => "oliver wendell holmes"
                     # Remove CR's added by hidden input field values with returns
                     facet => filter[1].gsub(/\r/, ""),

From fafb3a6db8d8c1746cf7e38de380f74635e54a72 Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Tue, 31 May 2022 12:14:30 -0500
Subject: [PATCH 14/36] check for deeper nesting of buckets

---
 app/services/search_item_res.rb | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/app/services/search_item_res.rb b/app/services/search_item_res.rb
index a82a199..04843d9 100644
--- a/app/services/search_item_res.rb
+++ b/app/services/search_item_res.rb
@@ -18,7 +18,6 @@ def build_response
     # strip out only the fields for the item response
     items = combine_highlights
     facets = reformat_facets
-
     {
       "code" => 200,
       "count" => count,
@@ -89,8 +88,7 @@ def reformat_facets
       facets = {}
       raw_facets.each do |field, info|
         facets[field] = {}
-        # nested fields do not have buckets at this level of response structure
-        buckets = info.key?("buckets") ? info["buckets"] : info.dig(field, "buckets")
+        buckets = get_buckets(info, field)
         if buckets
           buckets.each { |b| format_bucket_value(facets, field, b) }
         else
@@ -110,4 +108,18 @@ def remove_nonword_chars(term)
     transliterated.gsub(/<\/?(?:em|strong|u)>|\W/, "").downcase
   end
 
+  def get_buckets(info, field)
+    buckets = nil
+    # ordinary facet
+    if info.key?("buckets")
+      buckets = info["buckets"]
+    # nested facet
+    elsif info.dig(field, "buckets")
+      buckets = info.dig(field, "buckets")
+    # filtered facet
+    else
+      buckets = info.dig(field, field, "buckets")
+    end
+    buckets
+  end
 end

From 2ade65b6311e07d9654c9e5d7d6a939b77d2d080 Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Wed, 1 Jun 2022 09:35:50 -0500
Subject: [PATCH 15/36] Change separator

---
 app/services/search_item_req.rb | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb
index a3aa58d..e647119 100644
--- a/app/services/search_item_req.rb
+++ b/app/services/search_item_req.rb
@@ -113,8 +113,8 @@ def facets
         facet = original.split("[")[0]
         path = facet.split(".").first
         condition = original[/(?<=\[).+?(?=\])/]
-        subject = condition.split("|").first
-        predicate = condition.split("|").last
+        subject = condition.split("#").first
+        predicate = condition.split("#").last
         aggs[agg_name] = {
           "nested" => {
             "path" => path
@@ -212,8 +212,8 @@ def filters
         facet = original.split("[")[0]
         path = facet.split(".").first
         condition = original[/(?<=\[).+?(?=\])/]
-        subject = condition.split("|").first
-        predicate = condition.split("|").last
+        subject = condition.split("#").first
+        predicate = condition.split("#").last
         # this is a nested field and must be treated differently
         nested = {
           "nested" => {

From 3c223e4d94b11baf998c5370ee71b1c928b87987 Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Wed, 1 Jun 2022 09:36:21 -0500
Subject: [PATCH 16/36] Fix parsing and query for filter matching

---
 app/services/search_item_req.rb | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb
index e647119..2c39b5b 100644
--- a/app/services/search_item_req.rb
+++ b/app/services/search_item_req.rb
@@ -205,10 +205,9 @@ def filters
     # (type 2 will only be used for dates)
     filters = fields.map {|f| f.split(@@filter_separator, 3) }
     filters.each do |filter|
-      # NESTED matching
+      # filter aggregation with nesting
       if filter[0].include?("[")
-        options = JSON.parse(f)
-        original = options[1]
+        original = filter[0]
         facet = original.split("[")[0]
         path = facet.split(".").first
         condition = original[/(?<=\[).+?(?=\])/]
@@ -217,17 +216,14 @@ def filters
         # this is a nested field and must be treated differently
         nested = {
           "nested" => {
-      
             "path" => path,
             "query" => {
               "bool" => {
                 "must" => {
-                  "terms" => {
+                  "term" => {
                     # "person.name" => "oliver wendell holmes"
                     # Remove CR's added by hidden input field values with returns
-                    facet => filter[1].gsub(/\r/, ""),
-                    # "person.role" => "judge"
-                    subject => predicate
+                    facet => filter[1].gsub(/\r/, "")
                   }
                 }
               }

From ef223076cee6732c649d2f6e44db86067b445611 Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Thu, 2 Jun 2022 15:04:58 -0500
Subject: [PATCH 17/36] rewrite filtered aggregation to be either nested or not

---
 app/services/search_item_req.rb | 71 +++++++++++++++++++--------------
 1 file changed, 42 insertions(+), 29 deletions(-)

diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb
index 2c39b5b..81bd0ae 100644
--- a/app/services/search_item_req.rb
+++ b/app/services/search_item_req.rb
@@ -107,48 +107,61 @@ def facets
       
       elsif f.include?("[")
         # will be an array including the original, and an alternate aggregation name
+      
+
         options = JSON.parse(f)
         original = options[0]
         agg_name = options[1]
         facet = original.split("[")[0]
-        path = facet.split(".").first
+        # may or may not be nested
+        nested = facet.include?(".")
+        if nested
+          path = facet.split(".").first
+        end
         condition = original[/(?<=\[).+?(?=\])/]
         subject = condition.split("#").first
         predicate = condition.split("#").last
-        aggs[agg_name] = {
-          "nested" => {
-            "path" => path
-          },
-          "aggs" => {
-            agg_name => {
-              "filter" => {
-                "term" => {
-                  subject => predicate
-                }
-              },
-              "aggs" => {
-                agg_name => {
-                  "terms" => {
-                    "field" => facet,
-                    "order" => { type => dir },
-                    "size" => size
-                  },
-                  "aggs" => {
-                    "top_matches" => {
-                      "top_hits" => {
-                        "_source" => {
-                          "includes" => [ agg_name ]
-                        },
-                        "size" => 1
-                      }
+        aggregation = {
+            # common to nested and non-nested
+            "filter" => {
+              "term" => {
+                subject => predicate
+              }
+            },
+            "aggs" => {
+              agg_name => {
+                "terms" => {
+                  "field" => facet,
+                  "order" => { type => dir },
+                  "size" => size
+                },
+                "aggs" => {
+                  "top_matches" => {
+                    "top_hits" => {
+                      "_source" => {
+                        "includes" => [ agg_name ]
+                      },
+                      "size" => 1
                     }
                   }
                 }
               }
             }
           }
-        }
-      # ordinary nested facet
+        #interpolate above hash into nested query
+        if nested
+          aggs[agg_name] = {
+            "nested" => {
+              "path" => path
+            },
+            "aggs" => {
+              agg_name => aggregation
+            }
+          }
+        else
+          #otherwise it is the whole query
+          aggs[agg_name] = aggregation
+        end
       elsif f.include?(".")
         path = f.split(".").first
         aggs[f] = {

From 16a490d5d4b4749b85825740c491a7590b583ed7 Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Thu, 2 Jun 2022 16:05:44 -0500
Subject: [PATCH 18/36] filtering on a single item can either be nested or not

---
 app/services/search_item_req.rb | 35 +++++++++++++++++++--------------
 1 file changed, 20 insertions(+), 15 deletions(-)

diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb
index 81bd0ae..cd51f8f 100644
--- a/app/services/search_item_req.rb
+++ b/app/services/search_item_req.rb
@@ -222,28 +222,33 @@ def filters
       if filter[0].include?("[")
         original = filter[0]
         facet = original.split("[")[0]
-        path = facet.split(".").first
+        nested = facet.include?(".")
+        if nested
+          path = facet.split(".").first
+        end
         condition = original[/(?<=\[).+?(?=\])/]
         subject = condition.split("#").first
         predicate = condition.split("#").last
-        # this is a nested field and must be treated differently
-        nested = {
-          "nested" => {
-            "path" => path,
-            "query" => {
-              "bool" => {
-                "must" => {
-                  "term" => {
-                    # "person.name" => "oliver wendell holmes"
-                    # Remove CR's added by hidden input field values with returns
-                    facet => filter[1].gsub(/\r/, "")
-                  }
+        query = {
+          "term" => {
+            # "person.name" => "oliver wendell holmes"
+            # Remove CR's added by hidden input field values with returns
+            facet => filter[1].gsub(/\r/, "")
+          }
+        }
+        if nested
+          query = {
+            "nested" => {
+              "path" => path,
+              "query" => {
+                "bool" => {
+                  "must" => query
                 }
               }
             }
           }
-        }
-        filter_list << nested
+        end
+        filter_list << query
       #ordinary nested facet
       elsif filter[0].include?(".")
         path = filter[0].split(".").first

From b038be028989937f356c8e4f6f79d089b497e63a Mon Sep 17 00:00:00 2001
From: William Dewey <william.dewey@unl.edu>
Date: Mon, 26 Sep 2022 10:15:19 -0500
Subject: [PATCH 19/36] update config for server

---
 .ruby-gemset                       | 2 +-
 config/environments/development.rb | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.ruby-gemset b/.ruby-gemset
index eedd89b..fcf5595 100644
--- a/.ruby-gemset
+++ b/.ruby-gemset
@@ -1 +1 @@
-api
+api-v2
diff --git a/config/environments/development.rb b/config/environments/development.rb
index 1e22e09..14a0a40 100644
--- a/config/environments/development.rb
+++ b/config/environments/development.rb
@@ -61,4 +61,5 @@
   # CDRH CONFIGURATION
 
   config.hosts << "cdrhdev1.unl.edu"
+  config.hosts << "whitman-dev.unl.edu"
 end

From 6eaa38b74dedd8c2b84c1c98bb24549174518a7e Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Wed, 19 Oct 2022 12:03:51 -0500
Subject: [PATCH 20/36] revise query to match both the facet and the filter

---
 app/services/search_item_req.rb | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb
index cd51f8f..b34a75a 100644
--- a/app/services/search_item_req.rb
+++ b/app/services/search_item_req.rb
@@ -229,12 +229,13 @@ def filters
         condition = original[/(?<=\[).+?(?=\])/]
         subject = condition.split("#").first
         predicate = condition.split("#").last
-        query = {
-          "term" => {
-            # "person.name" => "oliver wendell holmes"
-            # Remove CR's added by hidden input field values with returns
-            facet => filter[1].gsub(/\r/, "")
-          }
+        term_match = {
+          # "person.name" => "oliver wendell holmes"
+          # Remove CR's added by hidden input field values with returns
+          facet => filter[1].gsub(/\r/, "")
+        }
+        term_filter = {
+          subject => predicate
         }
         if nested
           query = {
@@ -242,7 +243,10 @@ def filters
               "path" => path,
               "query" => {
                 "bool" => {
-                  "must" => query
+                  "must" => [
+                    { "match" => term_filter },
+                    { "match" => term_match }
+                  ]
                 }
               }
             }

From 88a8f80915f7528ded47734ff9328d4a651aaf6b Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Thu, 20 Oct 2022 09:57:32 -0500
Subject: [PATCH 21/36] use reverse nested agg for correct item count

---
 app/services/search_item_req.rb | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb
index b34a75a..e298836 100644
--- a/app/services/search_item_req.rb
+++ b/app/services/search_item_req.rb
@@ -51,6 +51,8 @@ def build_request
 
     # add bool to request body
     req["query"]["bool"] = bool
+    # uncomment below line to log ES query for debugging
+    # puts req.to_json()
     return req
   end
 
@@ -136,12 +138,17 @@ def facets
                   "size" => size
                 },
                 "aggs" => {
-                  "top_matches" => {
-                    "top_hits" => {
-                      "_source" => {
-                        "includes" => [ agg_name ]
-                      },
-                      "size" => 1
+                  "field_to_item" => {
+                    "reverse_nested" => {},
+                    "aggs" => {
+                      "top_matches" => {
+                        "top_hits" => {
+                          "_source" => {
+                            "includes" => [ agg_name ]
+                          },
+                          "size" => 1
+                        }
+                      }
                     }
                   }
                 }

From dd4bacdc941b14d8180e3307613b4099cf000621 Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Thu, 20 Oct 2022 10:01:28 -0500
Subject: [PATCH 22/36] used doc_count from reverse nested if it exists

---
 app/services/search_item_res.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/services/search_item_res.rb b/app/services/search_item_res.rb
index 04843d9..05210e7 100644
--- a/app/services/search_item_res.rb
+++ b/app/services/search_item_res.rb
@@ -65,7 +65,7 @@ def format_bucket_value(facets, field, bucket)
     # dates return in wonktastic ways, so grab key_as_string instead of gibberish number
     # but otherwise just grab the key if key_as_string unavailable
     key = bucket.key?("key_as_string") ? bucket["key_as_string"] : bucket["key"]
-    val = bucket["doc_count"]
+    val = bucket.key?("field_to_item") ? bucket["field_to_item"]["doc_count"] : bucket["doc_count"]
     source = key
     # top_matches is a top_hits aggregation which returns a list of terms
     # which were used for the facet.

From bd739e0fce4155b0df1588c583b0b5213314fa13 Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Fri, 21 Oct 2022 13:23:57 -0500
Subject: [PATCH 23/36] change key for new elasticsearch version

---
 app/services/search_item_req.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb
index e298836..497c84a 100644
--- a/app/services/search_item_req.rb
+++ b/app/services/search_item_req.rb
@@ -99,7 +99,7 @@ def facets
         aggs[f] = {
           "date_histogram" => {
             "field" => field,
-            "interval" => interval,
+            "calendar_interval" => interval,
             "format" => formatted,
             "min_doc_count" => 1,
             "order" => { f_type => dir },

From f0c3124cf43b982ea894a4bee00855944d65d589 Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Mon, 24 Oct 2022 12:51:22 -0500
Subject: [PATCH 24/36] change order query to avoid deprecated '_term'

---
 app/services/search_item_req.rb       | 10 +++++-----
 test/services/search_item_req_test.rb |  8 ++++----
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb
index 497c84a..56e6453 100644
--- a/app/services/search_item_req.rb
+++ b/app/services/search_item_req.rb
@@ -74,7 +74,7 @@ def facets
     dir = "desc"
     if @params["facet_sort"].present?
       sort_type, sort_dir = @params["facet_sort"].split(@@filter_separator)
-      type = "_term" if sort_type == "term"
+      type = "term" if sort_type == "term"
       dir = sort_dir if sort_dir == "asc"
     end
 
@@ -85,7 +85,7 @@ def facets
     aggs = {}
     Array.wrap(@params["facet"]).each do |f|
       # histograms use a different ordering terminology than normal aggs
-      f_type = type == "_term" ? "_key" : "_count"
+      f_type = (type == "term") ? "_key" : "_count"
       if f.include?("date") || f[/_d$/]
         # NOTE: if nested fields will ever have dates we will
         # need to refactor this to be available to both
@@ -134,7 +134,7 @@ def facets
               agg_name => {
                 "terms" => {
                   "field" => facet,
-                  "order" => { type => dir },
+                  "order" => {f_type => dir},
                   "size" => size
                 },
                 "aggs" => {
@@ -179,7 +179,7 @@ def facets
             f => {
               "terms" => {
                 "field" => f,
-                "order" => { type => dir },
+                "order" => {f_type => dir},
                 "size" => size
               },
               "aggs" => {
@@ -199,7 +199,7 @@ def facets
         aggs[f] = {
           "terms" => {
             "field" => f,
-            "order" => { type => dir },
+            "order" => { f_type => dir },
             "size" => size
           },
           "aggs" => {
diff --git a/test/services/search_item_req_test.rb b/test/services/search_item_req_test.rb
index 29bf323..c4d8197 100644
--- a/test/services/search_item_req_test.rb
+++ b/test/services/search_item_req_test.rb
@@ -44,7 +44,7 @@ def test_facets
       "facet" => [ "title", "subcategory" ]
     }).facets
     assert_equal(
-      {"title"=>{"terms"=>{"field"=>"title", "order"=>{"_term"=>"asc"}, "size"=>10}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["title"]}, "size"=>1}}}}, "subcategory"=>{"terms"=>{"field"=>"subcategory", "order"=>{"_term"=>"asc"}, "size"=>10}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["subcategory"]}, "size"=>1}}}}},
+      {"title"=>{"terms"=>{"field"=>"title", "order"=>"asc", "size"=>10}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["title"]}, "size"=>1}}}}, "subcategory"=>{"terms"=>{"field"=>"subcategory", "order"=>"asc", "size"=>10}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["subcategory"]}, "size"=>1}}}}},
       facets
     ) 
 
@@ -69,7 +69,7 @@ def test_facets
       "facet" => [ "creator.name" ]
     }).facets
     assert_equal(
-      {"creator.name"=>{"nested"=>{"path"=>"creator"}, "aggs"=>{"creator.name"=>{"terms"=>{"field"=>"creator.name", "order"=>{"_term"=>"desc"}, "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["creator.name"]}, "size"=>1}}}}}}},
+      {"creator.name"=>{"nested"=>{"path"=>"creator"}, "aggs"=>{"creator.name"=>{"terms"=>{"field"=>"creator.name", "order"=>"desc", "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["creator.name"]}, "size"=>1}}}}}}},
       facets
     )
 
@@ -83,14 +83,14 @@ def test_facets
     # sort term order specified
     facets = SearchItemReq.new({ "facet" => ["title", "format"], "facet_sort" => "term|desc" }).facets
     assert_equal(
-      {"title"=>{"terms"=>{"field"=>"title", "order"=>{"_term"=>"desc"}, "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["title"]}, "size"=>1}}}}, "format"=>{"terms"=>{"field"=>"format", "order"=>{"_term"=>"desc"}, "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["format"]}, "size"=>1}}}}},
+      {"title"=>{"terms"=>{"field"=>"title", "order"=>"desc", "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["title"]}, "size"=>1}}}}, "format"=>{"terms"=>{"field"=>"format", "order"=>"desc", "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["format"]}, "size"=>1}}}}},
       facets
     )
 
     # sort term no order specified
     facets = SearchItemReq.new({ "facet" => ["title", "format"], "facet_sort" => "term" }).facets
     assert_equal(
-      {"title"=>{"terms"=>{"field"=>"title", "order"=>{"_term"=>"desc"}, "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["title"]}, "size"=>1}}}}, "format"=>{"terms"=>{"field"=>"format", "order"=>{"_term"=>"desc"}, "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["format"]}, "size"=>1}}}}},
+      {"title"=>{"terms"=>{"field"=>"title", "order"=>"desc", "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["title"]}, "size"=>1}}}}, "format"=>{"terms"=>{"field"=>"format", "order"=>"desc", "size"=>20}, "aggs"=>{"top_matches"=>{"top_hits"=>{"_source"=>{"includes"=>["format"]}, "size"=>1}}}}},
       facets
     )
 

From 7441a9626ac5c223b5c6ac125d0538051b82ebac Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Wed, 26 Oct 2022 09:06:39 -0500
Subject: [PATCH 25/36] gitignore master key

---
 .gitignore | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitignore b/.gitignore
index 76fb9c4..a3bd71e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -48,3 +48,5 @@ bower.json
 .byebug_history
 
 .DS_Store
+
+/config/master.key

From 255f9db6ff73738cd528591d404f292f6fcce33f Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Wed, 26 Oct 2022 09:26:15 -0500
Subject: [PATCH 26/36] add basic auth to elasticsearch requests

---
 app/controllers/application_controller.rb | 3 ++-
 app/services/search_service.rb            | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index 15ff0d8..e97e006 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -3,7 +3,8 @@
 class ApplicationController < ActionController::API
 
   def post_search(json, error_method=method(:display_error))
-    res = RestClient.post("#{ES_URI}/_search", json.to_json, { "content-type" => "json" })
+    auth_hash = { "Authorization" => "Basic #{Base64::encode64("#{ES_USER}:#{ES_PASSWORD}")}" }
+    res = RestClient.post("#{ES_URI}/_search", json.to_json, auth_hash.merge({ "content-type" => "json" }))
     raise
     return JSON.parse(res.body)
   rescue => e
diff --git a/app/services/search_service.rb b/app/services/search_service.rb
index dbd8877..319b97b 100644
--- a/app/services/search_service.rb
+++ b/app/services/search_service.rb
@@ -11,7 +11,8 @@ def initialize(url, params={}, user_req)
   end
 
   def post(url_ending, json)
-    res = RestClient.post("#{@url}/#{url_ending}", json.to_json, { "content-type" => "json" } )
+    auth_hash = { "Authorization" => "Basic #{Base64::encode64("#{Rails.application.credentials.elasticsearch[:user]}:#{Rails.application.credentials.elasticsearch[:password]}")}" }
+    res = RestClient.post("#{@url}/#{url_ending}", json.to_json, auth_hash.merge({ "content-type" => "json" } ))
     JSON.parse(res.body)
   rescue => e
     e

From b8ab6b0ab42219f7daca11226f37caa610cdfef3 Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Fri, 28 Oct 2022 11:58:55 -0500
Subject: [PATCH 27/36] raise number of results per facet

---
 config/config.example.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/config.example.yml b/config/config.example.yml
index a06b13c..c0ce5d4 100644
--- a/config/config.example.yml
+++ b/config/config.example.yml
@@ -18,7 +18,7 @@ default: &default
     # highlight fragment number
     hl_num: 3
     # number of results returned per "page"
-    num: 20
+    num: 5000
     # default sorting field and direction (field|asc)
     sort_fl: "identifier|asc"
     # starting document in results list (0 == 1st result)

From 2c68524853eaac7cc2d21e7ff64c6f217a45fd72 Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Mon, 31 Oct 2022 11:40:32 -0500
Subject: [PATCH 28/36] use facet_limit instead of facet_num to match Orchid

---
 app/services/search_item_req.rb | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb
index 56e6453..749af2f 100644
--- a/app/services/search_item_req.rb
+++ b/app/services/search_item_req.rb
@@ -79,8 +79,7 @@ def facets
     end
 
     # FACET_SETTINGS["start"]
-    size = SETTINGS["num"]
-    size = @params["facet_num"].blank? ? SETTINGS["num"] : @params["facet_num"]
+    size = @params["facet_limit"].blank? ? SETTINGS["num"] : @params["facet_limit"]
 
     aggs = {}
     Array.wrap(@params["facet"]).each do |f|

From 303598f02ef88164194c09972eb80e2389b3609c Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Mon, 31 Oct 2022 11:40:50 -0500
Subject: [PATCH 29/36] revert, will set in Orchid

---
 config/config.example.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/config.example.yml b/config/config.example.yml
index c0ce5d4..a06b13c 100644
--- a/config/config.example.yml
+++ b/config/config.example.yml
@@ -18,7 +18,7 @@ default: &default
     # highlight fragment number
     hl_num: 3
     # number of results returned per "page"
-    num: 5000
+    num: 20
     # default sorting field and direction (field|asc)
     sort_fl: "identifier|asc"
     # starting document in results list (0 == 1st result)

From 2eed5b27505e5623bb7f6bb4abee4f7a535cc096 Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Tue, 1 Nov 2022 15:59:45 -0500
Subject: [PATCH 30/36] change facet_num to facet_limit

---
 test/services/search_item_req_test.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/services/search_item_req_test.rb b/test/services/search_item_req_test.rb
index c4d8197..a5b3eab 100644
--- a/test/services/search_item_req_test.rb
+++ b/test/services/search_item_req_test.rb
@@ -39,7 +39,7 @@ def test_facets
 
     # normal with pagination overrides, multiple facets
     facets = SearchItemReq.new({
-      "facet_num" => 10,
+      "facet_limit" => 10,
       "facet_sort" => "term|asc",
       "facet" => [ "title", "subcategory" ]
     }).facets
@@ -50,7 +50,7 @@ def test_facets
 
     # should be blank if there are no facets provided
     facets = SearchItemReq.new({
-      "facet_num" => 1,
+      "facet_limit" => 1,
       "facet_sort" => "nonterm|asc",
       "facet" => []
     }).facets

From a7c4a49eb4d9130000a59790f421ddd0475d1672 Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Thu, 10 Nov 2022 09:56:24 -0600
Subject: [PATCH 31/36] update nested facets documentation

---
 docs/README.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docs/README.md b/docs/README.md
index 8aa9ad6..cd2fc90 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -50,6 +50,12 @@ __Nested fields__
 facet[]=creator.name
 facet[]=creator.name&facet[]=creator.role
 ```
+you can also match on another nested field with the new API schema
+`facet[]=nested_field.keyword_field1[nested_field.keyword_field2#value]`
+```
+facet[]=person.name[person.role#judge]
+```
+the above will select all names of persons, where the role of that person is "judge".
 
 __Date ranges__ (currently supports days or years)
 

From 0512771a692ffbb6df4d988574b232a4b411de90 Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Thu, 10 Nov 2022 13:55:56 -0600
Subject: [PATCH 32/36] add links to more detailed documentation

---
 CHANGELOG.md | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ad8bf5d..2a0a34e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -35,7 +35,8 @@ Markdown Spec](https://github.github.com/gfm/).
 - "api_version" added to all response "res" objects
 
 ### Changed
-- upgraded to Rails 6
+- upgraded to Rails 6.1.7 and Ruby 3
+- changes reflect new api schemas in Datura, which make heavy use of nested fields
 - Added support for aggregating buckets by normalized keyword and returning
   the "top_hits" first document result for a non-normalized display
 - Changes response format of `facets` key
@@ -56,6 +57,20 @@ Markdown Spec](https://github.github.com/gfm/).
   Not only is the response format itself different, but there may be fewer
   facets returned since normalized values which match are combined
 
+### Migration
+- in Datura repos config `private.yml` api to `"api_version": "2.0"` to take advantage of new bucket aggregation functionality (or `"api_version": "1.0"` for legacy repos that have not been updated for the new schema). Please note that a running API index can only use one ES index at a time, and each ES index is restricted to one version of the schema. See new schema (2.0) documentation [here](https://github.com/CDRH/datura/docs/schema_v2.md)
+- Use Elasticsearch 8.5 or later. See [dev docs instructions](https://github.com/CDRH/cdrh_dev_docs/blob/update_elasticsearch_documentation/publishing/2_basic_requirements.md#downloading-elasticsearch).
+- If you are using ES with security enabled, you must configure credentials with Rails in the API repo. See https://guides.rubyonrails.org/v6.1/security.html. Configure the VSCode editor. Run `EDITOR="code --wait" rails credentials:edit` and add
+```
+elasticsearch:
+  user: username
+  password: *****
+```
+to the secrets file and then close the window to save. Do not commit `config/master.key` (it should be in `gitignore`)
+- Orchid apps that connect to the API should use `facet_limit` instead of `facet_num` in options.
+- Add nested facets as described above, if desired.
+
+
 ## [v1.0.4](https://github.com/CDRH/api/compare/v1.0....v1.0.4) - Updates & license
 
 ### Changed

From e87e40f22528d2178ef22f27907810f5cb1235aa Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Thu, 10 Nov 2022 14:24:33 -0600
Subject: [PATCH 33/36] clarify

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2a0a34e..420afa1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -58,7 +58,7 @@ Markdown Spec](https://github.github.com/gfm/).
   facets returned since normalized values which match are combined
 
 ### Migration
-- in Datura repos config `private.yml` api to `"api_version": "2.0"` to take advantage of new bucket aggregation functionality (or `"api_version": "1.0"` for legacy repos that have not been updated for the new schema). Please note that a running API index can only use one ES index at a time, and each ES index is restricted to one version of the schema. See new schema (2.0) documentation [here](https://github.com/CDRH/datura/docs/schema_v2.md)
+- in the config files of your Datura repos, (`private.yml` or `public.yml`, set the api to `"api_version": "2.0"` to take advantage of new bucket aggregation functionality (or `"api_version": "1.0"` for legacy repos that have not been updated for the new schema). Please note that a running API index can only use one ES index at a time, and each ES index is restricted to one version of the schema. See new schema (2.0) documentation [here](https://github.com/CDRH/datura/docs/schema_v2.md).
 - Use Elasticsearch 8.5 or later. See [dev docs instructions](https://github.com/CDRH/cdrh_dev_docs/blob/update_elasticsearch_documentation/publishing/2_basic_requirements.md#downloading-elasticsearch).
 - If you are using ES with security enabled, you must configure credentials with Rails in the API repo. See https://guides.rubyonrails.org/v6.1/security.html. Configure the VSCode editor. Run `EDITOR="code --wait" rails credentials:edit` and add
 ```

From 68f14debda1f13b213c250dad57c760c32dfec19 Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Thu, 25 May 2023 09:15:54 -0500
Subject: [PATCH 34/36] use reverse nested on simple nested aggregations

fixes #141
---
 app/services/search_item_req.rb | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb
index 749af2f..b76f5d3 100644
--- a/app/services/search_item_req.rb
+++ b/app/services/search_item_req.rb
@@ -182,12 +182,17 @@ def facets
                 "size" => size
               },
               "aggs" => {
-                "top_matches" => {
-                  "top_hits" => {
-                    "_source" => {
-                      "includes" => [ f ]
-                    },
-                    "size" => 1
+                "field_to_item" => {
+                  "reverse_nested" => {},
+                  "aggs" => {
+                    "top_matches" => {
+                      "top_hits" => {
+                        "_source" => {
+                          "includes" => [ agg_name ]
+                        },
+                        "size" => 1
+                      }
+                    }
                   }
                 }
               }

From b1632cf4cf4e99e5d22df87d26a406b67859db56 Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Fri, 26 May 2023 08:13:17 -0500
Subject: [PATCH 35/36] fix elasticsearch errors

---
 app/services/search_item_req.rb | 2 +-
 app/services/search_item_res.rb | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/services/search_item_req.rb b/app/services/search_item_req.rb
index b76f5d3..1475e87 100644
--- a/app/services/search_item_req.rb
+++ b/app/services/search_item_req.rb
@@ -188,7 +188,7 @@ def facets
                     "top_matches" => {
                       "top_hits" => {
                         "_source" => {
-                          "includes" => [ agg_name ]
+                          "includes" => [ f ]
                         },
                         "size" => 1
                       }
diff --git a/app/services/search_item_res.rb b/app/services/search_item_res.rb
index 05210e7..49b9c92 100644
--- a/app/services/search_item_res.rb
+++ b/app/services/search_item_res.rb
@@ -78,7 +78,7 @@ def format_bucket_value(facets, field, bucket)
     end
     facets[field][key] = {
       "num" => val,
-      "source" => source
+      "source" => source.to_s
     }
   end
 

From b2102340b4718b9b19e0355fd31b005a617d9e7c Mon Sep 17 00:00:00 2001
From: William Dewey <wkdewey@gmail.com>
Date: Fri, 21 Jul 2023 14:05:26 -0500
Subject: [PATCH 36/36] titleize bucket values because ES automatically
 lowercases them

---
 app/services/search_item_res.rb | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/app/services/search_item_res.rb b/app/services/search_item_res.rb
index 49b9c92..741e736 100644
--- a/app/services/search_item_res.rb
+++ b/app/services/search_item_res.rb
@@ -49,9 +49,9 @@ def find_source_from_top_hits(top_hits, field, key)
     if hit.class == Array
       # I don't love this, because we will have to match exactly the logic
       # that got us the key to get this to work
-      match_index = hit
-        .map { |s| remove_nonword_chars(s) }
-        .index(remove_nonword_chars(key))
+        match_index = hit
+          .map { |s| remove_nonword_chars(s) }
+          .index(remove_nonword_chars(key))
       # if nothing matches the original key, return the entire source hit
       # should return a string, regardless
       return match_index ? hit[match_index] : hit.join(" ")
@@ -64,7 +64,7 @@ def find_source_from_top_hits(top_hits, field, key)
   def format_bucket_value(facets, field, bucket)
     # dates return in wonktastic ways, so grab key_as_string instead of gibberish number
     # but otherwise just grab the key if key_as_string unavailable
-    key = bucket.key?("key_as_string") ? bucket["key_as_string"] : bucket["key"]
+    key = bucket.key?("key_as_string") ? bucket["key_as_string"].titleize : bucket["key"].titleize
     val = bucket.key?("field_to_item") ? bucket["field_to_item"]["doc_count"] : bucket["doc_count"]
     source = key
     # top_matches is a top_hits aggregation which returns a list of terms