Refine product check behaviour (#5913)

* Rework transient errors

* Update tests

* Update docs

* Cleanup

* Hide ProductCheckStatus on implementations

* Further cleanup during testing

* Reset scratch pad

* Fix comments

* Ensure we raise product check error as neccesary

This ensures that if the consumer catches the exception and reuses the client which has previously failed the check, the failure exception is raised.

* Update bulk all test which now fails due to product check

Author: stevejgordon

docs/client-concepts/connection-pooling/exceptions/unrecoverable-exceptions.asciidoc

@@ -58,6 +58,7 @@ foreach (var failure in failures)
         case PipelineFailure.MaxRetriesReached:
         case PipelineFailure.Unexpected:
         case PipelineFailure.NoNodesAttempted:
+        case PipelineFailure.FailedProductCheck:
             var unrecoverable = new PipelineException(failure);
             unrecoverable.Recoverable.Should().BeFalse(failure.GetStringValue());
             break;

docs/client-concepts/connection-pooling/product-checking/product-check-at-startup.asciidoc

@@ -18,14 +18,15 @@ please modify the original csharp file found at the link and submit the PR with
 Since v7.14.0, the client performs a required product check before the first call.
 This pre-flight product check allows the client to establish the version of Elasticsearch that it is communicating with.
 
-The product check requires one additional HTTP request to be sent to the server as part of the request pipeline before
-the main API call is sent. In most cases, this will succeed during the very first API call that the client sends.
+The product check generally requires a single additional HTTP request to be sent to the server as part of the request pipeline before
+the first API call is sent. In most cases, this will succeed during the first API call that the client sends.
 Once the product check succeeds, no further product check HTTP requests are sent for subsequent API calls.
 
 [source,csharp]
 ----
 var audit = new Auditor(() => VirtualClusterWith
     .Nodes(1)
+    .ProductCheck(r => r.SucceedAlways())
     .ClientCalls(r => r.SucceedAlways())
     .StaticConnectionPool()
     .Settings(s => s.DisablePing())
@@ -52,7 +53,7 @@ Here's an example with a single node cluster which fails for some reason during
 ----
 var audit = new Auditor(() => VirtualClusterWith
     .Nodes(1, productCheckAlwaysSucceeds: false)
-    .ProductCheck(r => r.Fails(TimesHelper.Once))
+    .ProductCheck(r => r.Fails(TimesHelper.Once, 429))
     .ProductCheck(r => r.SucceedAlways())
     .ClientCalls(r => r.SucceedAlways())
     .StaticConnectionPool()
@@ -62,31 +63,30 @@ var audit = new Auditor(() => VirtualClusterWith
 audit = await audit.TraceCalls(skipProductCheck: false,
     new ClientCall() {
         { ProductCheckOnStartup },
-        { ProductCheckFailure, 9200 }, <1>
-        { HealthyResponse, 9200 } <2>
+        { ProductCheckFailure, 9200 } <1>
     },
     new ClientCall() {
         { ProductCheckOnStartup },
-        { ProductCheckSuccess, 9200 }, <3>
-        { HealthyResponse, 9200 }
+        { ProductCheckSuccess, 9200 }, <2>
+        { HealthyResponse, 9200 } <3>
     },
     new ClientCall() {
         { HealthyResponse, 9200 } <4>
     }
 );
 ----
-<1> as this is the first call, the product check is executed, but fails
-<2> the actual request is still sent and succeeds
-<3> as the previous product check failed, it runs again on the second call
+<1> as this is the first call, the product check is executed, but times out
+<2> as the previous product check failed, it runs again on the second call and this time it succeeds
+<3> this time the main API call is sent and also succeeds
 <4> subsequent calls no longer perform product check
 
-Here's an example with a three node cluster which fails for some reason during the first and second product check attempts. 
+Here's an example with a three node cluster which fails (due to too many requests) during the first and second product check attempts. 
 
 [source,csharp]
 ----
 var audit = new Auditor(() => VirtualClusterWith
     .Nodes(3, productCheckAlwaysSucceeds: false)
-    .ProductCheck(r => r.FailAlways())
+    .ProductCheck(r => r.FailAlways(429))
     .ProductCheck(r => r.OnPort(9202).SucceedAlways())
     .ClientCalls(r => r.SucceedAlways())
     .StaticConnectionPool()

src/Elasticsearch.Net/ConnectionPool/SingleNodeConnectionPool.cs

@@ -27,9 +27,6 @@ public SingleNodeConnectionPool(Uri uri, IDateTimeProvider dateTimeProvider = nu
 		/// <inheritdoc />
 		public IReadOnlyCollection<Node> Nodes { get; }
 
-		/// <inheritdoc />
-		public ProductCheckStatus ProductCheckStatus { get; set; } = ProductCheckStatus.NotChecked;
-
 		/// <inheritdoc />
 		public bool SniffedOnStartup
 		{
@@ -46,6 +43,9 @@ public bool SniffedOnStartup
 		/// <inheritdoc />
 		public bool UsingSsl { get; }
 
+		/// <inheritdoc />
+		ProductCheckStatus IConnectionPool.ProductCheckStatus { get; set; } = ProductCheckStatus.NotChecked;
+
 		/// <inheritdoc />
 		public IEnumerable<Node> CreateView(Action<AuditEvent, Node> audit = null) => Nodes;
 

src/Elasticsearch.Net/ConnectionPool/StaticConnectionPool.cs

@@ -48,9 +48,6 @@ protected StaticConnectionPool(IEnumerable<Node> nodes, Func<Node, float> nodeSc
 		/// <inheritdoc />
 		public virtual IReadOnlyCollection<Node> Nodes => InternalNodes;
 
-		/// <inheritdoc />
-		public ProductCheckStatus ProductCheckStatus { get; set; } = ProductCheckStatus.NotChecked;
-
 		/// <inheritdoc />
 		public bool SniffedOnStartup { get; set; }
 
@@ -80,6 +77,9 @@ protected List<Node> AliveNodes
 		protected Random Random { get; }
 		protected bool Randomize { get; }
 
+		/// <inheritdoc />
+		ProductCheckStatus IConnectionPool.ProductCheckStatus { get; set; } = ProductCheckStatus.NotChecked;
+
 		/// <summary>
 		/// Creates a view of all the live nodes with changing starting positions that wraps over on each call
 		/// e.g Thread A might get 1,2,3,4,5 and thread B will get 2,3,4,5,1.

src/Elasticsearch.Net/Transport/Pipeline/InvalidProductException.cs

@@ -8,7 +8,10 @@ namespace Elasticsearch.Net
 {
 	public class InvalidProductException : Exception
 	{
-		public InvalidProductException()
-			: base(@"The client noticed that the server is not Elasticsearch and we do not support this unknown product.") { }
+		internal const string InvalidProductError = "The client noticed that the server is not Elasticsearch and we do not support this unknown product.";
+		internal const string InvalidBuildFlavorError = "The client noticed that the server is not a supported distribution of Elasticsearch.";
+
+		public InvalidProductException(string error)
+			: base(error) { }
 	}
 }

src/Elasticsearch.Net/Transport/Pipeline/PipelineException.cs

@@ -19,7 +19,7 @@ public PipelineException(PipelineFailure failure, Exception innerException)
 
 		public bool Recoverable =>
 			FailureReason is PipelineFailure.BadRequest or PipelineFailure.BadResponse or PipelineFailure.PingFailure;
-
+		
 		public IElasticsearchResponse Response { get; internal set; }
 
 		private static string GetMessage(PipelineFailure failure)
@@ -37,6 +37,8 @@ private static string GetMessage(PipelineFailure failure)
 				case PipelineFailure.MaxRetriesReached: return "The call was retried the configured maximum amount of times";
 				case PipelineFailure.NoNodesAttempted:
 					return "No nodes were attempted, this can happen when a node predicate does not match any nodes";
+				case PipelineFailure.FailedProductCheck:
+					return RequestPipeline.ProductCheckTransientErrorWarning;
 				case PipelineFailure.Unexpected:
 				default:
 					return "An unexpected error occurred. Try checking the original exception for more information.";

src/Elasticsearch.Net/Transport/Pipeline/PipelineFailure.cs

@@ -15,6 +15,7 @@ public enum PipelineFailure
 		MaxRetriesReached,
 		Unexpected,
 		BadRequest,
-		NoNodesAttempted
+		NoNodesAttempted,
+		FailedProductCheck
 	}
 }

src/Elasticsearch.Net/Transport/Pipeline/ProductCheckStatus.cs

@@ -13,6 +13,7 @@ public enum ProductCheckStatus
 		ValidProduct,
 		InvalidProduct,
 		UndeterminedProduct,
-		TransientFailure
+		TransientFailure,
+		UnsupportedBuildFlavor
 	}
 }