Add external cluster support to skip ephemeral startup

examples/Elastic.TUnit.ExampleComplex/Clusters.cs

@@ -2,6 +2,7 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using System;
 using Elastic.Clients.Elasticsearch;
 using Elastic.TUnit.Elasticsearch;
 using Elastic.TUnit.Elasticsearch.Core;
@@ -11,6 +12,8 @@ namespace Elastic.TUnit.ExampleComplex;
 
 /// <summary>
 ///     Shared base — inherits the default Client from <see cref="ElasticsearchCluster" />.
+///     Env-var detection (TEST_ELASTICSEARCH_URL / TEST_ELASTICSEARCH_API_KEY) is handled
+///     automatically by the base class — no code changes needed.
 /// </summary>
 public abstract class MyClusterBase : ElasticsearchCluster
 {
@@ -46,6 +49,10 @@ public TestGenericCluster() : base(new ElasticsearchConfiguration("latest-9"))
 		var settings = new ElasticsearchClientSettings(pool)
 			.EnableDebugMode()
 			.OnRequestCompleted(call => output.WriteLine(call.DebugInformation));
+
+		if (ExternalApiKey != null)
+			settings = settings.Authentication(new ApiKey(ExternalApiKey));
+
 		return new ElasticsearchClient(settings);
 	});
 
@@ -54,3 +61,27 @@ protected override void SeedCluster()
 		var response = Client.Info();
 	}
 }
+
+/// <summary>
+///     Demonstrates the programmatic hook — override <see cref="ElasticsearchCluster{TConfiguration}.TryUseExternalCluster" />
+///     to point at a specific cluster from code (e.g. read from a config file, service
+///     discovery, etc.). Falls through to ephemeral startup when returning null.
+/// </summary>
+public class ProgrammaticExternalCluster : ElasticsearchCluster
+{
+	public ProgrammaticExternalCluster() : base(new ElasticsearchConfiguration("latest-9"))
+	{
+	}
+
+	protected override ExternalClusterConfiguration TryUseExternalCluster()
+	{
+		var url = Environment.GetEnvironmentVariable("MY_DEV_CLUSTER_URL");
+		if (string.IsNullOrEmpty(url))
+			return null;
+
+		return new ExternalClusterConfiguration(
+			new Uri(url),
+			Environment.GetEnvironmentVariable("MY_DEV_CLUSTER_KEY")
+		);
+	}
+}

src/Elastic.TUnit.Elasticsearch.Core/ElasticsearchCluster{TConfiguration}.cs

@@ -4,6 +4,7 @@
 
 using System;
 using System.Collections.Concurrent;
+using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading;
@@ -21,6 +22,20 @@ namespace Elastic.TUnit.Elasticsearch.Core;
 ///     Base class for an Elasticsearch cluster that integrates with TUnit's lifecycle.
 ///     Implements <see cref="IAsyncInitializer" /> to start the cluster and
 ///     <see cref="IAsyncDisposable" /> to tear it down.
+///     <para>
+///         Supports skipping ephemeral cluster startup when an external cluster is
+///         available. The resolution order in <see cref="InitializeAsync" /> is:
+///         <list type="number">
+///             <item>
+///                 <see cref="TryUseExternalCluster" /> — override for programmatic control
+///             </item>
+///             <item>
+///                 <c>TEST_ELASTICSEARCH_URL</c> environment variable (with optional
+///                 <c>TEST_ELASTICSEARCH_API_KEY</c>)
+///             </item>
+///             <item>Start an ephemeral cluster as usual</item>
+///         </list>
+///     </para>
 /// </summary>
 public abstract class ElasticsearchCluster<TConfiguration> : EphemeralCluster<TConfiguration>,
 	IAsyncInitializer, IAsyncDisposable
@@ -35,14 +50,46 @@ public abstract class ElasticsearchCluster<TConfiguration> : EphemeralCluster<TC
 	/// </summary>
 	internal static ConcurrentDictionary<Type, ElasticVersion> ClusterVersions { get; } = new();
 
+	private ExternalClusterConfiguration _externalCluster;
+
 	protected ElasticsearchCluster(TConfiguration configuration) : base(configuration)
 	{
 	}
 
+	/// <summary>
+	///     Whether this cluster is backed by an external (remote) Elasticsearch instance
+	///     rather than a locally managed ephemeral process.
+	/// </summary>
+	public bool IsExternal => _externalCluster != null;
+
+	/// <summary>
+	///     The API key for the external cluster, or <c>null</c> if not applicable.
+	/// </summary>
+	public string ExternalApiKey => _externalCluster?.ApiKey;
+
+	/// <summary>
+	///     Override to programmatically provide an external Elasticsearch cluster,
+	///     skipping ephemeral cluster startup entirely. Return <c>null</c> to fall
+	///     through to environment variable detection and then ephemeral startup.
+	/// </summary>
+	protected virtual ExternalClusterConfiguration TryUseExternalCluster() => null;
+
+	/// <summary>
+	///     Returns the node URIs for this cluster. When connected to an external cluster,
+	///     returns the external URI; otherwise delegates to the ephemeral cluster's nodes.
+	/// </summary>
+	public override ICollection<Uri> NodesUris(string hostName = null) =>
+		_externalCluster != null ? [_externalCluster.Uri] : base.NodesUris(hostName);
+
 	/// <summary>
 	///     Starts the Elasticsearch cluster. Cluster startups are serialized via a
 	///     semaphore since Elasticsearch is resource-intensive.
 	///     <para>
+	///         Before starting an ephemeral cluster, checks for an external cluster
+	///         via <see cref="TryUseExternalCluster" /> and then the
+	///         <c>TEST_ELASTICSEARCH_URL</c> environment variable.
+	///     </para>
+	///     <para>
 	///         Bootstrap output mode is determined by
 	///         <see cref="ElasticsearchConfiguration.ShowBootstrapDiagnostics" />:
 	///         <list type="bullet">
@@ -57,6 +104,16 @@ protected ElasticsearchCluster(TConfiguration configuration) : base(configuratio
 	/// </summary>
 	public async Task InitializeAsync()
 	{
+		var external = ResolveExternalCluster();
+		if (external != null)
+		{
+			await external.ValidateAsync().ConfigureAwait(false);
+			_externalCluster = external;
+			ClusterVersions[GetType()] = ClusterConfiguration.Version;
+			WriteExternalClusterInfo(external);
+			return;
+		}
+
 		await StartupSemaphore.WaitAsync().ConfigureAwait(false);
 		try
 		{
@@ -87,11 +144,12 @@ public async Task InitializeAsync()
 	}
 
 	/// <summary>
-	///     Disposes the Elasticsearch cluster.
+	///     Disposes the Elasticsearch cluster. No-op when using an external cluster.
 	/// </summary>
 	public ValueTask DisposeAsync()
 	{
-		Dispose();
+		if (_externalCluster == null)
+			Dispose();
 		return default;
 	}
 
@@ -111,6 +169,35 @@ private IConsoleLineHandler CreateBootstrapWriter()
 		};
 	}
 
+	private ExternalClusterConfiguration ResolveExternalCluster()
+	{
+		var programmatic = TryUseExternalCluster();
+		if (programmatic != null)
+			return programmatic;
+
+		var url = Environment.GetEnvironmentVariable("TEST_ELASTICSEARCH_URL");
+		if (string.IsNullOrWhiteSpace(url))
+			return null;
+
+		if (!Uri.TryCreate(url, UriKind.Absolute, out var uri))
+			throw new InvalidOperationException(
+				$"TEST_ELASTICSEARCH_URL is set but is not a valid URI: {url}");
+
+		var apiKey = Environment.GetEnvironmentVariable("TEST_ELASTICSEARCH_API_KEY");
+		return new ExternalClusterConfiguration(uri, string.IsNullOrWhiteSpace(apiKey) ? null : apiKey);
+	}
+
+	private void WriteExternalClusterInfo(ExternalClusterConfiguration external)
+	{
+		var name = GetType().Name;
+		var auth = string.IsNullOrEmpty(external.ApiKey) ? "no auth" : "API key";
+		var message = $"[{name}] using external cluster at {external.Uri} ({auth}) — skipping ephemeral startup";
+
+		using var stdout = new System.IO.StreamWriter(Console.OpenStandardOutput()) { AutoFlush = true };
+		stdout.WriteLine(message);
+		TestContext.Current?.Output.WriteLine(message);
+	}
+
 	private void WriteBootstrapFailure(Exception ex)
 	{
 		var sb = new StringBuilder();

src/Elastic.TUnit.Elasticsearch.Core/ExternalClusterConfiguration.cs

@@ -0,0 +1,57 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Elastic.TUnit.Elasticsearch.Core;
+
+/// <summary>
+///     Connection details for an externally managed Elasticsearch cluster.
+///     Used by <see cref="ElasticsearchCluster{TConfiguration}" /> to skip
+///     ephemeral cluster startup when a remote cluster is available.
+/// </summary>
+public class ExternalClusterConfiguration
+{
+	public ExternalClusterConfiguration(Uri uri, string apiKey = null) =>
+		(Uri, ApiKey) = (uri ?? throw new ArgumentNullException(nameof(uri)), apiKey);
+
+	/// <summary> The base URI of the external Elasticsearch cluster. </summary>
+	public Uri Uri { get; }
+
+	/// <summary> An optional API key for authenticating with the cluster. </summary>
+	public string ApiKey { get; }
+
+	/// <summary>
+	///     Validates that the external cluster is reachable by issuing a GET request to the root endpoint.
+	///     Throws <see cref="InvalidOperationException" /> with a descriptive message on failure.
+	/// </summary>
+	public async Task ValidateAsync(TimeSpan? timeout = null, CancellationToken ctx = default)
+	{
+		timeout ??= TimeSpan.FromSeconds(10);
+
+		using var handler = new HttpClientHandler
+		{
+			ServerCertificateCustomValidationCallback = (_, _, _, _) => true
+		};
+		using var http = new HttpClient(handler) { Timeout = timeout.Value };
+
+		if (!string.IsNullOrEmpty(ApiKey))
+			http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("ApiKey", ApiKey);
+
+		try
+		{
+			var response = await http.GetAsync(Uri, ctx).ConfigureAwait(false);
+			response.EnsureSuccessStatusCode();
+		}
+		catch (Exception ex)
+		{
+			throw new InvalidOperationException(
+				$"External Elasticsearch cluster at {Uri} is not reachable: {ex.Message}", ex);
+		}
+	}
+}

src/Elastic.TUnit.Elasticsearch.Core/README.md

@@ -25,13 +25,36 @@ It depends on this package transitively and adds:
 
 - `ElasticsearchCluster<TConfiguration>` — generic cluster base with TUnit lifecycle integration
 - `ElasticsearchConfiguration` — cluster configuration with bootstrap diagnostics settings
+- `ExternalClusterConfiguration` — connection details for remote Elasticsearch clusters
 - `ElasticsearchClusterExtensions` — `GetOrAddClient` helpers for caching any client type
 - `ElasticsearchTestHooks` — global `[BeforeEvery(Test)]` hook for skip evaluation
 - `ElasticsearchTestBase<TCluster>` — optional convenience base class
 - `ElasticsearchParallelLimit` — default parallel limiter
 - `SkipVersionAttribute` / `SkipTestAttribute` — version-based and custom skip conditions
 - Bootstrap diagnostics writers (ANSI console, progress heartbeat)
 
+## External cluster support
+
+`ElasticsearchCluster<TConfiguration>` supports skipping ephemeral cluster startup when a
+remote Elasticsearch instance is already available. This significantly speeds up development
+feedback loops since you don't need to wait for Elasticsearch to bootstrap on every test run.
+
+The resolution order during `InitializeAsync()` is:
+
+1. **Programmatic hook** — override `TryUseExternalCluster()` to return an
+   `ExternalClusterConfiguration`, or `null` to fall through
+2. **Environment variables** — set `TEST_ELASTICSEARCH_URL` (and optionally
+   `TEST_ELASTICSEARCH_API_KEY`)
+3. **Ephemeral startup** — download, install, and start Elasticsearch locally
+
+When using an external cluster:
+
+- `NodesUris()` returns the external URI
+- `IsExternal` is `true`
+- `ExternalApiKey` contains the API key (if provided)
+- `Dispose()` is a no-op (the remote cluster is not managed)
+- Connectivity is validated with a GET `/` before tests run
+
 ## Dependencies
 
 - `TUnit.Core`

src/Elastic.TUnit.Elasticsearch/ElasticsearchCluster.cs

@@ -18,6 +18,11 @@ namespace Elastic.TUnit.Elasticsearch;
 ///         with debug mode enabled and per-request diagnostics routed to the
 ///         current TUnit test's output. Override to customize.
 ///     </para>
+///     <para>
+///         When connected to an external cluster (via <c>TEST_ELASTICSEARCH_URL</c> or
+///         <see cref="ElasticsearchCluster{TConfiguration}.TryUseExternalCluster" />),
+///         the client is automatically configured with the external API key if provided.
+///     </para>
 /// </summary>
 public class ElasticsearchCluster : ElasticsearchCluster<ElasticsearchConfiguration>
 {
@@ -40,12 +45,18 @@ public ElasticsearchCluster(ElasticsearchConfiguration configuration)
 	///     A default <see cref="ElasticsearchClient" /> configured with debug mode
 	///     and per-request diagnostics routed to the current TUnit test's output.
 	///     The client is cached for the cluster's lifetime.
+	///     When using an external cluster with an API key, authentication is
+	///     configured automatically.
 	///     Override to customize connection settings, authentication, serialization, etc.
 	/// </summary>
 	public virtual ElasticsearchClient Client => this.GetOrAddClient((c, output) =>
 	{
 		var settings = new ElasticsearchClientSettings(new StaticNodePool(c.NodesUris()))
 			.WireTUnitOutput(output);
+
+		if (ExternalApiKey != null)
+			settings = settings.Authentication(new ApiKey(ExternalApiKey));
+
 		return new ElasticsearchClient(settings);
 	});
 }