From 08abc7ebc25eca8b415aee4921aab1188303269f Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Wed, 21 Jan 2026 14:12:13 -0800 Subject: [PATCH 01/35] chore: Pipe headers through data sources. --- lib/sdk/server/build.gradle | 2 +- .../sdk/server/DefaultFDv2Requestor.java | 12 +- .../sdk/server/FDv2Requestor.java | 30 ++- .../sdk/server/HeaderConstants.java | 16 ++ .../launchdarkly/sdk/server/PollingBase.java | 82 ++++--- .../sdk/server/PollingInitializerImpl.java | 1 - .../sdk/server/StreamingSynchronizerImpl.java | 44 +++- .../server/datasources/FDv2SourceResult.java | 43 +++- .../sdk/server/DefaultFDv2RequestorTest.java | 49 ++-- .../server/PollingInitializerImplTest.java | 220 +++++++++++++++-- .../server/PollingSynchronizerImplTest.java | 227 +++++++++++++++++- .../server/StreamingSynchronizerImplTest.java | 178 ++++++++++++++ 12 files changed, 785 insertions(+), 119 deletions(-) create mode 100644 lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/HeaderConstants.java diff --git a/lib/sdk/server/build.gradle b/lib/sdk/server/build.gradle index a27e9c71..b91fe425 100644 --- a/lib/sdk/server/build.gradle +++ b/lib/sdk/server/build.gradle @@ -74,7 +74,7 @@ ext.versions = [ "launchdarklyJavaSdkInternal": "1.6.1", "launchdarklyLogging": "1.1.0", "okhttp": "4.12.0", // specify this for the SDK build instead of relying on the transitive dependency from okhttp-eventsource - "okhttpEventsource": "4.1.0", + "okhttpEventsource": "4.2.0", "reactorCore":"3.3.22.RELEASE", "slf4j": "1.7.36", "snakeyaml": "2.4", diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DefaultFDv2Requestor.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DefaultFDv2Requestor.java index 133a56aa..8d550597 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DefaultFDv2Requestor.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DefaultFDv2Requestor.java @@ -3,10 +3,8 @@ import com.launchdarkly.logging.LDLogger; import com.launchdarkly.sdk.internal.fdv2.payloads.FDv2Event; import com.launchdarkly.sdk.internal.fdv2.sources.Selector; -import com.launchdarkly.sdk.internal.http.HttpErrors; import com.launchdarkly.sdk.internal.http.HttpHelpers; import com.launchdarkly.sdk.internal.http.HttpProperties; -import com.launchdarkly.sdk.json.SerializationException; import okhttp3.Call; import okhttp3.Callback; @@ -104,17 +102,15 @@ public void onFailure(@Nonnull Call call, @Nonnull IOException e) { @Override public void onResponse(@Nonnull Call call, @Nonnull Response response) { try { - // Handle 304 Not Modified - no new data + // Handle 304 Not Modified - no new data, but return response with headers if (response.code() == 304) { logger.debug("FDv2 polling request returned 304: not modified"); - future.complete(null); + future.complete(FDv2PayloadResponse.none(response.code())); return; } if (!response.isSuccessful()) { - future.completeExceptionally( - new HttpErrors.HttpErrorException(response.code()) - ); + future.complete(FDv2PayloadResponse.failure(response.code(), response.headers())); return; } @@ -136,7 +132,7 @@ public void onResponse(@Nonnull Call call, @Nonnull Response response) { List events = FDv2Event.parseEventsArray(responseBody); // Create and return the response - FDv2PayloadResponse pollingResponse = new FDv2PayloadResponse(events, response.headers()); + FDv2PayloadResponse pollingResponse = FDv2PayloadResponse.success(events, response.headers(), response.code()); future.complete(pollingResponse); } catch (Exception e) { diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2Requestor.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2Requestor.java index 8a2297e4..dbebd9d8 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2Requestor.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2Requestor.java @@ -18,13 +18,31 @@ interface FDv2Requestor { * to get from one payload version to another. * This isn't intended for use for implementations which may require multiple executions to get an entire payload. */ - public static class FDv2PayloadResponse { +public static class FDv2PayloadResponse { private final List events; private final Headers headers; - public FDv2PayloadResponse(List events, Headers headers) { + private final boolean successful; + + private final int statusCode; + + private FDv2PayloadResponse(List events, Headers headers, boolean success, int statusCode) { this.events = events; this.headers = headers; + this.successful = success; + this.statusCode = statusCode; + } + + public static FDv2PayloadResponse failure(int statusCode, Headers headers) { + return new FDv2PayloadResponse(null, headers, false, statusCode); + } + + public static FDv2PayloadResponse success(List events, Headers headers, int statusCode) { + return new FDv2PayloadResponse(events, headers, true, statusCode); + } + + public static FDv2PayloadResponse none(int statusCode) { + return new FDv2PayloadResponse(null, null, true, statusCode); } public List getEvents() { @@ -34,6 +52,14 @@ public List getEvents() { public Headers getHeaders() { return headers; } + + public boolean isSuccess() { + return successful; + } + + public int getStatusCode() { + return statusCode; + } } CompletableFuture Poll(Selector selector); diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/HeaderConstants.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/HeaderConstants.java new file mode 100644 index 00000000..66ad1495 --- /dev/null +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/HeaderConstants.java @@ -0,0 +1,16 @@ +package com.launchdarkly.sdk.server; + +enum HeaderConstants { + ENVIRONMENT_ID("x-ld-envid"), + FDV1_FALLBACK("x-ld-fd-fallback"); + + private final String headerName; + + HeaderConstants(String headerName) { + this.headerName = headerName; + } + + public String getHeaderName() { + return headerName; + } +} diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingBase.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingBase.java index 20a5a1f1..f18309d2 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingBase.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingBase.java @@ -29,23 +29,33 @@ protected void internalShutdown() { requestor.close(); } + private static boolean getFallback(FDv2Requestor.FDv2PayloadResponse response) { + if (response != null && response.getHeaders() != null) { + String headerValue = response.getHeaders().get(HeaderConstants.FDV1_FALLBACK.getHeaderName()); + return headerValue != null && headerValue.equalsIgnoreCase("true"); + } +// if(ex != null) { +// if(ex instanceof HttpErrorException) { +// ((HttpErrors.HttpErrorException) ex). +// } +// } + + return false; + } + + private static String getEnvironmentId(FDv2Requestor.FDv2PayloadResponse response) { + if (response != null && response.getHeaders() != null) { + return response.getHeaders().get(HeaderConstants.ENVIRONMENT_ID.getHeaderName()); + } + return null; + } + protected CompletableFuture poll(Selector selector, boolean oneShot) { return requestor.Poll(selector).handle(((pollingResponse, ex) -> { + boolean fdv1Fallback = getFallback(pollingResponse); + String environmentId = getEnvironmentId(pollingResponse); if (ex != null) { - if (ex instanceof HttpErrors.HttpErrorException) { - HttpErrors.HttpErrorException e = (HttpErrors.HttpErrorException) ex; - DataSourceStatusProvider.ErrorInfo errorInfo = DataSourceStatusProvider.ErrorInfo.fromHttpError(e.getStatus()); - // Errors without an HTTP status are recoverable. If there is a status, then we check if the error - // is recoverable. - boolean recoverable = e.getStatus() <= 0 || isHttpErrorRecoverable(e.getStatus()); - logger.error("Polling request failed with HTTP error: {}", e.getStatus()); - // For a one-shot request all errors are terminal. - if (oneShot) { - return FDv2SourceResult.terminalError(errorInfo); - } else { - return recoverable ? FDv2SourceResult.interrupted(errorInfo) : FDv2SourceResult.terminalError(errorInfo); - } - } else if (ex instanceof IOException) { + if (ex instanceof IOException) { IOException e = (IOException) ex; logger.error("Polling request failed with network error: {}", e.toString()); DataSourceStatusProvider.ErrorInfo info = new DataSourceStatusProvider.ErrorInfo( @@ -54,7 +64,7 @@ protected CompletableFuture poll(Selector selector, boolean on e.toString(), new Date().toInstant() ); - return oneShot ? FDv2SourceResult.terminalError(info) : FDv2SourceResult.interrupted(info); + return oneShot ? FDv2SourceResult.terminalError(info, fdv1Fallback) : FDv2SourceResult.interrupted(info, fdv1Fallback); } else if (ex instanceof SerializationException) { SerializationException e = (SerializationException) ex; logger.error("Polling request received malformed data: {}", e.toString()); @@ -64,7 +74,7 @@ protected CompletableFuture poll(Selector selector, boolean on e.toString(), new Date().toInstant() ); - return oneShot ? FDv2SourceResult.terminalError(info) : FDv2SourceResult.interrupted(info); + return oneShot ? FDv2SourceResult.terminalError(info, fdv1Fallback) : FDv2SourceResult.interrupted(info, fdv1Fallback); } String msg = ex.toString(); logger.error("Polling request failed with an unknown error: {}", msg); @@ -74,17 +84,30 @@ protected CompletableFuture poll(Selector selector, boolean on msg, new Date().toInstant() ); - return oneShot ? FDv2SourceResult.terminalError(info) : FDv2SourceResult.interrupted(info); + return oneShot ? FDv2SourceResult.terminalError(info, fdv1Fallback) : FDv2SourceResult.interrupted(info, fdv1Fallback); } - // A null polling response indicates that we received a 304, which means nothing has changed. - if (pollingResponse == null) { + // If we get a 304, then that means nothing has changed. + if (pollingResponse.getStatusCode() == 304) { return FDv2SourceResult.changeSet( new DataStoreTypes.ChangeSet<>(DataStoreTypes.ChangeSetType.None, Selector.EMPTY, null, - // TODO: Implement environment ID support. - null - )); + null // Header derived values will have been handled on initial response. + ), + // Headers would have been processed from the initial response. + false); + } + if(!pollingResponse.isSuccess()) { + int statusCode = pollingResponse.getStatusCode(); + boolean recoverable = statusCode <= 0 || isHttpErrorRecoverable(statusCode); + DataSourceStatusProvider.ErrorInfo errorInfo = DataSourceStatusProvider.ErrorInfo.fromHttpError(statusCode); + logger.error("Polling request failed with HTTP error: {}", statusCode); + // For a one-shot request all errors are terminal. + if (oneShot) { + return FDv2SourceResult.terminalError(errorInfo, fdv1Fallback); + } else { + return recoverable ? FDv2SourceResult.interrupted(errorInfo, fdv1Fallback) : FDv2SourceResult.terminalError(errorInfo, fdv1Fallback); + } } FDv2ProtocolHandler handler = new FDv2ProtocolHandler(); for (FDv2Event event : pollingResponse.getEvents()) { @@ -96,10 +119,9 @@ protected CompletableFuture poll(Selector selector, boolean on DataStoreTypes.ChangeSet converted = FDv2ChangeSetTranslator.toChangeSet( ((FDv2ProtocolHandler.FDv2ActionChangeset) res).getChangeset(), logger, - // TODO: Implement environment ID support. - null + environmentId ); - return FDv2SourceResult.changeSet(converted); + return FDv2SourceResult.changeSet(converted, fdv1Fallback); } catch (Exception e) { // TODO: Do we need to be more specific about the exception type here? DataSourceStatusProvider.ErrorInfo info = new DataSourceStatusProvider.ErrorInfo( @@ -108,7 +130,7 @@ protected CompletableFuture poll(Selector selector, boolean on e.toString(), new Date().toInstant() ); - return oneShot ? FDv2SourceResult.terminalError(info) : FDv2SourceResult.interrupted(info); + return oneShot ? FDv2SourceResult.terminalError(info, fdv1Fallback) : FDv2SourceResult.interrupted(info, fdv1Fallback); } case ERROR: { FDv2ProtocolHandler.FDv2ActionError error = ((FDv2ProtocolHandler.FDv2ActionError) res); @@ -117,10 +139,10 @@ protected CompletableFuture poll(Selector selector, boolean on 0, error.getReason(), new Date().toInstant()); - return oneShot ? FDv2SourceResult.terminalError(info) : FDv2SourceResult.interrupted(info); + return oneShot ? FDv2SourceResult.terminalError(info, fdv1Fallback) : FDv2SourceResult.interrupted(info, fdv1Fallback); } case GOODBYE: - return FDv2SourceResult.goodbye(((FDv2ProtocolHandler.FDv2ActionGoodbye) res).getReason()); + return FDv2SourceResult.goodbye(((FDv2ProtocolHandler.FDv2ActionGoodbye) res).getReason(), fdv1Fallback); case NONE: break; case INTERNAL_ERROR: { @@ -141,7 +163,7 @@ protected CompletableFuture poll(Selector selector, boolean on 0, "Internal error occurred during polling", new Date().toInstant()); - return oneShot ? FDv2SourceResult.terminalError(info) : FDv2SourceResult.interrupted(info); + return oneShot ? FDv2SourceResult.terminalError(info, fdv1Fallback) : FDv2SourceResult.interrupted(info, fdv1Fallback); } } } @@ -152,7 +174,7 @@ protected CompletableFuture poll(Selector selector, boolean on "Unexpected end of polling response", new Date().toInstant() ); - return oneShot ? FDv2SourceResult.terminalError(info) : FDv2SourceResult.interrupted(info); + return oneShot ? FDv2SourceResult.terminalError(info, fdv1Fallback) : FDv2SourceResult.interrupted(info, fdv1Fallback); })); } } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingInitializerImpl.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingInitializerImpl.java index 9bc51659..856a118b 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingInitializerImpl.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingInitializerImpl.java @@ -1,7 +1,6 @@ package com.launchdarkly.sdk.server; import com.launchdarkly.logging.LDLogger; -import com.launchdarkly.sdk.internal.fdv2.sources.Selector; import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; import com.launchdarkly.sdk.server.datasources.Initializer; import com.launchdarkly.sdk.server.datasources.SelectorSource; diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java index c5d52f3d..6fa4487c 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java @@ -166,7 +166,7 @@ private Thread getRunThread() { ); // We aren't restarting the event source here. We aren't going to automatically recover, so the // data system will move to the next source when it determined this source is unhealthy. - resultQueue.put(FDv2SourceResult.interrupted(errorInfo)); + resultQueue.put(FDv2SourceResult.interrupted(errorInfo, getFallback(e))); } finally { eventSource.close(); } @@ -249,10 +249,12 @@ private void handleMessage(MessageEvent event) { case CHANGESET: FDv2ProtocolHandler.FDv2ActionChangeset changeset = (FDv2ProtocolHandler.FDv2ActionChangeset) action; try { - // TODO: Environment ID. DataStoreTypes.ChangeSet converted = - FDv2ChangeSetTranslator.toChangeSet(changeset.getChangeset(), logger, null); - result = FDv2SourceResult.changeSet(converted); + FDv2ChangeSetTranslator.toChangeSet( + changeset.getChangeset(), + logger, + event.getHeaders().value(HeaderConstants.ENVIRONMENT_ID.getHeaderName())); + result = FDv2SourceResult.changeSet(converted, getFallback(event)); } catch (Exception e) { logger.error("Failed to convert FDv2 changeset: {}", LogValues.exceptionSummary(e)); logger.debug(LogValues.exceptionTrace(e)); @@ -262,7 +264,7 @@ private void handleMessage(MessageEvent event) { e.toString(), Instant.now() ); - result = FDv2SourceResult.interrupted(conversionError); + result = FDv2SourceResult.interrupted(conversionError, getFallback(event)); restartStream(); } break; @@ -276,7 +278,7 @@ private void handleMessage(MessageEvent event) { case GOODBYE: FDv2ProtocolHandler.FDv2ActionGoodbye goodbye = (FDv2ProtocolHandler.FDv2ActionGoodbye) action; - result = FDv2SourceResult.goodbye(goodbye.getReason()); + result = FDv2SourceResult.goodbye(goodbye.getReason(), getFallback(event)); // We drop this current connection and attempt to restart the stream. restartStream(); break; @@ -300,7 +302,7 @@ private void handleMessage(MessageEvent event) { "Internal error during FDv2 event processing", Instant.now() ); - result = FDv2SourceResult.interrupted(internalError); + result = FDv2SourceResult.interrupted(internalError, getFallback(event)); restartStream(); break; @@ -322,7 +324,7 @@ private void interruptedWithException(Exception e, DataSourceStatusProvider.Erro e.toString(), Instant.now() ); - resultQueue.put(FDv2SourceResult.interrupted(errorInfo)); + resultQueue.put(FDv2SourceResult.interrupted(errorInfo, getFallback(e))); restartStream(); } @@ -343,11 +345,11 @@ private boolean handleError(StreamException e) { "will retry"); if (!recoverable) { - shutdownFuture.complete(FDv2SourceResult.terminalError(errorInfo)); + shutdownFuture.complete(FDv2SourceResult.terminalError(errorInfo, getFallback(e))); return false; } else { // Queue as INTERRUPTED to indicate temporary failure - resultQueue.put(FDv2SourceResult.interrupted(errorInfo)); + resultQueue.put(FDv2SourceResult.interrupted(errorInfo, getFallback(e))); return true; // allow reconnect } } @@ -361,7 +363,7 @@ private boolean handleError(StreamException e) { e.toString(), Instant.now() ); - resultQueue.put(FDv2SourceResult.interrupted(errorInfo)); + resultQueue.put(FDv2SourceResult.interrupted(errorInfo, getFallback(e))); return true; // allow reconnect } @@ -385,4 +387,24 @@ private FDv2Event parseFDv2Event(String eventName, Reader eventDataReader) throw throw new SerializationException(e); } } + + private static boolean getFallback(Exception ex) { + if(ex instanceof StreamHttpErrorException) { + String headerValue = ((StreamHttpErrorException) ex).getHeaders() + .value(HeaderConstants.FDV1_FALLBACK.getHeaderName()); + return headerValue != null && headerValue.equalsIgnoreCase("true"); + } + return false; + } + + private static boolean getFallback(StreamEvent event) { + String headerName = HeaderConstants.FDV1_FALLBACK.getHeaderName(); + String headerValue = null; + if(event instanceof FaultEvent) { + headerValue = ((FaultEvent) event).getHeaders().value(headerName); + } else if (event instanceof MessageEvent) { + headerValue = ((MessageEvent) event).getHeaders().value(headerName); + } + return headerValue != null && headerValue.equalsIgnoreCase("true"); + } } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/datasources/FDv2SourceResult.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/datasources/FDv2SourceResult.java index 3f7ad16f..6f440c63 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/datasources/FDv2SourceResult.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/datasources/FDv2SourceResult.java @@ -12,7 +12,7 @@ public class FDv2SourceResult { public enum State { /** * The data source has encountered an interruption and will attempt to reconnect. This isn't intended to be used - * with an initializer, and instead TERMINAL_ERROR should be used. When this status is used with an initializer + * with an initializer, and instead TERMINAL_ERROR should be used. When this status is used with an initializer, * it will still be a terminal state. */ INTERRUPTED, @@ -67,32 +67,49 @@ public Status(State state, DataSourceStatusProvider.ErrorInfo errorInfo) { private final Status status; private final ResultType resultType; + + private final boolean fdv1Fallback; - private FDv2SourceResult(DataStoreTypes.ChangeSet changeSet, Status status, ResultType resultType) { + private FDv2SourceResult(DataStoreTypes.ChangeSet changeSet, Status status, ResultType resultType, boolean fdv1Fallback) { this.changeSet = changeSet; this.status = status; this.resultType = resultType; + this.fdv1Fallback = fdv1Fallback; } - public static FDv2SourceResult interrupted(DataSourceStatusProvider.ErrorInfo errorInfo) { - return new FDv2SourceResult(null, new Status(State.INTERRUPTED, errorInfo), ResultType.STATUS); + public static FDv2SourceResult interrupted(DataSourceStatusProvider.ErrorInfo errorInfo, boolean fdv1Fallback) { + return new FDv2SourceResult( + null, + new Status(State.INTERRUPTED, errorInfo), + ResultType.STATUS, + fdv1Fallback); } public static FDv2SourceResult shutdown() { - return new FDv2SourceResult(null, new Status(State.SHUTDOWN, null), ResultType.STATUS); + return new FDv2SourceResult(null, + new Status(State.SHUTDOWN, null), + ResultType.STATUS, + false); } - public static FDv2SourceResult terminalError(DataSourceStatusProvider.ErrorInfo errorInfo) { - return new FDv2SourceResult(null, new Status(State.TERMINAL_ERROR, errorInfo), ResultType.STATUS); + public static FDv2SourceResult terminalError(DataSourceStatusProvider.ErrorInfo errorInfo, boolean fdv1Fallback) { + return new FDv2SourceResult(null, + new Status(State.TERMINAL_ERROR, errorInfo), + ResultType.STATUS, + fdv1Fallback); } - public static FDv2SourceResult changeSet(DataStoreTypes.ChangeSet changeSet) { - return new FDv2SourceResult(changeSet, null, ResultType.CHANGE_SET); + public static FDv2SourceResult changeSet(DataStoreTypes.ChangeSet changeSet, boolean fdv1Fallback) { + return new FDv2SourceResult(changeSet, null, ResultType.CHANGE_SET, fdv1Fallback); } - public static FDv2SourceResult goodbye(String reason) { + public static FDv2SourceResult goodbye(String reason, boolean fdv1Fallback) { // TODO: Goodbye reason. - return new FDv2SourceResult(null, new Status(State.GOODBYE, null), ResultType.STATUS); + return new FDv2SourceResult( + null, + new Status(State.GOODBYE, null), + ResultType.STATUS, + fdv1Fallback); } public ResultType getResultType() { @@ -106,4 +123,8 @@ public Status getStatus() { public DataStoreTypes.ChangeSet getChangeSet() { return changeSet; } + + public boolean isFdv1Fallback() { + return fdv1Fallback; + } } diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DefaultFDv2RequestorTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DefaultFDv2RequestorTest.java index 5ce321bb..dd5ec9d5 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DefaultFDv2RequestorTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DefaultFDv2RequestorTest.java @@ -2,6 +2,7 @@ import com.launchdarkly.sdk.internal.fdv2.payloads.FDv2Event; import com.launchdarkly.sdk.internal.fdv2.sources.Selector; +import com.launchdarkly.sdk.internal.http.HttpErrors; import com.launchdarkly.sdk.internal.http.HttpProperties; import com.launchdarkly.sdk.server.subsystems.ClientContext; import com.launchdarkly.testhelpers.httptest.Handler; @@ -18,12 +19,8 @@ import java.util.concurrent.TimeUnit; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.notNullValue; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.*; @SuppressWarnings("javadoc") public class DefaultFDv2RequestorTest extends BaseTest { @@ -216,14 +213,14 @@ public void etagCachingWith304NotModified() throws Exception { RequestInfo req1 = server.getRecorder().requireRequest(); assertEquals(REQUEST_PATH, req1.getPath()); - assertEquals(null, req1.getHeader("If-None-Match")); + assertNull(req1.getHeader("If-None-Match")); // Second request should send If-None-Match and receive 304 CompletableFuture future2 = requestor.Poll(Selector.EMPTY); FDv2Requestor.FDv2PayloadResponse response2 = future2.get(5, TimeUnit.SECONDS); - assertEquals(null, response2); + assertEquals(304, response2.getStatusCode()); RequestInfo req2 = server.getRecorder().requireRequest(); assertEquals(REQUEST_PATH, req2.getPath()); @@ -250,7 +247,7 @@ public void etagUpdatedOnNewResponse() throws Exception { // First request requestor.Poll(Selector.EMPTY).get(5, TimeUnit.SECONDS); RequestInfo req1 = server.getRecorder().requireRequest(); - assertEquals(null, req1.getHeader("If-None-Match")); + assertNull(req1.getHeader("If-None-Match")); // Second request should use etag-1 requestor.Poll(Selector.EMPTY).get(5, TimeUnit.SECONDS); @@ -289,13 +286,13 @@ public void etagRemovedWhenNotInResponse() throws Exception { // Third request should not send ETag (was removed) requestor.Poll(Selector.EMPTY).get(5, TimeUnit.SECONDS); RequestInfo req3 = server.getRecorder().requireRequest(); - assertEquals(null, req3.getHeader("If-None-Match")); + assertNull(req3.getHeader("If-None-Match")); } } } @Test - public void httpErrorCodeThrowsException() throws Exception { + public void httpErrorCodeReturnsFailureResponse() throws Exception { Handler resp = Handlers.status(500); try (HttpServer server = HttpServer.start(resp)) { @@ -303,19 +300,17 @@ public void httpErrorCodeThrowsException() throws Exception { CompletableFuture future = requestor.Poll(Selector.EMPTY); - try { - future.get(5, TimeUnit.SECONDS); - fail("Expected ExecutionException"); - } catch (ExecutionException e) { - assertThat(e.getCause(), notNullValue()); - assertThat(e.getCause().getMessage(), containsString("500")); - } + FDv2Requestor.FDv2PayloadResponse response = future.get(5, TimeUnit.SECONDS); + + assertNotNull(response); + assertEquals(500, response.getStatusCode()); + assertFalse(response.isSuccess()); } } } @Test - public void http404ThrowsException() throws Exception { + public void http404ReturnsFailureResponse() throws Exception { Handler resp = Handlers.status(404); try (HttpServer server = HttpServer.start(resp)) { @@ -323,13 +318,11 @@ public void http404ThrowsException() throws Exception { CompletableFuture future = requestor.Poll(Selector.EMPTY); - try { - future.get(5, TimeUnit.SECONDS); - fail("Expected ExecutionException"); - } catch (ExecutionException e) { - assertThat(e.getCause(), notNullValue()); - assertThat(e.getCause().getMessage(), containsString("404")); - } + FDv2Requestor.FDv2PayloadResponse response = future.get(5, TimeUnit.SECONDS); + + assertNotNull(response); + assertEquals(404, response.getStatusCode()); + assertFalse(response.isSuccess()); } } } @@ -408,7 +401,7 @@ public void differentSelectorsUseDifferentEtags() throws Exception { // First request with selector1 requestor.Poll(selector1).get(5, TimeUnit.SECONDS); RequestInfo req1 = server.getRecorder().requireRequest(); - assertEquals(null, req1.getHeader("If-None-Match")); + assertNull(req1.getHeader("If-None-Match")); // Second request with selector1 should use cached ETag requestor.Poll(selector1).get(5, TimeUnit.SECONDS); @@ -418,7 +411,7 @@ public void differentSelectorsUseDifferentEtags() throws Exception { // Request with selector2 should not have ETag (different URI) requestor.Poll(selector2).get(5, TimeUnit.SECONDS); RequestInfo req3 = server.getRecorder().requireRequest(); - assertEquals(null, req3.getHeader("If-None-Match")); + assertNull(req3.getHeader("If-None-Match")); } } } diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/PollingInitializerImplTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/PollingInitializerImplTest.java index c97194f7..c22e6f78 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/PollingInitializerImplTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/PollingInitializerImplTest.java @@ -67,9 +67,10 @@ private FDv2Requestor.FDv2PayloadResponse makeSuccessResponse() { "}"; try { - return new FDv2Requestor.FDv2PayloadResponse( + return FDv2Requestor.FDv2PayloadResponse.success( com.launchdarkly.sdk.internal.fdv2.payloads.FDv2Event.parseEventsArray(json), - okhttp3.Headers.of() + okhttp3.Headers.of(), + 200 ); } catch (Exception e) { throw new RuntimeException(e); @@ -102,8 +103,10 @@ public void httpRecoverableError() throws Exception { FDv2Requestor requestor = mockRequestor(); SelectorSource selectorSource = mockSelectorSource(); + FDv2Requestor.FDv2PayloadResponse errorResponse = + FDv2Requestor.FDv2PayloadResponse.failure(503, okhttp3.Headers.of()); when(requestor.Poll(any(Selector.class))) - .thenReturn(failedFuture(new HttpErrors.HttpErrorException(503))); + .thenReturn(CompletableFuture.completedFuture(errorResponse)); PollingInitializerImpl initializer = new PollingInitializerImpl(requestor, testLogger, selectorSource); @@ -116,7 +119,7 @@ public void httpRecoverableError() throws Exception { assertNotNull(result.getStatus().getErrorInfo()); assertEquals(DataSourceStatusProvider.ErrorKind.ERROR_RESPONSE, result.getStatus().getErrorInfo().getKind()); - + } @Test @@ -124,8 +127,10 @@ public void httpNonRecoverableError() throws Exception { FDv2Requestor requestor = mockRequestor(); SelectorSource selectorSource = mockSelectorSource(); + FDv2Requestor.FDv2PayloadResponse errorResponse = + FDv2Requestor.FDv2PayloadResponse.failure(401, okhttp3.Headers.of()); when(requestor.Poll(any(Selector.class))) - .thenReturn(failedFuture(new HttpErrors.HttpErrorException(401))); + .thenReturn(CompletableFuture.completedFuture(errorResponse)); PollingInitializerImpl initializer = new PollingInitializerImpl(requestor, testLogger, selectorSource); @@ -137,7 +142,7 @@ public void httpNonRecoverableError() throws Exception { assertEquals(FDv2SourceResult.State.TERMINAL_ERROR, result.getStatus().getState()); assertEquals(DataSourceStatusProvider.ErrorKind.ERROR_RESPONSE, result.getStatus().getErrorInfo().getKind()); - + } @Test @@ -248,9 +253,10 @@ public void errorEventInResponse() throws Exception { " ]\n" + "}"; - FDv2Requestor.FDv2PayloadResponse response = new FDv2Requestor.FDv2PayloadResponse( + FDv2Requestor.FDv2PayloadResponse response = FDv2Requestor.FDv2PayloadResponse.success( com.launchdarkly.sdk.internal.fdv2.payloads.FDv2Event.parseEventsArray(errorJson), - okhttp3.Headers.of() + okhttp3.Headers.of(), + 200 ); when(requestor.Poll(any(Selector.class))) @@ -284,9 +290,10 @@ public void goodbyeEventInResponse() throws Exception { " ]\n" + "}"; - FDv2Requestor.FDv2PayloadResponse response = new FDv2Requestor.FDv2PayloadResponse( + FDv2Requestor.FDv2PayloadResponse response = FDv2Requestor.FDv2PayloadResponse.success( com.launchdarkly.sdk.internal.fdv2.payloads.FDv2Event.parseEventsArray(goodbyeJson), - okhttp3.Headers.of() + okhttp3.Headers.of(), + 200 ); when(requestor.Poll(any(Selector.class))) @@ -311,9 +318,10 @@ public void emptyEventsArray() throws Exception { String emptyJson = "{\"events\": []}"; - FDv2Requestor.FDv2PayloadResponse response = new FDv2Requestor.FDv2PayloadResponse( + FDv2Requestor.FDv2PayloadResponse response = FDv2Requestor.FDv2PayloadResponse.success( com.launchdarkly.sdk.internal.fdv2.payloads.FDv2Event.parseEventsArray(emptyJson), - okhttp3.Headers.of() + okhttp3.Headers.of(), + 200 ); when(requestor.Poll(any(Selector.class))) @@ -366,9 +374,10 @@ public void internalErrorWithInvalidDataKind() throws Exception { " ]\n" + "}"; - FDv2Requestor.FDv2PayloadResponse response = new FDv2Requestor.FDv2PayloadResponse( + FDv2Requestor.FDv2PayloadResponse response = FDv2Requestor.FDv2PayloadResponse.success( com.launchdarkly.sdk.internal.fdv2.payloads.FDv2Event.parseEventsArray(malformedPayloadTransferred), - okhttp3.Headers.of() + okhttp3.Headers.of(), + 200 ); when(requestor.Poll(any(Selector.class))) @@ -403,9 +412,10 @@ public void internalErrorWithUnknownKind() throws Exception { " ]\n" + "}"; - FDv2Requestor.FDv2PayloadResponse response = new FDv2Requestor.FDv2PayloadResponse( + FDv2Requestor.FDv2PayloadResponse response = FDv2Requestor.FDv2PayloadResponse.success( com.launchdarkly.sdk.internal.fdv2.payloads.FDv2Event.parseEventsArray(unknownEventJson), - okhttp3.Headers.of() + okhttp3.Headers.of(), + 200 ); when(requestor.Poll(any(Selector.class))) @@ -423,4 +433,182 @@ public void internalErrorWithUnknownKind() throws Exception { } + + @Test + public void fdv1FallbackFlagSetToTrueInSuccessResponse() throws Exception { + FDv2Requestor requestor = mockRequestor(); + SelectorSource selectorSource = mockSelectorSource(); + + okhttp3.Headers headers = new okhttp3.Headers.Builder() + .add("x-ld-fd-fallback", "true") + .build(); + + FDv2Requestor.FDv2PayloadResponse response = FDv2Requestor.FDv2PayloadResponse.success( + com.launchdarkly.sdk.internal.fdv2.payloads.FDv2Event.parseEventsArray( + "{\"events\": [{\"event\": \"server-intent\", \"data\": {\"payloads\": [{\"id\": \"payload-1\", \"target\": 100, \"intentCode\": \"xfer-full\", \"reason\": \"payload-missing\"}]}}, {\"event\": \"payload-transferred\", \"data\": {\"state\": \"(p:payload-1:100)\", \"version\": 100}}]}" + ), + headers, + 200 + ); + + when(requestor.Poll(any(Selector.class))) + .thenReturn(CompletableFuture.completedFuture(response)); + + PollingInitializerImpl initializer = new PollingInitializerImpl(requestor, testLogger, selectorSource); + + CompletableFuture resultFuture = initializer.run(); + FDv2SourceResult result = resultFuture.get(5, TimeUnit.SECONDS); + + assertNotNull(result); + assertEquals(FDv2SourceResult.ResultType.CHANGE_SET, result.getResultType()); + assertEquals(true, result.isFdv1Fallback()); + } + + @Test + public void fdv1FallbackFlagSetToFalseWhenHeaderNotPresent() throws Exception { + FDv2Requestor requestor = mockRequestor(); + SelectorSource selectorSource = mockSelectorSource(); + + FDv2Requestor.FDv2PayloadResponse response = makeSuccessResponse(); + when(requestor.Poll(any(Selector.class))) + .thenReturn(CompletableFuture.completedFuture(response)); + + PollingInitializerImpl initializer = new PollingInitializerImpl(requestor, testLogger, selectorSource); + + CompletableFuture resultFuture = initializer.run(); + FDv2SourceResult result = resultFuture.get(5, TimeUnit.SECONDS); + + assertNotNull(result); + assertEquals(FDv2SourceResult.ResultType.CHANGE_SET, result.getResultType()); + assertEquals(false, result.isFdv1Fallback()); + } + + @Test + public void fdv1FallbackFlagSetToTrueInErrorResponse() throws Exception { + FDv2Requestor requestor = mockRequestor(); + SelectorSource selectorSource = mockSelectorSource(); + + okhttp3.Headers headers = new okhttp3.Headers.Builder() + .add("x-ld-fd-fallback", "true") + .build(); + + FDv2Requestor.FDv2PayloadResponse errorResponse = + FDv2Requestor.FDv2PayloadResponse.failure(503, headers); + when(requestor.Poll(any(Selector.class))) + .thenReturn(CompletableFuture.completedFuture(errorResponse)); + + PollingInitializerImpl initializer = new PollingInitializerImpl(requestor, testLogger, selectorSource); + + CompletableFuture resultFuture = initializer.run(); + FDv2SourceResult result = resultFuture.get(5, TimeUnit.SECONDS); + + assertNotNull(result); + assertEquals(FDv2SourceResult.ResultType.STATUS, result.getResultType()); + assertEquals(FDv2SourceResult.State.TERMINAL_ERROR, result.getStatus().getState()); + assertEquals(true, result.isFdv1Fallback()); + } + + @Test + public void fdv1FallbackFlagSetToFalseInNetworkError() throws Exception { + FDv2Requestor requestor = mockRequestor(); + SelectorSource selectorSource = mockSelectorSource(); + + when(requestor.Poll(any(Selector.class))) + .thenReturn(failedFuture(new IOException("Connection refused"))); + + PollingInitializerImpl initializer = new PollingInitializerImpl(requestor, testLogger, selectorSource); + + CompletableFuture resultFuture = initializer.run(); + FDv2SourceResult result = resultFuture.get(5, TimeUnit.SECONDS); + + assertNotNull(result); + assertEquals(FDv2SourceResult.ResultType.STATUS, result.getResultType()); + assertEquals(FDv2SourceResult.State.TERMINAL_ERROR, result.getStatus().getState()); + // Network errors don't have headers, so fallback should be false + assertEquals(false, result.isFdv1Fallback()); + } + + @Test + public void environmentIdExtractedFromHeaders() throws Exception { + FDv2Requestor requestor = mockRequestor(); + SelectorSource selectorSource = mockSelectorSource(); + + okhttp3.Headers headers = new okhttp3.Headers.Builder() + .add("x-ld-envid", "test-env-123") + .build(); + + FDv2Requestor.FDv2PayloadResponse response = FDv2Requestor.FDv2PayloadResponse.success( + com.launchdarkly.sdk.internal.fdv2.payloads.FDv2Event.parseEventsArray( + "{\"events\": [{\"event\": \"server-intent\", \"data\": {\"payloads\": [{\"id\": \"payload-1\", \"target\": 100, \"intentCode\": \"xfer-full\", \"reason\": \"payload-missing\"}]}}, {\"event\": \"payload-transferred\", \"data\": {\"state\": \"(p:payload-1:100)\", \"version\": 100}}]}" + ), + headers, + 200 + ); + + when(requestor.Poll(any(Selector.class))) + .thenReturn(CompletableFuture.completedFuture(response)); + + PollingInitializerImpl initializer = new PollingInitializerImpl(requestor, testLogger, selectorSource); + + CompletableFuture resultFuture = initializer.run(); + FDv2SourceResult result = resultFuture.get(5, TimeUnit.SECONDS); + + assertNotNull(result); + assertEquals(FDv2SourceResult.ResultType.CHANGE_SET, result.getResultType()); + assertNotNull(result.getChangeSet()); + assertEquals("test-env-123", result.getChangeSet().getEnvironmentId()); + } + + @Test + public void environmentIdNullWhenHeaderNotPresent() throws Exception { + FDv2Requestor requestor = mockRequestor(); + SelectorSource selectorSource = mockSelectorSource(); + + FDv2Requestor.FDv2PayloadResponse response = makeSuccessResponse(); + when(requestor.Poll(any(Selector.class))) + .thenReturn(CompletableFuture.completedFuture(response)); + + PollingInitializerImpl initializer = new PollingInitializerImpl(requestor, testLogger, selectorSource); + + CompletableFuture resultFuture = initializer.run(); + FDv2SourceResult result = resultFuture.get(5, TimeUnit.SECONDS); + + assertNotNull(result); + assertEquals(FDv2SourceResult.ResultType.CHANGE_SET, result.getResultType()); + assertNotNull(result.getChangeSet()); + assertNull(result.getChangeSet().getEnvironmentId()); + } + + @Test + public void bothFdv1FallbackAndEnvironmentIdExtractedFromHeaders() throws Exception { + FDv2Requestor requestor = mockRequestor(); + SelectorSource selectorSource = mockSelectorSource(); + + okhttp3.Headers headers = new okhttp3.Headers.Builder() + .add("x-ld-fd-fallback", "true") + .add("x-ld-envid", "test-env-456") + .build(); + + FDv2Requestor.FDv2PayloadResponse response = FDv2Requestor.FDv2PayloadResponse.success( + com.launchdarkly.sdk.internal.fdv2.payloads.FDv2Event.parseEventsArray( + "{\"events\": [{\"event\": \"server-intent\", \"data\": {\"payloads\": [{\"id\": \"payload-1\", \"target\": 100, \"intentCode\": \"xfer-full\", \"reason\": \"payload-missing\"}]}}, {\"event\": \"payload-transferred\", \"data\": {\"state\": \"(p:payload-1:100)\", \"version\": 100}}]}" + ), + headers, + 200 + ); + + when(requestor.Poll(any(Selector.class))) + .thenReturn(CompletableFuture.completedFuture(response)); + + PollingInitializerImpl initializer = new PollingInitializerImpl(requestor, testLogger, selectorSource); + + CompletableFuture resultFuture = initializer.run(); + FDv2SourceResult result = resultFuture.get(5, TimeUnit.SECONDS); + + assertNotNull(result); + assertEquals(FDv2SourceResult.ResultType.CHANGE_SET, result.getResultType()); + assertEquals(true, result.isFdv1Fallback()); + assertNotNull(result.getChangeSet()); + assertEquals("test-env-456", result.getChangeSet().getEnvironmentId()); + } } diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/PollingSynchronizerImplTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/PollingSynchronizerImplTest.java index 4dbccd02..da6a9fde 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/PollingSynchronizerImplTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/PollingSynchronizerImplTest.java @@ -67,9 +67,10 @@ private FDv2Requestor.FDv2PayloadResponse makeSuccessResponse() { "}"; try { - return new FDv2Requestor.FDv2PayloadResponse( + return FDv2Requestor.FDv2PayloadResponse.success( com.launchdarkly.sdk.internal.fdv2.payloads.FDv2Event.parseEventsArray(json), - okhttp3.Headers.of() + okhttp3.Headers.of(), + 200 ); } catch (Exception e) { throw new RuntimeException(e); @@ -535,7 +536,8 @@ public void nonRecoverableHttpErrorStopsPolling() throws Exception { int count = callCount.incrementAndGet(); // First call returns 401 (non-recoverable) if (count == 1) { - return failedFuture(new com.launchdarkly.sdk.internal.http.HttpErrors.HttpErrorException(401)); + return CompletableFuture.completedFuture( + FDv2Requestor.FDv2PayloadResponse.failure(401, okhttp3.Headers.of())); } else { // Subsequent calls should not happen, but return success if they do return CompletableFuture.completedFuture(makeSuccessResponse()); @@ -583,7 +585,8 @@ public void recoverableHttpErrorContinuesPolling() throws Exception { int count = callCount.incrementAndGet(); // First call returns 429 (recoverable - too many requests) if (count == 1) { - return failedFuture(new com.launchdarkly.sdk.internal.http.HttpErrors.HttpErrorException(429)); + return CompletableFuture.completedFuture( + FDv2Requestor.FDv2PayloadResponse.failure(429, okhttp3.Headers.of())); } else { // Subsequent calls succeed successCount.incrementAndGet(); @@ -635,9 +638,11 @@ public void multipleRecoverableErrorsContinuePolling() throws Exception { int count = callCount.incrementAndGet(); // Multiple recoverable errors: 408, 429, network error, success pattern if (count == 1) { - return failedFuture(new com.launchdarkly.sdk.internal.http.HttpErrors.HttpErrorException(408)); + return CompletableFuture.completedFuture( + FDv2Requestor.FDv2PayloadResponse.failure(408, okhttp3.Headers.of())); } else if (count == 2) { - return failedFuture(new com.launchdarkly.sdk.internal.http.HttpErrors.HttpErrorException(429)); + return CompletableFuture.completedFuture( + FDv2Requestor.FDv2PayloadResponse.failure(429, okhttp3.Headers.of())); } else if (count == 3) { return failedFuture(new IOException("Connection timeout")); } else { @@ -691,7 +696,8 @@ public void nonRecoverableThenRecoverableErrorStopsPolling() throws Exception { int count = callCount.incrementAndGet(); // First call returns 403 (non-recoverable) if (count == 1) { - return failedFuture(new com.launchdarkly.sdk.internal.http.HttpErrors.HttpErrorException(403)); + return CompletableFuture.completedFuture( + FDv2Requestor.FDv2PayloadResponse.failure(403, okhttp3.Headers.of())); } else { // Any subsequent calls should not happen return failedFuture(new IOException("Network error")); @@ -761,9 +767,10 @@ public void internalErrorWithInvalidDataKindContinuesPolling() throws Exception " ]\n" + "}"; - return CompletableFuture.completedFuture(new FDv2Requestor.FDv2PayloadResponse( + return CompletableFuture.completedFuture(FDv2Requestor.FDv2PayloadResponse.success( com.launchdarkly.sdk.internal.fdv2.payloads.FDv2Event.parseEventsArray(malformedPayloadTransferred), - okhttp3.Headers.of() + okhttp3.Headers.of(), + 200 )); } else { // Subsequent calls succeed @@ -824,9 +831,10 @@ public void internalErrorWithUnknownKindContinuesPolling() throws Exception { " ]\n" + "}"; - return CompletableFuture.completedFuture(new FDv2Requestor.FDv2PayloadResponse( + return CompletableFuture.completedFuture(FDv2Requestor.FDv2PayloadResponse.success( com.launchdarkly.sdk.internal.fdv2.payloads.FDv2Event.parseEventsArray(unknownEventJson), - okhttp3.Headers.of() + okhttp3.Headers.of(), + 200 )); } else { // Subsequent calls succeed @@ -866,4 +874,201 @@ public void internalErrorWithUnknownKindContinuesPolling() throws Exception { executor.shutdown(); } } + + @Test + public void fdv1FallbackFlagSetToTrueInSuccessResponse() throws Exception { + FDv2Requestor requestor = mockRequestor(); + SelectorSource selectorSource = mockSelectorSource(); + ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); + + okhttp3.Headers headers = new okhttp3.Headers.Builder() + .add("x-ld-fd-fallback", "true") + .build(); + + FDv2Requestor.FDv2PayloadResponse response = FDv2Requestor.FDv2PayloadResponse.success( + com.launchdarkly.sdk.internal.fdv2.payloads.FDv2Event.parseEventsArray( + "{\"events\": [{\"event\": \"server-intent\", \"data\": {\"payloads\": [{\"id\": \"payload-1\", \"target\": 100, \"intentCode\": \"xfer-full\", \"reason\": \"payload-missing\"}]}}, {\"event\": \"payload-transferred\", \"data\": {\"state\": \"(p:payload-1:100)\", \"version\": 100}}]}" + ), + headers, + 200 + ); + + when(requestor.Poll(any(Selector.class))) + .thenReturn(CompletableFuture.completedFuture(response)); + + try { + PollingSynchronizerImpl synchronizer = new PollingSynchronizerImpl( + requestor, + testLogger, + selectorSource, + executor, + Duration.ofMillis(100) + ); + + FDv2SourceResult result = synchronizer.next().get(1, TimeUnit.SECONDS); + + assertNotNull(result); + assertEquals(FDv2SourceResult.ResultType.CHANGE_SET, result.getResultType()); + assertEquals(true, result.isFdv1Fallback()); + + synchronizer.close(); + } finally { + executor.shutdown(); + } + } + + @Test + public void fdv1FallbackFlagSetToFalseWhenHeaderNotPresent() throws Exception { + FDv2Requestor requestor = mockRequestor(); + SelectorSource selectorSource = mockSelectorSource(); + ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); + + when(requestor.Poll(any(Selector.class))) + .thenReturn(CompletableFuture.completedFuture(makeSuccessResponse())); + + try { + PollingSynchronizerImpl synchronizer = new PollingSynchronizerImpl( + requestor, + testLogger, + selectorSource, + executor, + Duration.ofMillis(100) + ); + + FDv2SourceResult result = synchronizer.next().get(1, TimeUnit.SECONDS); + + assertNotNull(result); + assertEquals(FDv2SourceResult.ResultType.CHANGE_SET, result.getResultType()); + assertEquals(false, result.isFdv1Fallback()); + + synchronizer.close(); + } finally { + executor.shutdown(); + } + } + + @Test + public void fdv1FallbackFlagSetToTrueInErrorResponse() throws Exception { + FDv2Requestor requestor = mockRequestor(); + SelectorSource selectorSource = mockSelectorSource(); + ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); + + okhttp3.Headers headers = new okhttp3.Headers.Builder() + .add("x-ld-fd-fallback", "true") + .build(); + + FDv2Requestor.FDv2PayloadResponse errorResponse = + FDv2Requestor.FDv2PayloadResponse.failure(503, headers); + when(requestor.Poll(any(Selector.class))) + .thenReturn(CompletableFuture.completedFuture(errorResponse)); + + try { + PollingSynchronizerImpl synchronizer = new PollingSynchronizerImpl( + requestor, + testLogger, + selectorSource, + executor, + Duration.ofMillis(100) + ); + + FDv2SourceResult result = synchronizer.next().get(1, TimeUnit.SECONDS); + + assertNotNull(result); + assertEquals(FDv2SourceResult.ResultType.STATUS, result.getResultType()); + assertEquals(FDv2SourceResult.State.INTERRUPTED, result.getStatus().getState()); + assertEquals(true, result.isFdv1Fallback()); + + synchronizer.close(); + } finally { + executor.shutdown(); + } + } + + @Test + public void environmentIdExtractedFromHeaders() throws Exception { + FDv2Requestor requestor = mockRequestor(); + SelectorSource selectorSource = mockSelectorSource(); + ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); + + okhttp3.Headers headers = new okhttp3.Headers.Builder() + .add("x-ld-envid", "test-env-789") + .build(); + + FDv2Requestor.FDv2PayloadResponse response = FDv2Requestor.FDv2PayloadResponse.success( + com.launchdarkly.sdk.internal.fdv2.payloads.FDv2Event.parseEventsArray( + "{\"events\": [{\"event\": \"server-intent\", \"data\": {\"payloads\": [{\"id\": \"payload-1\", \"target\": 100, \"intentCode\": \"xfer-full\", \"reason\": \"payload-missing\"}]}}, {\"event\": \"payload-transferred\", \"data\": {\"state\": \"(p:payload-1:100)\", \"version\": 100}}]}" + ), + headers, + 200 + ); + + when(requestor.Poll(any(Selector.class))) + .thenReturn(CompletableFuture.completedFuture(response)); + + try { + PollingSynchronizerImpl synchronizer = new PollingSynchronizerImpl( + requestor, + testLogger, + selectorSource, + executor, + Duration.ofMillis(100) + ); + + FDv2SourceResult result = synchronizer.next().get(1, TimeUnit.SECONDS); + + assertNotNull(result); + assertEquals(FDv2SourceResult.ResultType.CHANGE_SET, result.getResultType()); + assertNotNull(result.getChangeSet()); + assertEquals("test-env-789", result.getChangeSet().getEnvironmentId()); + + synchronizer.close(); + } finally { + executor.shutdown(); + } + } + + @Test + public void bothFdv1FallbackAndEnvironmentIdExtractedFromHeaders() throws Exception { + FDv2Requestor requestor = mockRequestor(); + SelectorSource selectorSource = mockSelectorSource(); + ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); + + okhttp3.Headers headers = new okhttp3.Headers.Builder() + .add("x-ld-fd-fallback", "true") + .add("x-ld-envid", "test-env-combined") + .build(); + + FDv2Requestor.FDv2PayloadResponse response = FDv2Requestor.FDv2PayloadResponse.success( + com.launchdarkly.sdk.internal.fdv2.payloads.FDv2Event.parseEventsArray( + "{\"events\": [{\"event\": \"server-intent\", \"data\": {\"payloads\": [{\"id\": \"payload-1\", \"target\": 100, \"intentCode\": \"xfer-full\", \"reason\": \"payload-missing\"}]}}, {\"event\": \"payload-transferred\", \"data\": {\"state\": \"(p:payload-1:100)\", \"version\": 100}}]}" + ), + headers, + 200 + ); + + when(requestor.Poll(any(Selector.class))) + .thenReturn(CompletableFuture.completedFuture(response)); + + try { + PollingSynchronizerImpl synchronizer = new PollingSynchronizerImpl( + requestor, + testLogger, + selectorSource, + executor, + Duration.ofMillis(100) + ); + + FDv2SourceResult result = synchronizer.next().get(1, TimeUnit.SECONDS); + + assertNotNull(result); + assertEquals(FDv2SourceResult.ResultType.CHANGE_SET, result.getResultType()); + assertEquals(true, result.isFdv1Fallback()); + assertNotNull(result.getChangeSet()); + assertEquals("test-env-combined", result.getChangeSet().getEnvironmentId()); + + synchronizer.close(); + } finally { + executor.shutdown(); + } + } } diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/StreamingSynchronizerImplTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/StreamingSynchronizerImplTest.java index eaf305c3..48194f33 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/StreamingSynchronizerImplTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/StreamingSynchronizerImplTest.java @@ -823,4 +823,182 @@ public void nullPayloadFilterNotAddedToRequest() throws Exception { synchronizer.close(); } } + + @Test + public void fdv1FallbackFlagSetToTrueInSuccessResponse() throws Exception { + String serverIntent = makeEvent("server-intent", "{\"payloads\":[{\"id\":\"payload-1\",\"target\":100,\"intentCode\":\"xfer-full\",\"reason\":\"payload-missing\"}]}"); + String payloadTransferred = makeEvent("payload-transferred", "{\"state\":\"(p:payload-1:100)\",\"version\":100}"); + + try (HttpServer server = HttpServer.start(Handlers.all( + Handlers.header("x-ld-fd-fallback", "true"), + Handlers.SSE.start(), + Handlers.SSE.event(serverIntent), + Handlers.SSE.event(payloadTransferred), + Handlers.SSE.leaveOpen()))) { + + HttpProperties httpProperties = toHttpProperties(clientContext("sdk-key", baseConfig().build()).getHttp()); + SelectorSource selectorSource = mockSelectorSource(); + + StreamingSynchronizerImpl synchronizer = new StreamingSynchronizerImpl( + httpProperties, + server.getUri(), + "/stream", + testLogger, + selectorSource, + null, + Duration.ofMillis(100) + ); + + CompletableFuture resultFuture = synchronizer.next(); + FDv2SourceResult result = resultFuture.get(5, TimeUnit.SECONDS); + + assertNotNull(result); + assertEquals(FDv2SourceResult.ResultType.CHANGE_SET, result.getResultType()); + assertEquals(true, result.isFdv1Fallback()); + + synchronizer.close(); + } + } + + @Test + public void fdv1FallbackFlagSetToFalseWhenHeaderNotPresent() throws Exception { + String serverIntent = makeEvent("server-intent", "{\"payloads\":[{\"id\":\"payload-1\",\"target\":100,\"intentCode\":\"xfer-full\",\"reason\":\"payload-missing\"}]}"); + String payloadTransferred = makeEvent("payload-transferred", "{\"state\":\"(p:payload-1:100)\",\"version\":100}"); + + try (HttpServer server = HttpServer.start(Handlers.all( + Handlers.SSE.start(), + Handlers.SSE.event(serverIntent), + Handlers.SSE.event(payloadTransferred), + Handlers.SSE.leaveOpen()))) { + + HttpProperties httpProperties = toHttpProperties(clientContext("sdk-key", baseConfig().build()).getHttp()); + SelectorSource selectorSource = mockSelectorSource(); + + StreamingSynchronizerImpl synchronizer = new StreamingSynchronizerImpl( + httpProperties, + server.getUri(), + "/stream", + testLogger, + selectorSource, + null, + Duration.ofMillis(100) + ); + + CompletableFuture resultFuture = synchronizer.next(); + FDv2SourceResult result = resultFuture.get(5, TimeUnit.SECONDS); + + assertNotNull(result); + assertEquals(FDv2SourceResult.ResultType.CHANGE_SET, result.getResultType()); + assertEquals(false, result.isFdv1Fallback()); + + synchronizer.close(); + } + } + + @Test + public void fdv1FallbackFlagSetToTrueInErrorResponse() throws Exception { + try (HttpServer server = HttpServer.start(Handlers.all( + Handlers.status(503), + Handlers.header("x-ld-fd-fallback", "true")))) { + + HttpProperties httpProperties = toHttpProperties(clientContext("sdk-key", baseConfig().build()).getHttp()); + SelectorSource selectorSource = mockSelectorSource(); + + StreamingSynchronizerImpl synchronizer = new StreamingSynchronizerImpl( + httpProperties, + server.getUri(), + "/stream", + testLogger, + selectorSource, + null, + Duration.ofMillis(100) + ); + + CompletableFuture resultFuture = synchronizer.next(); + FDv2SourceResult result = resultFuture.get(5, TimeUnit.SECONDS); + + assertNotNull(result); + assertEquals(FDv2SourceResult.ResultType.STATUS, result.getResultType()); + assertEquals(FDv2SourceResult.State.INTERRUPTED, result.getStatus().getState()); + assertEquals(true, result.isFdv1Fallback()); + + synchronizer.close(); + } + } + + @Test + public void environmentIdExtractedFromHeaders() throws Exception { + String serverIntent = makeEvent("server-intent", "{\"payloads\":[{\"id\":\"payload-1\",\"target\":100,\"intentCode\":\"xfer-full\",\"reason\":\"payload-missing\"}]}"); + String payloadTransferred = makeEvent("payload-transferred", "{\"state\":\"(p:payload-1:100)\",\"version\":100}"); + + try (HttpServer server = HttpServer.start(Handlers.all( + Handlers.header("x-ld-envid", "test-env-streaming"), + Handlers.SSE.start(), + Handlers.SSE.event(serverIntent), + Handlers.SSE.event(payloadTransferred), + Handlers.SSE.leaveOpen()))) { + + HttpProperties httpProperties = toHttpProperties(clientContext("sdk-key", baseConfig().build()).getHttp()); + SelectorSource selectorSource = mockSelectorSource(); + + StreamingSynchronizerImpl synchronizer = new StreamingSynchronizerImpl( + httpProperties, + server.getUri(), + "/stream", + testLogger, + selectorSource, + null, + Duration.ofMillis(100) + ); + + CompletableFuture resultFuture = synchronizer.next(); + FDv2SourceResult result = resultFuture.get(5, TimeUnit.SECONDS); + + assertNotNull(result); + assertEquals(FDv2SourceResult.ResultType.CHANGE_SET, result.getResultType()); + assertNotNull(result.getChangeSet()); + assertEquals("test-env-streaming", result.getChangeSet().getEnvironmentId()); + + synchronizer.close(); + } + } + + @Test + public void bothFdv1FallbackAndEnvironmentIdExtractedFromHeaders() throws Exception { + String serverIntent = makeEvent("server-intent", "{\"payloads\":[{\"id\":\"payload-1\",\"target\":100,\"intentCode\":\"xfer-full\",\"reason\":\"payload-missing\"}]}"); + String payloadTransferred = makeEvent("payload-transferred", "{\"state\":\"(p:payload-1:100)\",\"version\":100}"); + + try (HttpServer server = HttpServer.start(Handlers.all( + Handlers.header("x-ld-fd-fallback", "true"), + Handlers.header("x-ld-envid", "test-env-combined-streaming"), + Handlers.SSE.start(), + Handlers.SSE.event(serverIntent), + Handlers.SSE.event(payloadTransferred), + Handlers.SSE.leaveOpen()))) { + + HttpProperties httpProperties = toHttpProperties(clientContext("sdk-key", baseConfig().build()).getHttp()); + SelectorSource selectorSource = mockSelectorSource(); + + StreamingSynchronizerImpl synchronizer = new StreamingSynchronizerImpl( + httpProperties, + server.getUri(), + "/stream", + testLogger, + selectorSource, + null, + Duration.ofMillis(100) + ); + + CompletableFuture resultFuture = synchronizer.next(); + FDv2SourceResult result = resultFuture.get(5, TimeUnit.SECONDS); + + assertNotNull(result); + assertEquals(FDv2SourceResult.ResultType.CHANGE_SET, result.getResultType()); + assertEquals(true, result.isFdv1Fallback()); + assertNotNull(result.getChangeSet()); + assertEquals("test-env-combined-streaming", result.getChangeSet().getEnvironmentId()); + + synchronizer.close(); + } + } } From b20c62102e1f3bd23468a9bc10f917d92187c2f4 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Wed, 21 Jan 2026 15:31:49 -0800 Subject: [PATCH 02/35] Cleanup --- .../com/launchdarkly/sdk/server/DefaultFDv2Requestor.java | 2 +- .../main/java/com/launchdarkly/sdk/server/PollingBase.java | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DefaultFDv2Requestor.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DefaultFDv2Requestor.java index 1eb75b11..d41be277 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DefaultFDv2Requestor.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DefaultFDv2Requestor.java @@ -111,7 +111,7 @@ public void onFailure(@Nonnull Call call, @Nonnull IOException e) { @Override public void onResponse(@Nonnull Call call, @Nonnull Response response) { try { - // Handle 304 Not Modified - no new data, but return response with headers + // Handle 304 Not Modified - no new data if (response.code() == 304) { logger.debug("FDv2 polling request returned 304: not modified"); future.complete(FDv2PayloadResponse.none(response.code())); diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingBase.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingBase.java index f18309d2..c1a42357 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingBase.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingBase.java @@ -4,7 +4,6 @@ import com.launchdarkly.sdk.internal.fdv2.payloads.FDv2Event; import com.launchdarkly.sdk.internal.fdv2.sources.FDv2ProtocolHandler; import com.launchdarkly.sdk.internal.fdv2.sources.Selector; -import com.launchdarkly.sdk.internal.http.HttpErrors; import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; import com.launchdarkly.sdk.server.interfaces.DataSourceStatusProvider; import com.launchdarkly.sdk.server.subsystems.DataStoreTypes; @@ -34,11 +33,6 @@ private static boolean getFallback(FDv2Requestor.FDv2PayloadResponse response) { String headerValue = response.getHeaders().get(HeaderConstants.FDV1_FALLBACK.getHeaderName()); return headerValue != null && headerValue.equalsIgnoreCase("true"); } -// if(ex != null) { -// if(ex instanceof HttpErrorException) { -// ((HttpErrors.HttpErrorException) ex). -// } -// } return false; } From b46ac5e11469de7ebd5b8c287a38f84b38a61c5c Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Fri, 23 Jan 2026 15:12:42 -0800 Subject: [PATCH 03/35] chore: Add fallback and recovery support for FDv2. --- .../sdk/server/FDv2DataSource.java | 370 +++++++++++++-- .../sdk/server/FDv2DataSystem.java | 5 +- .../com/launchdarkly/sdk/server/Loggers.java | 3 + .../sdk/server/PollingInitializerImpl.java | 2 +- .../sdk/server/PollingSynchronizerImpl.java | 2 +- .../sdk/server/StreamingSynchronizerImpl.java | 2 +- .../FDv2DataSourceFallbackConditionTest.java | 438 ++++++++++++++++++ 7 files changed, 790 insertions(+), 32 deletions(-) create mode 100644 lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceFallbackConditionTest.java diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index 906c4847..edaf130a 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -1,6 +1,7 @@ package com.launchdarkly.sdk.server; import com.google.common.collect.ImmutableList; +import com.launchdarkly.logging.LDLogger; import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; import com.launchdarkly.sdk.server.datasources.Initializer; import com.launchdarkly.sdk.server.datasources.Synchronizer; @@ -10,18 +11,29 @@ import java.io.Closeable; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; -import java.util.concurrent.CancellationException; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; class FDv2DataSource implements DataSource { + /** + * Default fallback timeout of 2 minutes. The timeout is only configurable for testing. + */ + private static final int defaultFallbackTimeout = 2 * 60; + + /** + * Default recovery timeout of 5 minutes. The timeout is only configurable for testing. + */ + private static final long defaultRecoveryTimeout = 5 * 60; + private final List> initializers; private final List synchronizers; + private final List conditionFactories; + private final DataSourceUpdateSinkV2 dataSourceUpdates; private final CompletableFuture startFuture = new CompletableFuture<>(); @@ -34,6 +46,168 @@ class FDv2DataSource implements DataSource { private Closeable activeSource; private boolean isShutdown = false; + private final int threadPriority; + + private final LDLogger logger; + + /** + * Package-private for testing. + */ + interface Condition { + enum ConditionType { + FALLBACK, + RECOVERY, + } + CompletableFuture execute(); + + void inform(FDv2SourceResult sourceResult); + + void close() throws IOException; + + ConditionType getType(); + } + + interface ConditionFactory { + Condition build(); + + Condition.ConditionType getType(); + } + + + static abstract class TimedCondition implements Condition { + protected final CompletableFuture resultFuture = new CompletableFuture<>(); + + protected final ScheduledExecutorService sharedExecutor; + + /** + * Future for the timeout task, if any. Will be null when no timeout is active. + */ + protected ScheduledFuture timerFuture; + + /** + * Timeout duration for the fallback operation. + */ + protected final long timeoutSeconds; + + public TimedCondition(ScheduledExecutorService sharedExecutor, long timeoutSeconds) { + this.sharedExecutor = sharedExecutor; + this.timeoutSeconds = timeoutSeconds; + } + + @Override + public CompletableFuture execute() { + return resultFuture; + } + + @Override + public void close() throws IOException { + if (timerFuture != null) { + timerFuture.cancel(false); + timerFuture = null; + } + } + + static abstract class Factory implements ConditionFactory { + protected final ScheduledExecutorService sharedExecutor; + protected final long timeoutSeconds; + + public Factory(ScheduledExecutorService sharedExecutor, long timeout) { + this.sharedExecutor = sharedExecutor; + this.timeoutSeconds = timeout; + } + } + } + + /** + * This condition is used to determine if a fallback should be performed. It is informed of each data source result + * via {@link #inform(FDv2SourceResult)}. Based on the results, it updates its internal state. When the fallback + * condition is met, then the {@link Future} returned by {@link #execute()} will complete. + *

+ * This is package-private, instead of private, for ease of testing. + */ + static class FallbackCondition extends TimedCondition { + static class Factory extends TimedCondition.Factory { + public Factory(ScheduledExecutorService sharedExecutor, long timeout) { + super(sharedExecutor, timeout); + } + @Override + public Condition build() { + return new FallbackCondition(sharedExecutor, timeoutSeconds); + } + + @Override + public ConditionType getType() { + return ConditionType.FALLBACK; + } + } + + public FallbackCondition(ScheduledExecutorService sharedExecutor, long timeoutSeconds) { + super(sharedExecutor, timeoutSeconds); + } + + @Override + public void inform(FDv2SourceResult sourceResult) { + if(sourceResult == null) { + return; + } + if(sourceResult.getResultType() == FDv2SourceResult.ResultType.CHANGE_SET) { + if(timerFuture != null) { + timerFuture.cancel(false); + timerFuture = null; + } + } + if(sourceResult.getResultType() == FDv2SourceResult.ResultType.STATUS && sourceResult.getStatus().getState() == FDv2SourceResult.State.INTERRUPTED) { + if (timerFuture == null) { + timerFuture = sharedExecutor.schedule(() -> { + resultFuture.complete(this); + return null; + }, timeoutSeconds, TimeUnit.SECONDS); + } + } + } + + @Override + public ConditionType getType() { + return ConditionType.FALLBACK; + } + } + + static class RecoveryCondition extends TimedCondition { + + static class Factory extends TimedCondition.Factory { + public Factory(ScheduledExecutorService sharedExecutor, long timeout) { + super(sharedExecutor, timeout); + } + @Override + public Condition build() { + return new RecoveryCondition(sharedExecutor, timeoutSeconds); + } + + @Override + public ConditionType getType() { + return ConditionType.RECOVERY; + } + } + + public RecoveryCondition(ScheduledExecutorService sharedExecutor, long timeoutSeconds) { + super(sharedExecutor, timeoutSeconds); + timerFuture = sharedExecutor.schedule(() -> { + resultFuture.complete(this); + return null; + }, timeoutSeconds, TimeUnit.SECONDS); + } + + @Override + public void inform(FDv2SourceResult sourceResult) { + // Time-based recovery. + } + + @Override + public ConditionType getType() { + return ConditionType.RECOVERY; + } + } + private static class SynchronizerFactoryWithState { public enum State { /** @@ -73,18 +247,39 @@ public interface DataSourceFactory { T build(); } + public FDv2DataSource( + ImmutableList> initializers, + ImmutableList> synchronizers, + DataSourceUpdateSinkV2 dataSourceUpdates, + int threadPriority, + LDLogger logger, + ScheduledExecutorService sharedExecutor + ) { + this(initializers, synchronizers, dataSourceUpdates, threadPriority, logger, sharedExecutor, defaultFallbackTimeout, defaultRecoveryTimeout); + } + public FDv2DataSource( - ImmutableList> initializers, - ImmutableList> synchronizers, - DataSourceUpdateSinkV2 dataSourceUpdates + ImmutableList> initializers, + ImmutableList> synchronizers, + DataSourceUpdateSinkV2 dataSourceUpdates, + int threadPriority, + LDLogger logger, + ScheduledExecutorService sharedExecutor, + long fallbackTimeout, + long recoveryTimeout ) { this.initializers = initializers; this.synchronizers = synchronizers - .stream() - .map(SynchronizerFactoryWithState::new) - .collect(Collectors.toList()); + .stream() + .map(SynchronizerFactoryWithState::new) + .collect(Collectors.toList()); this.dataSourceUpdates = dataSourceUpdates; + this.threadPriority = threadPriority; + this.logger = logger; + this.conditionFactories = new ArrayList<>(); + this.conditionFactories.add(new FallbackCondition.Factory(sharedExecutor, fallbackTimeout)); + this.conditionFactories.add(new RecoveryCondition.Factory(sharedExecutor, recoveryTimeout)); } private void run() { @@ -92,24 +287,64 @@ private void run() { if (!initializers.isEmpty()) { runInitializers(); } - runSynchronizers(); + boolean fdv1Fallback = runSynchronizers(); + if (fdv1Fallback) { + // TODO: Run FDv1 fallback. + } // TODO: Handle. We have ran out of sources or we are shutting down. }); runThread.setDaemon(true); - // TODO: Thread priority. - //thread.setPriority(threadPriority); + runThread.setPriority(threadPriority); runThread.start(); } - private SynchronizerFactoryWithState getFirstAvailableSynchronizer() { + /** + * We start at -1, so finding the next synchronizer can non-conditionally increment the index. + */ + private int sourceIndex = -1; + + /** + * Reset the source index to -1, indicating that we should start from the first synchronizer when looking for + * the next one to use. This is used when recovering from a non-primary synchronizer. + */ + private void resetSynchronizerSourceIndex() { + synchronized (activeSourceLock) { + sourceIndex = -1; + } + } + + /** + * Get the next synchronizer to use. This operates based on tracking the index of the currently active synchronizer, + * which will loop through all available synchronizers handling interruptions. Then a non-prime synchronizer recovers + * the source index will be reset, and we start at the beginning. + *

+ * Any given synchronizer can be marked as blocked, in which case that synchronizer is not eligible to be used again. + * Synchronizers that are not blocked are available, and this function will only return available synchronizers. + * @return the next synchronizer factory to use, or null if there are no more available synchronizers. + */ + private SynchronizerFactoryWithState getNextAvailableSynchronizer() { synchronized (synchronizers) { - for (SynchronizerFactoryWithState synchronizer : synchronizers) { - if (synchronizer.getState() == SynchronizerFactoryWithState.State.Available) { - return synchronizer; + SynchronizerFactoryWithState factory = null; + + // There is at least one available factory. + if(synchronizers.stream().anyMatch(s -> s.getState() == SynchronizerFactoryWithState.State.Available)) { + // Look for the next synchronizer starting at the position after the current one. (avoiding just re-using the same synchronizer.) + while(factory == null) { + sourceIndex++; + // We aren't using module here because we want to keep the stored index within range instead + // of increasing indefinitely. + if(sourceIndex >= synchronizers.size()) { + sourceIndex = 0; + } + SynchronizerFactoryWithState candidate = synchronizers.get(sourceIndex); + if (candidate.getState() == SynchronizerFactoryWithState.State.Available) { + factory = candidate; + } + } } - return null; + return factory; } } @@ -136,7 +371,9 @@ private void runInitializers() { break; } } catch (ExecutionException | InterruptedException | CancellationException e) { - // TODO: Log. + // TODO: Better messaging? + // TODO: Data source status? + logger.warn("Error running initializer: {}", e.toString()); } } // We received data without a selector, and we have exhausted initializers, so we are going to @@ -147,18 +384,88 @@ private void runInitializers() { } } - private void runSynchronizers() { - SynchronizerFactoryWithState availableSynchronizer = getFirstAvailableSynchronizer(); - // TODO: Add recovery handling. If there are no available synchronizers, but there are - // recovering ones, then we likely will want to wait for them to be available (or bypass recovery). + /** + * Determine conditions for the current synchronizer. Synchronizers require different conditions depending on if + * they are the 'prime' synchronizer or if there are other available synchronizers to use. + * @return a list of conditions to apply to the synchronizer + */ + private List getConditions() { + boolean isPrimeSynchronizer = false; + int availableSynchronizers = 0; + boolean firstAvailableSynchronizer = true; + + synchronized (activeSourceLock) { + for (int index = 0; index < synchronizers.size(); index++) { + + if (synchronizers.get(index).getState() == SynchronizerFactoryWithState.State.Available) { + if (firstAvailableSynchronizer && sourceIndex == index) { + // This is the first synchronizer that is available, and it also is the current one. + isPrimeSynchronizer = true; + } + // Subsequently encountered synchronizers that are available are not the first one. + firstAvailableSynchronizer = false; + availableSynchronizers++; + } + } + } + if(availableSynchronizers == 1) { + // If there is only 1 synchronizer, then we cannot fall back or recover, so we don't need any conditions. + return Collections.emptyList(); + } + if(isPrimeSynchronizer) { + // If there isn't a synchronizer to recover to, then don't add and recovery conditions. + return conditionFactories.stream() + .filter((ConditionFactory factory) -> factory.getType() != Condition.ConditionType.RECOVERY) + .map(ConditionFactory::build).collect(Collectors.toList()); + } + // The synchronizer can both fall back and recover. + return conditionFactories.stream().map(ConditionFactory::build).collect(Collectors.toList()); + } + + private boolean runSynchronizers() { + SynchronizerFactoryWithState availableSynchronizer = getNextAvailableSynchronizer(); while (availableSynchronizer != null) { Synchronizer synchronizer = availableSynchronizer.build(); + // Returns true if shutdown. - if (setActiveSource(synchronizer)) return; + if (setActiveSource(synchronizer)) return false; + try { boolean running = true; + // Conditions run once for the life of the synchronizer. + List conditions = getConditions(); + CompletableFuture conditionFutures = CompletableFuture.anyOf( + conditions.stream().map(Condition::execute).toArray(CompletableFuture[]::new)); + while (running) { - FDv2SourceResult result = synchronizer.next().get(); + CompletableFuture nextResultFuture = synchronizer.next(); + + Object res = CompletableFuture.anyOf(conditionFutures, nextResultFuture).get(); + + if(res instanceof Condition) { + Condition c = (Condition) res; + switch (c.getType()) { + case FALLBACK: + // For fallback, we will move to the next available synchronizer, which may loop. + // This is the default behavior of exiting the run loop, so we don't need to take + // any action. + break; + case RECOVERY: + // For recovery, we will start at the first available synchronizer. + // So we reset the source index, and finding the source will start at the beginning. + resetSynchronizerSourceIndex(); + break; + } + // A running synchronizer will only have fallback and recovery conditions that it can act on. + // So, if there are no synchronizers to recover to or fallback to, then we will not have + // those conditions. + break; + } + + + FDv2SourceResult result = (FDv2SourceResult) res; + conditions.forEach(c -> c.inform(result)); + switch (result.getResultType()) { case CHANGE_SET: dataSourceUpdates.apply(result.getChangeSet()); @@ -175,7 +482,7 @@ private void runSynchronizers() { case SHUTDOWN: // We should be overall shutting down. // TODO: We may need logging or to do a little more. - return; + return false; case TERMINAL_ERROR: availableSynchronizer.block(); running = false; @@ -186,13 +493,20 @@ private void runSynchronizers() { } break; } + // We have been requested to fall back to FDv1. We handle whatever message was associated, + // close the synchronizer, and then fallback. + if(result.isFdv1Fallback()) { + safeClose(synchronizer); + return true; + } } } catch (ExecutionException | InterruptedException | CancellationException e) { // TODO: Log. // Move to next synchronizer. } - availableSynchronizer = getFirstAvailableSynchronizer(); + availableSynchronizer = getNextAvailableSynchronizer(); } + return false; } private void safeClose(Closeable synchronizer) { @@ -239,7 +553,7 @@ public void close() throws IOException { // If there is an active source, we will shut it down, and that will result in the loop handling that source // exiting. // If we do not have an active source, then the loop will check isShutdown when attempting to set one. When - // it detects shutdown it will exit the loop. + // it detects shutdown, it will exit the loop. synchronized (activeSourceLock) { isShutdown = true; if (activeSource != null) { diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java index 70fc0048..c23e7168 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java @@ -138,7 +138,10 @@ static FDv2DataSystem create( DataSource dataSource = new FDv2DataSource( initializerFactories, synchronizerFactories, - dataSourceUpdates + dataSourceUpdates, + config.threadPriority, + clientContext.getBaseLogger().subLogger(Loggers.DATA_SOURCE_LOGGER_NAME), + clientContext.sharedExecutor ); DataSourceStatusProvider dataSourceStatusProvider = new DataSourceStatusProviderImpl( dataSourceStatusBroadcaster, diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/Loggers.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/Loggers.java index 823aa6a7..588ba3f3 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/Loggers.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/Loggers.java @@ -24,4 +24,7 @@ private Loggers() {} static final String EVALUATION_LOGGER_NAME = "Evaluation"; static final String EVENTS_LOGGER_NAME = "Events"; static final String HOOKS_LOGGER_NAME = "Hooks"; + static final String STREAMING_SYNCHRONIZER = "StreamingSynchronizer"; + static final String POLLING_SYNCHRONIZER = "PollingSynchronizer"; + static final String POLLING_INITIALIZER = "PollingInitializer"; } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingInitializerImpl.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingInitializerImpl.java index 856a118b..2e3b368f 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingInitializerImpl.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingInitializerImpl.java @@ -12,7 +12,7 @@ class PollingInitializerImpl extends PollingBase implements Initializer { private final SelectorSource selectorSource; public PollingInitializerImpl(FDv2Requestor requestor, LDLogger logger, SelectorSource selectorSource) { - super(requestor, logger); + super(requestor, logger.subLogger(Loggers.POLLING_INITIALIZER)); this.selectorSource = selectorSource; } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingSynchronizerImpl.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingSynchronizerImpl.java index 8bbc6a4e..43c95ee0 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingSynchronizerImpl.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingSynchronizerImpl.java @@ -23,7 +23,7 @@ public PollingSynchronizerImpl( ScheduledExecutorService sharedExecutor, Duration pollInterval ) { - super(requestor, logger); + super(requestor, logger.subLogger(Loggers.POLLING_SYNCHRONIZER)); this.selectorSource = selectorSource; synchronized (this) { diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java index 6fa4487c..e5099e5c 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java @@ -71,7 +71,7 @@ public StreamingSynchronizerImpl( ) { this.httpProperties = httpProperties; this.selectorSource = selectorSource; - this.logger = logger; + this.logger = logger.subLogger(Loggers.STREAMING_SYNCHRONIZER); this.payloadFilter = payloadFilter; this.streamUri = HttpHelpers.concatenateUriPath(baseUri, requestPath); this.initialReconnectDelay = initialReconnectDelaySeconds; diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceFallbackConditionTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceFallbackConditionTest.java new file mode 100644 index 00000000..b4577e1a --- /dev/null +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceFallbackConditionTest.java @@ -0,0 +1,438 @@ +package com.launchdarkly.sdk.server; + +import com.launchdarkly.sdk.internal.fdv2.sources.Selector; +import com.launchdarkly.sdk.server.FDv2DataSource.Condition; +import com.launchdarkly.sdk.server.FDv2DataSource.FallbackCondition; +import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; +import com.launchdarkly.sdk.server.interfaces.DataSourceStatusProvider; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes; + +import org.junit.After; +import org.junit.Test; + +import java.time.Instant; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +public class FDv2DataSourceFallbackConditionTest extends BaseTest { + + private ScheduledExecutorService executor; + + @After + public void tearDown() { + if (executor != null && !executor.isShutdown()) { + executor.shutdownNow(); + } + } + + private DataStoreTypes.ChangeSet makeChangeSet() { + return new DataStoreTypes.ChangeSet<>( + DataStoreTypes.ChangeSetType.None, + Selector.EMPTY, + null, + null + ); + } + + @Test + public void executeReturnsCompletableFuture() { + executor = Executors.newScheduledThreadPool(1); + FallbackCondition condition = new FallbackCondition(executor, 120); + + CompletableFuture result = condition.execute(); + + assertFalse(result.isDone()); + } + + @Test + public void getTypeReturnsFallback() { + executor = Executors.newScheduledThreadPool(1); + FallbackCondition condition = new FallbackCondition(executor, 120); + + assertEquals(Condition.ConditionType.FALLBACK, condition.getType()); + } + + @Test + public void interruptedStatusStartsTimerThatCompletesResultFuture() throws Exception { + executor = Executors.newScheduledThreadPool(1); + FallbackCondition condition = new FallbackCondition(executor, 1); + + CompletableFuture resultFuture = condition.execute(); + assertFalse(resultFuture.isDone()); + + // Inform with INTERRUPTED status + condition.inform( + FDv2SourceResult.interrupted( + new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), + false + ) + ); + + // Future should still not be done immediately + assertFalse(resultFuture.isDone()); + + // Wait for timeout to fire + Condition result = resultFuture.get(2, TimeUnit.SECONDS); + + // Now it should be done and return the condition instance + assertTrue(resultFuture.isDone()); + assertSame(condition, result); + } + + @Test + public void changeSetCancelsActiveTimer() throws Exception { + executor = Executors.newScheduledThreadPool(1); + FallbackCondition condition = new FallbackCondition(executor, 1); + + CompletableFuture resultFuture = condition.execute(); + + // Start timer with INTERRUPTED + condition.inform( + FDv2SourceResult.interrupted( + new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), + false + ) + ); + + // Cancel timer with CHANGE_SET + DataStoreTypes.ChangeSet changeSet = makeChangeSet(); + condition.inform(FDv2SourceResult.changeSet(changeSet, false)); + + // Wait longer than the timeout period + Thread.sleep(1500); + + // Future should still not be complete (timer was cancelled) + assertFalse(resultFuture.isDone()); + } + + @Test + public void changeSetWithoutActiveTimerDoesNothing() throws Exception { + executor = Executors.newScheduledThreadPool(1); + FallbackCondition condition = new FallbackCondition(executor, 1); + + CompletableFuture resultFuture = condition.execute(); + + // Inform with CHANGE_SET without starting a timer first + DataStoreTypes.ChangeSet changeSet = makeChangeSet(); + condition.inform(FDv2SourceResult.changeSet(changeSet, false)); + + // Wait to ensure nothing happens + Thread.sleep(100); + + // Future should still not be complete + assertFalse(resultFuture.isDone()); + } + + @Test + public void multipleInterruptedStatusesDoNotStartMultipleTimers() throws Exception { + executor = Executors.newScheduledThreadPool(1); + FallbackCondition condition = new FallbackCondition(executor, 2); + + CompletableFuture resultFuture = condition.execute(); + + // Inform with INTERRUPTED multiple times + condition.inform( + FDv2SourceResult.interrupted( + new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), + false + ) + ); + + Thread.sleep(100); + + condition.inform( + FDv2SourceResult.interrupted( + new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), + false + ) + ); + + Thread.sleep(100); + + condition.inform( + FDv2SourceResult.interrupted( + new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), + false + ) + ); + + // Wait for the timer (should only fire once) + Condition result = resultFuture.get(3, TimeUnit.SECONDS); + + assertTrue(resultFuture.isDone()); + assertSame(condition, result); + } + + @Test + public void terminalErrorStatusDoesNotStartTimer() throws Exception { + executor = Executors.newScheduledThreadPool(1); + FallbackCondition condition = new FallbackCondition(executor, 1); + + CompletableFuture resultFuture = condition.execute(); + + // Inform with TERMINAL_ERROR status + condition.inform( + FDv2SourceResult.terminalError( + new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.ERROR_RESPONSE, 401, null, Instant.now()), + false + ) + ); + + // Wait longer than timeout + Thread.sleep(1500); + + // Future should still not be complete (no timer started) + assertFalse(resultFuture.isDone()); + } + + @Test + public void shutdownStatusDoesNotStartTimer() throws Exception { + executor = Executors.newScheduledThreadPool(1); + FallbackCondition condition = new FallbackCondition(executor, 1); + + CompletableFuture resultFuture = condition.execute(); + + // Inform with SHUTDOWN status + condition.inform(FDv2SourceResult.shutdown()); + + // Wait longer than timeout + Thread.sleep(1500); + + // Future should still not be complete (no timer started) + assertFalse(resultFuture.isDone()); + } + + @Test + public void goodbyeStatusDoesNotStartTimer() throws Exception { + executor = Executors.newScheduledThreadPool(1); + FallbackCondition condition = new FallbackCondition(executor, 1); + + CompletableFuture resultFuture = condition.execute(); + + // Inform with GOODBYE status + condition.inform(FDv2SourceResult.goodbye("server-requested", false)); + + // Wait longer than timeout + Thread.sleep(1500); + + // Future should still not be complete (no timer started) + assertFalse(resultFuture.isDone()); + } + + @Test + public void closeCancelsActiveTimer() throws Exception { + executor = Executors.newScheduledThreadPool(1); + FallbackCondition condition = new FallbackCondition(executor, 1); + + CompletableFuture resultFuture = condition.execute(); + + // Start timer with INTERRUPTED + condition.inform( + FDv2SourceResult.interrupted( + new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), + false + ) + ); + + // Close the condition + condition.close(); + + // Wait longer than the timeout period + Thread.sleep(1500); + + // Future should still not be complete (timer was cancelled) + assertFalse(resultFuture.isDone()); + } + + @Test + public void closeWithoutActiveTimerDoesNotFail() throws Exception { + executor = Executors.newScheduledThreadPool(1); + FallbackCondition condition = new FallbackCondition(executor, 120); + + // Close without starting a timer + condition.close(); + + // Should not throw exception + } + + @Test + public void timerCanBeStartedAfterBeingCancelled() throws Exception { + executor = Executors.newScheduledThreadPool(1); + FallbackCondition condition = new FallbackCondition(executor, 1); + + CompletableFuture resultFuture = condition.execute(); + + // Start timer + condition.inform( + FDv2SourceResult.interrupted( + new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), + false + ) + ); + + // Cancel timer with CHANGE_SET + DataStoreTypes.ChangeSet changeSet = makeChangeSet(); + condition.inform(FDv2SourceResult.changeSet(changeSet, false)); + + // Start timer again + condition.inform( + FDv2SourceResult.interrupted( + new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), + false + ) + ); + + // Wait for second timer to fire + Condition result = resultFuture.get(2, TimeUnit.SECONDS); + + assertTrue(resultFuture.isDone()); + assertSame(condition, result); + } + + @Test + public void changeSetAfterTimerFiresDoesNotAffectCompletedFuture() throws Exception { + executor = Executors.newScheduledThreadPool(1); + FallbackCondition condition = new FallbackCondition(executor, 1); + + CompletableFuture resultFuture = condition.execute(); + + // Start timer + condition.inform( + FDv2SourceResult.interrupted( + new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), + false + ) + ); + + // Wait for timer to fire + Condition result = resultFuture.get(2, TimeUnit.SECONDS); + assertTrue(resultFuture.isDone()); + assertSame(condition, result); + + // Inform with CHANGE_SET after timer has fired + DataStoreTypes.ChangeSet changeSet = makeChangeSet(); + condition.inform(FDv2SourceResult.changeSet(changeSet, false)); + + // Future should remain complete + assertTrue(resultFuture.isDone()); + } + + @Test + public void factoryCreatesFallbackCondition() throws Exception { + executor = Executors.newScheduledThreadPool(1); + FallbackCondition.Factory factory = new FallbackCondition.Factory(executor, 1); + + FallbackCondition condition = (FallbackCondition) factory.build(); + + // Verify it works by using it + CompletableFuture resultFuture = condition.execute(); + assertFalse(resultFuture.isDone()); + + condition.inform( + FDv2SourceResult.interrupted( + new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), + false + ) + ); + + Condition result = resultFuture.get(2, TimeUnit.SECONDS); + assertTrue(resultFuture.isDone()); + assertSame(condition, result); + } + + @Test + public void executeReturnsTheSameFutureOnMultipleCalls() { + executor = Executors.newScheduledThreadPool(1); + FallbackCondition condition = new FallbackCondition(executor, 120); + + CompletableFuture first = condition.execute(); + CompletableFuture second = condition.execute(); + + assertSame(first, second); + } + + @Test + public void changeSetDuringTimerExecutionCancelsTimer() throws Exception { + executor = Executors.newScheduledThreadPool(1); + FallbackCondition condition = new FallbackCondition(executor, 1); + + CompletableFuture resultFuture = condition.execute(); + + // Start timer + condition.inform( + FDv2SourceResult.interrupted( + new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), + false + ) + ); + + // Wait partway through timeout period + Thread.sleep(500); + + // Cancel with CHANGE_SET + DataStoreTypes.ChangeSet changeSet = makeChangeSet(); + condition.inform(FDv2SourceResult.changeSet(changeSet, false)); + + // Wait past the original timeout + Thread.sleep(1000); + + // Future should still not be complete + assertFalse(resultFuture.isDone()); + } + + @Test + public void multipleChangeSetCallsWithActiveTimerAreHandled() throws Exception { + executor = Executors.newScheduledThreadPool(1); + FallbackCondition condition = new FallbackCondition(executor, 1); + + CompletableFuture resultFuture = condition.execute(); + + // Start timer + condition.inform( + FDv2SourceResult.interrupted( + new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), + false + ) + ); + + // Cancel with multiple CHANGE_SETs + DataStoreTypes.ChangeSet changeSet = makeChangeSet(); + condition.inform(FDv2SourceResult.changeSet(changeSet, false)); + condition.inform(FDv2SourceResult.changeSet(changeSet, false)); + condition.inform(FDv2SourceResult.changeSet(changeSet, false)); + + // Wait longer than timeout + Thread.sleep(1500); + + // Future should still not be complete + assertFalse(resultFuture.isDone()); + } + + @Test + public void closeCanBeCalledMultipleTimes() throws Exception { + executor = Executors.newScheduledThreadPool(1); + FallbackCondition condition = new FallbackCondition(executor, 1); + + // Start timer + condition.inform( + FDv2SourceResult.interrupted( + new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), + false + ) + ); + + // Close multiple times + condition.close(); + condition.close(); + condition.close(); + + // Should not throw exception + } +} \ No newline at end of file From fba5aa29b7a264d38fd1f65ed52c82bc10caed17 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Fri, 23 Jan 2026 15:27:34 -0800 Subject: [PATCH 04/35] Extract conditions from FDv2DataSource. --- .../sdk/server/FDv2DataSource.java | 198 +----------------- .../sdk/server/FDv2DataSourceConditions.java | 184 ++++++++++++++++ .../server/SynchronizerFactoryWithState.java | 38 ++++ .../FDv2DataSourceFallbackConditionTest.java | 4 +- 4 files changed, 229 insertions(+), 195 deletions(-) create mode 100644 lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java create mode 100644 lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerFactoryWithState.java diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index edaf130a..4ae0956b 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -18,6 +18,11 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; +import static com.launchdarkly.sdk.server.FDv2DataSourceConditions.Condition; +import static com.launchdarkly.sdk.server.FDv2DataSourceConditions.ConditionFactory; +import static com.launchdarkly.sdk.server.FDv2DataSourceConditions.FallbackCondition; +import static com.launchdarkly.sdk.server.FDv2DataSourceConditions.RecoveryCondition; + class FDv2DataSource implements DataSource { /** * Default fallback timeout of 2 minutes. The timeout is only configurable for testing. @@ -50,199 +55,6 @@ class FDv2DataSource implements DataSource { private final LDLogger logger; - /** - * Package-private for testing. - */ - interface Condition { - enum ConditionType { - FALLBACK, - RECOVERY, - } - CompletableFuture execute(); - - void inform(FDv2SourceResult sourceResult); - - void close() throws IOException; - - ConditionType getType(); - } - - interface ConditionFactory { - Condition build(); - - Condition.ConditionType getType(); - } - - - static abstract class TimedCondition implements Condition { - protected final CompletableFuture resultFuture = new CompletableFuture<>(); - - protected final ScheduledExecutorService sharedExecutor; - - /** - * Future for the timeout task, if any. Will be null when no timeout is active. - */ - protected ScheduledFuture timerFuture; - - /** - * Timeout duration for the fallback operation. - */ - protected final long timeoutSeconds; - - public TimedCondition(ScheduledExecutorService sharedExecutor, long timeoutSeconds) { - this.sharedExecutor = sharedExecutor; - this.timeoutSeconds = timeoutSeconds; - } - - @Override - public CompletableFuture execute() { - return resultFuture; - } - - @Override - public void close() throws IOException { - if (timerFuture != null) { - timerFuture.cancel(false); - timerFuture = null; - } - } - - static abstract class Factory implements ConditionFactory { - protected final ScheduledExecutorService sharedExecutor; - protected final long timeoutSeconds; - - public Factory(ScheduledExecutorService sharedExecutor, long timeout) { - this.sharedExecutor = sharedExecutor; - this.timeoutSeconds = timeout; - } - } - } - - /** - * This condition is used to determine if a fallback should be performed. It is informed of each data source result - * via {@link #inform(FDv2SourceResult)}. Based on the results, it updates its internal state. When the fallback - * condition is met, then the {@link Future} returned by {@link #execute()} will complete. - *

- * This is package-private, instead of private, for ease of testing. - */ - static class FallbackCondition extends TimedCondition { - static class Factory extends TimedCondition.Factory { - public Factory(ScheduledExecutorService sharedExecutor, long timeout) { - super(sharedExecutor, timeout); - } - @Override - public Condition build() { - return new FallbackCondition(sharedExecutor, timeoutSeconds); - } - - @Override - public ConditionType getType() { - return ConditionType.FALLBACK; - } - } - - public FallbackCondition(ScheduledExecutorService sharedExecutor, long timeoutSeconds) { - super(sharedExecutor, timeoutSeconds); - } - - @Override - public void inform(FDv2SourceResult sourceResult) { - if(sourceResult == null) { - return; - } - if(sourceResult.getResultType() == FDv2SourceResult.ResultType.CHANGE_SET) { - if(timerFuture != null) { - timerFuture.cancel(false); - timerFuture = null; - } - } - if(sourceResult.getResultType() == FDv2SourceResult.ResultType.STATUS && sourceResult.getStatus().getState() == FDv2SourceResult.State.INTERRUPTED) { - if (timerFuture == null) { - timerFuture = sharedExecutor.schedule(() -> { - resultFuture.complete(this); - return null; - }, timeoutSeconds, TimeUnit.SECONDS); - } - } - } - - @Override - public ConditionType getType() { - return ConditionType.FALLBACK; - } - } - - static class RecoveryCondition extends TimedCondition { - - static class Factory extends TimedCondition.Factory { - public Factory(ScheduledExecutorService sharedExecutor, long timeout) { - super(sharedExecutor, timeout); - } - @Override - public Condition build() { - return new RecoveryCondition(sharedExecutor, timeoutSeconds); - } - - @Override - public ConditionType getType() { - return ConditionType.RECOVERY; - } - } - - public RecoveryCondition(ScheduledExecutorService sharedExecutor, long timeoutSeconds) { - super(sharedExecutor, timeoutSeconds); - timerFuture = sharedExecutor.schedule(() -> { - resultFuture.complete(this); - return null; - }, timeoutSeconds, TimeUnit.SECONDS); - } - - @Override - public void inform(FDv2SourceResult sourceResult) { - // Time-based recovery. - } - - @Override - public ConditionType getType() { - return ConditionType.RECOVERY; - } - } - - private static class SynchronizerFactoryWithState { - public enum State { - /** - * This synchronizer is available to use. - */ - Available, - - /** - * This synchronizer is no longer available to use. - */ - Blocked - } - - private final DataSourceFactory factory; - - private State state = State.Available; - - - public SynchronizerFactoryWithState(DataSourceFactory factory) { - this.factory = factory; - } - - public State getState() { - return state; - } - - public void block() { - state = State.Blocked; - } - - public Synchronizer build() { - return factory.build(); - } - } - public interface DataSourceFactory { T build(); } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java new file mode 100644 index 00000000..714e5869 --- /dev/null +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java @@ -0,0 +1,184 @@ +package com.launchdarkly.sdk.server; + +import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; + +import java.io.Closeable; +import java.io.IOException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +/** + * Container class for FDv2 data source conditions and related types. + *

+ * This class is non-constructable and serves only as a namespace for condition-related types. + * Package-private for internal use and testing. + */ +class FDv2DataSourceConditions { + /** + * Private constructor to prevent instantiation. + */ + private FDv2DataSourceConditions() { + } + + /** + * Package-private for testing. + */ + interface Condition extends Closeable { + enum ConditionType { + FALLBACK, + RECOVERY, + } + + CompletableFuture execute(); + + void inform(FDv2SourceResult sourceResult); + + void close() throws IOException; + + ConditionType getType(); + } + + interface ConditionFactory { + Condition build(); + + Condition.ConditionType getType(); + } + + static abstract class TimedCondition implements Condition { + protected final CompletableFuture resultFuture = new CompletableFuture<>(); + + protected final ScheduledExecutorService sharedExecutor; + + /** + * Future for the timeout task, if any. Will be null when no timeout is active. + */ + protected ScheduledFuture timerFuture; + + /** + * Timeout duration for the fallback operation. + */ + protected final long timeoutSeconds; + + public TimedCondition(ScheduledExecutorService sharedExecutor, long timeoutSeconds) { + this.sharedExecutor = sharedExecutor; + this.timeoutSeconds = timeoutSeconds; + } + + @Override + public CompletableFuture execute() { + return resultFuture; + } + + @Override + public void close() throws IOException { + if (timerFuture != null) { + timerFuture.cancel(false); + timerFuture = null; + } + } + + static abstract class Factory implements ConditionFactory { + protected final ScheduledExecutorService sharedExecutor; + protected final long timeoutSeconds; + + public Factory(ScheduledExecutorService sharedExecutor, long timeout) { + this.sharedExecutor = sharedExecutor; + this.timeoutSeconds = timeout; + } + } + } + + /** + * This condition is used to determine if a fallback should be performed. It is informed of each data source result + * via {@link #inform(FDv2SourceResult)}. Based on the results, it updates its internal state. When the fallback + * condition is met, then the {@link java.util.concurrent.Future} returned by {@link #execute()} will complete. + *

+ * This is package-private, instead of private, for ease of testing. + */ + static class FallbackCondition extends TimedCondition { + static class Factory extends TimedCondition.Factory { + public Factory(ScheduledExecutorService sharedExecutor, long timeout) { + super(sharedExecutor, timeout); + } + + @Override + public Condition build() { + return new FallbackCondition(sharedExecutor, timeoutSeconds); + } + + @Override + public ConditionType getType() { + return ConditionType.FALLBACK; + } + } + + public FallbackCondition(ScheduledExecutorService sharedExecutor, long timeoutSeconds) { + super(sharedExecutor, timeoutSeconds); + } + + @Override + public void inform(FDv2SourceResult sourceResult) { + if (sourceResult == null) { + return; + } + if (sourceResult.getResultType() == FDv2SourceResult.ResultType.CHANGE_SET) { + if (timerFuture != null) { + timerFuture.cancel(false); + timerFuture = null; + } + } + if (sourceResult.getResultType() == FDv2SourceResult.ResultType.STATUS && sourceResult.getStatus().getState() == FDv2SourceResult.State.INTERRUPTED) { + if (timerFuture == null) { + timerFuture = sharedExecutor.schedule(() -> { + resultFuture.complete(this); + return null; + }, timeoutSeconds, TimeUnit.SECONDS); + } + } + } + + @Override + public ConditionType getType() { + return ConditionType.FALLBACK; + } + } + + static class RecoveryCondition extends TimedCondition { + + static class Factory extends TimedCondition.Factory { + public Factory(ScheduledExecutorService sharedExecutor, long timeout) { + super(sharedExecutor, timeout); + } + + @Override + public Condition build() { + return new RecoveryCondition(sharedExecutor, timeoutSeconds); + } + + @Override + public ConditionType getType() { + return ConditionType.RECOVERY; + } + } + + public RecoveryCondition(ScheduledExecutorService sharedExecutor, long timeoutSeconds) { + super(sharedExecutor, timeoutSeconds); + timerFuture = sharedExecutor.schedule(() -> { + resultFuture.complete(this); + return null; + }, timeoutSeconds, TimeUnit.SECONDS); + } + + @Override + public void inform(FDv2SourceResult sourceResult) { + // Time-based recovery. + } + + @Override + public ConditionType getType() { + return ConditionType.RECOVERY; + } + } +} \ No newline at end of file diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerFactoryWithState.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerFactoryWithState.java new file mode 100644 index 00000000..c0afa642 --- /dev/null +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerFactoryWithState.java @@ -0,0 +1,38 @@ +package com.launchdarkly.sdk.server; + +import com.launchdarkly.sdk.server.datasources.Synchronizer; + +class SynchronizerFactoryWithState { + public enum State { + /** + * This synchronizer is available to use. + */ + Available, + + /** + * This synchronizer is no longer available to use. + */ + Blocked + } + + private final FDv2DataSource.DataSourceFactory factory; + + private State state = State.Available; + + + public SynchronizerFactoryWithState(FDv2DataSource.DataSourceFactory factory) { + this.factory = factory; + } + + public State getState() { + return state; + } + + public void block() { + state = State.Blocked; + } + + public Synchronizer build() { + return factory.build(); + } +} diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceFallbackConditionTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceFallbackConditionTest.java index b4577e1a..ca3fb5fd 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceFallbackConditionTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceFallbackConditionTest.java @@ -1,8 +1,8 @@ package com.launchdarkly.sdk.server; import com.launchdarkly.sdk.internal.fdv2.sources.Selector; -import com.launchdarkly.sdk.server.FDv2DataSource.Condition; -import com.launchdarkly.sdk.server.FDv2DataSource.FallbackCondition; +import com.launchdarkly.sdk.server.FDv2DataSourceConditions.Condition; +import com.launchdarkly.sdk.server.FDv2DataSourceConditions.FallbackCondition; import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; import com.launchdarkly.sdk.server.interfaces.DataSourceStatusProvider; import com.launchdarkly.sdk.server.subsystems.DataStoreTypes; From 472a1d2b5847f8cc185e55f691e47e7e1ffd1dba Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Fri, 23 Jan 2026 15:40:03 -0800 Subject: [PATCH 05/35] Extract synchronizer state management from the FDv2DataSource. --- .../sdk/server/FDv2DataSource.java | 142 +++------------ .../sdk/server/SynchronizerStateManager.java | 164 ++++++++++++++++++ 2 files changed, 192 insertions(+), 114 deletions(-) create mode 100644 lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index 4ae0956b..1d2c75e4 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -9,7 +9,6 @@ import com.launchdarkly.sdk.server.subsystems.DataSource; import com.launchdarkly.sdk.server.subsystems.DataSourceUpdateSinkV2; -import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -35,7 +34,7 @@ class FDv2DataSource implements DataSource { private static final long defaultRecoveryTimeout = 5 * 60; private final List> initializers; - private final List synchronizers; + private final SynchronizerStateManager synchronizerStateManager; private final List conditionFactories; @@ -44,13 +43,6 @@ class FDv2DataSource implements DataSource { private final CompletableFuture startFuture = new CompletableFuture<>(); private final AtomicBoolean started = new AtomicBoolean(false); - /** - * Lock for active sources and shutdown state. - */ - private final Object activeSourceLock = new Object(); - private Closeable activeSource; - private boolean isShutdown = false; - private final int threadPriority; private final LDLogger logger; @@ -67,7 +59,15 @@ public FDv2DataSource( LDLogger logger, ScheduledExecutorService sharedExecutor ) { - this(initializers, synchronizers, dataSourceUpdates, threadPriority, logger, sharedExecutor, defaultFallbackTimeout, defaultRecoveryTimeout); + this(initializers, + synchronizers, + dataSourceUpdates, + threadPriority, + logger, + sharedExecutor, + defaultFallbackTimeout, + defaultRecoveryTimeout + ); } @@ -82,10 +82,11 @@ public FDv2DataSource( long recoveryTimeout ) { this.initializers = initializers; - this.synchronizers = synchronizers + List synchronizerFactories = synchronizers .stream() .map(SynchronizerFactoryWithState::new) .collect(Collectors.toList()); + this.synchronizerStateManager = new SynchronizerStateManager(synchronizerFactories); this.dataSourceUpdates = dataSourceUpdates; this.threadPriority = threadPriority; this.logger = logger; @@ -110,62 +111,13 @@ private void run() { runThread.start(); } - /** - * We start at -1, so finding the next synchronizer can non-conditionally increment the index. - */ - private int sourceIndex = -1; - - /** - * Reset the source index to -1, indicating that we should start from the first synchronizer when looking for - * the next one to use. This is used when recovering from a non-primary synchronizer. - */ - private void resetSynchronizerSourceIndex() { - synchronized (activeSourceLock) { - sourceIndex = -1; - } - } - - /** - * Get the next synchronizer to use. This operates based on tracking the index of the currently active synchronizer, - * which will loop through all available synchronizers handling interruptions. Then a non-prime synchronizer recovers - * the source index will be reset, and we start at the beginning. - *

- * Any given synchronizer can be marked as blocked, in which case that synchronizer is not eligible to be used again. - * Synchronizers that are not blocked are available, and this function will only return available synchronizers. - * @return the next synchronizer factory to use, or null if there are no more available synchronizers. - */ - private SynchronizerFactoryWithState getNextAvailableSynchronizer() { - synchronized (synchronizers) { - SynchronizerFactoryWithState factory = null; - - // There is at least one available factory. - if(synchronizers.stream().anyMatch(s -> s.getState() == SynchronizerFactoryWithState.State.Available)) { - // Look for the next synchronizer starting at the position after the current one. (avoiding just re-using the same synchronizer.) - while(factory == null) { - sourceIndex++; - // We aren't using module here because we want to keep the stored index within range instead - // of increasing indefinitely. - if(sourceIndex >= synchronizers.size()) { - sourceIndex = 0; - } - SynchronizerFactoryWithState candidate = synchronizers.get(sourceIndex); - if (candidate.getState() == SynchronizerFactoryWithState.State.Available) { - factory = candidate; - } - - } - } - - return factory; - } - } private void runInitializers() { boolean anyDataReceived = false; for (DataSourceFactory factory : initializers) { try { Initializer initializer = factory.build(); - if (setActiveSource(initializer)) return; + if (synchronizerStateManager.setActiveSource(initializer)) return; FDv2SourceResult result = initializer.run().get(); switch (result.getResultType()) { case CHANGE_SET: @@ -202,24 +154,9 @@ private void runInitializers() { * @return a list of conditions to apply to the synchronizer */ private List getConditions() { - boolean isPrimeSynchronizer = false; - int availableSynchronizers = 0; - boolean firstAvailableSynchronizer = true; - - synchronized (activeSourceLock) { - for (int index = 0; index < synchronizers.size(); index++) { + int availableSynchronizers = synchronizerStateManager.getAvailableSynchronizerCount(); + boolean isPrimeSynchronizer = synchronizerStateManager.isPrimeSynchronizer(); - if (synchronizers.get(index).getState() == SynchronizerFactoryWithState.State.Available) { - if (firstAvailableSynchronizer && sourceIndex == index) { - // This is the first synchronizer that is available, and it also is the current one. - isPrimeSynchronizer = true; - } - // Subsequently encountered synchronizers that are available are not the first one. - firstAvailableSynchronizer = false; - availableSynchronizers++; - } - } - } if(availableSynchronizers == 1) { // If there is only 1 synchronizer, then we cannot fall back or recover, so we don't need any conditions. return Collections.emptyList(); @@ -235,24 +172,27 @@ private List getConditions() { } private boolean runSynchronizers() { - SynchronizerFactoryWithState availableSynchronizer = getNextAvailableSynchronizer(); + SynchronizerFactoryWithState availableSynchronizer = synchronizerStateManager.getNextAvailableSynchronizer(); while (availableSynchronizer != null) { Synchronizer synchronizer = availableSynchronizer.build(); // Returns true if shutdown. - if (setActiveSource(synchronizer)) return false; + if (synchronizerStateManager.setActiveSource(synchronizer)) return false; try { boolean running = true; // Conditions run once for the life of the synchronizer. List conditions = getConditions(); - CompletableFuture conditionFutures = CompletableFuture.anyOf( + + // The conditionsFuture will complete if any condition is met. Meeting any condition means we will + // switch to a different synchronizer. + CompletableFuture conditionsFuture = CompletableFuture.anyOf( conditions.stream().map(Condition::execute).toArray(CompletableFuture[]::new)); while (running) { CompletableFuture nextResultFuture = synchronizer.next(); - Object res = CompletableFuture.anyOf(conditionFutures, nextResultFuture).get(); + Object res = CompletableFuture.anyOf(conditionsFuture, nextResultFuture).get(); if(res instanceof Condition) { Condition c = (Condition) res; @@ -265,7 +205,7 @@ private boolean runSynchronizers() { case RECOVERY: // For recovery, we will start at the first available synchronizer. // So we reset the source index, and finding the source will start at the beginning. - resetSynchronizerSourceIndex(); + synchronizerStateManager.resetSourceIndex(); break; } // A running synchronizer will only have fallback and recovery conditions that it can act on. @@ -308,7 +248,8 @@ private boolean runSynchronizers() { // We have been requested to fall back to FDv1. We handle whatever message was associated, // close the synchronizer, and then fallback. if(result.isFdv1Fallback()) { - safeClose(synchronizer); + // When falling back to FDv1, we are done with any FDv2 synchronizers. + synchronizerStateManager.shutdown(); return true; } } @@ -316,29 +257,7 @@ private boolean runSynchronizers() { // TODO: Log. // Move to next synchronizer. } - availableSynchronizer = getNextAvailableSynchronizer(); - } - return false; - } - - private void safeClose(Closeable synchronizer) { - try { - synchronizer.close(); - } catch (IOException e) { - // Ignore close exceptions. - } - } - - private boolean setActiveSource(Closeable synchronizer) { - synchronized (activeSourceLock) { - if (activeSource != null) { - safeClose(activeSource); - } - if (isShutdown) { - safeClose(synchronizer); - return true; - } - activeSource = synchronizer; + availableSynchronizer = synchronizerStateManager.getNextAvailableSynchronizer(); } return false; } @@ -361,17 +280,12 @@ public boolean isInitialized() { } @Override - public void close() throws IOException { + public void close() { // If there is an active source, we will shut it down, and that will result in the loop handling that source // exiting. // If we do not have an active source, then the loop will check isShutdown when attempting to set one. When // it detects shutdown, it will exit the loop. - synchronized (activeSourceLock) { - isShutdown = true; - if (activeSource != null) { - activeSource.close(); - } - } + synchronizerStateManager.shutdown(); // If this is already set, then this has no impact. startFuture.complete(false); diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java new file mode 100644 index 00000000..a1a143c9 --- /dev/null +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java @@ -0,0 +1,164 @@ +package com.launchdarkly.sdk.server; + +import java.io.Closeable; +import java.io.IOException; +import java.util.List; + +/** + * Manages the state of synchronizers including tracking which synchronizer is active, + * managing the list of available synchronizers, and handling source transitions. + *

+ * Package-private for internal use. + */ +class SynchronizerStateManager { + private final List synchronizers; + + /** + * Lock for active sources and shutdown state. + */ + private final Object activeSourceLock = new Object(); + private Closeable activeSource; + private boolean isShutdown = false; + + /** + * We start at -1, so finding the next synchronizer can non-conditionally increment the index. + */ + private int sourceIndex = -1; + + public SynchronizerStateManager(List synchronizers) { + this.synchronizers = synchronizers; + } + + /** + * Reset the source index to -1, indicating that we should start from the first synchronizer when looking for + * the next one to use. This is used when recovering from a non-primary synchronizer. + */ + public void resetSourceIndex() { + synchronized (activeSourceLock) { + sourceIndex = -1; + } + } + + /** + * Get the next synchronizer to use. This operates based on tracking the index of the currently active synchronizer, + * which will loop through all available synchronizers handling interruptions. Then a non-prime synchronizer recovers + * the source index will be reset, and we start at the beginning. + *

+ * Any given synchronizer can be marked as blocked, in which case that synchronizer is not eligible to be used again. + * Synchronizers that are not blocked are available, and this function will only return available synchronizers. + * @return the next synchronizer factory to use, or null if there are no more available synchronizers. + */ + public SynchronizerFactoryWithState getNextAvailableSynchronizer() { + synchronized (synchronizers) { + SynchronizerFactoryWithState factory = null; + + // There is at least one available factory. + if(synchronizers.stream().anyMatch(s -> s.getState() == SynchronizerFactoryWithState.State.Available)) { + // Look for the next synchronizer starting at the position after the current one. (avoiding just re-using the same synchronizer.) + while(factory == null) { + sourceIndex++; + // We aren't using module here because we want to keep the stored index within range instead + // of increasing indefinitely. + if(sourceIndex >= synchronizers.size()) { + sourceIndex = 0; + } + SynchronizerFactoryWithState candidate = synchronizers.get(sourceIndex); + if (candidate.getState() == SynchronizerFactoryWithState.State.Available) { + factory = candidate; + } + + } + } + + return factory; + } + } + + /** + * Determine if the currently active synchronizer is the prime (first available) synchronizer. + * @return true if the current synchronizer is the prime synchronizer, false otherwise + */ + public boolean isPrimeSynchronizer() { + synchronized (activeSourceLock) { + boolean firstAvailableSynchronizer = true; + for (int index = 0; index < synchronizers.size(); index++) { + if (synchronizers.get(index).getState() == SynchronizerFactoryWithState.State.Available) { + if (firstAvailableSynchronizer && sourceIndex == index) { + // This is the first synchronizer that is available, and it also is the current one. + return true; + } + // Subsequently encountered synchronizers that are available are not the first one. + firstAvailableSynchronizer = false; + } + } + } + return false; + } + + /** + * Get the count of available synchronizers. + * @return the number of available synchronizers + */ + public int getAvailableSynchronizerCount() { + synchronized (activeSourceLock) { + int count = 0; + for (int index = 0; index < synchronizers.size(); index++) { + if (synchronizers.get(index).getState() == SynchronizerFactoryWithState.State.Available) { + count++; + } + } + return count; + } + } + + /** + * Set the active source. If shutdown has been initiated, the source will be closed immediately. + * Any previously active source will be closed. + * @param source the source to set as active + * @return true if shutdown has been initiated, false otherwise + */ + public boolean setActiveSource(Closeable source) { + synchronized (activeSourceLock) { + if (activeSource != null) { + safeClose(activeSource); + } + if (isShutdown) { + safeClose(source); + return true; + } + activeSource = source; + } + return false; + } + + /** + * Initiate shutdown of the state manager. This will close any active source. + * @throws IOException if an error occurs closing the active source + */ + public void shutdown() { + synchronized (activeSourceLock) { + isShutdown = true; + if (activeSource != null) { + try { + activeSource.close(); + } catch (IOException e) { + // We are done with this synchronizer, so we don't care if it encounters + // an error condition. + } + activeSource = null; + } + } + } + + /** + * Safely close a closeable, ignoring any exceptions. + * @param closeable the closeable to close + */ + private void safeClose(Closeable closeable) { + try { + closeable.close(); + } catch (IOException e) { + // Ignore close exceptions. + } + } +} \ No newline at end of file From 0c150a13f2e59b1da4a48a4fd3e7edf422eb3330 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Fri, 23 Jan 2026 15:54:35 -0800 Subject: [PATCH 06/35] Add recovery condition tests. --- .../sdk/server/FDv2DataSourceConditions.java | 2 +- .../sdk/server/SynchronizerStateManager.java | 2 +- .../FDv2DataSourceRecoveryConditionTest.java | 322 ++++++++++++++++++ 3 files changed, 324 insertions(+), 2 deletions(-) create mode 100644 lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceRecoveryConditionTest.java diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java index 714e5869..c9a1fc0e 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java @@ -181,4 +181,4 @@ public ConditionType getType() { return ConditionType.RECOVERY; } } -} \ No newline at end of file +} diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java index a1a143c9..f0a191a0 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java @@ -161,4 +161,4 @@ private void safeClose(Closeable closeable) { // Ignore close exceptions. } } -} \ No newline at end of file +} diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceRecoveryConditionTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceRecoveryConditionTest.java new file mode 100644 index 00000000..481280df --- /dev/null +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceRecoveryConditionTest.java @@ -0,0 +1,322 @@ +package com.launchdarkly.sdk.server; + +import com.launchdarkly.sdk.internal.fdv2.sources.Selector; +import com.launchdarkly.sdk.server.FDv2DataSourceConditions.Condition; +import com.launchdarkly.sdk.server.FDv2DataSourceConditions.RecoveryCondition; +import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; +import com.launchdarkly.sdk.server.interfaces.DataSourceStatusProvider; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes; + +import org.junit.After; +import org.junit.Test; + +import java.time.Instant; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +@SuppressWarnings("javadoc") +public class FDv2DataSourceRecoveryConditionTest extends BaseTest { + + private ScheduledExecutorService executor; + + @After + public void tearDown() { + if (executor != null && !executor.isShutdown()) { + executor.shutdownNow(); + } + } + + private DataStoreTypes.ChangeSet makeChangeSet() { + return new DataStoreTypes.ChangeSet<>( + DataStoreTypes.ChangeSetType.None, + Selector.EMPTY, + null, + null + ); + } + + @Test + public void getTypeReturnsRecovery() { + executor = Executors.newScheduledThreadPool(1); + RecoveryCondition condition = new RecoveryCondition(executor, 120); + + assertEquals(Condition.ConditionType.RECOVERY, condition.getType()); + } + + @Test + public void timerStartsImmediatelyAndCompletesResultFuture() throws Exception { + executor = Executors.newScheduledThreadPool(1); + RecoveryCondition condition = new RecoveryCondition(executor, 1); + + CompletableFuture resultFuture = condition.execute(); + + // Future should not be done immediately + assertFalse(resultFuture.isDone()); + + // Wait for timeout to fire + Condition result = resultFuture.get(2, TimeUnit.SECONDS); + + // Now it should be done and return the condition instance + assertTrue(resultFuture.isDone()); + assertSame(condition, result); + } + + @Test + public void closeCancelsActiveTimer() throws Exception { + executor = Executors.newScheduledThreadPool(1); + RecoveryCondition condition = new RecoveryCondition(executor, 1); + + CompletableFuture resultFuture = condition.execute(); + + // Close the condition before timeout + condition.close(); + + // Wait longer than the timeout period + Thread.sleep(1500); + + // Future should still not be complete (timer was cancelled) + assertFalse(resultFuture.isDone()); + } + + @Test + public void closeAfterTimerFiresDoesNotCauseError() throws Exception { + executor = Executors.newScheduledThreadPool(1); + RecoveryCondition condition = new RecoveryCondition(executor, 1); + + CompletableFuture resultFuture = condition.execute(); + + // Wait for timer to fire + Condition result = resultFuture.get(2, TimeUnit.SECONDS); + assertTrue(resultFuture.isDone()); + assertSame(condition, result); + + // Close after timer has fired + condition.close(); + + // Should not throw exception + } + + @Test + public void closeCanBeCalledMultipleTimes() throws Exception { + executor = Executors.newScheduledThreadPool(1); + RecoveryCondition condition = new RecoveryCondition(executor, 1); + + // Close multiple times before timer fires + condition.close(); + condition.close(); + condition.close(); + + // Should not throw exception + } + + @Test + public void informWithChangeSetDoesNothing() throws Exception { + executor = Executors.newScheduledThreadPool(1); + RecoveryCondition condition = new RecoveryCondition(executor, 1); + + CompletableFuture resultFuture = condition.execute(); + + // Inform with CHANGE_SET + DataStoreTypes.ChangeSet changeSet = makeChangeSet(); + condition.inform(FDv2SourceResult.changeSet(changeSet, false)); + + // Timer should still fire after timeout (inform does nothing) + Condition result = resultFuture.get(2, TimeUnit.SECONDS); + assertTrue(resultFuture.isDone()); + assertSame(condition, result); + } + + @Test + public void informWithInterruptedStatusDoesNothing() throws Exception { + executor = Executors.newScheduledThreadPool(1); + RecoveryCondition condition = new RecoveryCondition(executor, 1); + + CompletableFuture resultFuture = condition.execute(); + + // Inform with INTERRUPTED status + condition.inform( + FDv2SourceResult.interrupted( + new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), + false + ) + ); + + // Timer should still fire after timeout (inform does nothing) + Condition result = resultFuture.get(2, TimeUnit.SECONDS); + assertTrue(resultFuture.isDone()); + assertSame(condition, result); + } + + @Test + public void informWithTerminalErrorStatusDoesNothing() throws Exception { + executor = Executors.newScheduledThreadPool(1); + RecoveryCondition condition = new RecoveryCondition(executor, 1); + + CompletableFuture resultFuture = condition.execute(); + + // Inform with TERMINAL_ERROR status + condition.inform( + FDv2SourceResult.terminalError( + new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.ERROR_RESPONSE, 401, null, Instant.now()), + false + ) + ); + + // Timer should still fire after timeout (inform does nothing) + Condition result = resultFuture.get(2, TimeUnit.SECONDS); + assertTrue(resultFuture.isDone()); + assertSame(condition, result); + } + + @Test + public void informWithShutdownStatusDoesNothing() throws Exception { + executor = Executors.newScheduledThreadPool(1); + RecoveryCondition condition = new RecoveryCondition(executor, 1); + + CompletableFuture resultFuture = condition.execute(); + + // Inform with SHUTDOWN status + condition.inform(FDv2SourceResult.shutdown()); + + // Timer should still fire after timeout (inform does nothing) + Condition result = resultFuture.get(2, TimeUnit.SECONDS); + assertTrue(resultFuture.isDone()); + assertSame(condition, result); + } + + @Test + public void informWithGoodbyeStatusDoesNothing() throws Exception { + executor = Executors.newScheduledThreadPool(1); + RecoveryCondition condition = new RecoveryCondition(executor, 1); + + CompletableFuture resultFuture = condition.execute(); + + // Inform with GOODBYE status + condition.inform(FDv2SourceResult.goodbye("server-requested", false)); + + // Timer should still fire after timeout (inform does nothing) + Condition result = resultFuture.get(2, TimeUnit.SECONDS); + assertTrue(resultFuture.isDone()); + assertSame(condition, result); + } + + @Test + public void informWithNullDoesNothing() throws Exception { + executor = Executors.newScheduledThreadPool(1); + RecoveryCondition condition = new RecoveryCondition(executor, 1); + + CompletableFuture resultFuture = condition.execute(); + + // Inform with null + condition.inform(null); + + // Timer should still fire after timeout (inform does nothing) + Condition result = resultFuture.get(2, TimeUnit.SECONDS); + assertTrue(resultFuture.isDone()); + assertSame(condition, result); + } + + @Test + public void multipleInformCallsDoNotAffectTimer() throws Exception { + executor = Executors.newScheduledThreadPool(1); + RecoveryCondition condition = new RecoveryCondition(executor, 1); + + CompletableFuture resultFuture = condition.execute(); + + // Multiple inform calls + DataStoreTypes.ChangeSet changeSet = makeChangeSet(); + condition.inform(FDv2SourceResult.changeSet(changeSet, false)); + condition.inform( + FDv2SourceResult.interrupted( + new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), + false + ) + ); + condition.inform(FDv2SourceResult.shutdown()); + + // Timer should still fire after timeout (all inform calls do nothing) + Condition result = resultFuture.get(2, TimeUnit.SECONDS); + assertTrue(resultFuture.isDone()); + assertSame(condition, result); + } + + @Test + public void factoryCreatesRecoveryCondition() throws Exception { + executor = Executors.newScheduledThreadPool(1); + RecoveryCondition.Factory factory = new RecoveryCondition.Factory(executor, 1); + + RecoveryCondition condition = (RecoveryCondition) factory.build(); + + // Verify it works by using it + CompletableFuture resultFuture = condition.execute(); + assertFalse(resultFuture.isDone()); + + Condition result = resultFuture.get(2, TimeUnit.SECONDS); + assertTrue(resultFuture.isDone()); + assertSame(condition, result); + } + + @Test + public void factoryGetTypeReturnsRecovery() { + executor = Executors.newScheduledThreadPool(1); + RecoveryCondition.Factory factory = new RecoveryCondition.Factory(executor, 1); + + assertEquals(Condition.ConditionType.RECOVERY, factory.getType()); + } + + @Test + public void executeReturnsTheSameFutureOnMultipleCalls() { + executor = Executors.newScheduledThreadPool(1); + RecoveryCondition condition = new RecoveryCondition(executor, 120); + + CompletableFuture first = condition.execute(); + CompletableFuture second = condition.execute(); + + assertSame(first, second); + } + + @Test + public void timerStartsImmediatelyOnConstruction() throws Exception { + executor = Executors.newScheduledThreadPool(1); + + // Create condition with very short timeout + RecoveryCondition condition = new RecoveryCondition(executor, 1); + + // Get the future + CompletableFuture resultFuture = condition.execute(); + + // Verify it's not done yet + assertFalse(resultFuture.isDone()); + + // Wait for it to complete + Condition result = resultFuture.get(2, TimeUnit.SECONDS); + + // Verify it completed with the condition + assertSame(condition, result); + } + + @Test + public void closeBeforeExecuteDoesNotPreventFutureAccess() throws Exception { + executor = Executors.newScheduledThreadPool(1); + RecoveryCondition condition = new RecoveryCondition(executor, 1); + + // Close immediately + condition.close(); + + // Should still be able to get the future + CompletableFuture resultFuture = condition.execute(); + + // Wait to ensure timer doesn't fire + Thread.sleep(1500); + + // Future should not be complete (timer was cancelled) + assertFalse(resultFuture.isDone()); + } +} \ No newline at end of file From 33fa6c584f617e913bd5d946a990c2ee43680c7d Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Mon, 26 Jan 2026 12:38:36 -0800 Subject: [PATCH 07/35] Closeable synchronizer state manager. --- .../sdk/server/FDv2DataSource.java | 30 +++++++++++++++---- .../sdk/server/FDv2DataSourceConditions.java | 4 +-- .../sdk/server/SynchronizerStateManager.java | 9 +++--- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index 1d2c75e4..f4b22ac9 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -105,6 +105,9 @@ private void run() { // TODO: Run FDv1 fallback. } // TODO: Handle. We have ran out of sources or we are shutting down. + + // If we had initialized at some point, then the future will already be complete and this will be ignored. + startFuture.complete(false); }); runThread.setDaemon(true); runThread.setPriority(threadPriority); @@ -181,6 +184,8 @@ private boolean runSynchronizers() { try { boolean running = true; + boolean fdv1Fallback = false; + // Conditions run once for the life of the synchronizer. List conditions = getConditions(); @@ -190,6 +195,9 @@ private boolean runSynchronizers() { conditions.stream().map(Condition::execute).toArray(CompletableFuture[]::new)); while (running) { + // If the loop needs to be exited, then running should be set to false, and the loop broken. + // We don't want to return within the loop because that would bypass cleanup. + CompletableFuture nextResultFuture = synchronizer.next(); Object res = CompletableFuture.anyOf(conditionsFuture, nextResultFuture).get(); @@ -234,7 +242,8 @@ private boolean runSynchronizers() { case SHUTDOWN: // We should be overall shutting down. // TODO: We may need logging or to do a little more. - return false; + running = false; + break; case TERMINAL_ERROR: availableSynchronizer.block(); running = false; @@ -248,17 +257,28 @@ private boolean runSynchronizers() { // We have been requested to fall back to FDv1. We handle whatever message was associated, // close the synchronizer, and then fallback. if(result.isFdv1Fallback()) { - // When falling back to FDv1, we are done with any FDv2 synchronizers. - synchronizerStateManager.shutdown(); - return true; + fdv1Fallback = true; + running = false; } } + // We are going to move to the next synchronizer or exit the synchronization loop, so we can close any + // conditions for this synchronizer. + conditions.forEach(Condition::close); + // If we are falling back, then we exit the synchronization process. + if(fdv1Fallback) { + // When falling back to FDv1, we are done with any FDv2 synchronizers. + synchronizerStateManager.close(); + return true; + } + } catch (ExecutionException | InterruptedException | CancellationException e) { // TODO: Log. // Move to next synchronizer. } availableSynchronizer = synchronizerStateManager.getNextAvailableSynchronizer(); } + + synchronizerStateManager.close(); return false; } @@ -285,7 +305,7 @@ public void close() { // exiting. // If we do not have an active source, then the loop will check isShutdown when attempting to set one. When // it detects shutdown, it will exit the loop. - synchronizerStateManager.shutdown(); + synchronizerStateManager.close(); // If this is already set, then this has no impact. startFuture.complete(false); diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java index c9a1fc0e..b5ed8b75 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java @@ -35,7 +35,7 @@ enum ConditionType { void inform(FDv2SourceResult sourceResult); - void close() throws IOException; + void close(); ConditionType getType(); } @@ -72,7 +72,7 @@ public CompletableFuture execute() { } @Override - public void close() throws IOException { + public void close() { if (timerFuture != null) { timerFuture.cancel(false); timerFuture = null; diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java index f0a191a0..fe8db810 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java @@ -10,7 +10,7 @@ *

* Package-private for internal use. */ -class SynchronizerStateManager { +class SynchronizerStateManager implements Closeable { private final List synchronizers; /** @@ -132,10 +132,11 @@ public boolean setActiveSource(Closeable source) { } /** - * Initiate shutdown of the state manager. This will close any active source. - * @throws IOException if an error occurs closing the active source + * Close the state manager and shut down any active source. + * Implements AutoCloseable to enable try-with-resources usage. */ - public void shutdown() { + @Override + public void close() { synchronized (activeSourceLock) { isShutdown = true; if (activeSource != null) { From ec0d0ecc8435577de1fd33c37b14d89eb5a001ec Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Mon, 26 Jan 2026 13:00:55 -0800 Subject: [PATCH 08/35] More clean shutdown model. --- .../sdk/server/FDv2DataSource.java | 208 ++++++++++-------- 1 file changed, 111 insertions(+), 97 deletions(-) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index f4b22ac9..5705efce 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -9,7 +9,6 @@ import com.launchdarkly.sdk.server.subsystems.DataSource; import com.launchdarkly.sdk.server.subsystems.DataSourceUpdateSinkV2; -import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -154,17 +153,18 @@ private void runInitializers() { /** * Determine conditions for the current synchronizer. Synchronizers require different conditions depending on if * they are the 'prime' synchronizer or if there are other available synchronizers to use. + * * @return a list of conditions to apply to the synchronizer */ private List getConditions() { int availableSynchronizers = synchronizerStateManager.getAvailableSynchronizerCount(); boolean isPrimeSynchronizer = synchronizerStateManager.isPrimeSynchronizer(); - if(availableSynchronizers == 1) { + if (availableSynchronizers == 1) { // If there is only 1 synchronizer, then we cannot fall back or recover, so we don't need any conditions. return Collections.emptyList(); } - if(isPrimeSynchronizer) { + if (isPrimeSynchronizer) { // If there isn't a synchronizer to recover to, then don't add and recovery conditions. return conditionFactories.stream() .filter((ConditionFactory factory) -> factory.getType() != Condition.ConditionType.RECOVERY) @@ -175,111 +175,96 @@ private List getConditions() { } private boolean runSynchronizers() { - SynchronizerFactoryWithState availableSynchronizer = synchronizerStateManager.getNextAvailableSynchronizer(); - while (availableSynchronizer != null) { - Synchronizer synchronizer = availableSynchronizer.build(); - - // Returns true if shutdown. - if (synchronizerStateManager.setActiveSource(synchronizer)) return false; - - try { - boolean running = true; - boolean fdv1Fallback = false; - - // Conditions run once for the life of the synchronizer. - List conditions = getConditions(); - - // The conditionsFuture will complete if any condition is met. Meeting any condition means we will - // switch to a different synchronizer. - CompletableFuture conditionsFuture = CompletableFuture.anyOf( - conditions.stream().map(Condition::execute).toArray(CompletableFuture[]::new)); - - while (running) { - // If the loop needs to be exited, then running should be set to false, and the loop broken. - // We don't want to return within the loop because that would bypass cleanup. - - CompletableFuture nextResultFuture = synchronizer.next(); - - Object res = CompletableFuture.anyOf(conditionsFuture, nextResultFuture).get(); - - if(res instanceof Condition) { - Condition c = (Condition) res; - switch (c.getType()) { - case FALLBACK: - // For fallback, we will move to the next available synchronizer, which may loop. - // This is the default behavior of exiting the run loop, so we don't need to take - // any action. - break; - case RECOVERY: - // For recovery, we will start at the first available synchronizer. - // So we reset the source index, and finding the source will start at the beginning. - synchronizerStateManager.resetSourceIndex(); + // When runSynchronizers exists, no matter how it exits, the synchronizerStateManager will be closed. + try { + SynchronizerFactoryWithState availableSynchronizer = synchronizerStateManager.getNextAvailableSynchronizer(); + + // We want to continue running synchronizers for as long as any are available. + while (availableSynchronizer != null) { + Synchronizer synchronizer = availableSynchronizer.build(); + + // Returns true if shutdown. + if (synchronizerStateManager.setActiveSource(synchronizer)) return false; + + try { + boolean running = true; + + try (Conditions conditions = new Conditions(getConditions())) { + while (running) { + CompletableFuture nextResultFuture = synchronizer.next(); + + // The conditionsFuture will complete if any condition is met. Meeting any condition means we will + // switch to a different synchronizer. + Object res = CompletableFuture.anyOf(conditions.getFuture(), nextResultFuture).get(); + + if (res instanceof Condition) { + Condition c = (Condition) res; + switch (c.getType()) { + case FALLBACK: + // For fallback, we will move to the next available synchronizer, which may loop. + // This is the default behavior of exiting the run loop, so we don't need to take + // any action. + break; + case RECOVERY: + // For recovery, we will start at the first available synchronizer. + // So we reset the source index, and finding the source will start at the beginning. + synchronizerStateManager.resetSourceIndex(); + break; + } + // A running synchronizer will only have fallback and recovery conditions that it can act on. + // So, if there are no synchronizers to recover to or fallback to, then we will not have + // those conditions. break; - } - // A running synchronizer will only have fallback and recovery conditions that it can act on. - // So, if there are no synchronizers to recover to or fallback to, then we will not have - // those conditions. - break; - } + } - FDv2SourceResult result = (FDv2SourceResult) res; - conditions.forEach(c -> c.inform(result)); + FDv2SourceResult result = (FDv2SourceResult) res; + conditions.inform(result); - switch (result.getResultType()) { - case CHANGE_SET: - dataSourceUpdates.apply(result.getChangeSet()); - // This could have been completed by any data source. But if it has not been completed before - // now, then we complete it. - startFuture.complete(true); - break; - case STATUS: - FDv2SourceResult.Status status = result.getStatus(); - switch (status.getState()) { - case INTERRUPTED: - // TODO: Track how long we are interrupted. - break; - case SHUTDOWN: - // We should be overall shutting down. - // TODO: We may need logging or to do a little more. - running = false; + switch (result.getResultType()) { + case CHANGE_SET: + dataSourceUpdates.apply(result.getChangeSet()); + // This could have been completed by any data source. But if it has not been completed before + // now, then we complete it. + startFuture.complete(true); break; - case TERMINAL_ERROR: - availableSynchronizer.block(); - running = false; - break; - case GOODBYE: - // We let the synchronizer handle this internally. + case STATUS: + FDv2SourceResult.Status status = result.getStatus(); + switch (status.getState()) { + case INTERRUPTED: + // TODO: Track how long we are interrupted. + break; + case SHUTDOWN: + // We should be overall shutting down. + // TODO: We may need logging or to do a little more. + return false; + case TERMINAL_ERROR: + availableSynchronizer.block(); + running = false; + break; + case GOODBYE: + // We let the synchronizer handle this internally. + break; + } break; } - break; - } - // We have been requested to fall back to FDv1. We handle whatever message was associated, - // close the synchronizer, and then fallback. - if(result.isFdv1Fallback()) { - fdv1Fallback = true; - running = false; + // We have been requested to fall back to FDv1. We handle whatever message was associated, + // close the synchronizer, and then fallback. + if (result.isFdv1Fallback()) { + return true; + } + } } + } catch (ExecutionException | InterruptedException | CancellationException e) { + // TODO: Log. + // Move to next synchronizer. } - // We are going to move to the next synchronizer or exit the synchronization loop, so we can close any - // conditions for this synchronizer. - conditions.forEach(Condition::close); - // If we are falling back, then we exit the synchronization process. - if(fdv1Fallback) { - // When falling back to FDv1, we are done with any FDv2 synchronizers. - synchronizerStateManager.close(); - return true; - } - - } catch (ExecutionException | InterruptedException | CancellationException e) { - // TODO: Log. - // Move to next synchronizer. + availableSynchronizer = synchronizerStateManager.getNextAvailableSynchronizer(); } - availableSynchronizer = synchronizerStateManager.getNextAvailableSynchronizer(); + return false; + } finally { + synchronizerStateManager.close(); } - - synchronizerStateManager.close(); - return false; } @Override @@ -310,4 +295,33 @@ public void close() { // If this is already set, then this has no impact. startFuture.complete(false); } + + /** + * Helper class to manage the lifecycle of conditions with automatic cleanup. + */ + private static class Conditions implements AutoCloseable { + private final List conditions; + private final CompletableFuture conditionsFuture; + + public Conditions(List conditions) { + this.conditions = conditions; + this.conditionsFuture = conditions.isEmpty() + ? new CompletableFuture<>() // Never completes if no conditions + : CompletableFuture.anyOf( + conditions.stream().map(Condition::execute).toArray(CompletableFuture[]::new)); + } + + public CompletableFuture getFuture() { + return conditionsFuture; + } + + public void inform(FDv2SourceResult result) { + conditions.forEach(c -> c.inform(result)); + } + + @Override + public void close() { + conditions.forEach(Condition::close); + } + } } From 5b6223869b59491fd42d001749a63b7a2e4398b8 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Mon, 26 Jan 2026 14:11:53 -0800 Subject: [PATCH 09/35] SynchronizerStateManager tests. --- .../sdk/server/FDv2DataSource.java | 3 +- .../FDv2DataSourceRecoveryConditionTest.java | 2 +- .../server/SynchronizerStateManagerTest.java | 425 ++++++++++++++++++ 3 files changed, 428 insertions(+), 2 deletions(-) create mode 100644 lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/SynchronizerStateManagerTest.java diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index 5705efce..dc9423d6 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -232,7 +232,8 @@ private boolean runSynchronizers() { FDv2SourceResult.Status status = result.getStatus(); switch (status.getState()) { case INTERRUPTED: - // TODO: Track how long we are interrupted. + // Handled by conditions. + // TODO: Data source status. break; case SHUTDOWN: // We should be overall shutting down. diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceRecoveryConditionTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceRecoveryConditionTest.java index 481280df..2c584f0a 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceRecoveryConditionTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceRecoveryConditionTest.java @@ -319,4 +319,4 @@ public void closeBeforeExecuteDoesNotPreventFutureAccess() throws Exception { // Future should not be complete (timer was cancelled) assertFalse(resultFuture.isDone()); } -} \ No newline at end of file +} diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/SynchronizerStateManagerTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/SynchronizerStateManagerTest.java new file mode 100644 index 00000000..ca19d351 --- /dev/null +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/SynchronizerStateManagerTest.java @@ -0,0 +1,425 @@ +package com.launchdarkly.sdk.server; + +import com.launchdarkly.sdk.server.datasources.Synchronizer; + +import org.junit.Test; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@SuppressWarnings("javadoc") +public class SynchronizerStateManagerTest extends BaseTest { + + private SynchronizerFactoryWithState createMockFactory() { + FDv2DataSource.DataSourceFactory factory = mock(FDv2DataSource.DataSourceFactory.class); + when(factory.build()).thenReturn(mock(Synchronizer.class)); + return new SynchronizerFactoryWithState(factory); + } + + @Test + public void getNextAvailableSynchronizerReturnsNullWhenEmpty() { + List synchronizers = new ArrayList<>(); + SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + + SynchronizerFactoryWithState result = manager.getNextAvailableSynchronizer(); + + assertNull(result); + } + + @Test + public void getNextAvailableSynchronizerReturnsFirstOnFirstCall() { + List synchronizers = new ArrayList<>(); + SynchronizerFactoryWithState sync1 = createMockFactory(); + synchronizers.add(sync1); + + SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + + SynchronizerFactoryWithState result = manager.getNextAvailableSynchronizer(); + + assertSame(sync1, result); + } + + @Test + public void getNextAvailableSynchronizerLoopsThroughAvailable() { + List synchronizers = new ArrayList<>(); + SynchronizerFactoryWithState sync1 = createMockFactory(); + SynchronizerFactoryWithState sync2 = createMockFactory(); + SynchronizerFactoryWithState sync3 = createMockFactory(); + synchronizers.add(sync1); + synchronizers.add(sync2); + synchronizers.add(sync3); + + SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + + // First call returns sync1 + assertSame(sync1, manager.getNextAvailableSynchronizer()); + // Second call returns sync2 + assertSame(sync2, manager.getNextAvailableSynchronizer()); + // Third call returns sync3 + assertSame(sync3, manager.getNextAvailableSynchronizer()); + } + + @Test + public void getNextAvailableSynchronizerWrapsAroundToBeginning() { + List synchronizers = new ArrayList<>(); + SynchronizerFactoryWithState sync1 = createMockFactory(); + SynchronizerFactoryWithState sync2 = createMockFactory(); + synchronizers.add(sync1); + synchronizers.add(sync2); + + SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + + // Get all synchronizers + manager.getNextAvailableSynchronizer(); // sync1 + manager.getNextAvailableSynchronizer(); // sync2 + + // Should wrap around to sync1 + assertSame(sync1, manager.getNextAvailableSynchronizer()); + } + + @Test + public void getNextAvailableSynchronizerSkipsBlockedSynchronizers() { + List synchronizers = new ArrayList<>(); + SynchronizerFactoryWithState sync1 = createMockFactory(); + SynchronizerFactoryWithState sync2 = createMockFactory(); + SynchronizerFactoryWithState sync3 = createMockFactory(); + synchronizers.add(sync1); + synchronizers.add(sync2); + synchronizers.add(sync3); + + SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + + // Block sync2 + sync2.block(); + + // First call returns sync1 + assertSame(sync1, manager.getNextAvailableSynchronizer()); + // Second call skips sync2 and returns sync3 + assertSame(sync3, manager.getNextAvailableSynchronizer()); + // Third call wraps and returns sync1 (skips sync2) + assertSame(sync1, manager.getNextAvailableSynchronizer()); + } + + @Test + public void getNextAvailableSynchronizerReturnsNullWhenAllBlocked() { + List synchronizers = new ArrayList<>(); + SynchronizerFactoryWithState sync1 = createMockFactory(); + SynchronizerFactoryWithState sync2 = createMockFactory(); + synchronizers.add(sync1); + synchronizers.add(sync2); + + SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + + // Block all synchronizers + sync1.block(); + sync2.block(); + + SynchronizerFactoryWithState result = manager.getNextAvailableSynchronizer(); + + assertNull(result); + } + + @Test + public void resetSourceIndexResetsToFirstSynchronizer() { + List synchronizers = new ArrayList<>(); + SynchronizerFactoryWithState sync1 = createMockFactory(); + SynchronizerFactoryWithState sync2 = createMockFactory(); + SynchronizerFactoryWithState sync3 = createMockFactory(); + synchronizers.add(sync1); + synchronizers.add(sync2); + synchronizers.add(sync3); + + SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + + // Advance to sync3 + manager.getNextAvailableSynchronizer(); // sync1 + manager.getNextAvailableSynchronizer(); // sync2 + manager.getNextAvailableSynchronizer(); // sync3 + + // Reset + manager.resetSourceIndex(); + + // Next call should return sync1 again + assertSame(sync1, manager.getNextAvailableSynchronizer()); + } + + @Test + public void isPrimeSynchronizerReturnsTrueForFirst() { + List synchronizers = new ArrayList<>(); + SynchronizerFactoryWithState sync1 = createMockFactory(); + SynchronizerFactoryWithState sync2 = createMockFactory(); + synchronizers.add(sync1); + synchronizers.add(sync2); + + SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + + // Get first synchronizer + manager.getNextAvailableSynchronizer(); + + assertTrue(manager.isPrimeSynchronizer()); + } + + @Test + public void isPrimeSynchronizerReturnsFalseForNonFirst() { + List synchronizers = new ArrayList<>(); + SynchronizerFactoryWithState sync1 = createMockFactory(); + SynchronizerFactoryWithState sync2 = createMockFactory(); + synchronizers.add(sync1); + synchronizers.add(sync2); + + SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + + // Get first then second synchronizer + manager.getNextAvailableSynchronizer(); + manager.getNextAvailableSynchronizer(); + + assertFalse(manager.isPrimeSynchronizer()); + } + + @Test + public void isPrimeSynchronizerReturnsFalseWhenNoSynchronizerSelected() { + List synchronizers = new ArrayList<>(); + SynchronizerFactoryWithState sync1 = createMockFactory(); + synchronizers.add(sync1); + + SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + + // Haven't called getNext yet + assertFalse(manager.isPrimeSynchronizer()); + } + + @Test + public void isPrimeSynchronizerHandlesBlockedFirstSynchronizer() { + List synchronizers = new ArrayList<>(); + SynchronizerFactoryWithState sync1 = createMockFactory(); + SynchronizerFactoryWithState sync2 = createMockFactory(); + SynchronizerFactoryWithState sync3 = createMockFactory(); + synchronizers.add(sync1); + synchronizers.add(sync2); + synchronizers.add(sync3); + + SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + + // Block first synchronizer + sync1.block(); + + // Get second synchronizer (which is now the prime) + manager.getNextAvailableSynchronizer(); + + assertTrue(manager.isPrimeSynchronizer()); + } + + @Test + public void getAvailableSynchronizerCountReturnsCorrectCount() { + List synchronizers = new ArrayList<>(); + SynchronizerFactoryWithState sync1 = createMockFactory(); + SynchronizerFactoryWithState sync2 = createMockFactory(); + SynchronizerFactoryWithState sync3 = createMockFactory(); + synchronizers.add(sync1); + synchronizers.add(sync2); + synchronizers.add(sync3); + + SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + + assertEquals(3, manager.getAvailableSynchronizerCount()); + } + + @Test + public void getAvailableSynchronizerCountUpdatesWhenBlocked() { + List synchronizers = new ArrayList<>(); + SynchronizerFactoryWithState sync1 = createMockFactory(); + SynchronizerFactoryWithState sync2 = createMockFactory(); + SynchronizerFactoryWithState sync3 = createMockFactory(); + synchronizers.add(sync1); + synchronizers.add(sync2); + synchronizers.add(sync3); + + SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + + assertEquals(3, manager.getAvailableSynchronizerCount()); + + sync2.block(); + assertEquals(2, manager.getAvailableSynchronizerCount()); + + sync1.block(); + assertEquals(1, manager.getAvailableSynchronizerCount()); + + sync3.block(); + assertEquals(0, manager.getAvailableSynchronizerCount()); + } + + @Test + public void setActiveSourceSetsNewSource() throws IOException { + List synchronizers = new ArrayList<>(); + SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + + Closeable source = mock(Closeable.class); + boolean shutdown = manager.setActiveSource(source); + + assertFalse(shutdown); + } + + @Test + public void setActiveSourceClosesPreviousSource() throws IOException { + List synchronizers = new ArrayList<>(); + SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + + Closeable firstSource = mock(Closeable.class); + Closeable secondSource = mock(Closeable.class); + + manager.setActiveSource(firstSource); + manager.setActiveSource(secondSource); + + verify(firstSource, times(1)).close(); + } + + @Test + public void setActiveSourceReturnsTrueAfterShutdown() throws IOException { + List synchronizers = new ArrayList<>(); + SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + + manager.close(); + + Closeable source = mock(Closeable.class); + boolean shutdown = manager.setActiveSource(source); + + assertTrue(shutdown); + verify(source, times(1)).close(); + } + + @Test + public void setActiveSourceIgnoresCloseExceptionFromPreviousSource() throws IOException { + List synchronizers = new ArrayList<>(); + SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + + Closeable firstSource = mock(Closeable.class); + doThrow(new IOException("test")).when(firstSource).close(); + + Closeable secondSource = mock(Closeable.class); + + manager.setActiveSource(firstSource); + // Should not throw + manager.setActiveSource(secondSource); + } + + @Test + public void shutdownClosesActiveSource() throws IOException { + List synchronizers = new ArrayList<>(); + SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + + Closeable source = mock(Closeable.class); + manager.setActiveSource(source); + + manager.close(); + + verify(source, times(1)).close(); + } + + @Test + public void shutdownCanBeCalledMultipleTimes() throws IOException { + List synchronizers = new ArrayList<>(); + SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + + Closeable source = mock(Closeable.class); + manager.setActiveSource(source); + + manager.close(); + manager.close(); + manager.close(); + + // Should only close once + verify(source, times(1)).close(); + } + + @Test + public void shutdownIgnoresCloseException() throws IOException { + List synchronizers = new ArrayList<>(); + SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + + Closeable source = mock(Closeable.class); + doThrow(new IOException("test")).when(source).close(); + + manager.setActiveSource(source); + + // Should not throw + manager.close(); + } + + @Test + public void shutdownWithoutActiveSourceDoesNotFail() { + List synchronizers = new ArrayList<>(); + SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + + // Should not throw + manager.close(); + } + + @Test + public void integrationTestFullCycle() throws IOException { + List synchronizers = new ArrayList<>(); + SynchronizerFactoryWithState sync1 = createMockFactory(); + SynchronizerFactoryWithState sync2 = createMockFactory(); + SynchronizerFactoryWithState sync3 = createMockFactory(); + synchronizers.add(sync1); + synchronizers.add(sync2); + synchronizers.add(sync3); + + SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + + // Initial state + assertEquals(3, manager.getAvailableSynchronizerCount()); + assertFalse(manager.isPrimeSynchronizer()); + + // Get first synchronizer + SynchronizerFactoryWithState first = manager.getNextAvailableSynchronizer(); + assertSame(sync1, first); + assertTrue(manager.isPrimeSynchronizer()); + + // Get second synchronizer + SynchronizerFactoryWithState second = manager.getNextAvailableSynchronizer(); + assertSame(sync2, second); + assertFalse(manager.isPrimeSynchronizer()); + + // Block second + sync2.block(); + assertEquals(2, manager.getAvailableSynchronizerCount()); + + // Get third synchronizer + SynchronizerFactoryWithState third = manager.getNextAvailableSynchronizer(); + assertSame(sync3, third); + assertFalse(manager.isPrimeSynchronizer()); + + // Reset and get first again + manager.resetSourceIndex(); + SynchronizerFactoryWithState firstAgain = manager.getNextAvailableSynchronizer(); + assertSame(sync1, firstAgain); + assertTrue(manager.isPrimeSynchronizer()); + + // Set active source + Closeable source = mock(Closeable.class); + assertFalse(manager.setActiveSource(source)); + + // Shutdown + manager.close(); + verify(source, times(1)).close(); + + // After shutdown, new sources are immediately closed + Closeable newSource = mock(Closeable.class); + assertTrue(manager.setActiveSource(newSource)); + verify(newSource, times(1)).close(); + } +} From 53ea3192071e9d8c6be119c7ad4e5e64b57c1255 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Mon, 26 Jan 2026 15:20:02 -0800 Subject: [PATCH 10/35] FDv2DataSource tests. --- .../sdk/server/FDv2DataSourceTest.java | 2200 +++++++++++++++++ 1 file changed, 2200 insertions(+) create mode 100644 lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java new file mode 100644 index 00000000..9c8d3b1e --- /dev/null +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java @@ -0,0 +1,2200 @@ +package com.launchdarkly.sdk.server; + +import com.google.common.collect.ImmutableList; +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.logging.Logs; +import com.launchdarkly.sdk.internal.fdv2.sources.Selector; +import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; +import com.launchdarkly.sdk.server.datasources.Initializer; +import com.launchdarkly.sdk.server.datasources.Synchronizer; +import com.launchdarkly.sdk.server.interfaces.DataSourceStatusProvider; +import com.launchdarkly.sdk.server.interfaces.DataStoreStatusProvider; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes; +import com.launchdarkly.sdk.server.subsystems.DataSourceUpdateSinkV2; + +import org.junit.After; +import org.junit.Test; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.*; + +@SuppressWarnings("javadoc") +public class FDv2DataSourceTest extends BaseTest { + + private ScheduledExecutorService executor; + private final LDLogger logger = LDLogger.withAdapter(Logs.none(), ""); + private final List resourcesToClose = new ArrayList<>(); + + @After + public void tearDown() { + if (executor != null && !executor.isShutdown()) { + executor.shutdownNow(); + } + for (AutoCloseable resource : resourcesToClose) { + try { + resource.close(); + } catch (Exception e) { + // Ignore cleanup exceptions + } + } + resourcesToClose.clear(); + } + + private DataStoreTypes.ChangeSet makeChangeSet(boolean withSelector) { + Selector selector = withSelector ? Selector.make(1, "test-state") : Selector.EMPTY; + return new DataStoreTypes.ChangeSet<>( + DataStoreTypes.ChangeSetType.None, + selector, + null, + null + ); + } + + private FDv2SourceResult makeInterruptedResult() { + return FDv2SourceResult.interrupted( + new DataSourceStatusProvider.ErrorInfo( + DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, + 500, + null, + Instant.now() + ), + false + ); + } + + private FDv2SourceResult makeTerminalErrorResult() { + return FDv2SourceResult.terminalError( + new DataSourceStatusProvider.ErrorInfo( + DataSourceStatusProvider.ErrorKind.ERROR_RESPONSE, + 401, + null, + Instant.now() + ), + false + ); + } + + // ============================================================================ + // Initializer Scenarios + // ============================================================================ + + @Test + public void firstInitializerFailsSecondInitializerSucceedsWithSelector() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + CompletableFuture firstInitializerFuture = new CompletableFuture<>(); + firstInitializerFuture.completeExceptionally(new RuntimeException("First initializer failed")); + + CompletableFuture secondInitializerFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(true), false) + ); + + ImmutableList> initializers = ImmutableList.of( + () -> new MockInitializer(firstInitializerFuture), + () -> new MockInitializer(secondInitializerFuture) + ); + + ImmutableList> synchronizers = ImmutableList.of(); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertTrue(dataSource.isInitialized()); + assertEquals(1, sink.getApplyCount()); + // TODO: Verify status updated to VALID when data source status is implemented + } + + @Test + public void firstInitializerFailsSecondInitializerSucceedsWithoutSelector() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + CompletableFuture firstInitializerFuture = new CompletableFuture<>(); + firstInitializerFuture.completeExceptionally(new RuntimeException("First initializer failed")); + + CompletableFuture secondInitializerFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(false), false) + ); + + BlockingQueue synchronizerCalledQueue = new LinkedBlockingQueue<>(); + ImmutableList> initializers = ImmutableList.of( + () -> new MockInitializer(firstInitializerFuture), + () -> new MockInitializer(secondInitializerFuture) + ); + + CompletableFuture synchronizerFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(false), false) + ); + + ImmutableList> synchronizers = ImmutableList.of( + () -> { + synchronizerCalledQueue.offer(true); + return new MockSynchronizer(synchronizerFuture); + } + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertTrue(dataSource.isInitialized()); + + // Wait for synchronizer to be called + Boolean synchronizerCalled = synchronizerCalledQueue.poll(2, TimeUnit.SECONDS); + assertNotNull("Synchronizer should be called", synchronizerCalled); + + // Wait for apply to be processed + sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); + assertEquals(2, sink.getApplyCount()); // One from initializer, one from synchronizer + // TODO: Verify status updated to VALID when data source status is implemented + } + + @Test + public void firstInitializerSucceedsWithSelectorSecondInitializerNotInvoked() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + CompletableFuture firstInitializerFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(true), false) + ); + + AtomicBoolean secondInitializerCalled = new AtomicBoolean(false); + + ImmutableList> initializers = ImmutableList.of( + () -> new MockInitializer(firstInitializerFuture), + () -> { + secondInitializerCalled.set(true); + return new MockInitializer(CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(true), false) + )); + } + ); + + ImmutableList> synchronizers = ImmutableList.of(); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertTrue(dataSource.isInitialized()); + assertFalse(secondInitializerCalled.get()); + assertEquals(1, sink.getApplyCount()); + // TODO: Verify status updated to VALID when data source status is implemented + } + + @Test + public void allInitializersFailSwitchesToSynchronizers() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + CompletableFuture firstInitializerFuture = new CompletableFuture<>(); + firstInitializerFuture.completeExceptionally(new RuntimeException("First failed")); + + CompletableFuture secondInitializerFuture = new CompletableFuture<>(); + secondInitializerFuture.completeExceptionally(new RuntimeException("Second failed")); + + ImmutableList> initializers = ImmutableList.of( + () -> new MockInitializer(firstInitializerFuture), + () -> new MockInitializer(secondInitializerFuture) + ); + + BlockingQueue synchronizerCalledQueue = new LinkedBlockingQueue<>(); + CompletableFuture synchronizerFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(false), false) + ); + + ImmutableList> synchronizers = ImmutableList.of( + () -> { + synchronizerCalledQueue.offer(true); + return new MockSynchronizer(synchronizerFuture); + } + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertTrue(dataSource.isInitialized()); + + // Wait for synchronizer to be called + Boolean synchronizerCalled = synchronizerCalledQueue.poll(2, TimeUnit.SECONDS); + assertNotNull("Synchronizer should be called", synchronizerCalled); + + // Wait for apply to be processed + sink.awaitApplyCount(1, 2, TimeUnit.SECONDS); + assertEquals(1, sink.getApplyCount()); + } + + @Test + public void allThreeInitializersFailWithNoSynchronizers() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of( + () -> { + CompletableFuture future = new CompletableFuture<>(); + future.completeExceptionally(new RuntimeException("Failed")); + return new MockInitializer(future); + }, + () -> { + CompletableFuture future = new CompletableFuture<>(); + future.completeExceptionally(new RuntimeException("Failed")); + return new MockInitializer(future); + }, + () -> { + CompletableFuture future = new CompletableFuture<>(); + future.completeExceptionally(new RuntimeException("Failed")); + return new MockInitializer(future); + } + ); + + ImmutableList> synchronizers = ImmutableList.of(); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertFalse(dataSource.isInitialized()); + assertEquals(0, sink.getApplyCount()); + // TODO: Verify status reflects exhausted sources when data source status is implemented + } + + @Test + public void oneInitializerNoSynchronizerIsWellBehaved() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + CompletableFuture initializerFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(true), false) + ); + + ImmutableList> initializers = ImmutableList.of( + () -> new MockInitializer(initializerFuture) + ); + + ImmutableList> synchronizers = ImmutableList.of(); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertTrue(dataSource.isInitialized()); + assertEquals(1, sink.getApplyCount()); + // TODO: Verify status updated to VALID when data source status is implemented + } + + // ============================================================================ + // Synchronizer Scenarios + // ============================================================================ + + @Test + public void noInitializersOneSynchronizerIsWellBehaved() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + CompletableFuture synchronizerFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(false), false) + ); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockSynchronizer(synchronizerFuture) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertTrue(dataSource.isInitialized()); + + // Wait for apply to be processed + sink.awaitApplyCount(1, 2, TimeUnit.SECONDS); + assertEquals(1, sink.getApplyCount()); + } + + @Test + public void oneInitializerOneSynchronizerIsWellBehaved() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + CompletableFuture initializerFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(false), false) + ); + + ImmutableList> initializers = ImmutableList.of( + () -> new MockInitializer(initializerFuture) + ); + + CompletableFuture synchronizerFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(false), false) + ); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockSynchronizer(synchronizerFuture) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertTrue(dataSource.isInitialized()); + + // Wait for both applies to be processed + sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); + assertEquals(2, sink.getApplyCount()); + } + + @Test + public void noInitializersAndNoSynchronizersIsWellBehaved() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + ImmutableList> synchronizers = ImmutableList.of(); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertFalse(dataSource.isInitialized()); + assertEquals(0, sink.getApplyCount()); + // TODO: Verify status reflects exhausted sources when data source status is implemented + } + + // ============================================================================ + // Fallback and Recovery + // ============================================================================ + + @Test + public void errorWithFDv1FallbackTriggersFallback() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + CompletableFuture synchronizerFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(false), true) // FDv1 fallback flag + ); + + AtomicBoolean synchronizerCalled = new AtomicBoolean(false); + ImmutableList> synchronizers = ImmutableList.of( + () -> { + synchronizerCalled.set(true); + return new MockSynchronizer(synchronizerFuture); + } + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertTrue(synchronizerCalled.get()); + assertEquals(1, sink.getApplyCount()); + // TODO: Verify FDv1 fallback behavior when implemented + } + + @Test + public void fallbackAndRecoveryTasksWellBehaved() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + // First synchronizer sends INTERRUPTED, triggering fallback after timeout + BlockingQueue firstSyncResults = new LinkedBlockingQueue<>(); + firstSyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + firstSyncResults.add(makeInterruptedResult()); + // Keep it alive so fallback timeout triggers + + // The second synchronizer works fine, sends data + BlockingQueue secondSyncResults = new LinkedBlockingQueue<>(); + secondSyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + // Keep alive for recovery + + AtomicInteger firstSyncCallCount = new AtomicInteger(0); + AtomicInteger secondSyncCallCount = new AtomicInteger(0); + + ImmutableList> synchronizers = ImmutableList.of( + () -> { + firstSyncCallCount.incrementAndGet(); + return new MockQueuedSynchronizer(firstSyncResults); + }, + () -> { + secondSyncCallCount.incrementAndGet(); + return new MockQueuedSynchronizer(secondSyncResults); + } + ); + + // Use short timeouts for testing + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(5, TimeUnit.SECONDS); + + assertTrue(dataSource.isInitialized()); + + // Wait for recovery timeout to trigger by waiting for multiple synchronizer calls + // Recovery brings us back to first, so we should see multiple calls eventually + for (int i = 0; i < 3; i++) { + sink.awaitApplyCount(i + 1, 5, TimeUnit.SECONDS); + } + + // Recovery should have brought us back to the first synchronizer multiple times + assertTrue(firstSyncCallCount.get() >= 1); + assertTrue(secondSyncCallCount.get() >= 1); + // TODO: Verify status transitions when data source status is implemented + } + + @Test + public void canDisposeWhenSynchronizersFallingBack() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + // Synchronizer that sends INTERRUPTED to trigger fallback + BlockingQueue syncResults = new LinkedBlockingQueue<>(); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + syncResults.add(makeInterruptedResult()); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(syncResults) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertTrue(dataSource.isInitialized()); + + // Close while the fallback condition is active + dataSource.close(); + + // Test passes if we reach here without hanging + } + + // ============================================================================ + // Source Blocking + // ============================================================================ + + @Test + public void terminalErrorBlocksSynchronizer() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + // First synchronizer sends terminal error + BlockingQueue firstSyncResults = new LinkedBlockingQueue<>(); + firstSyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + firstSyncResults.add(makeTerminalErrorResult()); + + // The second synchronizer works fine + BlockingQueue secondSyncResults = new LinkedBlockingQueue<>(); + secondSyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + BlockingQueue synchronizerCallQueue = new LinkedBlockingQueue<>(); + + ImmutableList> synchronizers = ImmutableList.of( + () -> { + synchronizerCallQueue.offer(1); + return new MockQueuedSynchronizer(firstSyncResults); + }, + () -> { + synchronizerCallQueue.offer(2); + return new MockQueuedSynchronizer(secondSyncResults); + } + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertTrue(dataSource.isInitialized()); + + // Wait for both synchronizers to be called + Integer firstCall = synchronizerCallQueue.poll(2, TimeUnit.SECONDS); + Integer secondCall = synchronizerCallQueue.poll(2, TimeUnit.SECONDS); + + assertNotNull("First synchronizer should be called", firstCall); + assertNotNull("Second synchronizer should be called after first is blocked", secondCall); + assertEquals(Integer.valueOf(1), firstCall); + assertEquals(Integer.valueOf(2), secondCall); + + // Wait for applies from both + sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); + // TODO: Verify status transitions when data source status is implemented + } + + @Test + public void allThreeSynchronizersFailReportsExhaustion() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + // All synchronizers send terminal errors + ImmutableList> synchronizers = ImmutableList.of( + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(makeTerminalErrorResult()); + return new MockQueuedSynchronizer(results); + }, + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(makeTerminalErrorResult()); + return new MockQueuedSynchronizer(results); + }, + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(makeTerminalErrorResult()); + return new MockQueuedSynchronizer(results); + } + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertFalse(dataSource.isInitialized()); + // TODO: Verify status reflects exhausted sources when data source status is implemented + } + + // ============================================================================ + // Disabled Source Prevention + // ============================================================================ + + @Test + public void disabledDataSourceCannotTriggerActions() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + // First synchronizer that we'll close and try to trigger + AtomicReference firstSyncRef = new AtomicReference<>(); + BlockingQueue firstSyncResults = new LinkedBlockingQueue<>(); + firstSyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + firstSyncResults.add(makeTerminalErrorResult()); + + // Second synchronizer + BlockingQueue secondSyncResults = new LinkedBlockingQueue<>(); + secondSyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + ImmutableList> synchronizers = ImmutableList.of( + () -> { + MockQueuedSynchronizer sync = new MockQueuedSynchronizer(firstSyncResults); + firstSyncRef.set(sync); + return sync; + }, + () -> new MockQueuedSynchronizer(secondSyncResults) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + // Wait for synchronizers to run and switch + sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); + int applyCountAfterSwitch = sink.getApplyCount(); + + // Try to send more data from the first (now closed) synchronizer + MockQueuedSynchronizer firstSync = firstSyncRef.get(); + if (firstSync != null) { + firstSync.addResult(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + } + + // Wait to ensure closed synchronizer's results aren't processed + try { + sink.awaitApplyCount(applyCountAfterSwitch + 1, 500, TimeUnit.MILLISECONDS); + } catch (Exception e) { + // Timeout expected + } + + // Apply count should not have increased from the closed synchronizer + assertEquals(applyCountAfterSwitch, sink.getApplyCount()); + } + + // ============================================================================ + // Disposal and Cleanup + // ============================================================================ + + @Test + public void disposeCompletesStartFuture() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + // Synchronizer that never completes + BlockingQueue syncResults = new LinkedBlockingQueue<>(); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(syncResults) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + dataSource.close(); + + assertTrue(startFuture.isDone()); + // TODO: Verify status updated to OFF when data source status is implemented + } + + @Test + public void noSourcesProvidedCompletesImmediately() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + ImmutableList> synchronizers = ImmutableList.of(); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertFalse(dataSource.isInitialized()); + // TODO: Verify status reflects exhausted sources when data source status is implemented + } + + // ============================================================================ + // Thread Safety and Concurrency + // ============================================================================ + + @Test + public void startFutureCompletesExactlyOnce() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + CompletableFuture initializerFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(true), false) + ); + + CompletableFuture synchronizerFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(false), false) + ); + + ImmutableList> initializers = ImmutableList.of( + () -> new MockInitializer(initializerFuture) + ); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockSynchronizer(synchronizerFuture) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertTrue(dataSource.isInitialized()); + // Multiple completions would throw, so if we get here, it's working correctly + } + + @Test + public void concurrentCloseAndStartHandledSafely() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + BlockingQueue syncResults = new LinkedBlockingQueue<>(); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + ImmutableList> initializers = ImmutableList.of(); + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(syncResults) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + + Future startFuture = dataSource.start(); + + // Close immediately after starting + dataSource.close(); + + // Should not throw or hang + startFuture.get(2, TimeUnit.SECONDS); + } + + @Test + public void multipleStartCallsEventuallyComplete() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + BlockingQueue syncResults = new LinkedBlockingQueue<>(); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + ImmutableList> initializers = ImmutableList.of(); + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(syncResults) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture1 = dataSource.start(); + Future startFuture2 = dataSource.start(); + Future startFuture3 = dataSource.start(); + + // All calls should complete successfully (even if they return different Future wrappers) + startFuture1.get(2, TimeUnit.SECONDS); + startFuture2.get(2, TimeUnit.SECONDS); + startFuture3.get(2, TimeUnit.SECONDS); + + assertTrue(dataSource.isInitialized()); + } + + @Test + public void isInitializedThreadSafe() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + BlockingQueue syncResults = new LinkedBlockingQueue<>(); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + ImmutableList> initializers = ImmutableList.of(); + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(syncResults) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + dataSource.start(); + + // Call isInitialized from multiple threads + CountDownLatch latch = new CountDownLatch(10); + for (int i = 0; i < 10; i++) { + new Thread(() -> { + try { + dataSource.isInitialized(); + } finally { + latch.countDown(); + } + }).start(); + } + + assertTrue(latch.await(2, TimeUnit.SECONDS)); + } + + @Test + public void dataSourceUpdatesApplyThreadSafe() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + BlockingQueue syncResults = new LinkedBlockingQueue<>(); + for (int i = 0; i < 10; i++) { + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + } + + ImmutableList> initializers = ImmutableList.of(); + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(syncResults) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + // Wait for all applies to process + sink.awaitApplyCount(10, 2, TimeUnit.SECONDS); + + // Should have received multiple applies without error + assertTrue(sink.getApplyCount() >= 10); + } + + // ============================================================================ + // Exception Handling + // ============================================================================ + + @Test + public void initializerThrowsExecutionException() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + CompletableFuture badFuture = new CompletableFuture<>(); + badFuture.completeExceptionally(new RuntimeException("Execution exception")); + + CompletableFuture goodFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(true), false) + ); + + ImmutableList> initializers = ImmutableList.of( + () -> new MockInitializer(badFuture), + () -> new MockInitializer(goodFuture) + ); + + ImmutableList> synchronizers = ImmutableList.of(); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertTrue(dataSource.isInitialized()); + assertEquals(1, sink.getApplyCount()); + } + + @Test + public void initializerThrowsInterruptedException() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + AtomicBoolean firstCalled = new AtomicBoolean(false); + CompletableFuture goodFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(true), false) + ); + + ImmutableList> initializers = ImmutableList.of( + () -> { + firstCalled.set(true); + return new MockInitializer(() -> { + throw new InterruptedException("Interrupted"); + }); + }, + () -> new MockInitializer(goodFuture) + ); + + ImmutableList> synchronizers = ImmutableList.of(); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertTrue(firstCalled.get()); + assertTrue(dataSource.isInitialized()); + assertEquals(1, sink.getApplyCount()); + } + + @Test + public void initializerThrowsCancellationException() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + CompletableFuture cancelledFuture = new CompletableFuture<>(); + cancelledFuture.cancel(true); + + CompletableFuture goodFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(true), false) + ); + + ImmutableList> initializers = ImmutableList.of( + () -> new MockInitializer(cancelledFuture), + () -> new MockInitializer(goodFuture) + ); + + ImmutableList> synchronizers = ImmutableList.of(); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertTrue(dataSource.isInitialized()); + assertEquals(1, sink.getApplyCount()); + } + + @Test + public void synchronizerNextThrowsExecutionException() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + CompletableFuture badFuture = new CompletableFuture<>(); + badFuture.completeExceptionally(new RuntimeException("Execution exception")); + + CompletableFuture goodFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(false), false) + ); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockSynchronizer(badFuture), + () -> new MockSynchronizer(goodFuture) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertTrue(dataSource.isInitialized()); + assertEquals(1, sink.getApplyCount()); + } + + @Test + public void synchronizerNextThrowsInterruptedException() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + AtomicBoolean firstCalled = new AtomicBoolean(false); + CompletableFuture goodFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(false), false) + ); + + ImmutableList> synchronizers = ImmutableList.of( + () -> { + firstCalled.set(true); + return new MockSynchronizer(() -> { + throw new InterruptedException("Interrupted"); + }); + }, + () -> new MockSynchronizer(goodFuture) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertTrue(firstCalled.get()); + assertTrue(dataSource.isInitialized()); + assertEquals(1, sink.getApplyCount()); + } + + @Test + public void synchronizerNextThrowsCancellationException() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + CompletableFuture cancelledFuture = new CompletableFuture<>(); + cancelledFuture.cancel(true); + + CompletableFuture goodFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(false), false) + ); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockSynchronizer(cancelledFuture), + () -> new MockSynchronizer(goodFuture) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertTrue(dataSource.isInitialized()); + assertEquals(1, sink.getApplyCount()); + } + + // ============================================================================ + // Resource Management + // ============================================================================ + + @Test + public void closeWithoutStartDoesNotHang() { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + ImmutableList> synchronizers = ImmutableList.of(); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + + dataSource.close(); + + // Test passes if we reach here without hanging + } + + @Test + public void closeAfterInitializersCompletesImmediately() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + CompletableFuture initializerFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(true), false) + ); + + ImmutableList> initializers = ImmutableList.of( + () -> new MockInitializer(initializerFuture) + ); + + ImmutableList> synchronizers = ImmutableList.of(); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + dataSource.close(); + + } + + @Test + public void closeWhileSynchronizerRunningShutdownsSource() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + BlockingQueue syncResults = new LinkedBlockingQueue<>(); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + AtomicBoolean synchronizerClosed = new AtomicBoolean(false); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(syncResults) { + @Override + public void close() { + synchronizerClosed.set(true); + super.close(); + } + } + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + dataSource.close(); + + assertTrue(synchronizerClosed.get()); + } + + @Test + public void multipleCloseCallsAreIdempotent() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + CompletableFuture initializerFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(true), false) + ); + + ImmutableList> initializers = ImmutableList.of( + () -> new MockInitializer(initializerFuture) + ); + + ImmutableList> synchronizers = ImmutableList.of(); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + dataSource.close(); + dataSource.close(); + dataSource.close(); + + // Test passes if we reach here without throwing + } + + @Test + public void closeInterruptsConditionWaiting() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + BlockingQueue syncResults = new LinkedBlockingQueue<>(); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + syncResults.add(makeInterruptedResult()); + // Don't add more, so it waits on condition + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(syncResults) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + // Close while condition is waiting + dataSource.close(); + + // Test passes if we reach here without hanging + } + + // ============================================================================ + // Active Source Management + // ============================================================================ + + @Test + public void setActiveSourceReturnsShutdownStatus() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + CompletableFuture initializerFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(true), false) + ); + + AtomicBoolean shutdownDetected = new AtomicBoolean(false); + + ImmutableList> initializers = ImmutableList.of( + () -> new MockInitializer(initializerFuture) { + @Override + public CompletableFuture run() { + // This won't be called because close() is called first + return super.run(); + } + } + ); + + ImmutableList> synchronizers = ImmutableList.of(); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + + dataSource.close(); + Future startFuture = dataSource.start(); + + // Should complete without hanging since shutdown was already called + startFuture.get(2, TimeUnit.SECONDS); + // Test passes if we reach here - shutdown was handled + } + + @Test + public void activeSourceClosedWhenSwitchingSynchronizers() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + BlockingQueue firstSyncResults = new LinkedBlockingQueue<>(); + firstSyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + firstSyncResults.add(makeTerminalErrorResult()); + + BlockingQueue secondSyncResults = new LinkedBlockingQueue<>(); + secondSyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + AtomicBoolean firstSyncClosed = new AtomicBoolean(false); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(firstSyncResults) { + @Override + public void close() { + firstSyncClosed.set(true); + super.close(); + } + }, + () -> new MockQueuedSynchronizer(secondSyncResults) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + // Wait for both synchronizers to run (switch happens after the first sends terminal error) + sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); + + assertTrue(firstSyncClosed.get()); + } + + @Test + public void activeSourceClosedOnShutdown() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + BlockingQueue syncResults = new LinkedBlockingQueue<>(); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + AtomicBoolean syncClosed = new AtomicBoolean(false); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(syncResults) { + @Override + public void close() { + syncClosed.set(true); + super.close(); + } + } + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + dataSource.close(); + + assertTrue(syncClosed.get()); + } + + @Test + public void setActiveSourceOnInitializerChecksShutdown() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + CountDownLatch initializerStarted = new CountDownLatch(1); + CompletableFuture slowResult = new CompletableFuture<>(); + + ImmutableList> initializers = ImmutableList.of( + () -> new MockInitializer(() -> { + initializerStarted.countDown(); + // Wait for the future to complete (will be completed by shutdown check) + try { + return slowResult.get(2, TimeUnit.SECONDS); + } catch (Exception e) { + return FDv2SourceResult.changeSet(makeChangeSet(true), false); + } + }) + ); + + ImmutableList> synchronizers = ImmutableList.of(); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + + Future startFuture = dataSource.start(); + + // Wait for initializer to start + assertTrue(initializerStarted.await(2, TimeUnit.SECONDS)); + + // Close while the initializer is running + dataSource.close(); + + // Complete the future so initializer can finish + slowResult.complete(FDv2SourceResult.changeSet(makeChangeSet(true), false)); + + // Wait for the start method to complete + startFuture.get(2, TimeUnit.SECONDS); + + // Test passes if we reach here - shutdown handled gracefully + } + + // ============================================================================ + // Synchronizer State Transitions + // ============================================================================ + + @Test + public void blockedSynchronizerSkippedInRotation() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + // First: terminal error (blocked) + BlockingQueue firstSyncResults = new LinkedBlockingQueue<>(); + firstSyncResults.add(makeTerminalErrorResult()); + + // Second: works + BlockingQueue secondSyncResults = new LinkedBlockingQueue<>(); + secondSyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + // Third: works + BlockingQueue thirdSyncResults = new LinkedBlockingQueue<>(); + thirdSyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + AtomicInteger firstCallCount = new AtomicInteger(0); + AtomicInteger secondCallCount = new AtomicInteger(0); + AtomicInteger thirdCallCount = new AtomicInteger(0); + + ImmutableList> synchronizers = ImmutableList.of( + () -> { + firstCallCount.incrementAndGet(); + return new MockQueuedSynchronizer(firstSyncResults); + }, + () -> { + secondCallCount.incrementAndGet(); + return new MockQueuedSynchronizer(secondSyncResults); + }, + () -> { + thirdCallCount.incrementAndGet(); + return new MockQueuedSynchronizer(thirdSyncResults); + } + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertEquals(1, firstCallCount.get()); // Called once, then blocked + assertTrue(secondCallCount.get() >= 1); // Called + } + + @Test + public void allSynchronizersBlockedReturnsNullAndExits() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + ImmutableList> synchronizers = ImmutableList.of( + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(makeTerminalErrorResult()); + return new MockQueuedSynchronizer(results); + }, + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(makeTerminalErrorResult()); + return new MockQueuedSynchronizer(results); + } + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertFalse(dataSource.isInitialized()); + } + + @Test + public void recoveryResetsToFirstAvailableSynchronizer() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + // First synchronizer: send data, then INTERRUPTED + BlockingQueue firstSyncResults = new LinkedBlockingQueue<>(); + firstSyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + firstSyncResults.add(makeInterruptedResult()); + + // Second synchronizer: send data + BlockingQueue secondSyncResults = new LinkedBlockingQueue<>(); + secondSyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + AtomicInteger firstCallCount = new AtomicInteger(0); + AtomicInteger secondCallCount = new AtomicInteger(0); + + ImmutableList> synchronizers = ImmutableList.of( + () -> { + firstCallCount.incrementAndGet(); + return new MockQueuedSynchronizer(firstSyncResults); + }, + () -> { + secondCallCount.incrementAndGet(); + return new MockQueuedSynchronizer(secondSyncResults); + } + ); + + // Short recovery timeout + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + // Wait for recovery timeout to trigger by waiting for multiple synchronizer calls + // Recovery brings us back to first, so we should see multiple calls eventually + for (int i = 0; i < 3; i++) { + sink.awaitApplyCount(i + 1, 5, TimeUnit.SECONDS); + } + + // Should have called first synchronizer again after recovery + assertTrue(firstCallCount.get() >= 2 || secondCallCount.get() >= 1); + } + + @Test + public void fallbackMovesToNextSynchronizer() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + // First: send INTERRUPTED to trigger fallback + BlockingQueue firstSyncResults = new LinkedBlockingQueue<>(); + firstSyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + firstSyncResults.add(makeInterruptedResult()); + + // Second: works + BlockingQueue secondSyncResults = new LinkedBlockingQueue<>(); + secondSyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + BlockingQueue secondCalledQueue = new LinkedBlockingQueue<>(); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(firstSyncResults), + () -> { + secondCalledQueue.offer(true); + return new MockQueuedSynchronizer(secondSyncResults); + } + ); + + // Short fallback timeout + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 1, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(3, TimeUnit.SECONDS); + + // Wait for the second synchronizer to be called after fallback timeout + Boolean secondCalled = secondCalledQueue.poll(3, TimeUnit.SECONDS); + assertNotNull("Second synchronizer should be called after fallback", secondCalled); + } + + // ============================================================================ + // Condition Lifecycle + // ============================================================================ + + @Test + public void conditionsClosedAfterSynchronizerLoop() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + BlockingQueue syncResults = new LinkedBlockingQueue<>(); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + syncResults.add(makeTerminalErrorResult()); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(syncResults) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + dataSource.close(); + + // If conditions weren't closed properly, we might see issues + } + + @Test + public void conditionsInformedOfAllResults() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + BlockingQueue syncResults = new LinkedBlockingQueue<>(); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + syncResults.add(makeInterruptedResult()); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(syncResults) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 10, 20); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + // All results should be processed + sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); + assertTrue(sink.getApplyCount() >= 2); + } + + @Test + public void conditionsClosedOnException() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + CompletableFuture exceptionFuture = new CompletableFuture<>(); + exceptionFuture.completeExceptionally(new RuntimeException("Error")); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockSynchronizer(exceptionFuture), + () -> new MockSynchronizer(CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(false), false) + )) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + // Conditions should be closed despite exception + } + + @Test + public void primeSynchronizerHasNoRecoveryCondition() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + BlockingQueue syncResults = new LinkedBlockingQueue<>(); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + // Keep alive + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(syncResults), + () -> new MockQueuedSynchronizer(new LinkedBlockingQueue<>()) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + // Prime synchronizer should not have a recovery condition + // This is tested implicitly by the implementation + } + + @Test + public void nonPrimeSynchronizerHasBothConditions() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + // First: send INTERRUPTED to trigger fallback + BlockingQueue firstSyncResults = new LinkedBlockingQueue<>(); + firstSyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + firstSyncResults.add(makeInterruptedResult()); + + // Second: will have both conditions + BlockingQueue secondSyncResults = new LinkedBlockingQueue<>(); + secondSyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(firstSyncResults), + () -> new MockQueuedSynchronizer(secondSyncResults) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + // Non-prime synchronizer should have both fallback and recovery + // This is tested implicitly by the implementation + } + + @Test + public void singleSynchronizerHasNoConditions() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + BlockingQueue syncResults = new LinkedBlockingQueue<>(); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(syncResults) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + // Single synchronizer should have no conditions + // This is tested implicitly by the implementation + } + + @Test + public void conditionFutureNeverCompletesWhenNoConditions() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + BlockingQueue syncResults = new LinkedBlockingQueue<>(); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(syncResults) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + // Should process both ChangeSet results without condition interruption + sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); + assertTrue(sink.getApplyCount() >= 2); + } + + // ============================================================================ + // Data Flow Verification + // ============================================================================ + + @Test + public void changeSetAppliedToDataSourceUpdates() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + CompletableFuture initializerFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(true), false) + ); + + ImmutableList> initializers = ImmutableList.of( + () -> new MockInitializer(initializerFuture) + ); + + ImmutableList> synchronizers = ImmutableList.of(); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertEquals(1, sink.getApplyCount()); + assertNotNull(sink.getLastChangeSet()); + } + + @Test + public void multipleChangeSetsAppliedInOrder() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + BlockingQueue syncResults = new LinkedBlockingQueue<>(); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(syncResults) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + // Wait for all 3 ChangeSets to be applied + sink.awaitApplyCount(3, 2, TimeUnit.SECONDS); + + assertEquals(3, sink.getApplyCount()); + } + + @Test + public void selectorEmptyStillCompletesIfAnyDataReceived() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + CompletableFuture initializerFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(false), false) + ); + + ImmutableList> initializers = ImmutableList.of( + () -> new MockInitializer(initializerFuture) + ); + + CompletableFuture synchronizerFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(false), false) + ); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockSynchronizer(synchronizerFuture) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertTrue(dataSource.isInitialized()); + + // Wait for the synchronizer to also run + sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); + assertEquals(2, sink.getApplyCount()); // Both initializer and synchronizer + } + + @Test + public void selectorNonEmptyCompletesInitialization() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + CompletableFuture initializerFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(true), false) + ); + + AtomicBoolean synchronizerCalled = new AtomicBoolean(false); + + ImmutableList> initializers = ImmutableList.of( + () -> new MockInitializer(initializerFuture) + ); + + ImmutableList> synchronizers = ImmutableList.of( + () -> { + synchronizerCalled.set(true); + return new MockSynchronizer(CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(false), false) + )); + } + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertTrue(dataSource.isInitialized()); + assertFalse(synchronizerCalled.get()); // Should not proceed to synchronizers + assertEquals(1, sink.getApplyCount()); + } + + @Test + public void initializerChangeSetWithoutSelectorCompletesIfLastInitializer() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + CompletableFuture initializerFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(false), false) + ); + + ImmutableList> initializers = ImmutableList.of( + () -> new MockInitializer(initializerFuture) + ); + + ImmutableList> synchronizers = ImmutableList.of(); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertTrue(dataSource.isInitialized()); + assertEquals(1, sink.getApplyCount()); + // TODO: Verify status updated to VALID when data source status is implemented + } + + @Test + public void synchronizerChangeSetAlwaysCompletesStartFuture() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + CompletableFuture synchronizerFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(false), false) + ); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockSynchronizer(synchronizerFuture) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertTrue(dataSource.isInitialized()); + } + + // ============================================================================ + // Status Result Handling + // ============================================================================ + + @Test + public void goodbyeStatusHandledGracefully() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + BlockingQueue syncResults = new LinkedBlockingQueue<>(); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + syncResults.add(FDv2SourceResult.goodbye("server-requested", false)); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(syncResults) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + // Wait for applies to be processed + sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); + + assertTrue(dataSource.isInitialized()); + assertTrue(sink.getApplyCount() >= 2); + } + + @Test + public void shutdownStatusExitsImmediately() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + BlockingQueue syncResults = new LinkedBlockingQueue<>(); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + syncResults.add(FDv2SourceResult.shutdown()); + + AtomicBoolean secondSynchronizerCalled = new AtomicBoolean(false); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(syncResults), + () -> { + secondSynchronizerCalled.set(true); + return new MockQueuedSynchronizer(new LinkedBlockingQueue<>()); + } + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + // Wait for first synchronizer's apply + sink.awaitApplyCount(1, 2, TimeUnit.SECONDS); + + // Verify the second synchronizer was not called (SHUTDOWN exits immediately) + assertFalse(secondSynchronizerCalled.get()); + } + + @Test + public void fdv1FallbackFlagHonored() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + BlockingQueue syncResults = new LinkedBlockingQueue<>(); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), true)); // FDv1 fallback + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(syncResults) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertTrue(dataSource.isInitialized()); + // TODO: Verify FDv1 fallback behavior when implemented + } + + // ============================================================================ + // Edge Cases and Initialization + // ============================================================================ + + @Test + public void emptyInitializerListSkipsToSynchronizers() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + AtomicBoolean synchronizerCalled = new AtomicBoolean(false); + CompletableFuture synchronizerFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(false), false) + ); + + ImmutableList> synchronizers = ImmutableList.of( + () -> { + synchronizerCalled.set(true); + return new MockSynchronizer(synchronizerFuture); + } + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + assertTrue(synchronizerCalled.get()); + assertTrue(dataSource.isInitialized()); + } + + @Test + public void startedFlagPreventsMultipleRuns() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + AtomicInteger runCount = new AtomicInteger(0); + + ImmutableList> initializers = ImmutableList.of( + () -> { + runCount.incrementAndGet(); + return new MockInitializer(CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(true), false) + )); + } + ); + + ImmutableList> synchronizers = ImmutableList.of(); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture1 = dataSource.start(); + Future startFuture2 = dataSource.start(); + Future startFuture3 = dataSource.start(); + + // Wait for all start futures to complete + // The data sources use Future instead of CompletableFuture, so we cannot use CompletableFuture.allOf. + startFuture1.get(2, TimeUnit.SECONDS); + startFuture2.get(2, TimeUnit.SECONDS); + startFuture3.get(2, TimeUnit.SECONDS); + + // Verify initializer was only called once despite multiple start() calls + assertEquals(1, runCount.get()); + } + + @Test + public void startBeforeRunCompletesAllComplete() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + BlockingQueue syncResults = new LinkedBlockingQueue<>(); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + ImmutableList> initializers = ImmutableList.of(); + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(syncResults) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + // Call start multiple times before completion + Future future1 = dataSource.start(); + Future future2 = dataSource.start(); + + // Both should complete successfully + future1.get(2, TimeUnit.SECONDS); + future2.get(2, TimeUnit.SECONDS); + + assertTrue(dataSource.isInitialized()); + } + + // ============================================================================ + // Mock Implementations + // ============================================================================ + + private static class MockDataSourceUpdateSink implements DataSourceUpdateSinkV2 { + private final AtomicInteger applyCount = new AtomicInteger(0); + private final AtomicReference> lastChangeSet = new AtomicReference<>(); + private final BlockingQueue applySignals = new LinkedBlockingQueue<>(); + + @Override + public boolean apply(DataStoreTypes.ChangeSet changeSet) { + applyCount.incrementAndGet(); + lastChangeSet.set(changeSet); + applySignals.offer(true); + return true; + } + + @Override + public void updateStatus(DataSourceStatusProvider.State newState, DataSourceStatusProvider.ErrorInfo errorInfo) { + // TODO: Track status updates when data source status is fully implemented + } + + @Override + public DataStoreStatusProvider getDataStoreStatusProvider() { + return null; // Not needed for these tests + } + + public int getApplyCount() { + return applyCount.get(); + } + + public DataStoreTypes.ChangeSet getLastChangeSet() { + return lastChangeSet.get(); + } + + public void awaitApplyCount(int expectedCount, long timeout, TimeUnit unit) throws InterruptedException { + long deadline = System.currentTimeMillis() + unit.toMillis(timeout); + while (applyCount.get() < expectedCount && System.currentTimeMillis() < deadline) { + long remaining = deadline - System.currentTimeMillis(); + if (remaining > 0) { + applySignals.poll(remaining, TimeUnit.MILLISECONDS); + } + } + } + } + + private static class MockInitializer implements Initializer { + private final CompletableFuture result; + private final ThrowingSupplier supplier; + + public MockInitializer(CompletableFuture result) { + this.result = result; + this.supplier = null; + } + + public MockInitializer(ThrowingSupplier supplier) { + this.result = null; + this.supplier = supplier; + } + + @Override + public CompletableFuture run() { + if (supplier != null) { + CompletableFuture future = new CompletableFuture<>(); + try { + future.complete(supplier.get()); + } catch (Exception e) { + future.completeExceptionally(e); + } + return future; + } + return result; + } + + @Override + public void close() { + // Nothing to close + } + } + + private static class MockSynchronizer implements Synchronizer { + private final CompletableFuture result; + private final ThrowingSupplier supplier; + private volatile boolean closed = false; + private volatile boolean resultReturned = false; + + public MockSynchronizer(CompletableFuture result) { + this.result = result; + this.supplier = null; + } + + public MockSynchronizer(ThrowingSupplier supplier) { + this.result = null; + this.supplier = supplier; + } + + @Override + public CompletableFuture next() { + if (closed) { + return CompletableFuture.completedFuture(FDv2SourceResult.shutdown()); + } + if (supplier != null) { + CompletableFuture future = new CompletableFuture<>(); + try { + future.complete(supplier.get()); + } catch (Exception e) { + future.completeExceptionally(e); + } + return future; + } + // Only return the result once, then return a never-completing future + if (!resultReturned) { + resultReturned = true; + return result; + } else { + return new CompletableFuture<>(); // Never completes + } + } + + @Override + public void close() { + closed = true; + } + } + + private static class MockQueuedSynchronizer implements Synchronizer { + private final BlockingQueue results; + private volatile boolean closed = false; + + public MockQueuedSynchronizer(BlockingQueue results) { + this.results = results; + } + + public void addResult(FDv2SourceResult result) { + if (!closed) { + results.add(result); + } + } + + @Override + public CompletableFuture next() { + if (closed) { + return CompletableFuture.completedFuture(FDv2SourceResult.shutdown()); + } + + // Try to get immediately, don't wait + FDv2SourceResult result = results.poll(); + if (result != null) { + return CompletableFuture.completedFuture(result); + } else { + // Queue is empty - return a never-completing future to simulate waiting for more data + return new CompletableFuture<>(); + } + } + + @Override + public void close() { + closed = true; + } + } + + @FunctionalInterface + private interface ThrowingSupplier { + T get() throws Exception; + } +} \ No newline at end of file From 147556d55d6abd7523ff966a766f76f2f6688efb Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Mon, 26 Jan 2026 15:25:48 -0800 Subject: [PATCH 11/35] Add documentation to long-running test. --- .../sdk/server/FDv2DataSourceTest.java | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java index 9c8d3b1e..2ed52c6e 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java @@ -460,19 +460,24 @@ public void fallbackAndRecoveryTasksWellBehaved() throws Exception { resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); - startFuture.get(5, TimeUnit.SECONDS); + startFuture.get(2, TimeUnit.SECONDS); assertTrue(dataSource.isInitialized()); - // Wait for recovery timeout to trigger by waiting for multiple synchronizer calls - // Recovery brings us back to first, so we should see multiple calls eventually - for (int i = 0; i < 3; i++) { - sink.awaitApplyCount(i + 1, 5, TimeUnit.SECONDS); - } + // Expected sequence: + // 1. First sync sends apply (1) + // 2. First sync sends INTERRUPTED, fallback timer starts (1 second) + // 3. After fallback, second sync sends apply (2) + // 4. Recovery timer starts (2 seconds) + // 5. After recovery, first sync sends apply again (3) + // Total time: ~3-4 seconds (1s fallback + 2s recovery + processing) + + // Wait for 3 applies with enough time for fallback (1s) + recovery (2s) + overhead + sink.awaitApplyCount(3, 5, TimeUnit.SECONDS); - // Recovery should have brought us back to the first synchronizer multiple times - assertTrue(firstSyncCallCount.get() >= 1); - assertTrue(secondSyncCallCount.get() >= 1); + // Both synchronizers should have been called due to fallback and recovery + assertTrue(firstSyncCallCount.get() >= 2); // Called initially and after recovery + assertTrue(secondSyncCallCount.get() >= 1); // Called after fallback // TODO: Verify status transitions when data source status is implemented } From 321fb4f6e00350dd5a0662dc100fc38653fba77c Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Mon, 26 Jan 2026 15:42:11 -0800 Subject: [PATCH 12/35] Fix test expectations. --- .../sdk/server/FDv2DataSourceTest.java | 66 +++++-------------- 1 file changed, 18 insertions(+), 48 deletions(-) diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java index 2ed52c6e..dc354ac9 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java @@ -17,6 +17,7 @@ import java.time.Instant; import java.util.ArrayList; +import java.util.LinkedList; import java.util.List; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; @@ -1437,11 +1438,8 @@ public void recoveryResetsToFirstAvailableSynchronizer() throws Exception { Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); - // Wait for recovery timeout to trigger by waiting for multiple synchronizer calls - // Recovery brings us back to first, so we should see multiple calls eventually - for (int i = 0; i < 3; i++) { - sink.awaitApplyCount(i + 1, 5, TimeUnit.SECONDS); - } + // Wait for 3 applies with enough time for recovery (2s) + overhead + sink.awaitApplyCount(3, 5, TimeUnit.SECONDS); // Should have called first synchronizer again after recovery assertTrue(firstCallCount.get() >= 2 || secondCallCount.get() >= 1); @@ -1732,64 +1730,33 @@ public void multipleChangeSetsAppliedInOrder() throws Exception { assertEquals(3, sink.getApplyCount()); } - @Test - public void selectorEmptyStillCompletesIfAnyDataReceived() throws Exception { - executor = Executors.newScheduledThreadPool(2); - MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); - - CompletableFuture initializerFuture = CompletableFuture.completedFuture( - FDv2SourceResult.changeSet(makeChangeSet(false), false) - ); - - ImmutableList> initializers = ImmutableList.of( - () -> new MockInitializer(initializerFuture) - ); - - CompletableFuture synchronizerFuture = CompletableFuture.completedFuture( - FDv2SourceResult.changeSet(makeChangeSet(false), false) - ); - - ImmutableList> synchronizers = ImmutableList.of( - () -> new MockSynchronizer(synchronizerFuture) - ); - - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); - resourcesToClose.add(dataSource); - - Future startFuture = dataSource.start(); - startFuture.get(2, TimeUnit.SECONDS); - - assertTrue(dataSource.isInitialized()); - - // Wait for the synchronizer to also run - sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); - assertEquals(2, sink.getApplyCount()); // Both initializer and synchronizer - } - @Test public void selectorNonEmptyCompletesInitialization() throws Exception { executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); - CompletableFuture initializerFuture = CompletableFuture.completedFuture( + CompletableFuture firstInitializerFuture = CompletableFuture.completedFuture( FDv2SourceResult.changeSet(makeChangeSet(true), false) ); - AtomicBoolean synchronizerCalled = new AtomicBoolean(false); + BlockingQueue secondInitializerCalledQueue = new LinkedBlockingQueue<>(); ImmutableList> initializers = ImmutableList.of( - () -> new MockInitializer(initializerFuture) - ); - - ImmutableList> synchronizers = ImmutableList.of( + () -> new MockInitializer(firstInitializerFuture), () -> { - synchronizerCalled.set(true); - return new MockSynchronizer(CompletableFuture.completedFuture( + secondInitializerCalledQueue.offer(true); + return new MockInitializer(CompletableFuture.completedFuture( FDv2SourceResult.changeSet(makeChangeSet(false), false) )); } ); + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockSynchronizer(CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(false), false) + )) + ); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); @@ -1797,8 +1764,11 @@ public void selectorNonEmptyCompletesInitialization() throws Exception { startFuture.get(2, TimeUnit.SECONDS); assertTrue(dataSource.isInitialized()); - assertFalse(synchronizerCalled.get()); // Should not proceed to synchronizers assertEquals(1, sink.getApplyCount()); + + // Second initializer should not be called since first had non-empty selector + Boolean secondInitializerCalled = secondInitializerCalledQueue.poll(500, TimeUnit.MILLISECONDS); + assertNull("Second initializer should not be called when first returns non-empty selector", secondInitializerCalled); } @Test From 1234bc9218b12409a78ba87281f6b5e643150be9 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Mon, 26 Jan 2026 16:32:12 -0800 Subject: [PATCH 13/35] Remove un-needed synchronizer. --- .../com/launchdarkly/sdk/server/FDv2DataSourceTest.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java index dc354ac9..757c3374 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java @@ -1751,13 +1751,7 @@ public void selectorNonEmptyCompletesInitialization() throws Exception { } ); - ImmutableList> synchronizers = ImmutableList.of( - () -> new MockSynchronizer(CompletableFuture.completedFuture( - FDv2SourceResult.changeSet(makeChangeSet(false), false) - )) - ); - - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, ImmutableList.of(), sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); From 10cab33db42d085dc5753a9a52edf1bdf9bd7463 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 27 Jan 2026 08:44:05 -0800 Subject: [PATCH 14/35] Correct lock on getNextAvailableSynchronizer. --- .../com/launchdarkly/sdk/server/SynchronizerStateManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java index fe8db810..41da2f64 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java @@ -49,7 +49,7 @@ public void resetSourceIndex() { * @return the next synchronizer factory to use, or null if there are no more available synchronizers. */ public SynchronizerFactoryWithState getNextAvailableSynchronizer() { - synchronized (synchronizers) { + synchronized (activeSourceLock) { SynchronizerFactoryWithState factory = null; // There is at least one available factory. From 1ade2f774514d3ba257eff5d50e268429d4ad564 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 27 Jan 2026 09:28:01 -0800 Subject: [PATCH 15/35] chore: Add support for FDv1 fallback. --- .../java/com/launchdarkly/sdk/server/FDv2DataSource.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index dc9423d6..f9e987ce 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -101,9 +101,10 @@ private void run() { } boolean fdv1Fallback = runSynchronizers(); if (fdv1Fallback) { - // TODO: Run FDv1 fallback. + runFdv1Fallback(); } // TODO: Handle. We have ran out of sources or we are shutting down. + dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.OFF, new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.UNKNOWN, 0, "", new Date().instant())); // If we had initialized at some point, then the future will already be complete and this will be ignored. startFuture.complete(false); @@ -268,6 +269,10 @@ private boolean runSynchronizers() { } } + private void runFdv1Fallback() { + + } + @Override public Future start() { if (!started.getAndSet(true)) { From 382246d97d96c8b52dd465cfccd96f88e03b8615 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 27 Jan 2026 10:02:02 -0800 Subject: [PATCH 16/35] PR Feedback. --- .../sdk/server/FDv2DataSource.java | 8 +++-- .../sdk/server/FDv2DataSourceConditions.java | 3 -- .../sdk/server/SynchronizerStateManager.java | 34 ++++++++----------- 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index dc9423d6..bedf0eb6 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -25,7 +25,7 @@ class FDv2DataSource implements DataSource { /** * Default fallback timeout of 2 minutes. The timeout is only configurable for testing. */ - private static final int defaultFallbackTimeout = 2 * 60; + private static final int defaultFallbackTimeoutSeconds = 2 * 60; /** * Default recovery timeout of 5 minutes. The timeout is only configurable for testing. @@ -64,7 +64,7 @@ public FDv2DataSource( threadPriority, logger, sharedExecutor, - defaultFallbackTimeout, + defaultFallbackTimeoutSeconds, defaultRecoveryTimeout ); } @@ -217,6 +217,10 @@ private boolean runSynchronizers() { break; } + if(!(res instanceof FDv2SourceResult)) { + logger.error("Unexpected result type from synchronizer: {}", res.getClass().getName()); + continue; + } FDv2SourceResult result = (FDv2SourceResult) res; conditions.inform(result); diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java index b5ed8b75..1eb3abff 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java @@ -120,9 +120,6 @@ public FallbackCondition(ScheduledExecutorService sharedExecutor, long timeoutSe @Override public void inform(FDv2SourceResult sourceResult) { - if (sourceResult == null) { - return; - } if (sourceResult.getResultType() == FDv2SourceResult.ResultType.CHANGE_SET) { if (timerFuture != null) { timerFuture.cancel(false); diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java index 41da2f64..6a4d829c 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java @@ -52,24 +52,24 @@ public SynchronizerFactoryWithState getNextAvailableSynchronizer() { synchronized (activeSourceLock) { SynchronizerFactoryWithState factory = null; - // There is at least one available factory. - if(synchronizers.stream().anyMatch(s -> s.getState() == SynchronizerFactoryWithState.State.Available)) { + int visited = 0; + while(visited < synchronizers.size()) { // Look for the next synchronizer starting at the position after the current one. (avoiding just re-using the same synchronizer.) - while(factory == null) { - sourceIndex++; - // We aren't using module here because we want to keep the stored index within range instead - // of increasing indefinitely. - if(sourceIndex >= synchronizers.size()) { - sourceIndex = 0; - } - SynchronizerFactoryWithState candidate = synchronizers.get(sourceIndex); - if (candidate.getState() == SynchronizerFactoryWithState.State.Available) { - factory = candidate; - } + sourceIndex++; + // We aren't using module here because we want to keep the stored index within range instead + // of increasing indefinitely. + if(sourceIndex >= synchronizers.size()) { + sourceIndex = 0; } - } + SynchronizerFactoryWithState candidate = synchronizers.get(sourceIndex); + if (candidate.getState() == SynchronizerFactoryWithState.State.Available) { + factory = candidate; + break; + } + visited++; + } return factory; } } @@ -80,15 +80,11 @@ public SynchronizerFactoryWithState getNextAvailableSynchronizer() { */ public boolean isPrimeSynchronizer() { synchronized (activeSourceLock) { - boolean firstAvailableSynchronizer = true; for (int index = 0; index < synchronizers.size(); index++) { if (synchronizers.get(index).getState() == SynchronizerFactoryWithState.State.Available) { - if (firstAvailableSynchronizer && sourceIndex == index) { - // This is the first synchronizer that is available, and it also is the current one. + if (sourceIndex == index) { return true; } - // Subsequently encountered synchronizers that are available are not the first one. - firstAvailableSynchronizer = false; } } } From e6b032e3de4769ee3fff17e7d805ebc9373cdeb0 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 27 Jan 2026 10:11:40 -0800 Subject: [PATCH 17/35] Correct prime synchronizer logic. --- .../com/launchdarkly/sdk/server/SynchronizerStateManager.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java index 6a4d829c..5ff754ed 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java @@ -83,8 +83,11 @@ public boolean isPrimeSynchronizer() { for (int index = 0; index < synchronizers.size(); index++) { if (synchronizers.get(index).getState() == SynchronizerFactoryWithState.State.Available) { if (sourceIndex == index) { + // This is the first synchronizer that is available, and it also is the current one. return true; } + break; + // Subsequently encountered synchronizers that are available are not the first one. } } } From 615ce13180d0350d05b911eda1c6b7145d61ac87 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 27 Jan 2026 12:23:29 -0800 Subject: [PATCH 18/35] WIP --- .../server/DataSourceSynchronizerAdapter.java | 178 ++++++++++++++++++ .../sdk/server/FDv2DataSource.java | 35 ++-- 2 files changed, 199 insertions(+), 14 deletions(-) create mode 100644 lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java new file mode 100644 index 00000000..48133935 --- /dev/null +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java @@ -0,0 +1,178 @@ +package com.launchdarkly.sdk.server; + +import com.launchdarkly.sdk.internal.fdv2.sources.Selector; +import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; +import com.launchdarkly.sdk.server.datasources.Synchronizer; +import com.launchdarkly.sdk.server.interfaces.DataSourceStatusProvider; +import com.launchdarkly.sdk.server.interfaces.DataStoreStatusProvider; +import com.launchdarkly.sdk.server.subsystems.DataSource; +import com.launchdarkly.sdk.server.subsystems.DataSourceUpdateSink; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes; + +import java.io.IOException; +import java.time.Instant; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +/** + * Adapter that wraps a DataSource (FDv1 protocol) and exposes it as a Synchronizer (FDv2 protocol). + *

+ * This adapter bridges the push-based DataSource interface with the pull-based Synchronizer interface + * by intercepting updates through a custom DataSourceUpdateSink and queueing them as FDv2SourceResult objects. + *

+ * The adapter is constructed with a factory function that receives the intercepting update sink and + * creates the DataSource. This ensures the DataSource uses the adapter's internal sink without exposing it. + */ +class DataSourceSynchronizerAdapter implements Synchronizer { + private final DataSource dataSource; + private final IterableAsyncQueue resultQueue = new IterableAsyncQueue<>(); + private final CompletableFuture shutdownFuture = new CompletableFuture<>(); + private final Object startLock = new Object(); + private boolean started = false; + private boolean closed = false; + private Future startFuture; + + /** + * Functional interface for creating a DataSource with a given update sink. + */ + @FunctionalInterface + public interface DataSourceFactory { + DataSource create(DataSourceUpdateSink updateSink); + } + + /** + * Creates a new adapter that wraps a DataSource. + * + * @param dataSourceFactory factory that creates the DataSource with the provided update sink + * @param originalUpdateSink the original update sink to delegate to + */ + public DataSourceSynchronizerAdapter(DataSourceFactory dataSourceFactory, DataSourceUpdateSink originalUpdateSink) { + InterceptingUpdateSink interceptingSink = new InterceptingUpdateSink(originalUpdateSink, resultQueue); + this.dataSource = dataSourceFactory.create(interceptingSink); + } + + @Override + public CompletableFuture next() { + synchronized (startLock) { + if (!started && !closed) { + started = true; + startFuture = dataSource.start(); + + // Monitor the start future for errors + // The data source will emit updates through the intercepting sink + CompletableFuture.runAsync(() -> { + try { + startFuture.get(); + } catch (ExecutionException e) { + // Initialization failed - emit an interrupted status + DataSourceStatusProvider.ErrorInfo errorInfo = new DataSourceStatusProvider.ErrorInfo( + DataSourceStatusProvider.ErrorKind.UNKNOWN, + 0, + e.getCause() != null ? e.getCause().toString() : e.toString(), + Instant.now() + ); + resultQueue.put(FDv2SourceResult.interrupted(errorInfo, false)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + } + } + + return CompletableFuture.anyOf(shutdownFuture, resultQueue.take()) + .thenApply(result -> (FDv2SourceResult) result); + } + + @Override + public void close() throws IOException { + synchronized (startLock) { + if (closed) { + return; + } + closed = true; + } + + dataSource.close(); + shutdownFuture.complete(FDv2SourceResult.shutdown()); + } + + /** + * An intercepting DataSourceUpdateSink that converts DataSource updates into FDv2SourceResult objects. + */ + private static class InterceptingUpdateSink implements DataSourceUpdateSink { + private final DataSourceUpdateSink delegate; + private final IterableAsyncQueue resultQueue; + + public InterceptingUpdateSink(DataSourceUpdateSink delegate, IterableAsyncQueue resultQueue) { + this.delegate = delegate; + this.resultQueue = resultQueue; + } + + @Override + public boolean init(DataStoreTypes.FullDataSet allData) { + boolean success = delegate.init(allData); + if (success) { + // Convert the full data set into a ChangeSet and emit it + DataStoreTypes.ChangeSet changeSet = + new DataStoreTypes.ChangeSet<>( + DataStoreTypes.ChangeSetType.Full, + Selector.EMPTY, + allData.getData(), + null); + resultQueue.put(FDv2SourceResult.changeSet(changeSet, false)); + } + return success; + } + + @Override + public boolean upsert(DataStoreTypes.DataKind kind, String key, DataStoreTypes.ItemDescriptor item) { + boolean success = delegate.upsert(kind, key, item); + if (success) { + // Convert the upsert into a ChangeSet with a single item and emit it + DataStoreTypes.KeyedItems items = + new DataStoreTypes.KeyedItems<>(Collections.singletonList( + Map.entry(key, item))); + Iterable>> data = + Collections.singletonList(Map.entry(kind, items)); + + DataStoreTypes.ChangeSet changeSet = + new DataStoreTypes.ChangeSet<>( + DataStoreTypes.ChangeSetType.Partial, + Selector.EMPTY, + data, + null); + resultQueue.put(FDv2SourceResult.changeSet(changeSet, false)); + } + return success; + } + + @Override + public DataStoreStatusProvider getDataStoreStatusProvider() { + return delegate.getDataStoreStatusProvider(); + } + + @Override + public void updateStatus(DataSourceStatusProvider.State newState, DataSourceStatusProvider.ErrorInfo newError) { + delegate.updateStatus(newState, newError); + + // Convert state changes to FDv2SourceResult status events + switch (newState) { + case INTERRUPTED: + resultQueue.put(FDv2SourceResult.interrupted(newError, false)); + break; + case OFF: + if (newError != null) { + resultQueue.put(FDv2SourceResult.terminalError(newError, false)); + } + break; + case VALID: + case INITIALIZING: + // These states don't map to FDv2SourceResult status events + break; + } + } + } +} diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index f9e987ce..2b919562 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -9,8 +9,10 @@ import com.launchdarkly.sdk.server.subsystems.DataSource; import com.launchdarkly.sdk.server.subsystems.DataSourceUpdateSinkV2; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.Date; import java.util.List; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; @@ -46,6 +48,8 @@ class FDv2DataSource implements DataSource { private final LDLogger logger; + private final DataSourceFactory fdv1DataSourceFactory; + public interface DataSourceFactory { T build(); } @@ -53,6 +57,7 @@ public interface DataSourceFactory { public FDv2DataSource( ImmutableList> initializers, ImmutableList> synchronizers, + DataSourceFactory fdv1DataSourceFactory, DataSourceUpdateSinkV2 dataSourceUpdates, int threadPriority, LDLogger logger, @@ -60,6 +65,7 @@ public FDv2DataSource( ) { this(initializers, synchronizers, + fdv1DataSourceFactory, dataSourceUpdates, threadPriority, logger, @@ -73,6 +79,7 @@ public FDv2DataSource( public FDv2DataSource( ImmutableList> initializers, ImmutableList> synchronizers, + DataSourceFactory fdv1DataSourceFactory, DataSourceUpdateSinkV2 dataSourceUpdates, int threadPriority, LDLogger logger, @@ -85,6 +92,7 @@ public FDv2DataSource( .stream() .map(SynchronizerFactoryWithState::new) .collect(Collectors.toList()); + this.fdv1DataSourceFactory = fdv1DataSourceFactory; this.synchronizerStateManager = new SynchronizerStateManager(synchronizerFactories); this.dataSourceUpdates = dataSourceUpdates; this.threadPriority = threadPriority; @@ -99,16 +107,20 @@ private void run() { if (!initializers.isEmpty()) { runInitializers(); } - boolean fdv1Fallback = runSynchronizers(); - if (fdv1Fallback) { - runFdv1Fallback(); - } - // TODO: Handle. We have ran out of sources or we are shutting down. - dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.OFF, new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.UNKNOWN, 0, "", new Date().instant())); + runSynchronizers(); + dataSourceUpdates.updateStatus( + DataSourceStatusProvider.State.OFF, + new DataSourceStatusProvider.ErrorInfo( + DataSourceStatusProvider.ErrorKind.UNKNOWN, + 0, + "", + new Date().toInstant()) + ); // If we had initialized at some point, then the future will already be complete and this will be ignored. startFuture.complete(false); }); + runThread.setName("LaunchDarkly-SDK-Server-FDv2DataSource"); runThread.setDaemon(true); runThread.setPriority(threadPriority); runThread.start(); @@ -175,7 +187,7 @@ private List getConditions() { return conditionFactories.stream().map(ConditionFactory::build).collect(Collectors.toList()); } - private boolean runSynchronizers() { + private void runSynchronizers() { // When runSynchronizers exists, no matter how it exits, the synchronizerStateManager will be closed. try { SynchronizerFactoryWithState availableSynchronizer = synchronizerStateManager.getNextAvailableSynchronizer(); @@ -185,7 +197,7 @@ private boolean runSynchronizers() { Synchronizer synchronizer = availableSynchronizer.build(); // Returns true if shutdown. - if (synchronizerStateManager.setActiveSource(synchronizer)) return false; + if (synchronizerStateManager.setActiveSource(synchronizer)) return; try { boolean running = true; @@ -253,7 +265,7 @@ private boolean runSynchronizers() { // We have been requested to fall back to FDv1. We handle whatever message was associated, // close the synchronizer, and then fallback. if (result.isFdv1Fallback()) { - return true; +// return true; } } } @@ -263,16 +275,11 @@ private boolean runSynchronizers() { } availableSynchronizer = synchronizerStateManager.getNextAvailableSynchronizer(); } - return false; } finally { synchronizerStateManager.close(); } } - private void runFdv1Fallback() { - - } - @Override public Future start() { if (!started.getAndSet(true)) { From cce4e8b1d2cc5aadf9fffdce21013b204d4ce651 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 27 Jan 2026 15:50:18 -0800 Subject: [PATCH 19/35] Fallback and data source status. --- .../server/DataSourceSynchronizerAdapter.java | 83 ++-- .../sdk/server/FDv2DataSource.java | 96 +++- .../sdk/server/FDv2DataSystem.java | 12 + .../server/SynchronizerFactoryWithState.java | 16 + .../sdk/server/SynchronizerStateManager.java | 29 +- .../sdk/server/FDv2DataSourceTest.java | 431 +++++++++++++++--- 6 files changed, 539 insertions(+), 128 deletions(-) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java index 48133935..04dcc1c8 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java @@ -11,6 +11,7 @@ import java.io.IOException; import java.time.Instant; +import java.util.AbstractMap; import java.util.Collections; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -21,9 +22,9 @@ * Adapter that wraps a DataSource (FDv1 protocol) and exposes it as a Synchronizer (FDv2 protocol). *

* This adapter bridges the push-based DataSource interface with the pull-based Synchronizer interface - * by intercepting updates through a custom DataSourceUpdateSink and queueing them as FDv2SourceResult objects. + * by listening to updates through a custom DataSourceUpdateSink and queueing them as FDv2SourceResult objects. *

- * The adapter is constructed with a factory function that receives the intercepting update sink and + * The adapter is constructed with a factory function that receives the listening update sink and * creates the DataSource. This ensures the DataSource uses the adapter's internal sink without exposing it. */ class DataSourceSynchronizerAdapter implements Synchronizer { @@ -47,11 +48,10 @@ public interface DataSourceFactory { * Creates a new adapter that wraps a DataSource. * * @param dataSourceFactory factory that creates the DataSource with the provided update sink - * @param originalUpdateSink the original update sink to delegate to */ - public DataSourceSynchronizerAdapter(DataSourceFactory dataSourceFactory, DataSourceUpdateSink originalUpdateSink) { - InterceptingUpdateSink interceptingSink = new InterceptingUpdateSink(originalUpdateSink, resultQueue); - this.dataSource = dataSourceFactory.create(interceptingSink); + public DataSourceSynchronizerAdapter(DataSourceFactory dataSourceFactory) { + ConvertingUpdateSink convertingSink = new ConvertingUpdateSink(resultQueue); + this.dataSource = dataSourceFactory.create(convertingSink); } @Override @@ -62,7 +62,7 @@ public CompletableFuture next() { startFuture = dataSource.start(); // Monitor the start future for errors - // The data source will emit updates through the intercepting sink + // The data source will emit updates through the listening sink CompletableFuture.runAsync(() -> { try { startFuture.get(); @@ -100,64 +100,57 @@ public void close() throws IOException { } /** - * An intercepting DataSourceUpdateSink that converts DataSource updates into FDv2SourceResult objects. + * A DataSourceUpdateSink that converts DataSource updates into FDv2SourceResult objects. + * This sink doesn't delegate to any other sink - it exists solely to convert FDv1 updates to FDv2 results. */ - private static class InterceptingUpdateSink implements DataSourceUpdateSink { - private final DataSourceUpdateSink delegate; + private static class ConvertingUpdateSink implements DataSourceUpdateSink { private final IterableAsyncQueue resultQueue; - public InterceptingUpdateSink(DataSourceUpdateSink delegate, IterableAsyncQueue resultQueue) { - this.delegate = delegate; + public ConvertingUpdateSink(IterableAsyncQueue resultQueue) { this.resultQueue = resultQueue; } @Override public boolean init(DataStoreTypes.FullDataSet allData) { - boolean success = delegate.init(allData); - if (success) { - // Convert the full data set into a ChangeSet and emit it - DataStoreTypes.ChangeSet changeSet = - new DataStoreTypes.ChangeSet<>( - DataStoreTypes.ChangeSetType.Full, - Selector.EMPTY, - allData.getData(), - null); - resultQueue.put(FDv2SourceResult.changeSet(changeSet, false)); - } - return success; + // Convert the full data set into a ChangeSet and emit it + DataStoreTypes.ChangeSet changeSet = + new DataStoreTypes.ChangeSet<>( + DataStoreTypes.ChangeSetType.Full, + Selector.EMPTY, + allData.getData(), + null); + resultQueue.put(FDv2SourceResult.changeSet(changeSet, false)); + return true; } @Override public boolean upsert(DataStoreTypes.DataKind kind, String key, DataStoreTypes.ItemDescriptor item) { - boolean success = delegate.upsert(kind, key, item); - if (success) { - // Convert the upsert into a ChangeSet with a single item and emit it - DataStoreTypes.KeyedItems items = - new DataStoreTypes.KeyedItems<>(Collections.singletonList( - Map.entry(key, item))); - Iterable>> data = - Collections.singletonList(Map.entry(kind, items)); - - DataStoreTypes.ChangeSet changeSet = - new DataStoreTypes.ChangeSet<>( - DataStoreTypes.ChangeSetType.Partial, - Selector.EMPTY, - data, - null); - resultQueue.put(FDv2SourceResult.changeSet(changeSet, false)); - } - return success; + // Convert the upsert into a ChangeSet with a single item and emit it + DataStoreTypes.KeyedItems items = + new DataStoreTypes.KeyedItems<>(Collections.>singletonList( + new AbstractMap.SimpleEntry<>(key, item))); + Iterable>> data = + Collections.>>singletonList( + new AbstractMap.SimpleEntry<>(kind, items)); + + DataStoreTypes.ChangeSet changeSet = + new DataStoreTypes.ChangeSet<>( + DataStoreTypes.ChangeSetType.Partial, + Selector.EMPTY, + data, + null); + resultQueue.put(FDv2SourceResult.changeSet(changeSet, false)); + return true; } @Override public DataStoreStatusProvider getDataStoreStatusProvider() { - return delegate.getDataStoreStatusProvider(); + // This adapter doesn't use a data store + return null; } @Override public void updateStatus(DataSourceStatusProvider.State newState, DataSourceStatusProvider.ErrorInfo newError) { - delegate.updateStatus(newState, newError); - // Convert state changes to FDv2SourceResult status events switch (newState) { case INTERRUPTED: diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index dd0d137d..5c01b81d 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -9,7 +9,6 @@ import com.launchdarkly.sdk.server.subsystems.DataSource; import com.launchdarkly.sdk.server.subsystems.DataSourceUpdateSinkV2; -import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.Date; @@ -48,8 +47,6 @@ class FDv2DataSource implements DataSource { private final LDLogger logger; - private final DataSourceFactory fdv1DataSourceFactory; - public interface DataSourceFactory { T build(); } @@ -92,7 +89,19 @@ public FDv2DataSource( .stream() .map(SynchronizerFactoryWithState::new) .collect(Collectors.toList()); - this.fdv1DataSourceFactory = fdv1DataSourceFactory; + + // If we have a fdv1 data source factory, then add that to the synchronizer factories in a blocked state. + // If we receive a request to fallback, then we will unblock it and block all other synchronizers. + if (fdv1DataSourceFactory != null) { + SynchronizerFactoryWithState wrapped = new SynchronizerFactoryWithState(fdv1DataSourceFactory, + true); + wrapped.block(); + synchronizerFactories.add(wrapped); + + // Currently, we only support 1 fdv1 fallback synchronizer, but that limitation is introduced by the + // configuration. + } + this.synchronizerStateManager = new SynchronizerStateManager(synchronizerFactories); this.dataSourceUpdates = dataSourceUpdates; this.threadPriority = threadPriority; @@ -104,6 +113,11 @@ public FDv2DataSource( private void run() { Thread runThread = new Thread(() -> { + if (initializers.isEmpty() && synchronizerStateManager.getAvailableSynchronizerCount() == 0) { + dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.VALID, null); + startFuture.complete(true); + return; + } if (!initializers.isEmpty()) { runInitializers(); } @@ -113,7 +127,7 @@ private void run() { new DataSourceStatusProvider.ErrorInfo( DataSourceStatusProvider.ErrorKind.UNKNOWN, 0, - "", + "All data source acquisition methods have been exhausted.", new Date().toInstant()) ); @@ -146,12 +160,37 @@ private void runInitializers() { } break; case STATUS: - // TODO: Implement. + FDv2SourceResult.Status status = result.getStatus(); + switch(status.getState()) { + case INTERRUPTED: + case TERMINAL_ERROR: + // The data source updates handler will filter the state during initializing, but this + // will make the error information available. + dataSourceUpdates.updateStatus( + // While the error was terminal to the individual initializer, it isn't terminal + // to the data source as a whole. + DataSourceStatusProvider.State.INTERRUPTED, + status.getErrorInfo()); + break; + case SHUTDOWN: + case GOODBYE: + // We don't need to inform anyone of these statuses. + logger.debug("Ignoring status {} from initializer", result.getStatus().getState()); + break; + } break; } } catch (ExecutionException | InterruptedException | CancellationException e) { - // TODO: Better messaging? - // TODO: Data source status? + // We don't expect these conditions to happen in practice. + + // The data source updates handler will filter the state during initializing, but this + // will make the error information available. + dataSourceUpdates.updateStatus( + DataSourceStatusProvider.State.INTERRUPTED, + new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.UNKNOWN, + 0, + e.toString(), + new Date().toInstant())); logger.warn("Error running initializer: {}", e.toString()); } } @@ -161,6 +200,8 @@ private void runInitializers() { dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.VALID, null); startFuture.complete(true); } + // If no data was received, then it is possible initialization will complete from synchronizers, so we give + // them an opportunity to run before reporting any issues. } /** @@ -217,11 +258,13 @@ private void runSynchronizers() { // For fallback, we will move to the next available synchronizer, which may loop. // This is the default behavior of exiting the run loop, so we don't need to take // any action. + logger.debug("A synchronizer has experienced an interruption and we are falling back."); break; case RECOVERY: // For recovery, we will start at the first available synchronizer. // So we reset the source index, and finding the source will start at the beginning. synchronizerStateManager.resetSourceIndex(); + logger.debug("The data source is attempting to recover to a higher priority synchronizer."); break; } // A running synchronizer will only have fallback and recovery conditions that it can act on. @@ -230,7 +273,7 @@ private void runSynchronizers() { break; } - if(!(res instanceof FDv2SourceResult)) { + if (!(res instanceof FDv2SourceResult)) { logger.error("Unexpected result type from synchronizer: {}", res.getClass().getName()); continue; } @@ -241,6 +284,7 @@ private void runSynchronizers() { switch (result.getResultType()) { case CHANGE_SET: dataSourceUpdates.apply(result.getChangeSet()); + dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.VALID, null); // This could have been completed by any data source. But if it has not been completed before // now, then we complete it. startFuture.complete(true); @@ -250,15 +294,20 @@ private void runSynchronizers() { switch (status.getState()) { case INTERRUPTED: // Handled by conditions. - // TODO: Data source status. + dataSourceUpdates.updateStatus( + DataSourceStatusProvider.State.INTERRUPTED, + status.getErrorInfo()); break; case SHUTDOWN: // We should be overall shutting down. - // TODO: We may need logging or to do a little more. - return false; + logger.debug("Synchronizer shutdown."); + return; case TERMINAL_ERROR: availableSynchronizer.block(); running = false; + dataSourceUpdates.updateStatus( + DataSourceStatusProvider.State.INTERRUPTED, + status.getErrorInfo()); break; case GOODBYE: // We let the synchronizer handle this internally. @@ -268,14 +317,29 @@ private void runSynchronizers() { } // We have been requested to fall back to FDv1. We handle whatever message was associated, // close the synchronizer, and then fallback. - if (result.isFdv1Fallback()) { -// return true; + // Only trigger fallback if we're not already running the FDv1 fallback synchronizer. + if ( + result.isFdv1Fallback() && + synchronizerStateManager.hasFDv1Fallback() && + // This shouldn't happen in practice, an FDv1 source shouldn't request fallback + // to FDv1. But if it does, then we will discard its request. + !availableSynchronizer.isFDv1Fallback() + ) { + synchronizerStateManager.fdv1Fallback(); + running = false; } } } } catch (ExecutionException | InterruptedException | CancellationException e) { - // TODO: Log. - // Move to next synchronizer. + dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.INTERRUPTED, + new DataSourceStatusProvider.ErrorInfo( + DataSourceStatusProvider.ErrorKind.UNKNOWN, + 0, + e.toString(), + new Date().toInstant() + )); + logger.warn("Error running synchronizer: {}, will try next synchronizer, or retry.", e.toString()); + // Move to the next synchronizer. } availableSynchronizer = synchronizerStateManager.getNextAvailableSynchronizer(); } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java index 77aeb4e8..189155d6 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java @@ -154,9 +154,21 @@ static FDv2DataSystem create( .map(synchronizer -> new FactoryWrapper<>(synchronizer, builderContext)) .collect(ImmutableList.toImmutableList()); + // Create FDv1 fallback synchronizer factory if configured + FDv2DataSource.DataSourceFactory fdv1FallbackFactory = null; + if (dataSystemConfiguration.getFDv1FallbackSynchronizer() != null) { + fdv1FallbackFactory = () -> { + // Wrap the FDv1 DataSource as a Synchronizer using the adapter + return new DataSourceSynchronizerAdapter( + updateSink -> dataSystemConfiguration.getFDv1FallbackSynchronizer().build(clientContext) + ); + }; + } + DataSource dataSource = new FDv2DataSource( initializerFactories, synchronizerFactories, + fdv1FallbackFactory, dataSourceUpdates, config.threadPriority, clientContext.getBaseLogger().subLogger(Loggers.DATA_SOURCE_LOGGER_NAME), diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerFactoryWithState.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerFactoryWithState.java index c0afa642..31f02cc1 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerFactoryWithState.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerFactoryWithState.java @@ -19,11 +19,19 @@ public enum State { private State state = State.Available; + private boolean isFDv1Fallback = false; + public SynchronizerFactoryWithState(FDv2DataSource.DataSourceFactory factory) { this.factory = factory; } + public SynchronizerFactoryWithState(FDv2DataSource.DataSourceFactory factory, boolean isFDv1Fallback) { + this.factory = factory; + this.isFDv1Fallback = isFDv1Fallback; + } + + public State getState() { return state; } @@ -35,4 +43,12 @@ public void block() { public Synchronizer build() { return factory.build(); } + + public void unblock() { + state = State.Available; + } + + public boolean isFDv1Fallback() { + return isFDv1Fallback; + } } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java index 5ff754ed..0bf77dd6 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java @@ -39,6 +39,28 @@ public void resetSourceIndex() { } } + public boolean hasFDv1Fallback() { + for (SynchronizerFactoryWithState factory : synchronizers) { + if (factory.isFDv1Fallback()) { + return true; + } + } + return false; + } + + /** + * Block all synchronizers aside from the fdv1 fallback and unblock the fdv1 fallback. + */ + public void fdv1Fallback() { + for (SynchronizerFactoryWithState factory : synchronizers) { + if(factory.isFDv1Fallback()) { + factory.unblock(); + } else { + factory.block(); + } + } + } + /** * Get the next synchronizer to use. This operates based on tracking the index of the currently active synchronizer, * which will loop through all available synchronizers handling interruptions. Then a non-prime synchronizer recovers @@ -82,12 +104,7 @@ public boolean isPrimeSynchronizer() { synchronized (activeSourceLock) { for (int index = 0; index < synchronizers.size(); index++) { if (synchronizers.get(index).getState() == SynchronizerFactoryWithState.State.Available) { - if (sourceIndex == index) { - // This is the first synchronizer that is available, and it also is the current one. - return true; - } - break; - // Subsequently encountered synchronizers that are available are not the first one. + return sourceIndex == index; } } } diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java index 757c3374..14a3cce2 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java @@ -105,7 +105,7 @@ public void firstInitializerFailsSecondInitializerSucceedsWithSelector() throws ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -145,7 +145,7 @@ public void firstInitializerFailsSecondInitializerSucceedsWithoutSelector() thro } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -186,7 +186,7 @@ public void firstInitializerSucceedsWithSelectorSecondInitializerNotInvoked() th ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -226,7 +226,7 @@ public void allInitializersFailSwitchesToSynchronizers() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -268,7 +268,7 @@ public void allThreeInitializersFailWithNoSynchronizers() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -294,7 +294,7 @@ public void oneInitializerNoSynchronizerIsWellBehaved() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -324,7 +324,7 @@ public void noInitializersOneSynchronizerIsWellBehaved() throws Exception { () -> new MockSynchronizer(synchronizerFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -358,7 +358,7 @@ public void oneInitializerOneSynchronizerIsWellBehaved() throws Exception { () -> new MockSynchronizer(synchronizerFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -379,13 +379,13 @@ public void noInitializersAndNoSynchronizersIsWellBehaved() throws Exception { ImmutableList> initializers = ImmutableList.of(); ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); - assertFalse(dataSource.isInitialized()); + assertTrue(dataSource.isInitialized()); assertEquals(0, sink.getApplyCount()); // TODO: Verify status reflects exhausted sources when data source status is implemented } @@ -413,7 +413,7 @@ public void errorWithFDv1FallbackTriggersFallback() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -457,7 +457,7 @@ public void fallbackAndRecoveryTasksWellBehaved() throws Exception { ); // Use short timeouts for testing - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -498,7 +498,7 @@ public void canDisposeWhenSynchronizersFallingBack() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -545,7 +545,7 @@ public void terminalErrorBlocksSynchronizer() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -593,7 +593,7 @@ public void allThreeSynchronizersFailReportsExhaustion() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -633,7 +633,7 @@ public void disabledDataSourceCannotTriggerActions() throws Exception { () -> new MockQueuedSynchronizer(secondSyncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -679,7 +679,7 @@ public void disposeCompletesStartFuture() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -698,13 +698,13 @@ public void noSourcesProvidedCompletesImmediately() throws Exception { ImmutableList> initializers = ImmutableList.of(); ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); - assertFalse(dataSource.isInitialized()); + assertTrue(dataSource.isInitialized()); // TODO: Verify status reflects exhausted sources when data source status is implemented } @@ -733,7 +733,7 @@ public void startFutureCompletesExactlyOnce() throws Exception { () -> new MockSynchronizer(synchronizerFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -756,7 +756,7 @@ public void concurrentCloseAndStartHandledSafely() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); Future startFuture = dataSource.start(); @@ -780,7 +780,7 @@ public void multipleStartCallsEventuallyComplete() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture1 = dataSource.start(); @@ -808,7 +808,7 @@ public void isInitializedThreadSafe() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); dataSource.start(); @@ -843,7 +843,7 @@ public void dataSourceUpdatesApplyThreadSafe() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -879,7 +879,7 @@ public void initializerThrowsExecutionException() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -911,7 +911,7 @@ public void initializerThrowsInterruptedException() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -941,7 +941,7 @@ public void initializerThrowsCancellationException() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -970,7 +970,7 @@ public void synchronizerNextThrowsExecutionException() throws Exception { () -> new MockSynchronizer(goodFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1002,7 +1002,7 @@ public void synchronizerNextThrowsInterruptedException() throws Exception { () -> new MockSynchronizer(goodFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1032,7 +1032,7 @@ public void synchronizerNextThrowsCancellationException() throws Exception { () -> new MockSynchronizer(goodFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1054,7 +1054,7 @@ public void closeWithoutStartDoesNotHang() { ImmutableList> initializers = ImmutableList.of(); ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); dataSource.close(); @@ -1076,7 +1076,7 @@ public void closeAfterInitializersCompletesImmediately() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1107,7 +1107,7 @@ public void close() { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1132,7 +1132,7 @@ public void multipleCloseCallsAreIdempotent() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1160,7 +1160,7 @@ public void closeInterruptsConditionWaiting() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1198,7 +1198,7 @@ public CompletableFuture run() { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); dataSource.close(); Future startFuture = dataSource.start(); @@ -1235,7 +1235,7 @@ public void close() { () -> new MockQueuedSynchronizer(secondSyncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1269,7 +1269,7 @@ public void close() { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1301,7 +1301,7 @@ public void setActiveSourceOnInitializerChecksShutdown() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); Future startFuture = dataSource.start(); @@ -1362,7 +1362,7 @@ public void blockedSynchronizerSkippedInRotation() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1392,7 +1392,7 @@ public void allSynchronizersBlockedReturnsNullAndExits() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1432,7 +1432,7 @@ public void recoveryResetsToFirstAvailableSynchronizer() throws Exception { ); // Short recovery timeout - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1472,7 +1472,7 @@ public void fallbackMovesToNextSynchronizer() throws Exception { ); // Short fallback timeout - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 1, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1502,7 +1502,7 @@ public void conditionsClosedAfterSynchronizerLoop() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1529,7 +1529,7 @@ public void conditionsInformedOfAllResults() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 10, 20); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 10, 20); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1557,7 +1557,7 @@ public void conditionsClosedOnException() throws Exception { )) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1582,7 +1582,7 @@ public void primeSynchronizerHasNoRecoveryCondition() throws Exception { () -> new MockQueuedSynchronizer(new LinkedBlockingQueue<>()) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1613,7 +1613,7 @@ public void nonPrimeSynchronizerHasBothConditions() throws Exception { () -> new MockQueuedSynchronizer(secondSyncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1637,7 +1637,7 @@ public void singleSynchronizerHasNoConditions() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1662,7 +1662,7 @@ public void conditionFutureNeverCompletesWhenNoConditions() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1692,7 +1692,7 @@ public void changeSetAppliedToDataSourceUpdates() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1718,7 +1718,7 @@ public void multipleChangeSetsAppliedInOrder() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1751,7 +1751,7 @@ public void selectorNonEmptyCompletesInitialization() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, ImmutableList.of(), sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, ImmutableList.of(), null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1780,7 +1780,7 @@ public void initializerChangeSetWithoutSelectorCompletesIfLastInitializer() thro ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1806,7 +1806,7 @@ public void synchronizerChangeSetAlwaysCompletesStartFuture() throws Exception { () -> new MockSynchronizer(synchronizerFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1835,7 +1835,7 @@ public void goodbyeStatusHandledGracefully() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1869,7 +1869,7 @@ public void shutdownStatusExitsImmediately() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1896,7 +1896,7 @@ public void fdv1FallbackFlagHonored() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1929,7 +1929,7 @@ public void emptyInitializerListSkipsToSynchronizers() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1957,7 +1957,7 @@ public void startedFlagPreventsMultipleRuns() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); Future startFuture1 = dataSource.start(); @@ -1987,7 +1987,7 @@ public void startBeforeRunCompletesAllComplete() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); resourcesToClose.add(dataSource); // Call start multiple times before completion @@ -2001,6 +2001,315 @@ public void startBeforeRunCompletesAllComplete() throws Exception { assertTrue(dataSource.isInitialized()); } + // ============================================================================ + // FDv1 Fallback Tests + // ============================================================================ + + @Test + public void fdv1FallbackActivatesWhenFlagSet() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + // First synchronizer sends a result with FDv1 fallback flag set + BlockingQueue fdv2SyncResults = new LinkedBlockingQueue<>(); + fdv2SyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + fdv2SyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), true)); // FDv1 fallback triggered + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(fdv2SyncResults) + ); + + // Create FDv1 fallback synchronizer + BlockingQueue fdv1SyncResults = new LinkedBlockingQueue<>(); + fdv1SyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + BlockingQueue fdv1CalledQueue = new LinkedBlockingQueue<>(); + FDv2DataSource.DataSourceFactory fdv1Fallback = () -> { + fdv1CalledQueue.offer(true); + return new MockQueuedSynchronizer(fdv1SyncResults); + }; + + FDv2DataSource dataSource = new FDv2DataSource( + initializers, + synchronizers, + fdv1Fallback, + sink, + Thread.NORM_PRIORITY, + logger, + executor, + 120, + 300 + ); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + // Wait for FDv1 to be called + Boolean fdv1Called = fdv1CalledQueue.poll(2, TimeUnit.SECONDS); + assertNotNull("FDv1 fallback synchronizer should be activated", fdv1Called); + + // Wait for changesets from both FDv2 and FDv1 + sink.awaitApplyCount(3, 2, TimeUnit.SECONDS); + assertTrue("Should have at least 2 changesets", sink.getApplyCount() >= 2); + } + + @Test + public void fdv1FallbackNotCalledWithoutFlag() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + // Synchronizer sends normal results without FDv1 fallback flag + BlockingQueue fdv2SyncResults = new LinkedBlockingQueue<>(); + fdv2SyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + fdv2SyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(fdv2SyncResults) + ); + + // Create FDv1 fallback synchronizer + BlockingQueue fdv1CalledQueue = new LinkedBlockingQueue<>(); + FDv2DataSource.DataSourceFactory fdv1Fallback = () -> { + fdv1CalledQueue.offer(true); + return new MockQueuedSynchronizer(new LinkedBlockingQueue<>()); + }; + + FDv2DataSource dataSource = new FDv2DataSource( + initializers, + synchronizers, + fdv1Fallback, + sink, + Thread.NORM_PRIORITY, + logger, + executor, + 120, + 300 + ); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + // Wait to see if FDv1 gets called (it shouldn't) + Boolean fdv1Called = fdv1CalledQueue.poll(500, TimeUnit.MILLISECONDS); + assertNull("FDv1 fallback should not be activated without flag", fdv1Called); + + sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); + assertEquals(2, sink.getApplyCount()); + } + + @Test + public void fdv1FallbackWorksAfterInterruption() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + // First synchronizer sends data, then INTERRUPTED with fallback flag + BlockingQueue fdv2SyncResults = new LinkedBlockingQueue<>(); + fdv2SyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + fdv2SyncResults.add(FDv2SourceResult.interrupted( + new DataSourceStatusProvider.ErrorInfo( + DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, + 500, + "Network error", + Instant.now() + ), + true // FDv1 fallback flag + )); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(fdv2SyncResults) + ); + + // Create FDv1 fallback synchronizer that sends data + BlockingQueue fdv1SyncResults = new LinkedBlockingQueue<>(); + fdv1SyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + BlockingQueue fdv1CalledQueue = new LinkedBlockingQueue<>(); + FDv2DataSource.DataSourceFactory fdv1Fallback = () -> { + fdv1CalledQueue.offer(true); + return new MockQueuedSynchronizer(fdv1SyncResults); + }; + + FDv2DataSource dataSource = new FDv2DataSource( + initializers, + synchronizers, + fdv1Fallback, + sink, + Thread.NORM_PRIORITY, + logger, + executor, + 120, + 300 + ); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + // Wait for FDv1 to be called + Boolean fdv1Called = fdv1CalledQueue.poll(2, TimeUnit.SECONDS); + assertNotNull("FDv1 fallback should be activated after interruption with flag", fdv1Called); + + // Wait for changesets from both FDv2 and FDv1 + sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); + assertTrue("Should have at least 2 changesets", sink.getApplyCount() >= 2); + } + + @Test + public void fdv1FallbackWithoutConfiguredFallbackIgnoresFlag() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + // Synchronizer sends result with FDv1 fallback flag + BlockingQueue fdv2SyncResults = new LinkedBlockingQueue<>(); + fdv2SyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + fdv2SyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), true)); // FDv1 fallback flag + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(fdv2SyncResults) + ); + + // No FDv1 fallback configured (null) + FDv2DataSource dataSource = new FDv2DataSource( + initializers, + synchronizers, + null, // No FDv1 fallback + sink, + Thread.NORM_PRIORITY, + logger, + executor, + 120, + 300 + ); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + // Should receive both changesets even though fallback flag was set + sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); + assertEquals(2, sink.getApplyCount()); + } + + @Test + public void fdv1FallbackBlocksOtherSynchronizers() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + // First synchronizer sends result with FDv1 fallback flag + BlockingQueue fdv2SyncResults = new LinkedBlockingQueue<>(); + fdv2SyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + fdv2SyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), true)); // FDv1 fallback + + // Second synchronizer that should not be called after fallback + BlockingQueue secondSyncCalledQueue = new LinkedBlockingQueue<>(); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(fdv2SyncResults), + () -> { + secondSyncCalledQueue.offer(true); + return new MockQueuedSynchronizer(new LinkedBlockingQueue<>()); + } + ); + + // Create FDv1 fallback synchronizer + BlockingQueue fdv1SyncResults = new LinkedBlockingQueue<>(); + fdv1SyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + BlockingQueue fdv1CalledQueue = new LinkedBlockingQueue<>(); + FDv2DataSource.DataSourceFactory fdv1Fallback = () -> { + fdv1CalledQueue.offer(true); + return new MockQueuedSynchronizer(fdv1SyncResults); + }; + + FDv2DataSource dataSource = new FDv2DataSource( + initializers, + synchronizers, + fdv1Fallback, + sink, + Thread.NORM_PRIORITY, + logger, + executor, + 120, + 300 + ); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + // Wait for FDv1 to be called + Boolean fdv1Called = fdv1CalledQueue.poll(2, TimeUnit.SECONDS); + assertNotNull("FDv1 fallback should be activated", fdv1Called); + + // Second synchronizer should not be called + Boolean secondSyncCalled = secondSyncCalledQueue.poll(500, TimeUnit.MILLISECONDS); + assertNull("Second synchronizer should not be called after FDv1 fallback", secondSyncCalled); + } + + @Test + public void fdv1FallbackOnlyCalledOncePerDataSource() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + // Synchronizer sends multiple results with FDv1 fallback flags + BlockingQueue fdv2SyncResults = new LinkedBlockingQueue<>(); + fdv2SyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + fdv2SyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), true)); // First fallback + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(fdv2SyncResults) + ); + + // Create FDv1 fallback synchronizer that also sends a fallback flag + BlockingQueue fdv1SyncResults = new LinkedBlockingQueue<>(); + fdv1SyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + fdv1SyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), true)); // Second fallback attempt + + BlockingQueue fdv1CalledQueue = new LinkedBlockingQueue<>(); + FDv2DataSource.DataSourceFactory fdv1Fallback = () -> { + fdv1CalledQueue.offer(true); + return new MockQueuedSynchronizer(fdv1SyncResults); + }; + + FDv2DataSource dataSource = new FDv2DataSource( + initializers, + synchronizers, + fdv1Fallback, + sink, + Thread.NORM_PRIORITY, + logger, + executor, + 120, + 300 + ); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + // Wait for first FDv1 call + Boolean firstCall = fdv1CalledQueue.poll(2, TimeUnit.SECONDS); + assertNotNull("FDv1 fallback should be called once", firstCall); + + // Should not be called again even if FDv1 sends fallback flag + Boolean secondCall = fdv1CalledQueue.poll(500, TimeUnit.MILLISECONDS); + assertNull("FDv1 fallback should only be called once", secondCall); + } + // ============================================================================ // Mock Implementations // ============================================================================ From 5b4ed39210e7feb1539ac9ccc509c8a1924f4c91 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Tue, 27 Jan 2026 16:30:33 -0800 Subject: [PATCH 20/35] More nuanced status. --- .../sdk/server/FDv2DataSource.java | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index 5c01b81d..3bf0a33f 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -114,14 +114,34 @@ public FDv2DataSource( private void run() { Thread runThread = new Thread(() -> { if (initializers.isEmpty() && synchronizerStateManager.getAvailableSynchronizerCount() == 0) { - dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.VALID, null); - startFuture.complete(true); - return; + // There are not any initializer or synchronizers, so we are at the best state that + // can be achieved. + dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.VALID, null); + startFuture.complete(true); + return; } if (!initializers.isEmpty()) { runInitializers(); } + boolean synchronizersAvailable = synchronizerStateManager.getAvailableSynchronizerCount() != 0; + if(!synchronizersAvailable) { + // If already completed by the initializers, then this will have no effect. + startFuture.complete(false); + if (!isInitialized()) { + dataSourceUpdates.updateStatus( + DataSourceStatusProvider.State.OFF, + new DataSourceStatusProvider.ErrorInfo( + DataSourceStatusProvider.ErrorKind.UNKNOWN, + 0, + "Initializers exhausted and there are no synchronizers", + new Date().toInstant()) + ); + } + } + runSynchronizers(); + // If we had synchronizers, and we ran out of them, then we are off. + dataSourceUpdates.updateStatus( DataSourceStatusProvider.State.OFF, new DataSourceStatusProvider.ErrorInfo( From d9e3182ec824aa7791dbca80b2e5393128984a8c Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Wed, 28 Jan 2026 10:05:01 -0800 Subject: [PATCH 21/35] Data source status tests. --- .../sdk/server/FDv2DataSource.java | 1 + .../sdk/server/FDv2DataSourceTest.java | 241 ++++++++++++++++-- 2 files changed, 227 insertions(+), 15 deletions(-) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index 3bf0a33f..fa84112d 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -137,6 +137,7 @@ private void run() { new Date().toInstant()) ); } + return; } runSynchronizers(); diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java index 14a3cce2..75409c33 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java @@ -113,7 +113,7 @@ public void firstInitializerFailsSecondInitializerSucceedsWithSelector() throws assertTrue(dataSource.isInitialized()); assertEquals(1, sink.getApplyCount()); - // TODO: Verify status updated to VALID when data source status is implemented + assertEquals(DataSourceStatusProvider.State.VALID, sink.getLastState()); } @Test @@ -160,7 +160,7 @@ public void firstInitializerFailsSecondInitializerSucceedsWithoutSelector() thro // Wait for apply to be processed sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); assertEquals(2, sink.getApplyCount()); // One from initializer, one from synchronizer - // TODO: Verify status updated to VALID when data source status is implemented + assertEquals(DataSourceStatusProvider.State.VALID, sink.getLastState()); } @Test @@ -195,7 +195,7 @@ public void firstInitializerSucceedsWithSelectorSecondInitializerNotInvoked() th assertTrue(dataSource.isInitialized()); assertFalse(secondInitializerCalled.get()); assertEquals(1, sink.getApplyCount()); - // TODO: Verify status updated to VALID when data source status is implemented + assertEquals(DataSourceStatusProvider.State.VALID, sink.getLastState()); } @Test @@ -276,7 +276,8 @@ public void allThreeInitializersFailWithNoSynchronizers() throws Exception { assertFalse(dataSource.isInitialized()); assertEquals(0, sink.getApplyCount()); - // TODO: Verify status reflects exhausted sources when data source status is implemented + assertEquals(DataSourceStatusProvider.State.OFF, sink.getLastState()); + assertNotNull(sink.getLastError()); } @Test @@ -298,11 +299,11 @@ public void oneInitializerNoSynchronizerIsWellBehaved() throws Exception { resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); - startFuture.get(2, TimeUnit.SECONDS); + startFuture.get(2000, TimeUnit.SECONDS); assertTrue(dataSource.isInitialized()); assertEquals(1, sink.getApplyCount()); - // TODO: Verify status updated to VALID when data source status is implemented + assertEquals(DataSourceStatusProvider.State.VALID, sink.getLastState()); } // ============================================================================ @@ -387,7 +388,7 @@ public void noInitializersAndNoSynchronizersIsWellBehaved() throws Exception { assertTrue(dataSource.isInitialized()); assertEquals(0, sink.getApplyCount()); - // TODO: Verify status reflects exhausted sources when data source status is implemented + assertEquals(DataSourceStatusProvider.State.VALID, sink.getLastState()); } // ============================================================================ @@ -479,7 +480,7 @@ public void fallbackAndRecoveryTasksWellBehaved() throws Exception { // Both synchronizers should have been called due to fallback and recovery assertTrue(firstSyncCallCount.get() >= 2); // Called initially and after recovery assertTrue(secondSyncCallCount.get() >= 1); // Called after fallback - // TODO: Verify status transitions when data source status is implemented + assertEquals(DataSourceStatusProvider.State.VALID, sink.getLastState()); } @Test @@ -564,7 +565,7 @@ public void terminalErrorBlocksSynchronizer() throws Exception { // Wait for applies from both sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); - // TODO: Verify status transitions when data source status is implemented + assertEquals(DataSourceStatusProvider.State.VALID, sink.getLastState()); } @Test @@ -600,7 +601,8 @@ public void allThreeSynchronizersFailReportsExhaustion() throws Exception { startFuture.get(2, TimeUnit.SECONDS); assertFalse(dataSource.isInitialized()); - // TODO: Verify status reflects exhausted sources when data source status is implemented + assertEquals(DataSourceStatusProvider.State.OFF, sink.getLastState()); + assertNotNull(sink.getLastError()); } // ============================================================================ @@ -687,11 +689,12 @@ public void disposeCompletesStartFuture() throws Exception { dataSource.close(); assertTrue(startFuture.isDone()); - // TODO: Verify status updated to OFF when data source status is implemented + // Status remains VALID after close - close doesn't change status + assertEquals(DataSourceStatusProvider.State.VALID, sink.getLastState()); } @Test - public void noSourcesProvidedCompletesImmediately() throws Exception { + public void noSourcesProvidedCompletesImmediately() throws Exception{ executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); @@ -705,7 +708,7 @@ public void noSourcesProvidedCompletesImmediately() throws Exception { startFuture.get(2, TimeUnit.SECONDS); assertTrue(dataSource.isInitialized()); - // TODO: Verify status reflects exhausted sources when data source status is implemented + assertEquals(DataSourceStatusProvider.State.VALID, sink.getLastState()); } // ============================================================================ @@ -1788,7 +1791,7 @@ public void initializerChangeSetWithoutSelectorCompletesIfLastInitializer() thro assertTrue(dataSource.isInitialized()); assertEquals(1, sink.getApplyCount()); - // TODO: Verify status updated to VALID when data source status is implemented + assertEquals(DataSourceStatusProvider.State.VALID, sink.getLastState()); } @Test @@ -2001,6 +2004,197 @@ public void startBeforeRunCompletesAllComplete() throws Exception { assertTrue(dataSource.isInitialized()); } + // ============================================================================ + // Data Source Status Tests + // ============================================================================ + + @Test + public void statusTransitionsToValidAfterInitialization() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + CompletableFuture initializerFuture = CompletableFuture.completedFuture( + FDv2SourceResult.changeSet(makeChangeSet(false), false) + ); + + ImmutableList> initializers = ImmutableList.of( + () -> new MockInitializer(initializerFuture) + ); + + FDv2DataSource dataSource = new FDv2DataSource( + initializers, + ImmutableList.of(), + null, + sink, + Thread.NORM_PRIORITY, + logger, + executor, + 120, + 300 + ); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + // After initializers complete with data (no selector), VALID status is emitted + // Since we initialized successfully and there are no synchronizers, we stay VALID + DataSourceStatusProvider.State status = sink.awaitStatus(2, TimeUnit.SECONDS); + assertNotNull("Should receive status update", status); + assertEquals(DataSourceStatusProvider.State.VALID, status); + assertEquals(DataSourceStatusProvider.State.VALID, sink.getLastState()); + assertNull("Should not have error when VALID", sink.getLastError()); + } + + @Test + public void statusIncludesErrorInfoOnFailure() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + // Synchronizer that sends terminal error + BlockingQueue syncResults = new LinkedBlockingQueue<>(); + syncResults.add(FDv2SourceResult.terminalError( + new DataSourceStatusProvider.ErrorInfo( + DataSourceStatusProvider.ErrorKind.ERROR_RESPONSE, + 401, + "Unauthorized", + Instant.now() + ), + false + )); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(syncResults) + ); + + FDv2DataSource dataSource = new FDv2DataSource( + initializers, + synchronizers, + null, + sink, + Thread.NORM_PRIORITY, + logger, + executor, + 120, + 300 + ); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + // Should receive INTERRUPTED first (from terminal error), then OFF (from exhausted synchronizers) + DataSourceStatusProvider.State firstStatus = sink.awaitStatus(2, TimeUnit.SECONDS); + assertNotNull("Should receive first status update", firstStatus); + assertEquals(DataSourceStatusProvider.State.INTERRUPTED, firstStatus); + + DataSourceStatusProvider.State secondStatus = sink.awaitStatus(2, TimeUnit.SECONDS); + assertNotNull("Should receive second status update", secondStatus); + assertEquals(DataSourceStatusProvider.State.OFF, secondStatus); + + // Final state should be OFF with error info from the last status update + assertEquals(DataSourceStatusProvider.State.OFF, sink.getLastState()); + assertNotNull("Should have error info", sink.getLastError()); + assertEquals(DataSourceStatusProvider.ErrorKind.UNKNOWN, sink.getLastError().getKind()); + } + + @Test + public void statusRemainsValidDuringSynchronizerOperation() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + // Synchronizer that sends multiple changesets + BlockingQueue syncResults = new LinkedBlockingQueue<>(); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(syncResults) + ); + + FDv2DataSource dataSource = new FDv2DataSource( + initializers, + synchronizers, + null, + sink, + Thread.NORM_PRIORITY, + logger, + executor, + 120, + 300 + ); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + // Wait for all changesets to be applied + sink.awaitApplyCount(3, 2, TimeUnit.SECONDS); + + // Status should be VALID throughout + assertEquals(DataSourceStatusProvider.State.VALID, sink.getLastState()); + assertEquals(3, sink.getApplyCount()); + } + + @Test + public void statusTransitionsFromValidToOffWhenAllSynchronizersFail() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + // First synchronizer sends a changeset then terminal error + BlockingQueue syncResults = new LinkedBlockingQueue<>(); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + syncResults.add(FDv2SourceResult.terminalError( + new DataSourceStatusProvider.ErrorInfo( + DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, + 500, + "Server error", + Instant.now() + ), + false + )); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(syncResults) + ); + + FDv2DataSource dataSource = new FDv2DataSource( + initializers, + synchronizers, + null, + sink, + Thread.NORM_PRIORITY, + logger, + executor, + 120, + 300 + ); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + // Should transition: VALID (from changeset) → INTERRUPTED (from terminal error) → OFF (from exhausted sources) + DataSourceStatusProvider.State firstStatus = sink.awaitStatus(2, TimeUnit.SECONDS); + assertEquals(DataSourceStatusProvider.State.VALID, firstStatus); + + DataSourceStatusProvider.State secondStatus = sink.awaitStatus(2, TimeUnit.SECONDS); + assertEquals(DataSourceStatusProvider.State.INTERRUPTED, secondStatus); + + DataSourceStatusProvider.State thirdStatus = sink.awaitStatus(2, TimeUnit.SECONDS); + assertEquals(DataSourceStatusProvider.State.OFF, thirdStatus); + + assertEquals(DataSourceStatusProvider.State.OFF, sink.getLastState()); + assertNotNull("Should have error info when OFF", sink.getLastError()); + } + // ============================================================================ // FDv1 Fallback Tests // ============================================================================ @@ -2318,6 +2512,9 @@ private static class MockDataSourceUpdateSink implements DataSourceUpdateSinkV2 private final AtomicInteger applyCount = new AtomicInteger(0); private final AtomicReference> lastChangeSet = new AtomicReference<>(); private final BlockingQueue applySignals = new LinkedBlockingQueue<>(); + private final AtomicReference lastState = new AtomicReference<>(); + private final AtomicReference lastError = new AtomicReference<>(); + private final BlockingQueue statusUpdates = new LinkedBlockingQueue<>(); @Override public boolean apply(DataStoreTypes.ChangeSet changeSet) { @@ -2329,7 +2526,9 @@ public boolean apply(DataStoreTypes.ChangeSet cha @Override public void updateStatus(DataSourceStatusProvider.State newState, DataSourceStatusProvider.ErrorInfo errorInfo) { - // TODO: Track status updates when data source status is fully implemented + lastState.set(newState); + lastError.set(errorInfo); + statusUpdates.offer(newState); } @Override @@ -2345,6 +2544,18 @@ public DataStoreTypes.ChangeSet getLastChangeSet( return lastChangeSet.get(); } + public DataSourceStatusProvider.State getLastState() { + return lastState.get(); + } + + public DataSourceStatusProvider.ErrorInfo getLastError() { + return lastError.get(); + } + + public DataSourceStatusProvider.State awaitStatus(long timeout, TimeUnit unit) throws InterruptedException { + return statusUpdates.poll(timeout, unit); + } + public void awaitApplyCount(int expectedCount, long timeout, TimeUnit unit) throws InterruptedException { long deadline = System.currentTimeMillis() + unit.toMillis(timeout); while (applyCount.get() < expectedCount && System.currentTimeMillis() < deadline) { From 0f063bef6dbd83d51757cf4bf89c85f15e12e25b Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Wed, 28 Jan 2026 12:35:02 -0800 Subject: [PATCH 22/35] Fix remaining contract tests. --- .../main/java/sdktest/SdkClientEntity.java | 75 +++++++++++++++++++ .../contract-tests/test-suppressions-fdv2.txt | 4 - .../sdk/server/FDv2DataSource.java | 3 + .../sdk/server/StreamingSynchronizerImpl.java | 8 +- 4 files changed, 85 insertions(+), 5 deletions(-) diff --git a/lib/sdk/server/contract-tests/service/src/main/java/sdktest/SdkClientEntity.java b/lib/sdk/server/contract-tests/service/src/main/java/sdktest/SdkClientEntity.java index 17697fb4..ee8db8be 100644 --- a/lib/sdk/server/contract-tests/service/src/main/java/sdktest/SdkClientEntity.java +++ b/lib/sdk/server/contract-tests/service/src/main/java/sdktest/SdkClientEntity.java @@ -11,6 +11,7 @@ import com.launchdarkly.sdk.server.FlagsStateOption; import com.launchdarkly.sdk.server.LDClient; import com.launchdarkly.sdk.server.LDConfig; +import com.launchdarkly.sdk.server.interfaces.ServiceEndpoints; import com.launchdarkly.sdk.server.migrations.Migration; import com.launchdarkly.sdk.server.migrations.MigrationBuilder; import com.launchdarkly.sdk.server.migrations.MigrationExecution; @@ -25,12 +26,15 @@ import com.launchdarkly.sdk.server.integrations.HooksConfigurationBuilder; import com.launchdarkly.sdk.server.integrations.ServiceEndpointsBuilder; import com.launchdarkly.sdk.server.integrations.StreamingDataSourceBuilder; +import com.launchdarkly.sdk.server.integrations.PollingDataSourceBuilder; import com.launchdarkly.sdk.server.integrations.DataSystemBuilder; import com.launchdarkly.sdk.server.DataSystemComponents; import com.launchdarkly.sdk.server.integrations.FDv2PollingInitializerBuilder; import com.launchdarkly.sdk.server.integrations.FDv2PollingSynchronizerBuilder; import com.launchdarkly.sdk.server.integrations.FDv2StreamingSynchronizerBuilder; import com.launchdarkly.sdk.server.interfaces.BigSegmentStoreStatusProvider; +import com.launchdarkly.sdk.server.subsystems.ComponentConfigurer; +import com.launchdarkly.sdk.server.subsystems.DataSource; import com.launchdarkly.sdk.server.subsystems.DataSourceBuilder; import com.launchdarkly.sdk.server.datasources.Initializer; import com.launchdarkly.sdk.server.datasources.Synchronizer; @@ -563,6 +567,22 @@ private LDConfig buildSdkConfig(SdkConfigParams params, String tag) { } } + // Configure FDv1 fallback synchronizer + SdkConfigSynchronizerParams fallbackSynchronizer = + selectFallbackSynchronizer(params.dataSystem); + if (fallbackSynchronizer != null) { + // Set global polling endpoints if the fallback synchronizer has polling with custom base URI + if (fallbackSynchronizer.polling != null && + fallbackSynchronizer.polling.baseUri != null) { + endpoints.polling(fallbackSynchronizer.polling.baseUri); + } + + // Create and configure FDv1 fallback + ComponentConfigurer fdv1Fallback = + createFDv1FallbackSynchronizer(fallbackSynchronizer, endpoints); + dataSystemBuilder.fDv1FallbackSynchronizer(fdv1Fallback); + } + builder.dataSystem(dataSystemBuilder); } @@ -601,4 +621,59 @@ private DataSourceBuilder createSynchronizer( } return null; } + + /** + * Selects the best synchronizer configuration to use for FDv1 fallback. + * Prefers polling synchronizers, falls back to primary synchronizer. + */ + private static SdkConfigSynchronizerParams selectFallbackSynchronizer( + SdkConfigDataSystemParams dataSystemParams) { + + // Prefer secondary polling synchronizer + if (dataSystemParams.synchronizers != null && + dataSystemParams.synchronizers.secondary != null && + dataSystemParams.synchronizers.secondary.polling != null) { + return dataSystemParams.synchronizers.secondary; + } + + // Fall back to primary polling synchronizer + if (dataSystemParams.synchronizers != null && + dataSystemParams.synchronizers.primary != null && + dataSystemParams.synchronizers.primary.polling != null) { + return dataSystemParams.synchronizers.primary; + } + + // Fall back to primary synchronizer (even if streaming) + if (dataSystemParams.synchronizers != null && + dataSystemParams.synchronizers.primary != null) { + return dataSystemParams.synchronizers.primary; + } + + return null; + } + + /** + * Creates the FDv1 fallback synchronizer based on the selected synchronizer config. + * FDv1 fallback is always polling-based. + */ + private static ComponentConfigurer createFDv1FallbackSynchronizer( + SdkConfigSynchronizerParams synchronizer, + ServiceEndpointsBuilder endpoints) { + + // FDv1 fallback is always polling-based + PollingDataSourceBuilder fdv1Polling = Components.pollingDataSource(); + + // Configure polling interval if the synchronizer has polling configuration + if (synchronizer.polling != null) { + if (synchronizer.polling.pollIntervalMs != null) { + fdv1Polling.pollInterval(Duration.ofMillis(synchronizer.polling.pollIntervalMs)); + } + // Note: FDv1 polling doesn't support per-source service endpoints override, + // so it will use the global service endpoints configuration + } + // If streaming synchronizer, use default polling interval + // (no additional configuration needed) + + return fdv1Polling; + } } diff --git a/lib/sdk/server/contract-tests/test-suppressions-fdv2.txt b/lib/sdk/server/contract-tests/test-suppressions-fdv2.txt index 637ace30..e69de29b 100644 --- a/lib/sdk/server/contract-tests/test-suppressions-fdv2.txt +++ b/lib/sdk/server/contract-tests/test-suppressions-fdv2.txt @@ -1,4 +0,0 @@ -streaming/validation/unrecognized data that can be safely ignored/unknown event name with JSON body -streaming/validation/unrecognized data that can be safely ignored/unknown event name with non-JSON body -streaming/validation/unrecognized data that can be safely ignored/patch event with unrecognized path kind -streaming/fdv2/fallback to FDv1 handling diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index fa84112d..81099721 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -336,6 +336,9 @@ private void runSynchronizers() { } break; } + if(result.isFdv1Fallback()) { + System.out.println("fdv1 fallback"); + } // We have been requested to fall back to FDv1. We handle whatever message was associated, // close the synchronizer, and then fallback. // Only trigger fallback if we're not already running the FDv1 fallback synchronizer. diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java index 0145396a..4ca430de 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java @@ -212,6 +212,10 @@ public void close() { } private boolean handleEvent(StreamEvent event) { + System.out.println(event); + if(event instanceof MessageEvent && ((MessageEvent) event).getEventName().equals("whatever")) { + System.out.println("stop"); + } if (event instanceof MessageEvent) { handleMessage((MessageEvent) event); return true; @@ -303,7 +307,9 @@ private void handleMessage(MessageEvent event) { Instant.now() ); result = FDv2SourceResult.interrupted(internalError, getFallback(event)); - restartStream(); + if(kind == DataSourceStatusProvider.ErrorKind.INVALID_DATA) { + restartStream(); + } break; case NONE: From fb198476695a3e9e32af63ddf044140db280c651 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Wed, 28 Jan 2026 12:48:35 -0800 Subject: [PATCH 23/35] More robust data source status testing. --- .../sdk/server/FDv2DataSourceTest.java | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java index 75409c33..714e3f52 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java @@ -111,6 +111,14 @@ public void firstInitializerFailsSecondInitializerSucceedsWithSelector() throws Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); + // In practice this intermediate status will be supressed by the data source updates sink. + + // Should receive INTERRUPTED from the first failed initializer, then VALID from second successful initializer + List statuses = sink.awaitStatuses(2, 2, TimeUnit.SECONDS); + assertEquals("Should receive 2 status updates", 2, statuses.size()); + assertEquals(DataSourceStatusProvider.State.INTERRUPTED, statuses.get(0)); + assertEquals(DataSourceStatusProvider.State.VALID, statuses.get(1)); + assertTrue(dataSource.isInitialized()); assertEquals(1, sink.getApplyCount()); assertEquals(DataSourceStatusProvider.State.VALID, sink.getLastState()); @@ -157,6 +165,14 @@ public void firstInitializerFailsSecondInitializerSucceedsWithoutSelector() thro Boolean synchronizerCalled = synchronizerCalledQueue.poll(2, TimeUnit.SECONDS); assertNotNull("Synchronizer should be called", synchronizerCalled); + // Expected status sequence: + // 1. INTERRUPTED when first initializer fails + // 2. VALID after synchronizer completes (second initializer has no selector, so must wait for synchronizer) + List statuses = sink.awaitStatuses(2, 2, TimeUnit.SECONDS); + assertEquals("Should receive 2 status updates", 2, statuses.size()); + assertEquals(DataSourceStatusProvider.State.INTERRUPTED, statuses.get(0)); + assertEquals(DataSourceStatusProvider.State.VALID, statuses.get(1)); + // Wait for apply to be processed sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); assertEquals(2, sink.getApplyCount()); // One from initializer, one from synchronizer @@ -274,6 +290,19 @@ public void allThreeInitializersFailWithNoSynchronizers() throws Exception { Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); + // Should receive INTERRUPTED for each failing initializer, then OFF when all sources exhausted + // Expected: 3 INTERRUPTED statuses + 1 OFF status = 4 total + List statuses = sink.awaitStatuses(4, 2, TimeUnit.SECONDS); + assertEquals("Should receive 4 status updates", 4, statuses.size()); + + // First 3 should be INTERRUPTED (one per failed initializer) + assertEquals(DataSourceStatusProvider.State.INTERRUPTED, statuses.get(0)); + assertEquals(DataSourceStatusProvider.State.INTERRUPTED, statuses.get(1)); + assertEquals(DataSourceStatusProvider.State.INTERRUPTED, statuses.get(2)); + + // Final status should be OFF (all sources exhausted, no synchronizers) + assertEquals(DataSourceStatusProvider.State.OFF, statuses.get(3)); + assertFalse(dataSource.isInitialized()); assertEquals(0, sink.getApplyCount()); assertEquals(DataSourceStatusProvider.State.OFF, sink.getLastState()); @@ -301,6 +330,11 @@ public void oneInitializerNoSynchronizerIsWellBehaved() throws Exception { Future startFuture = dataSource.start(); startFuture.get(2000, TimeUnit.SECONDS); + // Expected status: VALID (initializer succeeds with selector, no need to wait for synchronizer) + List statuses = sink.awaitStatuses(1, 2, TimeUnit.SECONDS); + assertEquals("Should receive 1 status update", 1, statuses.size()); + assertEquals(DataSourceStatusProvider.State.VALID, statuses.get(0)); + assertTrue(dataSource.isInitialized()); assertEquals(1, sink.getApplyCount()); assertEquals(DataSourceStatusProvider.State.VALID, sink.getLastState()); @@ -2556,6 +2590,23 @@ public DataSourceStatusProvider.State awaitStatus(long timeout, TimeUnit unit) t return statusUpdates.poll(timeout, unit); } + public List awaitStatuses(int count, long timeout, TimeUnit unit) throws InterruptedException { + List statuses = new ArrayList<>(); + long deadline = System.currentTimeMillis() + unit.toMillis(timeout); + for (int i = 0; i < count; i++) { + long remaining = deadline - System.currentTimeMillis(); + if (remaining <= 0) { + break; + } + DataSourceStatusProvider.State status = statusUpdates.poll(remaining, TimeUnit.MILLISECONDS); + if (status == null) { + break; + } + statuses.add(status); + } + return statuses; + } + public void awaitApplyCount(int expectedCount, long timeout, TimeUnit unit) throws InterruptedException { long deadline = System.currentTimeMillis() + unit.toMillis(timeout); while (applyCount.get() < expectedCount && System.currentTimeMillis() < deadline) { From cb8d9ad16551377279d86115ae989185bc1aaa61 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Wed, 28 Jan 2026 12:51:42 -0800 Subject: [PATCH 24/35] Remove debug code and make tests more robust. --- .../sdk/server/FDv2DataSource.java | 3 -- .../sdk/server/StreamingSynchronizerImpl.java | 4 -- .../sdk/server/FDv2DataSourceTest.java | 54 +++++++++++++++++++ 3 files changed, 54 insertions(+), 7 deletions(-) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index 81099721..fa84112d 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -336,9 +336,6 @@ private void runSynchronizers() { } break; } - if(result.isFdv1Fallback()) { - System.out.println("fdv1 fallback"); - } // We have been requested to fall back to FDv1. We handle whatever message was associated, // close the synchronizer, and then fallback. // Only trigger fallback if we're not already running the FDv1 fallback synchronizer. diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java index 4ca430de..69c0cd93 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java @@ -212,10 +212,6 @@ public void close() { } private boolean handleEvent(StreamEvent event) { - System.out.println(event); - if(event instanceof MessageEvent && ((MessageEvent) event).getEventName().equals("whatever")) { - System.out.println("stop"); - } if (event instanceof MessageEvent) { handleMessage((MessageEvent) event); return true; diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java index 714e3f52..7405a045 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java @@ -208,6 +208,11 @@ public void firstInitializerSucceedsWithSelectorSecondInitializerNotInvoked() th Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); + // Expected status: VALID (first initializer succeeds with selector, second not called) + List statuses = sink.awaitStatuses(1, 2, TimeUnit.SECONDS); + assertEquals("Should receive 1 status update", 1, statuses.size()); + assertEquals(DataSourceStatusProvider.State.VALID, statuses.get(0)); + assertTrue(dataSource.isInitialized()); assertFalse(secondInitializerCalled.get()); assertEquals(1, sink.getApplyCount()); @@ -420,6 +425,11 @@ public void noInitializersAndNoSynchronizersIsWellBehaved() throws Exception { Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); + // Expected status: VALID (no sources but data source initializes immediately) + List statuses = sink.awaitStatuses(1, 2, TimeUnit.SECONDS); + assertEquals("Should receive 1 status update", 1, statuses.size()); + assertEquals(DataSourceStatusProvider.State.VALID, statuses.get(0)); + assertTrue(dataSource.isInitialized()); assertEquals(0, sink.getApplyCount()); assertEquals(DataSourceStatusProvider.State.VALID, sink.getLastState()); @@ -508,6 +518,18 @@ public void fallbackAndRecoveryTasksWellBehaved() throws Exception { // 5. After recovery, first sync sends apply again (3) // Total time: ~3-4 seconds (1s fallback + 2s recovery + processing) + // Expected status sequence: + // 1. VALID when first sync sends initial changeset + // 2. INTERRUPTED when first sync sends interrupted result + // 3. VALID when second sync (fallback) sends changeset + // 4. VALID when first sync recovers and sends changeset again + // Wait for at least the first 3 statuses that happen during initialization + List statuses = sink.awaitStatuses(3, 6, TimeUnit.SECONDS); + assertTrue("Should receive at least 3 status updates", statuses.size() >= 3); + assertEquals(DataSourceStatusProvider.State.VALID, statuses.get(0)); + assertEquals(DataSourceStatusProvider.State.INTERRUPTED, statuses.get(1)); + assertEquals(DataSourceStatusProvider.State.VALID, statuses.get(2)); + // Wait for 3 applies with enough time for fallback (1s) + recovery (2s) + overhead sink.awaitApplyCount(3, 5, TimeUnit.SECONDS); @@ -597,6 +619,16 @@ public void terminalErrorBlocksSynchronizer() throws Exception { assertEquals(Integer.valueOf(1), firstCall); assertEquals(Integer.valueOf(2), secondCall); + // Expected status sequence: + // 1. VALID when first synchronizer sends initial changeset + // 2. Terminal error from first synchronizer (blocks it) + // 3. VALID when second synchronizer sends changeset + List statuses = sink.awaitStatuses(3, 2, TimeUnit.SECONDS); + assertEquals("Should receive 3 status updates", 3, statuses.size()); + assertEquals(DataSourceStatusProvider.State.VALID, statuses.get(0)); + // Note: terminal error might be suppressed or show as INTERRUPTED + assertEquals(DataSourceStatusProvider.State.VALID, statuses.get(2)); + // Wait for applies from both sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); assertEquals(DataSourceStatusProvider.State.VALID, sink.getLastState()); @@ -634,6 +666,13 @@ public void allThreeSynchronizersFailReportsExhaustion() throws Exception { Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); + // Expected status sequence: 3 terminal errors (one per synchronizer) → OFF when all exhausted + // Terminal errors might show as INTERRUPTED status + List statuses = sink.awaitStatuses(4, 2, TimeUnit.SECONDS); + assertEquals("Should receive 4 status updates", 4, statuses.size()); + // Last status should be OFF + assertEquals(DataSourceStatusProvider.State.OFF, statuses.get(3)); + assertFalse(dataSource.isInitialized()); assertEquals(DataSourceStatusProvider.State.OFF, sink.getLastState()); assertNotNull(sink.getLastError()); @@ -720,6 +759,11 @@ public void disposeCompletesStartFuture() throws Exception { Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); + // Expected status: VALID when synchronizer sends initial changeset + List statuses = sink.awaitStatuses(1, 2, TimeUnit.SECONDS); + assertEquals("Should receive 1 status update", 1, statuses.size()); + assertEquals(DataSourceStatusProvider.State.VALID, statuses.get(0)); + dataSource.close(); assertTrue(startFuture.isDone()); @@ -741,6 +785,11 @@ public void noSourcesProvidedCompletesImmediately() throws Exception{ Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); + // Expected status: VALID (no sources but data source initializes immediately) + List statuses = sink.awaitStatuses(1, 2, TimeUnit.SECONDS); + assertEquals("Should receive 1 status update", 1, statuses.size()); + assertEquals(DataSourceStatusProvider.State.VALID, statuses.get(0)); + assertTrue(dataSource.isInitialized()); assertEquals(DataSourceStatusProvider.State.VALID, sink.getLastState()); } @@ -1823,6 +1872,11 @@ public void initializerChangeSetWithoutSelectorCompletesIfLastInitializer() thro Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); + // Expected status: VALID (single initializer without selector completes when it's the last initializer) + List statuses = sink.awaitStatuses(1, 2, TimeUnit.SECONDS); + assertEquals("Should receive 1 status update", 1, statuses.size()); + assertEquals(DataSourceStatusProvider.State.VALID, statuses.get(0)); + assertTrue(dataSource.isInitialized()); assertEquals(1, sink.getApplyCount()); assertEquals(DataSourceStatusProvider.State.VALID, sink.getLastState()); From 6673954d3cde0615582c68df4bbde761cc80f12b Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Wed, 28 Jan 2026 13:08:11 -0800 Subject: [PATCH 25/35] PR feedback first pass. --- .../main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java | 4 +++- .../java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java index 189155d6..5c63a654 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java @@ -160,7 +160,9 @@ static FDv2DataSystem create( fdv1FallbackFactory = () -> { // Wrap the FDv1 DataSource as a Synchronizer using the adapter return new DataSourceSynchronizerAdapter( - updateSink -> dataSystemConfiguration.getFDv1FallbackSynchronizer().build(clientContext) + updateSink -> dataSystemConfiguration + .getFDv1FallbackSynchronizer() + .build(clientContext.withDataSourceUpdateSink(updateSink)) ); }; } diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java index 7405a045..b8b60b1d 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java @@ -333,7 +333,7 @@ public void oneInitializerNoSynchronizerIsWellBehaved() throws Exception { resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); - startFuture.get(2000, TimeUnit.SECONDS); + startFuture.get(2, TimeUnit.SECONDS); // Expected status: VALID (initializer succeeds with selector, no need to wait for synchronizer) List statuses = sink.awaitStatuses(1, 2, TimeUnit.SECONDS); From 5b3c66943be0860e3c6de5372f5d940f2fe94f54 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Wed, 28 Jan 2026 13:44:23 -0800 Subject: [PATCH 26/35] More robust DataSourceSynchronizerAdapter. --- .../server/DataSourceSynchronizerAdapter.java | 8 + .../DataSourceSynchronizerAdapterTest.java | 296 ++++++++++++++++++ 2 files changed, 304 insertions(+) create mode 100644 lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapterTest.java diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java index 04dcc1c8..2e9c91c9 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java @@ -14,6 +14,7 @@ import java.util.AbstractMap; import java.util.Collections; import java.util.Map; +import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -75,6 +76,8 @@ public CompletableFuture next() { Instant.now() ); resultQueue.put(FDv2SourceResult.interrupted(errorInfo, false)); + } catch (CancellationException e) { + // Start future was cancelled (during close) - exit cleanly } catch (InterruptedException e) { Thread.currentThread().interrupt(); } @@ -97,6 +100,11 @@ public void close() throws IOException { dataSource.close(); shutdownFuture.complete(FDv2SourceResult.shutdown()); + if(startFuture != null) { + // If the start future is done, this has no effect. + // If it is not, then this will unblock the code waiting on start. + startFuture.cancel(true); + } } /** diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapterTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapterTest.java new file mode 100644 index 00000000..8f8f7824 --- /dev/null +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapterTest.java @@ -0,0 +1,296 @@ +package com.launchdarkly.sdk.server; + +import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; +import com.launchdarkly.sdk.server.subsystems.DataSource; +import com.launchdarkly.sdk.server.subsystems.DataSourceUpdateSink; + +import org.junit.After; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static org.junit.Assert.*; + +@SuppressWarnings("javadoc") +public class DataSourceSynchronizerAdapterTest extends BaseTest { + + private final List resourcesToClose = new ArrayList<>(); + + @After + public void tearDown() { + for (AutoCloseable resource : resourcesToClose) { + try { + resource.close(); + } catch (Exception e) { + // Ignore cleanup exceptions + } + } + resourcesToClose.clear(); + } + + /** + * Test that closing the adapter before initialization completes does not leak threads. + * This is the main test for the bug fix - verifies that cancelling startFuture unblocks the monitoring task. + */ + @Test + public void closeBeforeInitializationDoesNotLeakThread() throws Exception { + CountDownLatch blockInitLatch = new CountDownLatch(1); + CountDownLatch futureGetCalledLatch = new CountDownLatch(1); + + // Create an adapter with a data source that blocks during initialization + // The MockDataSource will signal futureGetCalledLatch when get() is called on the returned future + DataSourceSynchronizerAdapter adapter = new DataSourceSynchronizerAdapter(sink -> + new MockDataSource(blockInitLatch, null, null, futureGetCalledLatch) + ); + resourcesToClose.add(adapter); + + // Start the adapter (launches monitoring task) + CompletableFuture nextFuture = adapter.next(); + + // Wait for the monitoring task to actually call get() on the startFuture and block + // This ensures we're testing the exact scenario: monitoring task is blocked when cancel() is called + assertTrue("Future.get() should have been called", + futureGetCalledLatch.await(1, TimeUnit.SECONDS)); + + // Close before initialization completes - this should cancel startFuture and unblock the monitoring task + adapter.close(); + + // Verify next() completes with shutdown result (should be nearly immediate) + FDv2SourceResult result = nextFuture.get(2, TimeUnit.SECONDS); + assertEquals(FDv2SourceResult.ResultType.STATUS, result.getResultType()); + assertEquals(FDv2SourceResult.State.SHUTDOWN, result.getStatus().getState()); + + // Signal the blocked initialization (should already be cancelled/irrelevant) + blockInitLatch.countDown(); + + // Test passes if we reach here without hanging. + } + + /** + * Test that normal initialization (without premature close) still works correctly. + * This ensures the fix doesn't break the happy path. + */ + @Test + public void normalInitializationCompletes() throws Exception { + CountDownLatch allowInitLatch = new CountDownLatch(1); + + DataSourceSynchronizerAdapter adapter = new DataSourceSynchronizerAdapter(sink -> + new MockDataSource(allowInitLatch, null) + ); + resourcesToClose.add(adapter); + + CompletableFuture nextFuture = adapter.next(); + + // Allow initialization to complete + allowInitLatch.countDown(); + + // Wait briefly for the monitoring task to process completion + Thread.sleep(200); + + // Close normally + adapter.close(); + + // Verify shutdown result is received + FDv2SourceResult result = nextFuture.get(1, TimeUnit.SECONDS); + assertEquals(FDv2SourceResult.ResultType.STATUS, result.getResultType()); + assertEquals(FDv2SourceResult.State.SHUTDOWN, result.getStatus().getState()); + } + + /** + * Test that initialization errors are properly reported. + * Ensures the exception handling in the monitoring task still works correctly. + */ + @Test + public void initializationErrorIsReported() throws Exception { + // Create an adapter with a data source that fails during initialization + DataSourceSynchronizerAdapter adapter = new DataSourceSynchronizerAdapter(sink -> + new MockDataSource(new RuntimeException("Init failed")) + ); + resourcesToClose.add(adapter); + + CompletableFuture nextFuture = adapter.next(); + + // Wait for the error to be reported + FDv2SourceResult result = nextFuture.get(2, TimeUnit.SECONDS); + + // Should receive an interrupted status with error info + assertEquals(FDv2SourceResult.ResultType.STATUS, result.getResultType()); + assertEquals(FDv2SourceResult.State.INTERRUPTED, result.getStatus().getState()); + assertNotNull(result.getStatus().getErrorInfo()); + assertTrue(result.getStatus().getErrorInfo().getMessage().contains("Init failed")); + + adapter.close(); + } + + /** + * Test that close() can be called before start()/next() without issues. + */ + @Test + public void closeBeforeStartDoesNotFail() throws Exception { + DataSourceSynchronizerAdapter adapter = new DataSourceSynchronizerAdapter(sink -> + new MockDataSource(new CountDownLatch(1), null) + ); + + // Close before calling next() + adapter.close(); + + // next() should still work and return shutdown immediately + CompletableFuture nextFuture = adapter.next(); + FDv2SourceResult result = nextFuture.get(1, TimeUnit.SECONDS); + assertEquals(FDv2SourceResult.ResultType.STATUS, result.getResultType()); + assertEquals(FDv2SourceResult.State.SHUTDOWN, result.getStatus().getState()); + } + + /** + * Test multiple rapid close/next cycles to ensure no race conditions. + */ + @Test + public void rapidCloseDoesNotCauseIssues() throws Exception { + for (int i = 0; i < 10; i++) { + CountDownLatch blockLatch = new CountDownLatch(1); + DataSourceSynchronizerAdapter adapter = new DataSourceSynchronizerAdapter(sink -> + new MockDataSource(blockLatch, null) + ); + + CompletableFuture nextFuture = adapter.next(); + Thread.sleep(10); // Brief delay to let init start + adapter.close(); + + FDv2SourceResult result = nextFuture.get(1, TimeUnit.SECONDS); + assertEquals(FDv2SourceResult.ResultType.STATUS, result.getResultType()); + assertEquals(FDv2SourceResult.State.SHUTDOWN, result.getStatus().getState()); + + blockLatch.countDown(); + } + } + + /** + * Mock DataSource implementation for testing. + * Allows controlling when initialization completes or fails. + */ + private static class MockDataSource implements DataSource { + private final CountDownLatch blockLatch; + private final CountDownLatch signalLatch; + private final Exception initException; + private final CountDownLatch futureGetCalledLatch; + private final CompletableFuture startFuture = new CompletableFuture<>(); + private volatile boolean closed = false; + + // Constructor for blocking init + public MockDataSource(CountDownLatch blockLatch, CountDownLatch signalLatch) { + this(blockLatch, signalLatch, null, null); + } + + // Constructor for init that fails + public MockDataSource(Exception initException) { + this(null, null, initException, null); + } + + public MockDataSource(CountDownLatch blockLatch, CountDownLatch signalLatch, Exception initException) { + this(blockLatch, signalLatch, initException, null); + } + + public MockDataSource(CountDownLatch blockLatch, CountDownLatch signalLatch, Exception initException, CountDownLatch futureGetCalledLatch) { + this.blockLatch = blockLatch; + this.signalLatch = signalLatch; + this.initException = initException; + this.futureGetCalledLatch = futureGetCalledLatch; + } + + @Override + public Future start() { + // Start initialization in background thread + CompletableFuture.runAsync(() -> { + try { + // Signal that init has started + if (signalLatch != null) { + signalLatch.countDown(); + } + + // If there's an exception to throw, throw it + if (initException != null) { + startFuture.completeExceptionally(initException); + return; + } + + // If there's a latch, wait for it (simulating slow initialization) + if (blockLatch != null) { + blockLatch.await(); + } + + // Complete successfully + startFuture.complete(null); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + startFuture.completeExceptionally(e); + } + }); + + // If we need to signal when get() is called, wrap the future + if (futureGetCalledLatch != null) { + return new SignalingFuture<>(startFuture, futureGetCalledLatch); + } + return startFuture; + } + + @Override + public void close() { + closed = true; + // Note: Like PollingProcessor and StreamProcessor, we do NOT complete the startFuture here + // This is what originally caused the thread leak that we're fixing in the adapter + } + + @Override + public boolean isInitialized() { + return startFuture.isDone() && !startFuture.isCompletedExceptionally(); + } + } + + /** + * Wrapper around a Future that signals a latch when get() is called. + * Used to precisely detect when the monitoring task calls get() and blocks. + */ + private static class SignalingFuture implements Future { + private final Future delegate; + private final CountDownLatch getCalledLatch; + + public SignalingFuture(Future delegate, CountDownLatch getCalledLatch) { + this.delegate = delegate; + this.getCalledLatch = getCalledLatch; + } + + @Override + public T get() throws InterruptedException, ExecutionException { + getCalledLatch.countDown(); + return delegate.get(); + } + + @Override + public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { + getCalledLatch.countDown(); + return delegate.get(timeout, unit); + } + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return delegate.cancel(mayInterruptIfRunning); + } + + @Override + public boolean isCancelled() { + return delegate.isCancelled(); + } + + @Override + public boolean isDone() { + return delegate.isDone(); + } + } +} From 86aa10efc99b3ff0b390e80b3e5e11167bf4891f Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Wed, 28 Jan 2026 15:24:07 -0800 Subject: [PATCH 27/35] More graceful shutdown. --- .../main/java/sdktest/SdkClientEntity.java | 10 +- .../sdk/server/FDv2DataSource.java | 58 +++++---- ...teManager.java => SourceStateManager.java} | 56 +++++++-- .../sdk/server/FDv2DataSourceTest.java | 117 +++++++++++++++--- ...rTest.java => SourceStateManagerTest.java} | 46 +++---- 5 files changed, 207 insertions(+), 80 deletions(-) rename lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/{SynchronizerStateManager.java => SourceStateManager.java} (79%) rename lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/{SynchronizerStateManagerTest.java => SourceStateManagerTest.java} (86%) diff --git a/lib/sdk/server/contract-tests/service/src/main/java/sdktest/SdkClientEntity.java b/lib/sdk/server/contract-tests/service/src/main/java/sdktest/SdkClientEntity.java index ee8db8be..22180a59 100644 --- a/lib/sdk/server/contract-tests/service/src/main/java/sdktest/SdkClientEntity.java +++ b/lib/sdk/server/contract-tests/service/src/main/java/sdktest/SdkClientEntity.java @@ -579,7 +579,7 @@ private LDConfig buildSdkConfig(SdkConfigParams params, String tag) { // Create and configure FDv1 fallback ComponentConfigurer fdv1Fallback = - createFDv1FallbackSynchronizer(fallbackSynchronizer, endpoints); + createFDv1FallbackSynchronizer(fallbackSynchronizer); dataSystemBuilder.fDv1FallbackSynchronizer(fdv1Fallback); } @@ -654,11 +654,10 @@ private static SdkConfigSynchronizerParams selectFallbackSynchronizer( /** * Creates the FDv1 fallback synchronizer based on the selected synchronizer config. - * FDv1 fallback is always polling-based. + * FDv1 fallback is always polling-based and uses the global service endpoints configuration. */ private static ComponentConfigurer createFDv1FallbackSynchronizer( - SdkConfigSynchronizerParams synchronizer, - ServiceEndpointsBuilder endpoints) { + SdkConfigSynchronizerParams synchronizer) { // FDv1 fallback is always polling-based PollingDataSourceBuilder fdv1Polling = Components.pollingDataSource(); @@ -669,7 +668,8 @@ private static ComponentConfigurer createFDv1FallbackSynchronizer( fdv1Polling.pollInterval(Duration.ofMillis(synchronizer.polling.pollIntervalMs)); } // Note: FDv1 polling doesn't support per-source service endpoints override, - // so it will use the global service endpoints configuration + // so it will use the global service endpoints configuration (which is set + // by the caller before this method is invoked) } // If streaming synchronizer, use default polling interval // (no additional configuration needed) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index fa84112d..878ef6ad 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -33,8 +33,7 @@ class FDv2DataSource implements DataSource { */ private static final long defaultRecoveryTimeout = 5 * 60; - private final List> initializers; - private final SynchronizerStateManager synchronizerStateManager; + private final SourceStateManager sourceStateManager; private final List conditionFactories; @@ -47,6 +46,8 @@ class FDv2DataSource implements DataSource { private final LDLogger logger; + private volatile boolean closed = false; + public interface DataSourceFactory { T build(); } @@ -84,7 +85,6 @@ public FDv2DataSource( long fallbackTimeout, long recoveryTimeout ) { - this.initializers = initializers; List synchronizerFactories = synchronizers .stream() .map(SynchronizerFactoryWithState::new) @@ -102,7 +102,7 @@ public FDv2DataSource( // configuration. } - this.synchronizerStateManager = new SynchronizerStateManager(synchronizerFactories); + this.sourceStateManager = new SourceStateManager(synchronizerFactories, initializers); this.dataSourceUpdates = dataSourceUpdates; this.threadPriority = threadPriority; this.logger = logger; @@ -113,23 +113,24 @@ public FDv2DataSource( private void run() { Thread runThread = new Thread(() -> { - if (initializers.isEmpty() && synchronizerStateManager.getAvailableSynchronizerCount() == 0) { + if (!sourceStateManager.hasAvailableSources()) { // There are not any initializer or synchronizers, so we are at the best state that // can be achieved. dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.VALID, null); startFuture.complete(true); return; } - if (!initializers.isEmpty()) { + if (sourceStateManager.hasInitializers()) { runInitializers(); } - boolean synchronizersAvailable = synchronizerStateManager.getAvailableSynchronizerCount() != 0; + boolean synchronizersAvailable = sourceStateManager.hasAvailableSynchronizers(); if(!synchronizersAvailable) { // If already completed by the initializers, then this will have no effect. - startFuture.complete(false); - if (!isInitialized()) { + if (!isInitialized() && !closed) { + // If we were closed, then closing would have handled our terminal update. dataSourceUpdates.updateStatus( DataSourceStatusProvider.State.OFF, + // If we were shutdown during initialization, then we don't need to include an error. new DataSourceStatusProvider.ErrorInfo( DataSourceStatusProvider.ErrorKind.UNKNOWN, 0, @@ -137,6 +138,8 @@ private void run() { new Date().toInstant()) ); } + // If already completed has no effect. + startFuture.complete(false); return; } @@ -145,7 +148,9 @@ private void run() { dataSourceUpdates.updateStatus( DataSourceStatusProvider.State.OFF, - new DataSourceStatusProvider.ErrorInfo( + // If the data source was closed, then we just report we are OFF without an + // associated error. + closed? null : new DataSourceStatusProvider.ErrorInfo( DataSourceStatusProvider.ErrorKind.UNKNOWN, 0, "All data source acquisition methods have been exhausted.", @@ -164,10 +169,11 @@ private void run() { private void runInitializers() { boolean anyDataReceived = false; - for (DataSourceFactory factory : initializers) { + DataSourceFactory factory = sourceStateManager.getNextInitializer(); + while(factory != null) { try { Initializer initializer = factory.build(); - if (synchronizerStateManager.setActiveSource(initializer)) return; + if (sourceStateManager.setActiveSource(initializer)) return; FDv2SourceResult result = initializer.run().get(); switch (result.getResultType()) { case CHANGE_SET: @@ -214,6 +220,7 @@ private void runInitializers() { new Date().toInstant())); logger.warn("Error running initializer: {}", e.toString()); } + factory = sourceStateManager.getNextInitializer(); } // We received data without a selector, and we have exhausted initializers, so we are going to // consider ourselves initialized. @@ -232,8 +239,8 @@ private void runInitializers() { * @return a list of conditions to apply to the synchronizer */ private List getConditions() { - int availableSynchronizers = synchronizerStateManager.getAvailableSynchronizerCount(); - boolean isPrimeSynchronizer = synchronizerStateManager.isPrimeSynchronizer(); + int availableSynchronizers = sourceStateManager.getAvailableSynchronizerCount(); + boolean isPrimeSynchronizer = sourceStateManager.isPrimeSynchronizer(); if (availableSynchronizers == 1) { // If there is only 1 synchronizer, then we cannot fall back or recover, so we don't need any conditions. @@ -252,14 +259,14 @@ private List getConditions() { private void runSynchronizers() { // When runSynchronizers exists, no matter how it exits, the synchronizerStateManager will be closed. try { - SynchronizerFactoryWithState availableSynchronizer = synchronizerStateManager.getNextAvailableSynchronizer(); + SynchronizerFactoryWithState availableSynchronizer = sourceStateManager.getNextAvailableSynchronizer(); // We want to continue running synchronizers for as long as any are available. while (availableSynchronizer != null) { Synchronizer synchronizer = availableSynchronizer.build(); // Returns true if shutdown. - if (synchronizerStateManager.setActiveSource(synchronizer)) return; + if (sourceStateManager.setActiveSource(synchronizer)) return; try { boolean running = true; @@ -284,7 +291,7 @@ private void runSynchronizers() { case RECOVERY: // For recovery, we will start at the first available synchronizer. // So we reset the source index, and finding the source will start at the beginning. - synchronizerStateManager.resetSourceIndex(); + sourceStateManager.resetSourceIndex(); logger.debug("The data source is attempting to recover to a higher priority synchronizer."); break; } @@ -341,12 +348,12 @@ private void runSynchronizers() { // Only trigger fallback if we're not already running the FDv1 fallback synchronizer. if ( result.isFdv1Fallback() && - synchronizerStateManager.hasFDv1Fallback() && + sourceStateManager.hasFDv1Fallback() && // This shouldn't happen in practice, an FDv1 source shouldn't request fallback // to FDv1. But if it does, then we will discard its request. !availableSynchronizer.isFDv1Fallback() ) { - synchronizerStateManager.fdv1Fallback(); + sourceStateManager.fdv1Fallback(); running = false; } } @@ -362,10 +369,12 @@ private void runSynchronizers() { logger.warn("Error running synchronizer: {}, will try next synchronizer, or retry.", e.toString()); // Move to the next synchronizer. } - availableSynchronizer = synchronizerStateManager.getNextAvailableSynchronizer(); + availableSynchronizer = sourceStateManager.getNextAvailableSynchronizer(); } - } finally { - synchronizerStateManager.close(); + } catch(Exception e) { + logger.error("Unexpected error in DataSource: {}", e.toString()); + }finally { + sourceStateManager.close(); } } @@ -388,11 +397,14 @@ public boolean isInitialized() { @Override public void close() { + closed = true; // If there is an active source, we will shut it down, and that will result in the loop handling that source // exiting. // If we do not have an active source, then the loop will check isShutdown when attempting to set one. When // it detects shutdown, it will exit the loop. - synchronizerStateManager.close(); + sourceStateManager.close(); + + dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.OFF, null); // If this is already set, then this has no impact. startFuture.complete(false); diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceStateManager.java similarity index 79% rename from lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java rename to lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceStateManager.java index bec5726a..d4112d94 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerStateManager.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceStateManager.java @@ -1,5 +1,7 @@ package com.launchdarkly.sdk.server; +import com.launchdarkly.sdk.server.datasources.Initializer; + import java.io.Closeable; import java.io.IOException; import java.util.List; @@ -10,9 +12,11 @@ *

* Package-private for internal use. */ -class SynchronizerStateManager implements Closeable { +class SourceStateManager implements Closeable { private final List synchronizers; + private final List> initializers; + /** * Lock for active sources and shutdown state. */ @@ -23,10 +27,13 @@ class SynchronizerStateManager implements Closeable { /** * We start at -1, so finding the next synchronizer can non-conditionally increment the index. */ - private int sourceIndex = -1; + private int synchronizerIndex = -1; + + private int initializerIndex = -1; - public SynchronizerStateManager(List synchronizers) { + public SourceStateManager(List synchronizers, List> initializers) { this.synchronizers = synchronizers; + this.initializers = initializers; } /** @@ -35,7 +42,7 @@ public SynchronizerStateManager(List synchronizers */ public void resetSourceIndex() { synchronized (activeSourceLock) { - sourceIndex = -1; + synchronizerIndex = -1; } } @@ -74,18 +81,24 @@ public SynchronizerFactoryWithState getNextAvailableSynchronizer() { synchronized (activeSourceLock) { SynchronizerFactoryWithState factory = null; + if(isShutdown) { + safeClose(activeSource); + activeSource = null; + return factory; + } + int visited = 0; while(visited < synchronizers.size()) { // Look for the next synchronizer starting at the position after the current one. (avoiding just re-using the same synchronizer.) - sourceIndex++; + synchronizerIndex++; // We aren't using module here because we want to keep the stored index within range instead // of increasing indefinitely. - if(sourceIndex >= synchronizers.size()) { - sourceIndex = 0; + if(synchronizerIndex >= synchronizers.size()) { + synchronizerIndex = 0; } - SynchronizerFactoryWithState candidate = synchronizers.get(sourceIndex); + SynchronizerFactoryWithState candidate = synchronizers.get(synchronizerIndex); if (candidate.getState() == SynchronizerFactoryWithState.State.Available) { factory = candidate; break; @@ -96,6 +109,31 @@ public SynchronizerFactoryWithState getNextAvailableSynchronizer() { } } + public boolean hasAvailableSources() { + return hasInitializers() || getAvailableSynchronizerCount() > 0; + } + + public boolean hasInitializers() { + return !initializers.isEmpty(); + } + + public boolean hasAvailableSynchronizers() { + return getAvailableSynchronizerCount() > 0; + } + + public FDv2DataSource.DataSourceFactory getNextInitializer() { + synchronized (activeSourceLock) { + if(isShutdown) { + return null; + } + initializerIndex++; + if (initializerIndex >= initializers.size()) { + return null; + } + return initializers.get(initializerIndex); + } + } + /** * Determine if the currently active synchronizer is the prime (first available) synchronizer. * @return true if the current synchronizer is the prime synchronizer, false otherwise @@ -104,7 +142,7 @@ public boolean isPrimeSynchronizer() { synchronized (activeSourceLock) { for (int index = 0; index < synchronizers.size(); index++) { if (synchronizers.get(index).getState() == SynchronizerFactoryWithState.State.Available) { - if (sourceIndex == index) { + if (synchronizerIndex == index) { // This is the first synchronizer that is available, and it also is the current one. return true; } diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java index b8b60b1d..e9fd8a49 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java @@ -17,7 +17,6 @@ import java.time.Instant; import java.util.ArrayList; -import java.util.LinkedList; import java.util.List; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; @@ -767,8 +766,10 @@ public void disposeCompletesStartFuture() throws Exception { dataSource.close(); assertTrue(startFuture.isDone()); - // Status remains VALID after close - close doesn't change status - assertEquals(DataSourceStatusProvider.State.VALID, sink.getLastState()); + + statuses = sink.awaitStatuses(1, 2, TimeUnit.SECONDS); + assertEquals("Should receive 1 status update", 1, statuses.size()); + assertEquals(DataSourceStatusProvider.State.OFF, statuses.get(0)); } @Test @@ -1230,6 +1231,81 @@ public void multipleCloseCallsAreIdempotent() throws Exception { // Test passes if we reach here without throwing } + @Test + public void closingDataSourceDuringInitializationReportsOffWithoutErrors() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + CompletableFuture initializerFuture = new CompletableFuture<>(); + + + ImmutableList> initializers = ImmutableList.of( + () -> new MockInitializer(initializerFuture) + ); + + ImmutableList> synchronizers = ImmutableList.of(); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + resourcesToClose.add(dataSource); + + Future startFuture = dataSource.start(); + + // Close the data source - this sets closed=true + dataSource.close(); + // Result shouldn't be used. +// initializerFuture.complete(FDv2SourceResult.changeSet(makeChangeSet(true), false)); + + // Wait for start future (completes when exhaustion happens) + startFuture.get(2, TimeUnit.SECONDS); + System.out.println("Start future completed"); + + // Wait for the OFF status to be reported + DataSourceStatusProvider.State status = sink.awaitStatus(2, TimeUnit.SECONDS); + assertNotNull("Should receive status update", status); + assertEquals(DataSourceStatusProvider.State.OFF, status); + + // The data source should report OFF with null error because it was closed + assertNull("Error should be null when closed data source exhausts initializers", sink.getLastError()); + assertFalse(dataSource.isInitialized()); + } + + @Test + public void dataSourceClosedDuringSynchronizationReportsOffWithoutError() throws Exception { + executor = Executors.newScheduledThreadPool(2); + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + // Synchronizer that emits a changeset to initialize, then waits. + ImmutableList> synchronizers = ImmutableList.of( + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + return new MockQueuedSynchronizer(results); + } + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + + Future startFuture = dataSource.start(); + startFuture.get(2, TimeUnit.SECONDS); + + // Close the data source - this sets closed=true + dataSource.close(); + + // Expected status sequence: + // 1. VALID (from first synchronizer's changeset) + // 4. OFF (from exhaustion, with null error because closed=true) + List statuses = sink.awaitStatuses(2, 2, TimeUnit.SECONDS); + assertEquals("Should receive 4 status updates", 2, statuses.size()); + + assertEquals(DataSourceStatusProvider.State.VALID, statuses.get(0)); + assertEquals(DataSourceStatusProvider.State.OFF, statuses.get(1)); + + assertNull("Error should be null when closed data source exhausts synchronizers", sink.getLastError()); + assertTrue(dataSource.isInitialized()); // Was initialized before close + } + @Test public void closeInterruptsConditionWaiting() throws Exception { executor = Executors.newScheduledThreadPool(2); @@ -1522,10 +1598,10 @@ public void recoveryResetsToFirstAvailableSynchronizer() throws Exception { resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); - startFuture.get(2, TimeUnit.SECONDS); + startFuture.get(2000, TimeUnit.SECONDS); // Wait for 3 applies with enough time for recovery (2s) + overhead - sink.awaitApplyCount(3, 5, TimeUnit.SECONDS); + sink.awaitApplyCount(30000, 5, TimeUnit.SECONDS); // Should have called first synchronizer again after recovery assertTrue(firstCallCount.get() >= 2 || secondCallCount.get() >= 1); @@ -2752,38 +2828,39 @@ public void close() { } private static class MockQueuedSynchronizer implements Synchronizer { - private final BlockingQueue results; + private final IterableAsyncQueue results; private volatile boolean closed = false; public MockQueuedSynchronizer(BlockingQueue results) { + // Convert BlockingQueue to IterableAsyncQueue by draining it + this.results = new IterableAsyncQueue<>(); + java.util.ArrayList temp = new java.util.ArrayList<>(); + results.drainTo(temp); + temp.forEach(this.results::put); + } + + public MockQueuedSynchronizer(IterableAsyncQueue results) { this.results = results; } public void addResult(FDv2SourceResult result) { if (!closed) { - results.add(result); + results.put(result); } } @Override public CompletableFuture next() { - if (closed) { - return CompletableFuture.completedFuture(FDv2SourceResult.shutdown()); - } - - // Try to get immediately, don't wait - FDv2SourceResult result = results.poll(); - if (result != null) { - return CompletableFuture.completedFuture(result); - } else { - // Queue is empty - return a never-completing future to simulate waiting for more data - return new CompletableFuture<>(); - } + return results.take(); } @Override public void close() { - closed = true; + if (!closed) { + closed = true; + // Emit shutdown result - this will complete any pending take() or queue it for next take() + results.put(FDv2SourceResult.shutdown()); + } } } diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/SynchronizerStateManagerTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/SourceStateManagerTest.java similarity index 86% rename from lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/SynchronizerStateManagerTest.java rename to lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/SourceStateManagerTest.java index ca19d351..9ad73cec 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/SynchronizerStateManagerTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/SourceStateManagerTest.java @@ -22,7 +22,7 @@ import static org.mockito.Mockito.when; @SuppressWarnings("javadoc") -public class SynchronizerStateManagerTest extends BaseTest { +public class SourceStateManagerTest extends BaseTest { private SynchronizerFactoryWithState createMockFactory() { FDv2DataSource.DataSourceFactory factory = mock(FDv2DataSource.DataSourceFactory.class); @@ -33,7 +33,7 @@ private SynchronizerFactoryWithState createMockFactory() { @Test public void getNextAvailableSynchronizerReturnsNullWhenEmpty() { List synchronizers = new ArrayList<>(); - SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); SynchronizerFactoryWithState result = manager.getNextAvailableSynchronizer(); @@ -46,7 +46,7 @@ public void getNextAvailableSynchronizerReturnsFirstOnFirstCall() { SynchronizerFactoryWithState sync1 = createMockFactory(); synchronizers.add(sync1); - SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); SynchronizerFactoryWithState result = manager.getNextAvailableSynchronizer(); @@ -63,7 +63,7 @@ public void getNextAvailableSynchronizerLoopsThroughAvailable() { synchronizers.add(sync2); synchronizers.add(sync3); - SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); // First call returns sync1 assertSame(sync1, manager.getNextAvailableSynchronizer()); @@ -81,7 +81,7 @@ public void getNextAvailableSynchronizerWrapsAroundToBeginning() { synchronizers.add(sync1); synchronizers.add(sync2); - SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); // Get all synchronizers manager.getNextAvailableSynchronizer(); // sync1 @@ -101,7 +101,7 @@ public void getNextAvailableSynchronizerSkipsBlockedSynchronizers() { synchronizers.add(sync2); synchronizers.add(sync3); - SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); // Block sync2 sync2.block(); @@ -122,7 +122,7 @@ public void getNextAvailableSynchronizerReturnsNullWhenAllBlocked() { synchronizers.add(sync1); synchronizers.add(sync2); - SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); // Block all synchronizers sync1.block(); @@ -143,7 +143,7 @@ public void resetSourceIndexResetsToFirstSynchronizer() { synchronizers.add(sync2); synchronizers.add(sync3); - SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); // Advance to sync3 manager.getNextAvailableSynchronizer(); // sync1 @@ -165,7 +165,7 @@ public void isPrimeSynchronizerReturnsTrueForFirst() { synchronizers.add(sync1); synchronizers.add(sync2); - SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); // Get first synchronizer manager.getNextAvailableSynchronizer(); @@ -181,7 +181,7 @@ public void isPrimeSynchronizerReturnsFalseForNonFirst() { synchronizers.add(sync1); synchronizers.add(sync2); - SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); // Get first then second synchronizer manager.getNextAvailableSynchronizer(); @@ -196,7 +196,7 @@ public void isPrimeSynchronizerReturnsFalseWhenNoSynchronizerSelected() { SynchronizerFactoryWithState sync1 = createMockFactory(); synchronizers.add(sync1); - SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); // Haven't called getNext yet assertFalse(manager.isPrimeSynchronizer()); @@ -212,7 +212,7 @@ public void isPrimeSynchronizerHandlesBlockedFirstSynchronizer() { synchronizers.add(sync2); synchronizers.add(sync3); - SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); // Block first synchronizer sync1.block(); @@ -233,7 +233,7 @@ public void getAvailableSynchronizerCountReturnsCorrectCount() { synchronizers.add(sync2); synchronizers.add(sync3); - SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); assertEquals(3, manager.getAvailableSynchronizerCount()); } @@ -248,7 +248,7 @@ public void getAvailableSynchronizerCountUpdatesWhenBlocked() { synchronizers.add(sync2); synchronizers.add(sync3); - SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); assertEquals(3, manager.getAvailableSynchronizerCount()); @@ -265,7 +265,7 @@ public void getAvailableSynchronizerCountUpdatesWhenBlocked() { @Test public void setActiveSourceSetsNewSource() throws IOException { List synchronizers = new ArrayList<>(); - SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); Closeable source = mock(Closeable.class); boolean shutdown = manager.setActiveSource(source); @@ -276,7 +276,7 @@ public void setActiveSourceSetsNewSource() throws IOException { @Test public void setActiveSourceClosesPreviousSource() throws IOException { List synchronizers = new ArrayList<>(); - SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); Closeable firstSource = mock(Closeable.class); Closeable secondSource = mock(Closeable.class); @@ -290,7 +290,7 @@ public void setActiveSourceClosesPreviousSource() throws IOException { @Test public void setActiveSourceReturnsTrueAfterShutdown() throws IOException { List synchronizers = new ArrayList<>(); - SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); manager.close(); @@ -304,7 +304,7 @@ public void setActiveSourceReturnsTrueAfterShutdown() throws IOException { @Test public void setActiveSourceIgnoresCloseExceptionFromPreviousSource() throws IOException { List synchronizers = new ArrayList<>(); - SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); Closeable firstSource = mock(Closeable.class); doThrow(new IOException("test")).when(firstSource).close(); @@ -319,7 +319,7 @@ public void setActiveSourceIgnoresCloseExceptionFromPreviousSource() throws IOEx @Test public void shutdownClosesActiveSource() throws IOException { List synchronizers = new ArrayList<>(); - SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); Closeable source = mock(Closeable.class); manager.setActiveSource(source); @@ -332,7 +332,7 @@ public void shutdownClosesActiveSource() throws IOException { @Test public void shutdownCanBeCalledMultipleTimes() throws IOException { List synchronizers = new ArrayList<>(); - SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); Closeable source = mock(Closeable.class); manager.setActiveSource(source); @@ -348,7 +348,7 @@ public void shutdownCanBeCalledMultipleTimes() throws IOException { @Test public void shutdownIgnoresCloseException() throws IOException { List synchronizers = new ArrayList<>(); - SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); Closeable source = mock(Closeable.class); doThrow(new IOException("test")).when(source).close(); @@ -362,7 +362,7 @@ public void shutdownIgnoresCloseException() throws IOException { @Test public void shutdownWithoutActiveSourceDoesNotFail() { List synchronizers = new ArrayList<>(); - SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); // Should not throw manager.close(); @@ -378,7 +378,7 @@ public void integrationTestFullCycle() throws IOException { synchronizers.add(sync2); synchronizers.add(sync3); - SynchronizerStateManager manager = new SynchronizerStateManager(synchronizers); + SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); // Initial state assertEquals(3, manager.getAvailableSynchronizerCount()); From 16ecc8fa9ac431840342f77ad1a1923a9c070866 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Wed, 28 Jan 2026 15:27:01 -0800 Subject: [PATCH 28/35] Rename sourceManager. --- .../sdk/server/FDv2DataSource.java | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index 878ef6ad..e072151c 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -33,7 +33,7 @@ class FDv2DataSource implements DataSource { */ private static final long defaultRecoveryTimeout = 5 * 60; - private final SourceStateManager sourceStateManager; + private final SourceStateManager sourceManager; private final List conditionFactories; @@ -102,7 +102,7 @@ public FDv2DataSource( // configuration. } - this.sourceStateManager = new SourceStateManager(synchronizerFactories, initializers); + this.sourceManager = new SourceStateManager(synchronizerFactories, initializers); this.dataSourceUpdates = dataSourceUpdates; this.threadPriority = threadPriority; this.logger = logger; @@ -113,17 +113,17 @@ public FDv2DataSource( private void run() { Thread runThread = new Thread(() -> { - if (!sourceStateManager.hasAvailableSources()) { + if (!sourceManager.hasAvailableSources()) { // There are not any initializer or synchronizers, so we are at the best state that // can be achieved. dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.VALID, null); startFuture.complete(true); return; } - if (sourceStateManager.hasInitializers()) { + if (sourceManager.hasInitializers()) { runInitializers(); } - boolean synchronizersAvailable = sourceStateManager.hasAvailableSynchronizers(); + boolean synchronizersAvailable = sourceManager.hasAvailableSynchronizers(); if(!synchronizersAvailable) { // If already completed by the initializers, then this will have no effect. if (!isInitialized() && !closed) { @@ -169,11 +169,11 @@ private void run() { private void runInitializers() { boolean anyDataReceived = false; - DataSourceFactory factory = sourceStateManager.getNextInitializer(); + DataSourceFactory factory = sourceManager.getNextInitializer(); while(factory != null) { try { Initializer initializer = factory.build(); - if (sourceStateManager.setActiveSource(initializer)) return; + if (sourceManager.setActiveSource(initializer)) return; FDv2SourceResult result = initializer.run().get(); switch (result.getResultType()) { case CHANGE_SET: @@ -220,7 +220,7 @@ private void runInitializers() { new Date().toInstant())); logger.warn("Error running initializer: {}", e.toString()); } - factory = sourceStateManager.getNextInitializer(); + factory = sourceManager.getNextInitializer(); } // We received data without a selector, and we have exhausted initializers, so we are going to // consider ourselves initialized. @@ -239,8 +239,8 @@ private void runInitializers() { * @return a list of conditions to apply to the synchronizer */ private List getConditions() { - int availableSynchronizers = sourceStateManager.getAvailableSynchronizerCount(); - boolean isPrimeSynchronizer = sourceStateManager.isPrimeSynchronizer(); + int availableSynchronizers = sourceManager.getAvailableSynchronizerCount(); + boolean isPrimeSynchronizer = sourceManager.isPrimeSynchronizer(); if (availableSynchronizers == 1) { // If there is only 1 synchronizer, then we cannot fall back or recover, so we don't need any conditions. @@ -259,14 +259,14 @@ private List getConditions() { private void runSynchronizers() { // When runSynchronizers exists, no matter how it exits, the synchronizerStateManager will be closed. try { - SynchronizerFactoryWithState availableSynchronizer = sourceStateManager.getNextAvailableSynchronizer(); + SynchronizerFactoryWithState availableSynchronizer = sourceManager.getNextAvailableSynchronizer(); // We want to continue running synchronizers for as long as any are available. while (availableSynchronizer != null) { Synchronizer synchronizer = availableSynchronizer.build(); // Returns true if shutdown. - if (sourceStateManager.setActiveSource(synchronizer)) return; + if (sourceManager.setActiveSource(synchronizer)) return; try { boolean running = true; @@ -291,7 +291,7 @@ private void runSynchronizers() { case RECOVERY: // For recovery, we will start at the first available synchronizer. // So we reset the source index, and finding the source will start at the beginning. - sourceStateManager.resetSourceIndex(); + sourceManager.resetSourceIndex(); logger.debug("The data source is attempting to recover to a higher priority synchronizer."); break; } @@ -348,12 +348,12 @@ private void runSynchronizers() { // Only trigger fallback if we're not already running the FDv1 fallback synchronizer. if ( result.isFdv1Fallback() && - sourceStateManager.hasFDv1Fallback() && + sourceManager.hasFDv1Fallback() && // This shouldn't happen in practice, an FDv1 source shouldn't request fallback // to FDv1. But if it does, then we will discard its request. !availableSynchronizer.isFDv1Fallback() ) { - sourceStateManager.fdv1Fallback(); + sourceManager.fdv1Fallback(); running = false; } } @@ -369,12 +369,12 @@ private void runSynchronizers() { logger.warn("Error running synchronizer: {}, will try next synchronizer, or retry.", e.toString()); // Move to the next synchronizer. } - availableSynchronizer = sourceStateManager.getNextAvailableSynchronizer(); + availableSynchronizer = sourceManager.getNextAvailableSynchronizer(); } } catch(Exception e) { logger.error("Unexpected error in DataSource: {}", e.toString()); }finally { - sourceStateManager.close(); + sourceManager.close(); } } @@ -402,7 +402,7 @@ public void close() { // exiting. // If we do not have an active source, then the loop will check isShutdown when attempting to set one. When // it detects shutdown, it will exit the loop. - sourceStateManager.close(); + sourceManager.close(); dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.OFF, null); From 54df2d54970a1f568a8bb1e831bf6c50ece4a75f Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Wed, 28 Jan 2026 16:04:32 -0800 Subject: [PATCH 29/35] Iterator pattern for SourceManager. --- .../sdk/server/FDv2DataSource.java | 54 ++- ...ceStateManager.java => SourceManager.java} | 171 +++++-- .../sdk/server/SourceManagerTest.java | 409 +++++++++++++++++ .../sdk/server/SourceStateManagerTest.java | 425 ------------------ 4 files changed, 556 insertions(+), 503 deletions(-) rename lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/{SourceStateManager.java => SourceManager.java} (53%) create mode 100644 lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/SourceManagerTest.java delete mode 100644 lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/SourceStateManagerTest.java diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index e072151c..2d2cab02 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -33,7 +33,7 @@ class FDv2DataSource implements DataSource { */ private static final long defaultRecoveryTimeout = 5 * 60; - private final SourceStateManager sourceManager; + private final SourceManager sourceManager; private final List conditionFactories; @@ -102,7 +102,7 @@ public FDv2DataSource( // configuration. } - this.sourceManager = new SourceStateManager(synchronizerFactories, initializers); + this.sourceManager = new SourceManager(synchronizerFactories, initializers); this.dataSourceUpdates = dataSourceUpdates; this.threadPriority = threadPriority; this.logger = logger; @@ -123,8 +123,8 @@ private void run() { if (sourceManager.hasInitializers()) { runInitializers(); } - boolean synchronizersAvailable = sourceManager.hasAvailableSynchronizers(); - if(!synchronizersAvailable) { + + if(!sourceManager.hasAvailableSynchronizers()) { // If already completed by the initializers, then this will have no effect. if (!isInitialized() && !closed) { // If we were closed, then closing would have handled our terminal update. @@ -146,16 +146,18 @@ private void run() { runSynchronizers(); // If we had synchronizers, and we ran out of them, then we are off. - dataSourceUpdates.updateStatus( - DataSourceStatusProvider.State.OFF, - // If the data source was closed, then we just report we are OFF without an - // associated error. - closed? null : new DataSourceStatusProvider.ErrorInfo( - DataSourceStatusProvider.ErrorKind.UNKNOWN, - 0, - "All data source acquisition methods have been exhausted.", - new Date().toInstant()) - ); + if(!closed) { + dataSourceUpdates.updateStatus( + DataSourceStatusProvider.State.OFF, + // If the data source was closed, then we just report we are OFF without an + // associated error. + new DataSourceStatusProvider.ErrorInfo( + DataSourceStatusProvider.ErrorKind.UNKNOWN, + 0, + "All data source acquisition methods have been exhausted.", + new Date().toInstant()) + ); + } // If we had initialized at some point, then the future will already be complete and this will be ignored. startFuture.complete(false); @@ -169,11 +171,9 @@ private void run() { private void runInitializers() { boolean anyDataReceived = false; - DataSourceFactory factory = sourceManager.getNextInitializer(); - while(factory != null) { + Initializer initializer = sourceManager.getNextInitializerAndSetActive(); + while(initializer != null) { try { - Initializer initializer = factory.build(); - if (sourceManager.setActiveSource(initializer)) return; FDv2SourceResult result = initializer.run().get(); switch (result.getResultType()) { case CHANGE_SET: @@ -220,7 +220,7 @@ private void runInitializers() { new Date().toInstant())); logger.warn("Error running initializer: {}", e.toString()); } - factory = sourceManager.getNextInitializer(); + initializer = sourceManager.getNextInitializerAndSetActive(); } // We received data without a selector, and we have exhausted initializers, so we are going to // consider ourselves initialized. @@ -259,15 +259,10 @@ private List getConditions() { private void runSynchronizers() { // When runSynchronizers exists, no matter how it exits, the synchronizerStateManager will be closed. try { - SynchronizerFactoryWithState availableSynchronizer = sourceManager.getNextAvailableSynchronizer(); + Synchronizer synchronizer = sourceManager.getNextAvailableSynchronizerAndSetActive(); // We want to continue running synchronizers for as long as any are available. - while (availableSynchronizer != null) { - Synchronizer synchronizer = availableSynchronizer.build(); - - // Returns true if shutdown. - if (sourceManager.setActiveSource(synchronizer)) return; - + while (synchronizer != null) { try { boolean running = true; @@ -331,7 +326,7 @@ private void runSynchronizers() { logger.debug("Synchronizer shutdown."); return; case TERMINAL_ERROR: - availableSynchronizer.block(); + sourceManager.blockCurrentSynchronizer(); running = false; dataSourceUpdates.updateStatus( DataSourceStatusProvider.State.INTERRUPTED, @@ -351,7 +346,7 @@ private void runSynchronizers() { sourceManager.hasFDv1Fallback() && // This shouldn't happen in practice, an FDv1 source shouldn't request fallback // to FDv1. But if it does, then we will discard its request. - !availableSynchronizer.isFDv1Fallback() + !sourceManager.isCurrentSynchronizerFDv1Fallback() ) { sourceManager.fdv1Fallback(); running = false; @@ -369,7 +364,8 @@ private void runSynchronizers() { logger.warn("Error running synchronizer: {}, will try next synchronizer, or retry.", e.toString()); // Move to the next synchronizer. } - availableSynchronizer = sourceManager.getNextAvailableSynchronizer(); + // Get the next available synchronizer and set it active + synchronizer = sourceManager.getNextAvailableSynchronizerAndSetActive(); } } catch(Exception e) { logger.error("Unexpected error in DataSource: {}", e.toString()); diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceStateManager.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceManager.java similarity index 53% rename from lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceStateManager.java rename to lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceManager.java index d4112d94..dd093d9a 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceStateManager.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceManager.java @@ -12,7 +12,7 @@ *

* Package-private for internal use. */ -class SourceStateManager implements Closeable { +class SourceManager implements Closeable { private final List synchronizers; private final List> initializers; @@ -31,7 +31,12 @@ class SourceStateManager implements Closeable { private int initializerIndex = -1; - public SourceStateManager(List synchronizers, List> initializers) { + /** + * The current synchronizer factory (for checking FDv1 fallback status and blocking) + */ + private SynchronizerFactoryWithState currentSynchronizerFactory; + + public SourceManager(List synchronizers, List> initializers) { this.synchronizers = synchronizers; this.initializers = initializers; } @@ -75,40 +80,76 @@ public void fdv1Fallback() { *

* Any given synchronizer can be marked as blocked, in which case that synchronizer is not eligible to be used again. * Synchronizers that are not blocked are available, and this function will only return available synchronizers. + *

+ * Note: This is an internal method that must be called while holding activeSourceLock. + * It does not check shutdown status or handle locking - that's done by the caller. + * * @return the next synchronizer factory to use, or null if there are no more available synchronizers. */ - public SynchronizerFactoryWithState getNextAvailableSynchronizer() { - synchronized (activeSourceLock) { - SynchronizerFactoryWithState factory = null; + private SynchronizerFactoryWithState getNextAvailableSynchronizer() { + SynchronizerFactoryWithState factory = null; - if(isShutdown) { - safeClose(activeSource); - activeSource = null; - return factory; + int visited = 0; + while(visited < synchronizers.size()) { + // Look for the next synchronizer starting at the position after the current one. (avoiding just re-using the same synchronizer.) + synchronizerIndex++; + + // We aren't using module here because we want to keep the stored index within range instead + // of increasing indefinitely. + if(synchronizerIndex >= synchronizers.size()) { + synchronizerIndex = 0; } - int visited = 0; - while(visited < synchronizers.size()) { - // Look for the next synchronizer starting at the position after the current one. (avoiding just re-using the same synchronizer.) - synchronizerIndex++; + SynchronizerFactoryWithState candidate = synchronizers.get(synchronizerIndex); + if (candidate.getState() == SynchronizerFactoryWithState.State.Available) { + factory = candidate; + break; + } + visited++; + } + return factory; + } - // We aren't using module here because we want to keep the stored index within range instead - // of increasing indefinitely. - if(synchronizerIndex >= synchronizers.size()) { - synchronizerIndex = 0; - } + /** + * Get the next available synchronizer, build it, and set it as the active source in one atomic operation. + * This combines the two-step process of getting the next synchronizer and setting it active. + *

+ * If shutdown has been initiated, returns null without building or setting a source. + * Any previously active source will be closed before setting the new one. + *

+ * The current synchronizer factory can be retrieved with {@link #blockCurrentSynchronizer()} + * or {@link #isCurrentSynchronizerFDv1Fallback()} to interact with it. + * + * @return the built synchronizer that is now active, or null if no more synchronizers are available or shutdown has been initiated + */ + public com.launchdarkly.sdk.server.datasources.Synchronizer getNextAvailableSynchronizerAndSetActive() { + synchronized (activeSourceLock) { + // Handle shutdown first - if shutdown, don't do any work + if (isShutdown) { + currentSynchronizerFactory = null; + return null; + } - SynchronizerFactoryWithState candidate = synchronizers.get(synchronizerIndex); - if (candidate.getState() == SynchronizerFactoryWithState.State.Available) { - factory = candidate; - break; - } - visited++; + SynchronizerFactoryWithState factory = getNextAvailableSynchronizer(); + if (factory == null) { + currentSynchronizerFactory = null; + return null; } - return factory; + + currentSynchronizerFactory = factory; + com.launchdarkly.sdk.server.datasources.Synchronizer synchronizer = factory.build(); + + // Close any previously active source + if (activeSource != null) { + safeClose(activeSource); + } + + activeSource = synchronizer; + return synchronizer; } } + public boolean hasAvailableSources() { return hasInitializers() || getAvailableSynchronizerCount() > 0; } @@ -121,16 +162,64 @@ public boolean hasAvailableSynchronizers() { return getAvailableSynchronizerCount() > 0; } - public FDv2DataSource.DataSourceFactory getNextInitializer() { + /** + * Get the next initializer factory. This is an internal method that must be called while holding activeSourceLock. + * It does not check shutdown status or handle locking - that's done by the caller. + * + * @return the next initializer factory, or null if no more initializers are available + */ + private FDv2DataSource.DataSourceFactory getNextInitializer() { + initializerIndex++; + if (initializerIndex >= initializers.size()) { + return null; + } + return initializers.get(initializerIndex); + } + + public void blockCurrentSynchronizer() { synchronized (activeSourceLock) { - if(isShutdown) { + if (currentSynchronizerFactory != null) { + currentSynchronizerFactory.block(); + } + } + } + + public boolean isCurrentSynchronizerFDv1Fallback() { + synchronized (activeSourceLock) { + return currentSynchronizerFactory != null && currentSynchronizerFactory.isFDv1Fallback(); + } + } + + /** + * Get the next initializer, build it, and set it as the active source in one atomic operation. + * This combines the two-step process of getting the next initializer and setting it active. + *

+ * If shutdown has been initiated, returns null without building or setting a source. + * Any previously active source will be closed before setting the new one. + * + * @return the built initializer that is now active, or null if no more initializers are available or shutdown has been initiated + */ + public Initializer getNextInitializerAndSetActive() { + synchronized (activeSourceLock) { + // Handle shutdown first - if shutdown, don't do any work + if (isShutdown) { return null; } - initializerIndex++; - if (initializerIndex >= initializers.size()) { + + FDv2DataSource.DataSourceFactory factory = getNextInitializer(); + if (factory == null) { return null; } - return initializers.get(initializerIndex); + + Initializer initializer = factory.build(); + + // Close any previously active source + if (activeSource != null) { + safeClose(activeSource); + } + + activeSource = initializer; + return initializer; } } @@ -170,25 +259,6 @@ public int getAvailableSynchronizerCount() { } } - /** - * Set the active source. If shutdown has been initiated, the source will be closed immediately. - * Any previously active source will be closed. - * @param source the source to set as active - * @return true if shutdown has been initiated, false otherwise - */ - public boolean setActiveSource(Closeable source) { - synchronized (activeSourceLock) { - if (activeSource != null) { - safeClose(activeSource); - } - if (isShutdown) { - safeClose(source); - return true; - } - activeSource = source; - } - return false; - } /** * Close the state manager and shut down any active source. @@ -215,6 +285,9 @@ public void close() { * @param closeable the closeable to close */ private void safeClose(Closeable closeable) { + if (closeable == null) { + return; + } try { closeable.close(); } catch (IOException e) { diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/SourceManagerTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/SourceManagerTest.java new file mode 100644 index 00000000..d0a20e66 --- /dev/null +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/SourceManagerTest.java @@ -0,0 +1,409 @@ +package com.launchdarkly.sdk.server; + +import com.launchdarkly.sdk.server.datasources.Synchronizer; + +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@SuppressWarnings("javadoc") +public class SourceManagerTest extends BaseTest { + + private static class TestSynchronizerFactory extends SynchronizerFactoryWithState { + private final FDv2DataSource.DataSourceFactory mockFactory; + + public TestSynchronizerFactory(FDv2DataSource.DataSourceFactory mockFactory) { + super(mockFactory); + this.mockFactory = mockFactory; + } + + public FDv2DataSource.DataSourceFactory getFactory() { + return mockFactory; + } + } + + private TestSynchronizerFactory createMockFactory() { + FDv2DataSource.DataSourceFactory factory = mock(FDv2DataSource.DataSourceFactory.class); + // Return a new mock each time build() is called to avoid reusing the same instance + when(factory.build()).thenAnswer(invocation -> mock(Synchronizer.class)); + return new TestSynchronizerFactory(factory); + } + + @Test + public void getNextAvailableSynchronizerReturnsNullWhenEmpty() { + List synchronizers = new ArrayList<>(); + SourceManager manager = new SourceManager(synchronizers, new ArrayList<>()); + + Synchronizer result = manager.getNextAvailableSynchronizerAndSetActive(); + + assertNull(result); + } + + @Test + public void getNextAvailableSynchronizerReturnsFirstOnFirstCall() { + List synchronizers = new ArrayList<>(); + TestSynchronizerFactory sync1 = createMockFactory(); + synchronizers.add(sync1); + + SourceManager manager = new SourceManager(synchronizers, new ArrayList<>()); + + Synchronizer result = manager.getNextAvailableSynchronizerAndSetActive(); + + assertNotNull(result); + // Verify it was built from sync1 + verify(sync1.getFactory(), times(1)).build(); + } + + @Test + public void getNextAvailableSynchronizerLoopsThroughAvailable() { + List synchronizers = new ArrayList<>(); + TestSynchronizerFactory sync1 = createMockFactory(); + TestSynchronizerFactory sync2 = createMockFactory(); + TestSynchronizerFactory sync3 = createMockFactory(); + synchronizers.add(sync1); + synchronizers.add(sync2); + synchronizers.add(sync3); + + SourceManager manager = new SourceManager(synchronizers, new ArrayList<>()); + + // First call builds from sync1 + manager.getNextAvailableSynchronizerAndSetActive(); + verify(sync1.getFactory(), times(1)).build(); + + // Second call builds from sync2 + manager.getNextAvailableSynchronizerAndSetActive(); + verify(sync2.getFactory(), times(1)).build(); + + // Third call builds from sync3 + manager.getNextAvailableSynchronizerAndSetActive(); + verify(sync3.getFactory(), times(1)).build(); + } + + @Test + public void getNextAvailableSynchronizerWrapsAroundToBeginning() { + List synchronizers = new ArrayList<>(); + TestSynchronizerFactory sync1 = createMockFactory(); + TestSynchronizerFactory sync2 = createMockFactory(); + synchronizers.add(sync1); + synchronizers.add(sync2); + + SourceManager manager = new SourceManager(synchronizers, new ArrayList<>()); + + // Get all synchronizers + manager.getNextAvailableSynchronizerAndSetActive(); // sync1 + manager.getNextAvailableSynchronizerAndSetActive(); // sync2 + + // Should wrap around to sync1 + manager.getNextAvailableSynchronizerAndSetActive(); + + // sync1 should have been built twice + verify(sync1.getFactory(), times(2)).build(); + } + + @Test + public void getNextAvailableSynchronizerSkipsBlockedSynchronizers() { + List synchronizers = new ArrayList<>(); + TestSynchronizerFactory sync1 = createMockFactory(); + TestSynchronizerFactory sync2 = createMockFactory(); + TestSynchronizerFactory sync3 = createMockFactory(); + synchronizers.add(sync1); + synchronizers.add(sync2); + synchronizers.add(sync3); + + SourceManager manager = new SourceManager(synchronizers, new ArrayList<>()); + + // Block sync2 + sync2.block(); + + // First call builds from sync1 + manager.getNextAvailableSynchronizerAndSetActive(); + verify(sync1.getFactory(), times(1)).build(); + + // Second call skips sync2 and builds from sync3 + manager.getNextAvailableSynchronizerAndSetActive(); + verify(sync3.getFactory(), times(1)).build(); + verify(sync2.getFactory(), times(0)).build(); + + // Third call wraps and builds from sync1 (skips sync2) + manager.getNextAvailableSynchronizerAndSetActive(); + verify(sync1.getFactory(), times(2)).build(); + } + + @Test + public void getNextAvailableSynchronizerReturnsNullWhenAllBlocked() { + List synchronizers = new ArrayList<>(); + SynchronizerFactoryWithState sync1 = createMockFactory(); + SynchronizerFactoryWithState sync2 = createMockFactory(); + synchronizers.add(sync1); + synchronizers.add(sync2); + + SourceManager manager = new SourceManager(synchronizers, new ArrayList<>()); + + // Block all synchronizers + sync1.block(); + sync2.block(); + + Synchronizer result = manager.getNextAvailableSynchronizerAndSetActive(); + + assertNull(result); + } + + @Test + public void resetSourceIndexResetsToFirstSynchronizer() { + List synchronizers = new ArrayList<>(); + TestSynchronizerFactory sync1 = createMockFactory(); + TestSynchronizerFactory sync2 = createMockFactory(); + TestSynchronizerFactory sync3 = createMockFactory(); + synchronizers.add(sync1); + synchronizers.add(sync2); + synchronizers.add(sync3); + + SourceManager manager = new SourceManager(synchronizers, new ArrayList<>()); + + // Advance to sync3 + manager.getNextAvailableSynchronizerAndSetActive(); // sync1 + manager.getNextAvailableSynchronizerAndSetActive(); // sync2 + manager.getNextAvailableSynchronizerAndSetActive(); // sync3 + + // Reset + manager.resetSourceIndex(); + + // Next call should build from sync1 again + manager.getNextAvailableSynchronizerAndSetActive(); + verify(sync1.getFactory(), times(2)).build(); + } + + @Test + public void isPrimeSynchronizerReturnsTrueForFirst() { + List synchronizers = new ArrayList<>(); + TestSynchronizerFactory sync1 = createMockFactory(); + TestSynchronizerFactory sync2 = createMockFactory(); + synchronizers.add(sync1); + synchronizers.add(sync2); + + SourceManager manager = new SourceManager(synchronizers, new ArrayList<>()); + + // Get first synchronizer + manager.getNextAvailableSynchronizerAndSetActive(); + + assertTrue(manager.isPrimeSynchronizer()); + } + + @Test + public void isPrimeSynchronizerReturnsFalseForNonFirst() { + List synchronizers = new ArrayList<>(); + TestSynchronizerFactory sync1 = createMockFactory(); + TestSynchronizerFactory sync2 = createMockFactory(); + synchronizers.add(sync1); + synchronizers.add(sync2); + + SourceManager manager = new SourceManager(synchronizers, new ArrayList<>()); + + // Get first then second synchronizer + manager.getNextAvailableSynchronizerAndSetActive(); + manager.getNextAvailableSynchronizerAndSetActive(); + + assertFalse(manager.isPrimeSynchronizer()); + } + + @Test + public void isPrimeSynchronizerReturnsFalseWhenNoSynchronizerSelected() { + List synchronizers = new ArrayList<>(); + SynchronizerFactoryWithState sync1 = createMockFactory(); + synchronizers.add(sync1); + + SourceManager manager = new SourceManager(synchronizers, new ArrayList<>()); + + // Haven't called getNext yet + assertFalse(manager.isPrimeSynchronizer()); + } + + @Test + public void isPrimeSynchronizerHandlesBlockedFirstSynchronizer() { + List synchronizers = new ArrayList<>(); + TestSynchronizerFactory sync1 = createMockFactory(); + TestSynchronizerFactory sync2 = createMockFactory(); + TestSynchronizerFactory sync3 = createMockFactory(); + synchronizers.add(sync1); + synchronizers.add(sync2); + synchronizers.add(sync3); + + SourceManager manager = new SourceManager(synchronizers, new ArrayList<>()); + + // Block first synchronizer + sync1.block(); + + // Get second synchronizer (which is now the prime) + manager.getNextAvailableSynchronizerAndSetActive(); + + assertTrue(manager.isPrimeSynchronizer()); + } + + @Test + public void getAvailableSynchronizerCountReturnsCorrectCount() { + List synchronizers = new ArrayList<>(); + SynchronizerFactoryWithState sync1 = createMockFactory(); + SynchronizerFactoryWithState sync2 = createMockFactory(); + SynchronizerFactoryWithState sync3 = createMockFactory(); + synchronizers.add(sync1); + synchronizers.add(sync2); + synchronizers.add(sync3); + + SourceManager manager = new SourceManager(synchronizers, new ArrayList<>()); + + assertEquals(3, manager.getAvailableSynchronizerCount()); + } + + @Test + public void getAvailableSynchronizerCountUpdatesWhenBlocked() { + List synchronizers = new ArrayList<>(); + SynchronizerFactoryWithState sync1 = createMockFactory(); + SynchronizerFactoryWithState sync2 = createMockFactory(); + SynchronizerFactoryWithState sync3 = createMockFactory(); + synchronizers.add(sync1); + synchronizers.add(sync2); + synchronizers.add(sync3); + + SourceManager manager = new SourceManager(synchronizers, new ArrayList<>()); + + assertEquals(3, manager.getAvailableSynchronizerCount()); + + sync2.block(); + assertEquals(2, manager.getAvailableSynchronizerCount()); + + sync1.block(); + assertEquals(1, manager.getAvailableSynchronizerCount()); + + sync3.block(); + assertEquals(0, manager.getAvailableSynchronizerCount()); + } + + + @Test + public void shutdownClosesActiveSource() throws IOException { + List synchronizers = new ArrayList<>(); + SynchronizerFactoryWithState sync = createMockFactory(); + synchronizers.add(sync); + SourceManager manager = new SourceManager(synchronizers, new ArrayList<>()); + + Synchronizer source = manager.getNextAvailableSynchronizerAndSetActive(); + assertNotNull(source); + + manager.close(); + + verify(source, times(1)).close(); + } + + @Test + public void shutdownCanBeCalledMultipleTimes() throws IOException { + List synchronizers = new ArrayList<>(); + SynchronizerFactoryWithState sync = createMockFactory(); + synchronizers.add(sync); + SourceManager manager = new SourceManager(synchronizers, new ArrayList<>()); + + Synchronizer source = manager.getNextAvailableSynchronizerAndSetActive(); + assertNotNull(source); + + manager.close(); + manager.close(); + manager.close(); + + // Should only close once + verify(source, times(1)).close(); + } + + @Test + public void shutdownIgnoresCloseException() throws IOException { + List synchronizers = new ArrayList<>(); + SynchronizerFactoryWithState sync = createMockFactory(); + synchronizers.add(sync); + SourceManager manager = new SourceManager(synchronizers, new ArrayList<>()); + + Synchronizer source = manager.getNextAvailableSynchronizerAndSetActive(); + assertNotNull(source); + doThrow(new IOException("test")).when(source).close(); + + // Should not throw + manager.close(); + } + + @Test + public void shutdownWithoutActiveSourceDoesNotFail() { + List synchronizers = new ArrayList<>(); + SourceManager manager = new SourceManager(synchronizers, new ArrayList<>()); + + // Should not throw + manager.close(); + } + + @Test + public void integrationTestFullCycle() throws IOException { + List synchronizers = new ArrayList<>(); + TestSynchronizerFactory sync1 = createMockFactory(); + TestSynchronizerFactory sync2 = createMockFactory(); + TestSynchronizerFactory sync3 = createMockFactory(); + synchronizers.add(sync1); + synchronizers.add(sync2); + synchronizers.add(sync3); + + SourceManager manager = new SourceManager(synchronizers, new ArrayList<>()); + + // Initial state + assertEquals(3, manager.getAvailableSynchronizerCount()); + assertFalse(manager.isPrimeSynchronizer()); + + // Get first synchronizer + Synchronizer first = manager.getNextAvailableSynchronizerAndSetActive(); + assertNotNull(first); + verify(sync1.getFactory(), times(1)).build(); + assertTrue(manager.isPrimeSynchronizer()); + + // Get second synchronizer + Synchronizer second = manager.getNextAvailableSynchronizerAndSetActive(); + assertNotNull(second); + verify(sync2.getFactory(), times(1)).build(); + assertFalse(manager.isPrimeSynchronizer()); + + // Block second + sync2.block(); + assertEquals(2, manager.getAvailableSynchronizerCount()); + + // Get third synchronizer + Synchronizer third = manager.getNextAvailableSynchronizerAndSetActive(); + assertNotNull(third); + verify(sync3.getFactory(), times(1)).build(); + assertFalse(manager.isPrimeSynchronizer()); + + // Reset and get first again + manager.resetSourceIndex(); + Synchronizer firstAgain = manager.getNextAvailableSynchronizerAndSetActive(); + assertNotNull(firstAgain); + verify(sync1.getFactory(), times(2)).build(); + assertTrue(manager.isPrimeSynchronizer()); + + // Verify latest active source is set + Synchronizer source = manager.getNextAvailableSynchronizerAndSetActive(); + assertNotNull(source); + + // Shutdown + manager.close(); + verify(source, times(1)).close(); + + // After shutdown, new sources are immediately closed + Synchronizer newSource = manager.getNextAvailableSynchronizerAndSetActive(); + assertNull(newSource); // Returns null after shutdown + } +} diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/SourceStateManagerTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/SourceStateManagerTest.java deleted file mode 100644 index 9ad73cec..00000000 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/SourceStateManagerTest.java +++ /dev/null @@ -1,425 +0,0 @@ -package com.launchdarkly.sdk.server; - -import com.launchdarkly.sdk.server.datasources.Synchronizer; - -import org.junit.Test; - -import java.io.Closeable; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -@SuppressWarnings("javadoc") -public class SourceStateManagerTest extends BaseTest { - - private SynchronizerFactoryWithState createMockFactory() { - FDv2DataSource.DataSourceFactory factory = mock(FDv2DataSource.DataSourceFactory.class); - when(factory.build()).thenReturn(mock(Synchronizer.class)); - return new SynchronizerFactoryWithState(factory); - } - - @Test - public void getNextAvailableSynchronizerReturnsNullWhenEmpty() { - List synchronizers = new ArrayList<>(); - SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); - - SynchronizerFactoryWithState result = manager.getNextAvailableSynchronizer(); - - assertNull(result); - } - - @Test - public void getNextAvailableSynchronizerReturnsFirstOnFirstCall() { - List synchronizers = new ArrayList<>(); - SynchronizerFactoryWithState sync1 = createMockFactory(); - synchronizers.add(sync1); - - SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); - - SynchronizerFactoryWithState result = manager.getNextAvailableSynchronizer(); - - assertSame(sync1, result); - } - - @Test - public void getNextAvailableSynchronizerLoopsThroughAvailable() { - List synchronizers = new ArrayList<>(); - SynchronizerFactoryWithState sync1 = createMockFactory(); - SynchronizerFactoryWithState sync2 = createMockFactory(); - SynchronizerFactoryWithState sync3 = createMockFactory(); - synchronizers.add(sync1); - synchronizers.add(sync2); - synchronizers.add(sync3); - - SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); - - // First call returns sync1 - assertSame(sync1, manager.getNextAvailableSynchronizer()); - // Second call returns sync2 - assertSame(sync2, manager.getNextAvailableSynchronizer()); - // Third call returns sync3 - assertSame(sync3, manager.getNextAvailableSynchronizer()); - } - - @Test - public void getNextAvailableSynchronizerWrapsAroundToBeginning() { - List synchronizers = new ArrayList<>(); - SynchronizerFactoryWithState sync1 = createMockFactory(); - SynchronizerFactoryWithState sync2 = createMockFactory(); - synchronizers.add(sync1); - synchronizers.add(sync2); - - SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); - - // Get all synchronizers - manager.getNextAvailableSynchronizer(); // sync1 - manager.getNextAvailableSynchronizer(); // sync2 - - // Should wrap around to sync1 - assertSame(sync1, manager.getNextAvailableSynchronizer()); - } - - @Test - public void getNextAvailableSynchronizerSkipsBlockedSynchronizers() { - List synchronizers = new ArrayList<>(); - SynchronizerFactoryWithState sync1 = createMockFactory(); - SynchronizerFactoryWithState sync2 = createMockFactory(); - SynchronizerFactoryWithState sync3 = createMockFactory(); - synchronizers.add(sync1); - synchronizers.add(sync2); - synchronizers.add(sync3); - - SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); - - // Block sync2 - sync2.block(); - - // First call returns sync1 - assertSame(sync1, manager.getNextAvailableSynchronizer()); - // Second call skips sync2 and returns sync3 - assertSame(sync3, manager.getNextAvailableSynchronizer()); - // Third call wraps and returns sync1 (skips sync2) - assertSame(sync1, manager.getNextAvailableSynchronizer()); - } - - @Test - public void getNextAvailableSynchronizerReturnsNullWhenAllBlocked() { - List synchronizers = new ArrayList<>(); - SynchronizerFactoryWithState sync1 = createMockFactory(); - SynchronizerFactoryWithState sync2 = createMockFactory(); - synchronizers.add(sync1); - synchronizers.add(sync2); - - SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); - - // Block all synchronizers - sync1.block(); - sync2.block(); - - SynchronizerFactoryWithState result = manager.getNextAvailableSynchronizer(); - - assertNull(result); - } - - @Test - public void resetSourceIndexResetsToFirstSynchronizer() { - List synchronizers = new ArrayList<>(); - SynchronizerFactoryWithState sync1 = createMockFactory(); - SynchronizerFactoryWithState sync2 = createMockFactory(); - SynchronizerFactoryWithState sync3 = createMockFactory(); - synchronizers.add(sync1); - synchronizers.add(sync2); - synchronizers.add(sync3); - - SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); - - // Advance to sync3 - manager.getNextAvailableSynchronizer(); // sync1 - manager.getNextAvailableSynchronizer(); // sync2 - manager.getNextAvailableSynchronizer(); // sync3 - - // Reset - manager.resetSourceIndex(); - - // Next call should return sync1 again - assertSame(sync1, manager.getNextAvailableSynchronizer()); - } - - @Test - public void isPrimeSynchronizerReturnsTrueForFirst() { - List synchronizers = new ArrayList<>(); - SynchronizerFactoryWithState sync1 = createMockFactory(); - SynchronizerFactoryWithState sync2 = createMockFactory(); - synchronizers.add(sync1); - synchronizers.add(sync2); - - SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); - - // Get first synchronizer - manager.getNextAvailableSynchronizer(); - - assertTrue(manager.isPrimeSynchronizer()); - } - - @Test - public void isPrimeSynchronizerReturnsFalseForNonFirst() { - List synchronizers = new ArrayList<>(); - SynchronizerFactoryWithState sync1 = createMockFactory(); - SynchronizerFactoryWithState sync2 = createMockFactory(); - synchronizers.add(sync1); - synchronizers.add(sync2); - - SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); - - // Get first then second synchronizer - manager.getNextAvailableSynchronizer(); - manager.getNextAvailableSynchronizer(); - - assertFalse(manager.isPrimeSynchronizer()); - } - - @Test - public void isPrimeSynchronizerReturnsFalseWhenNoSynchronizerSelected() { - List synchronizers = new ArrayList<>(); - SynchronizerFactoryWithState sync1 = createMockFactory(); - synchronizers.add(sync1); - - SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); - - // Haven't called getNext yet - assertFalse(manager.isPrimeSynchronizer()); - } - - @Test - public void isPrimeSynchronizerHandlesBlockedFirstSynchronizer() { - List synchronizers = new ArrayList<>(); - SynchronizerFactoryWithState sync1 = createMockFactory(); - SynchronizerFactoryWithState sync2 = createMockFactory(); - SynchronizerFactoryWithState sync3 = createMockFactory(); - synchronizers.add(sync1); - synchronizers.add(sync2); - synchronizers.add(sync3); - - SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); - - // Block first synchronizer - sync1.block(); - - // Get second synchronizer (which is now the prime) - manager.getNextAvailableSynchronizer(); - - assertTrue(manager.isPrimeSynchronizer()); - } - - @Test - public void getAvailableSynchronizerCountReturnsCorrectCount() { - List synchronizers = new ArrayList<>(); - SynchronizerFactoryWithState sync1 = createMockFactory(); - SynchronizerFactoryWithState sync2 = createMockFactory(); - SynchronizerFactoryWithState sync3 = createMockFactory(); - synchronizers.add(sync1); - synchronizers.add(sync2); - synchronizers.add(sync3); - - SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); - - assertEquals(3, manager.getAvailableSynchronizerCount()); - } - - @Test - public void getAvailableSynchronizerCountUpdatesWhenBlocked() { - List synchronizers = new ArrayList<>(); - SynchronizerFactoryWithState sync1 = createMockFactory(); - SynchronizerFactoryWithState sync2 = createMockFactory(); - SynchronizerFactoryWithState sync3 = createMockFactory(); - synchronizers.add(sync1); - synchronizers.add(sync2); - synchronizers.add(sync3); - - SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); - - assertEquals(3, manager.getAvailableSynchronizerCount()); - - sync2.block(); - assertEquals(2, manager.getAvailableSynchronizerCount()); - - sync1.block(); - assertEquals(1, manager.getAvailableSynchronizerCount()); - - sync3.block(); - assertEquals(0, manager.getAvailableSynchronizerCount()); - } - - @Test - public void setActiveSourceSetsNewSource() throws IOException { - List synchronizers = new ArrayList<>(); - SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); - - Closeable source = mock(Closeable.class); - boolean shutdown = manager.setActiveSource(source); - - assertFalse(shutdown); - } - - @Test - public void setActiveSourceClosesPreviousSource() throws IOException { - List synchronizers = new ArrayList<>(); - SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); - - Closeable firstSource = mock(Closeable.class); - Closeable secondSource = mock(Closeable.class); - - manager.setActiveSource(firstSource); - manager.setActiveSource(secondSource); - - verify(firstSource, times(1)).close(); - } - - @Test - public void setActiveSourceReturnsTrueAfterShutdown() throws IOException { - List synchronizers = new ArrayList<>(); - SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); - - manager.close(); - - Closeable source = mock(Closeable.class); - boolean shutdown = manager.setActiveSource(source); - - assertTrue(shutdown); - verify(source, times(1)).close(); - } - - @Test - public void setActiveSourceIgnoresCloseExceptionFromPreviousSource() throws IOException { - List synchronizers = new ArrayList<>(); - SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); - - Closeable firstSource = mock(Closeable.class); - doThrow(new IOException("test")).when(firstSource).close(); - - Closeable secondSource = mock(Closeable.class); - - manager.setActiveSource(firstSource); - // Should not throw - manager.setActiveSource(secondSource); - } - - @Test - public void shutdownClosesActiveSource() throws IOException { - List synchronizers = new ArrayList<>(); - SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); - - Closeable source = mock(Closeable.class); - manager.setActiveSource(source); - - manager.close(); - - verify(source, times(1)).close(); - } - - @Test - public void shutdownCanBeCalledMultipleTimes() throws IOException { - List synchronizers = new ArrayList<>(); - SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); - - Closeable source = mock(Closeable.class); - manager.setActiveSource(source); - - manager.close(); - manager.close(); - manager.close(); - - // Should only close once - verify(source, times(1)).close(); - } - - @Test - public void shutdownIgnoresCloseException() throws IOException { - List synchronizers = new ArrayList<>(); - SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); - - Closeable source = mock(Closeable.class); - doThrow(new IOException("test")).when(source).close(); - - manager.setActiveSource(source); - - // Should not throw - manager.close(); - } - - @Test - public void shutdownWithoutActiveSourceDoesNotFail() { - List synchronizers = new ArrayList<>(); - SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); - - // Should not throw - manager.close(); - } - - @Test - public void integrationTestFullCycle() throws IOException { - List synchronizers = new ArrayList<>(); - SynchronizerFactoryWithState sync1 = createMockFactory(); - SynchronizerFactoryWithState sync2 = createMockFactory(); - SynchronizerFactoryWithState sync3 = createMockFactory(); - synchronizers.add(sync1); - synchronizers.add(sync2); - synchronizers.add(sync3); - - SourceStateManager manager = new SourceStateManager(synchronizers, new ArrayList<>()); - - // Initial state - assertEquals(3, manager.getAvailableSynchronizerCount()); - assertFalse(manager.isPrimeSynchronizer()); - - // Get first synchronizer - SynchronizerFactoryWithState first = manager.getNextAvailableSynchronizer(); - assertSame(sync1, first); - assertTrue(manager.isPrimeSynchronizer()); - - // Get second synchronizer - SynchronizerFactoryWithState second = manager.getNextAvailableSynchronizer(); - assertSame(sync2, second); - assertFalse(manager.isPrimeSynchronizer()); - - // Block second - sync2.block(); - assertEquals(2, manager.getAvailableSynchronizerCount()); - - // Get third synchronizer - SynchronizerFactoryWithState third = manager.getNextAvailableSynchronizer(); - assertSame(sync3, third); - assertFalse(manager.isPrimeSynchronizer()); - - // Reset and get first again - manager.resetSourceIndex(); - SynchronizerFactoryWithState firstAgain = manager.getNextAvailableSynchronizer(); - assertSame(sync1, firstAgain); - assertTrue(manager.isPrimeSynchronizer()); - - // Set active source - Closeable source = mock(Closeable.class); - assertFalse(manager.setActiveSource(source)); - - // Shutdown - manager.close(); - verify(source, times(1)).close(); - - // After shutdown, new sources are immediately closed - Closeable newSource = mock(Closeable.class); - assertTrue(manager.setActiveSource(newSource)); - verify(newSource, times(1)).close(); - } -} From af8a3e2726f8fad3f5a6eb80bf1a832ef9c932a4 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Wed, 28 Jan 2026 16:16:17 -0800 Subject: [PATCH 30/35] Cleanup off state emission. --- .../sdk/server/FDv2DataSource.java | 49 +++++++++---------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index 2d2cab02..12b55d1d 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -120,23 +120,17 @@ private void run() { startFuture.complete(true); return; } + if (sourceManager.hasInitializers()) { runInitializers(); } if(!sourceManager.hasAvailableSynchronizers()) { // If already completed by the initializers, then this will have no effect. - if (!isInitialized() && !closed) { - // If we were closed, then closing would have handled our terminal update. - dataSourceUpdates.updateStatus( - DataSourceStatusProvider.State.OFF, - // If we were shutdown during initialization, then we don't need to include an error. - new DataSourceStatusProvider.ErrorInfo( - DataSourceStatusProvider.ErrorKind.UNKNOWN, - 0, - "Initializers exhausted and there are no synchronizers", - new Date().toInstant()) - ); + if (!isInitialized()) { + // If we have no synchronizers, and we didn't manage to initialize, and we aren't shutting down, + // then that was unexpected, and we will report it. + maybeReportUnexpectedExhaustion("All initializers exhausted and there are no available synchronizers."); } // If already completed has no effect. startFuture.complete(false); @@ -144,20 +138,10 @@ private void run() { } runSynchronizers(); - // If we had synchronizers, and we ran out of them, then we are off. - if(!closed) { - dataSourceUpdates.updateStatus( - DataSourceStatusProvider.State.OFF, - // If the data source was closed, then we just report we are OFF without an - // associated error. - new DataSourceStatusProvider.ErrorInfo( - DataSourceStatusProvider.ErrorKind.UNKNOWN, - 0, - "All data source acquisition methods have been exhausted.", - new Date().toInstant()) - ); - } + // If we had synchronizers, and we ran out of them, and we aren't shutting down, then that was unexpected, + // and we will report it. + maybeReportUnexpectedExhaustion("All data source acquisition methods have been exhausted."); // If we had initialized at some point, then the future will already be complete and this will be ignored. startFuture.complete(false); @@ -168,7 +152,6 @@ private void run() { runThread.start(); } - private void runInitializers() { boolean anyDataReceived = false; Initializer initializer = sourceManager.getNextInitializerAndSetActive(); @@ -368,6 +351,7 @@ private void runSynchronizers() { synchronizer = sourceManager.getNextAvailableSynchronizerAndSetActive(); } } catch(Exception e) { + // We are not expecting to encounter this situation, but if we do, then we should log it. logger.error("Unexpected error in DataSource: {}", e.toString()); }finally { sourceManager.close(); @@ -406,6 +390,21 @@ public void close() { startFuture.complete(false); } + private void maybeReportUnexpectedExhaustion(String message) { + if(!closed) { + dataSourceUpdates.updateStatus( + DataSourceStatusProvider.State.OFF, + // If the data source was closed, then we just report we are OFF without an + // associated error. + new DataSourceStatusProvider.ErrorInfo( + DataSourceStatusProvider.ErrorKind.UNKNOWN, + 0, + message, + new Date().toInstant()) + ); + } + } + /** * Helper class to manage the lifecycle of conditions with automatic cleanup. */ From cf7fc3fb32db6e204cb92eb3a0011e1647cb4d5f Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Wed, 28 Jan 2026 16:28:54 -0800 Subject: [PATCH 31/35] Remove debug timeouts. --- .../java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java index e9fd8a49..5dea1f24 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java @@ -1598,10 +1598,10 @@ public void recoveryResetsToFirstAvailableSynchronizer() throws Exception { resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); - startFuture.get(2000, TimeUnit.SECONDS); + startFuture.get(2, TimeUnit.SECONDS); // Wait for 3 applies with enough time for recovery (2s) + overhead - sink.awaitApplyCount(30000, 5, TimeUnit.SECONDS); + sink.awaitApplyCount(3, 5, TimeUnit.SECONDS); // Should have called first synchronizer again after recovery assertTrue(firstCallCount.get() >= 2 || secondCallCount.get() >= 1); From beb300c060e20528d8fd112982d80efeabb50dea Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Wed, 28 Jan 2026 16:50:33 -0800 Subject: [PATCH 32/35] Handle IO exceptions in adapter shutdown. --- .../sdk/server/DataSourceSynchronizerAdapter.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java index 2e9c91c9..0786de7f 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java @@ -90,7 +90,7 @@ public CompletableFuture next() { } @Override - public void close() throws IOException { + public void close() { synchronized (startLock) { if (closed) { return; @@ -98,7 +98,11 @@ public void close() throws IOException { closed = true; } - dataSource.close(); + try { + dataSource.close(); + } catch (IOException e) { + // Ignore as we are shutting down. + } shutdownFuture.complete(FDv2SourceResult.shutdown()); if(startFuture != null) { // If the start future is done, this has no effect. From 26f5ea7468f701604f97e24096ea81f74f7ee5ee Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Thu, 29 Jan 2026 09:37:25 -0800 Subject: [PATCH 33/35] Use 'data source' instead of 'DataSource' in logs for the DataSource. --- .../main/java/com/launchdarkly/sdk/server/FDv2DataSource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index 12b55d1d..1bed2ec9 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -352,7 +352,7 @@ private void runSynchronizers() { } } catch(Exception e) { // We are not expecting to encounter this situation, but if we do, then we should log it. - logger.error("Unexpected error in DataSource: {}", e.toString()); + logger.error("Unexpected error in data source: {}", e.toString()); }finally { sourceManager.close(); } From 546fa87cb26a4301e118aa8e2e30cbe91cb4919c Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Thu, 29 Jan 2026 09:43:48 -0800 Subject: [PATCH 34/35] Use a thread insead of default thread pool for watiing for a FDv1 data source to start. --- .../sdk/server/DataSourceSynchronizerAdapter.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java index 0786de7f..fdd0ccea 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java @@ -64,7 +64,7 @@ public CompletableFuture next() { // Monitor the start future for errors // The data source will emit updates through the listening sink - CompletableFuture.runAsync(() -> { + Thread monitorThread = new Thread(() -> { try { startFuture.get(); } catch (ExecutionException e) { @@ -77,11 +77,14 @@ public CompletableFuture next() { ); resultQueue.put(FDv2SourceResult.interrupted(errorInfo, false)); } catch (CancellationException e) { - // Start future was cancelled (during close) - exit cleanly + // Start future was canceled (during close) - exit cleanly } catch (InterruptedException e) { Thread.currentThread().interrupt(); } }); + monitorThread.setName("LaunchDarkly-SDK-Server-DataSourceAdapter-Monitor"); + monitorThread.setDaemon(true); + monitorThread.start(); } } From dfe0534a7da742a97a428a2c07f7aeb62071b8a9 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Thu, 29 Jan 2026 15:06:34 -0800 Subject: [PATCH 35/35] Mutability PR feedback. --- .../main/java/com/launchdarkly/sdk/server/FDv2DataSource.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index 1bed2ec9..25b6ddab 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -88,7 +88,8 @@ public FDv2DataSource( List synchronizerFactories = synchronizers .stream() .map(SynchronizerFactoryWithState::new) - .collect(Collectors.toList()); + // Collect to an ArrayList to ensure mutability. + .collect(Collectors.toCollection(ArrayList::new)); // If we have a fdv1 data source factory, then add that to the synchronizer factories in a blocked state. // If we receive a request to fallback, then we will unblock it and block all other synchronizers.