Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,10 @@ default ResourceEventRecorder eventRecorder() {
* reconciliation is already scheduled, which would in turn trigger another status update, thus
* rendering the current one moot.
*
* <p>This holds regardless of whether the current reconciliation succeeds or throws, so with
* {@link ControllerConfiguration#triggerReconcilerOnAllEvents()} it also covers an event that
* arrived after a delete event.
*
* @return {@code true} is another reconciliation is already scheduled, {@code false} otherwise
*/
boolean isNextReconciliationImminent();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -380,9 +380,7 @@ private void handleRetryOnException(
ExecutionScope<P> executionScope, Exception exception, boolean errorHandledByReconciler) {
final var state = getOrInitRetryExecution(executionScope);
var resourceID = state.getId();
boolean eventPresent =
state.eventPresent()
|| (triggerOnAllEvents() && state.isAdditionalEventPresentAfterDeleteEvent());
boolean eventPresent = isNextReconciliationImminent(state);
state.markEventReceived();
retryAwareErrorLogging(
state.getRetry(), eventPresent, errorHandledByReconciler, exception, executionScope);
Expand Down Expand Up @@ -510,8 +508,20 @@ public synchronized void start() throws OperatorException {
handleAlreadyMarkedEvents();
}

public boolean isNextReconciliationImminent(ResourceID resourceID) {
return resourceStateManager.getOrCreate(resourceID).eventPresent();
public synchronized boolean isNextReconciliationImminent(ResourceID resourceID) {
return isNextReconciliationImminent(resourceStateManager.getOrCreate(resourceID));
}

/**
* An event that arrives after a delete event is tracked in a dedicated state, so {@link
* ResourceState#eventPresent()} alone does not cover it. Such an event triggers a new
* reconciliation right after the current one, both when it succeeds (see {@link
* #eventProcessingFinished}) and when it fails (see {@link #handleRetryOnException}), so it has
* to be reported as imminent too.
*/
private boolean isNextReconciliationImminent(ResourceState state) {
return state.eventPresent()
|| (triggerOnAllEvents() && state.isAdditionalEventPresentAfterDeleteEvent());
Comment thread
csviri marked this conversation as resolved.
}

private void handleAlreadyMarkedEvents() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -192,14 +192,16 @@ void additionalEventDuringRetryOnDeleteEvent() {
await()
.untilAsserted(
() -> {
assertThat(reconciler.isWaiting());
assertThat(reconciler.isWaiting()).isTrue();
});

// trigger reconciliation while waiting in reconciler
res = getResource();
res.getMetadata().getAnnotations().put("my-annotation", "true");
extension.update(res);
// continue reconciliation
// continue reconciliation; the reconciler additionally waits until the framework actually
// registered the event above, otherwise the failure below would consume a retry attempt
// instead of being instantly re-triggered by the superseding event
reconciler.setContinuerOnRetryWait(true);

await()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@ public class TriggerReconcilerOnAllEventReconciler
public static final String ADDITIONAL_FINALIZER = "all.event.mode/finalizer2";
public static final String NO_MORE_EXCEPTION_ANNOTATION_KEY = "no.more.exception";

// safety net so a missing event does not block the reconciler thread forever, the test assertions
// fail long before this elapses
private static final long MAX_WAIT_FOR_SUPERSEDING_EVENT_MILLIS = 30_000;

private static final Logger log =
LoggerFactory.getLogger(TriggerReconcilerOnAllEventReconciler.class);

Expand All @@ -47,6 +51,7 @@ public class TriggerReconcilerOnAllEventReconciler
private volatile boolean waitAfterFirstRetry = false;
private volatile boolean continuerOnRetryWait = false;
private volatile boolean waiting = false;
private volatile boolean alreadyWaitedAfterFirstRetry = false;

// control flag to throw an exception on first delete event
private volatile boolean isFirstDeleteEvent = true;
Expand Down Expand Up @@ -79,10 +84,24 @@ public UpdateControl<TriggerReconcilerOnAllEventCustomResource> reconcile(
}

if (waitAfterFirstRetry
&& !alreadyWaitedAfterFirstRetry
&& context.getRetryInfo().isPresent()
&& context.getRetryInfo().orElseThrow().getAttemptCount() == 1) {
// The reconciliation triggered by the superseding event below reuses the same retry
// execution, so its attempt count is still 1. Wait only on the very first one, otherwise that
// follow-up reconciliation would block here too and never be released.
alreadyWaitedAfterFirstRetry = true;
waiting = true;
while (!continuerOnRetryWait) {
// Releasing on continuerOnRetryWait alone is racy: the test sets that flag right after the
// update call returns, but the update event still has to travel back through the informer.
// If this reconciliation failed before the event was registered, the framework would treat
// the failure as a plain retry (consuming the last attempt) instead of instantly
// re-triggering because of a superseding event. isNextReconciliationImminent() reports the
// same condition the framework evaluates after this reconciliation fails, and it cannot be
// unset while this reconciliation is in progress.
var waitUntil = System.currentTimeMillis() + MAX_WAIT_FOR_SUPERSEDING_EVENT_MILLIS;
while ((!continuerOnRetryWait || !context.isNextReconciliationImminent())
&& System.currentTimeMillis() < waitUntil) {
Comment thread
csviri marked this conversation as resolved.
Thread.sleep(50);
}
waiting = false;
Expand Down
Loading