fix(watchdog): fire once and log the device's last error
CI / Backend tests (Maven / Java 21) (push) Successful in 1m3s

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
rootandClaude Sonnet 5.5 committed 2026-10-24 00:46:02 +02:00
1 parent eb0b23637b
commit 1a8e8e8585
2 files changed
+31 -4

No files matched your search

@@ -9,10 +9,12 @@ import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
/**
* Ends the process when a critical device stays silent, so systemd restarts the service. It runs
* on its own schedule and needs no HTTP traffic (D1).
* on its own schedule and needs no HTTP traffic (D1). Silence is counted from the start of this
* process, so a restart never fires again before a full limit has passed, and it fires once.
*/
public class Watchdog implements AutoCloseable {
@@ -23,6 +25,7 @@ public class Watchdog implements AutoCloseable {
private final Set<String> critical;
private final int allowedIntervals;
private final Runnable exit;
private final AtomicBoolean fired = new AtomicBoolean();
private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(
task -> Thread.ofPlatform().daemon().name("watchdog").unstarted(task));
@@ -45,9 +48,12 @@ public class Watchdog implements AutoCloseable {
for (DeviceState state : snapshot.states()) {
var limit = state.interval().multipliedBy(allowedIntervals);
if (critical.contains(state.name()) && state.silentFor(now).compareTo(limit) > 0) {
LOGGER.error("{} has had no success for {}, limit {}; exiting for restart",
state.name(), state.silentFor(now), limit);
exit.run();
if (fired.compareAndSet(false, true)) {
LOGGER.error("{} has had no success for {}, limit {}, last error: {}; "
+ "exiting for restart", state.name(), state.silentFor(now), limit,
state.lastError());
exit.run();
}
return;
}
}
@@ -50,6 +50,27 @@ class WatchdogTest {
assertEquals(1, exitsAt(T0.plusSeconds(131), Set.of("pump")));
}
@Test
void firesOnlyOnceEvenWhenExitReturns() {
snapshot.register("pump", INTERVAL, T0);
var watchdog = new Watchdog(snapshot, Clock.fixed(T0.plusSeconds(60), ZoneOffset.UTC),
Set.of("pump"), 3, exits::incrementAndGet);
watchdog.check();
watchdog.check();
assertEquals(1, exits.get());
}
@Test
void restartedProcessGetsAFullGracePeriodEvenIfTheDeviceFailedBefore() {
snapshot.register("pump", INTERVAL, T0.plusSeconds(1000));
snapshot.recordFailure("pump", T0.plusSeconds(1010), "busy: app connected");
assertEquals(0, exitsAt(T0.plusSeconds(1030), Set.of("pump")));
assertEquals(1, exitsAt(T0.plusSeconds(1031), Set.of("pump")));
}
@Test
void neverFiresForNonCriticalDevice() {
snapshot.register("meter", INTERVAL, T0);