fix(watchdog): fire once and log the device's last error
CI / Backend tests (Maven / Java 21) (push) Successful in 1m3s
CI / Backend tests (Maven / Java 21) (push) Successful in 1m3s
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
eb0b23637b
commit
1a8e8e8585
2 files changed
+31
-4
No files matched your search
@@ -9,10 +9,12 @@ import java.util.Set;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
/**
|
||||
* Ends the process when a critical device stays silent, so systemd restarts the service. It runs
|
||||
* on its own schedule and needs no HTTP traffic (D1).
|
||||
* on its own schedule and needs no HTTP traffic (D1). Silence is counted from the start of this
|
||||
* process, so a restart never fires again before a full limit has passed, and it fires once.
|
||||
*/
|
||||
public class Watchdog implements AutoCloseable {
|
||||
|
||||
@@ -23,6 +25,7 @@ public class Watchdog implements AutoCloseable {
|
||||
private final Set<String> critical;
|
||||
private final int allowedIntervals;
|
||||
private final Runnable exit;
|
||||
private final AtomicBoolean fired = new AtomicBoolean();
|
||||
private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(
|
||||
task -> Thread.ofPlatform().daemon().name("watchdog").unstarted(task));
|
||||
|
||||
@@ -45,9 +48,12 @@ public class Watchdog implements AutoCloseable {
|
||||
for (DeviceState state : snapshot.states()) {
|
||||
var limit = state.interval().multipliedBy(allowedIntervals);
|
||||
if (critical.contains(state.name()) && state.silentFor(now).compareTo(limit) > 0) {
|
||||
LOGGER.error("{} has had no success for {}, limit {}; exiting for restart",
|
||||
state.name(), state.silentFor(now), limit);
|
||||
exit.run();
|
||||
if (fired.compareAndSet(false, true)) {
|
||||
LOGGER.error("{} has had no success for {}, limit {}, last error: {}; "
|
||||
+ "exiting for restart", state.name(), state.silentFor(now), limit,
|
||||
state.lastError());
|
||||
exit.run();
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,6 +50,27 @@ class WatchdogTest {
|
||||
assertEquals(1, exitsAt(T0.plusSeconds(131), Set.of("pump")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void firesOnlyOnceEvenWhenExitReturns() {
|
||||
snapshot.register("pump", INTERVAL, T0);
|
||||
var watchdog = new Watchdog(snapshot, Clock.fixed(T0.plusSeconds(60), ZoneOffset.UTC),
|
||||
Set.of("pump"), 3, exits::incrementAndGet);
|
||||
|
||||
watchdog.check();
|
||||
watchdog.check();
|
||||
|
||||
assertEquals(1, exits.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void restartedProcessGetsAFullGracePeriodEvenIfTheDeviceFailedBefore() {
|
||||
snapshot.register("pump", INTERVAL, T0.plusSeconds(1000));
|
||||
snapshot.recordFailure("pump", T0.plusSeconds(1010), "busy: app connected");
|
||||
|
||||
assertEquals(0, exitsAt(T0.plusSeconds(1030), Set.of("pump")));
|
||||
assertEquals(1, exitsAt(T0.plusSeconds(1031), Set.of("pump")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void neverFiresForNonCriticalDevice() {
|
||||
snapshot.register("meter", INTERVAL, T0);
|
||||
|
||||
Reference in new issue
Block a user