feat(gateway): add watchdog independent of HTTP (fixes D1)
Self-restart no longer depends on something polling /health. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3be02ed3a7
commit
d2873efc7c
2 files changed
+134
No files matched your search
@@ -0,0 +1,60 @@
|
||||
package home.gateway.application;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
/**
|
||||
* Ends the process when a critical device stays silent, so systemd restarts the service. It runs
|
||||
* on its own schedule and needs no HTTP traffic (D1).
|
||||
*/
|
||||
public class Watchdog implements AutoCloseable {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(Watchdog.class);
|
||||
|
||||
private final Snapshot snapshot;
|
||||
private final Clock clock;
|
||||
private final Set<String> critical;
|
||||
private final int allowedIntervals;
|
||||
private final Runnable exit;
|
||||
private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(
|
||||
task -> Thread.ofPlatform().daemon().name("watchdog").unstarted(task));
|
||||
|
||||
public Watchdog(Snapshot snapshot, Clock clock, Set<String> critical, int allowedIntervals,
|
||||
Runnable exit) {
|
||||
this.snapshot = snapshot;
|
||||
this.clock = clock;
|
||||
this.critical = critical;
|
||||
this.allowedIntervals = allowedIntervals;
|
||||
this.exit = exit;
|
||||
}
|
||||
|
||||
public void start(Duration period) {
|
||||
scheduler.scheduleWithFixedDelay(this::check, period.toMillis(), period.toMillis(),
|
||||
TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
void check() {
|
||||
var now = clock.instant();
|
||||
for (DeviceState state : snapshot.states()) {
|
||||
var limit = state.interval().multipliedBy(allowedIntervals);
|
||||
if (critical.contains(state.name()) && state.silentFor(now).compareTo(limit) > 0) {
|
||||
LOGGER.error("{} has had no success for {}, limit {}; exiting for restart",
|
||||
state.name(), state.silentFor(now), limit);
|
||||
exit.run();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
scheduler.shutdownNow();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package home.gateway.application;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
class WatchdogTest {
|
||||
|
||||
private static final Instant T0 = Instant.parse("2026-10-24T10:00:00Z");
|
||||
private static final Duration INTERVAL = Duration.ofSeconds(10);
|
||||
|
||||
private final Snapshot snapshot = new Snapshot();
|
||||
private final AtomicInteger exits = new AtomicInteger();
|
||||
|
||||
private int exitsAt(Instant now, Set<String> critical) {
|
||||
new Watchdog(snapshot, Clock.fixed(now, ZoneOffset.UTC), critical, 3, exits::incrementAndGet)
|
||||
.check();
|
||||
return exits.get();
|
||||
}
|
||||
|
||||
@Test
|
||||
void doesNotFireWithinTheLimit() {
|
||||
snapshot.register("pump", INTERVAL, T0);
|
||||
|
||||
assertEquals(0, exitsAt(T0.plusSeconds(30), Set.of("pump")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void firesForCriticalDeviceThatNeverSucceeded() {
|
||||
snapshot.register("pump", INTERVAL, T0);
|
||||
|
||||
assertEquals(1, exitsAt(T0.plusSeconds(31), Set.of("pump")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void countsFromTheLastSuccess() {
|
||||
snapshot.register("pump", INTERVAL, T0);
|
||||
snapshot.recordSuccess("pump", T0.plusSeconds(100), List.of());
|
||||
|
||||
assertEquals(0, exitsAt(T0.plusSeconds(130), Set.of("pump")));
|
||||
assertEquals(1, exitsAt(T0.plusSeconds(131), Set.of("pump")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void neverFiresForNonCriticalDevice() {
|
||||
snapshot.register("meter", INTERVAL, T0);
|
||||
|
||||
assertEquals(0, exitsAt(T0.plusSeconds(3600), Set.of("pump")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void firesOnItsOwnScheduleWithoutAnyHttpTraffic() throws Exception {
|
||||
snapshot.register("pump", Duration.ofMillis(1), Instant.now().minusSeconds(60));
|
||||
try (var watchdog = new Watchdog(snapshot, Clock.systemUTC(), Set.of("pump"), 3,
|
||||
exits::incrementAndGet)) {
|
||||
watchdog.start(Duration.ofMillis(5));
|
||||
long deadline = System.nanoTime() + Duration.ofSeconds(5).toNanos();
|
||||
while (exits.get() == 0 && System.nanoTime() < deadline) {
|
||||
Thread.sleep(5);
|
||||
}
|
||||
}
|
||||
|
||||
assertTrue(exits.get() > 0);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user