113 lines
3.4 KiB
Bash
Executable File
113 lines
3.4 KiB
Bash
Executable File
#!/bin/bash
|
|
# Health check and restart script for home-pump service
|
|
# Run this via cron every 2 minutes: */2 * * * * /home/pi/home-pump/scripts/healthcheck-monitor.sh
|
|
|
|
SERVICE_NAME="home-pump.service"
|
|
HEALTH_ENDPOINT="http://localhost:8080/health"
|
|
LOG_FILE="/var/log/home-pump-monitor.log"
|
|
MAX_RESTART_ATTEMPTS=3
|
|
RESTART_WINDOW_SECONDS=600
|
|
|
|
RESTART_COUNT_FILE="/tmp/home-pump-restart-count"
|
|
RESTART_TIMESTAMP_FILE="/tmp/home-pump-restart-timestamp"
|
|
|
|
log() {
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
|
|
}
|
|
|
|
# Function to check if service is running
|
|
is_service_running() {
|
|
systemctl is-active --quiet "$SERVICE_NAME"
|
|
}
|
|
|
|
# Function to get restart count in the current window
|
|
get_restart_count() {
|
|
if [ -f "$RESTART_COUNT_FILE" ]; then
|
|
cat "$RESTART_COUNT_FILE"
|
|
else
|
|
echo "0"
|
|
fi
|
|
}
|
|
|
|
# Function to get last restart timestamp
|
|
get_last_restart_time() {
|
|
if [ -f "$RESTART_TIMESTAMP_FILE" ]; then
|
|
cat "$RESTART_TIMESTAMP_FILE"
|
|
else
|
|
echo "0"
|
|
fi
|
|
}
|
|
|
|
# Function to reset restart counter if window expired
|
|
reset_counter_if_expired() {
|
|
local last_restart=$(get_last_restart_time)
|
|
local now=$(date +%s)
|
|
local diff=$((now - last_restart))
|
|
|
|
if [ "$diff" -gt "$RESTART_WINDOW_SECONDS" ]; then
|
|
echo "0" > "$RESTART_COUNT_FILE"
|
|
fi
|
|
}
|
|
|
|
# Function to increment restart counter
|
|
increment_restart_count() {
|
|
local count=$(get_restart_count)
|
|
count=$((count + 1))
|
|
echo "$count" > "$RESTART_COUNT_FILE"
|
|
date +%s > "$RESTART_TIMESTAMP_FILE"
|
|
}
|
|
|
|
# Check if service is running
|
|
if ! is_service_running; then
|
|
log "ERROR: Service $SERVICE_NAME is not running. Attempting to start..."
|
|
sudo systemctl start "$SERVICE_NAME"
|
|
sleep 5
|
|
if is_service_running; then
|
|
log "SUCCESS: Service restarted successfully"
|
|
else
|
|
log "CRITICAL: Failed to start service $SERVICE_NAME"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# Check health endpoint
|
|
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "$HEALTH_ENDPOINT" --max-time 10)
|
|
|
|
if [ "$HTTP_CODE" != "200" ]; then
|
|
log "WARNING: Health check failed with HTTP code: $HTTP_CODE"
|
|
|
|
# Reset counter if restart window expired
|
|
reset_counter_if_expired
|
|
|
|
# Check if we can restart
|
|
restart_count=$(get_restart_count)
|
|
|
|
if [ "$restart_count" -ge "$MAX_RESTART_ATTEMPTS" ]; then
|
|
log "CRITICAL: Maximum restart attempts ($MAX_RESTART_ATTEMPTS) reached in the last $RESTART_WINDOW_SECONDS seconds. Manual intervention required."
|
|
# Send alert (could be email, Pushover, etc.)
|
|
# Example: echo "Home pump service needs manual intervention" | mail -s "Critical: Home Pump Alert" admin@example.com
|
|
exit 2
|
|
fi
|
|
|
|
log "Attempting to restart service (attempt $((restart_count + 1))/$MAX_RESTART_ATTEMPTS)..."
|
|
increment_restart_count
|
|
|
|
sudo systemctl restart "$SERVICE_NAME"
|
|
sleep 10
|
|
|
|
# Verify restart
|
|
if is_service_running; then
|
|
NEW_HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "$HEALTH_ENDPOINT" --max-time 10)
|
|
if [ "$NEW_HTTP_CODE" == "200" ]; then
|
|
log "SUCCESS: Service restarted and health check passed"
|
|
else
|
|
log "WARNING: Service restarted but health check still failing (HTTP $NEW_HTTP_CODE)"
|
|
fi
|
|
else
|
|
log "ERROR: Failed to restart service"
|
|
fi
|
|
else
|
|
# Health check passed, reset counter
|
|
echo "0" > "$RESTART_COUNT_FILE"
|
|
fi
|