879f97c843
- docs/incident-log: mark x4078 recovered remotely via Tailscale (no physical access needed); document the recovery steps and root cause - connect-daemon.sh: remove bare `wait` that wedged the retry loop forever on the setsid'd tailscaled child — once the tunnel failed the daemon could not self-heal until the package was restarted; now the main loop falls through start_tunnel (already blocking+retrying) and restarts the whole cycle cleanly - SPK rebuilt (32839680 bytes) with the daemon fix included - Also documents the two latent lessons: (1) a field router is only inaccessible when ALL three paths fail; (2) always check Tailscale + QuickConnect before assuming a truck roll is required Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
216 lines
8.3 KiB
Bash
Executable File
216 lines
8.3 KiB
Bash
Executable File
#!/bin/sh
|
|
# kit-connect daemon — manages Tailscale + reverse SSH tunnel.
|
|
# Config: /etc/kit-connect/connect.conf (fetched from hub on install)
|
|
# One SPK, all routers. Hub assigns ports and keys.
|
|
|
|
PKG="kit-connect"
|
|
PKG_DIR="/var/packages/${PKG}/target"
|
|
CONF="/etc/${PKG}/connect.conf"
|
|
KEY="$PKG_DIR/bin/connect_id_ed25519"
|
|
DAEMON_PID_FILE="$PKG_DIR/var/daemon.pid"
|
|
LOG_FILE="$PKG_DIR/var/connect.log"
|
|
LOG_TAG="kit-connect"
|
|
|
|
log() {
|
|
logger -t "$LOG_TAG" -p local0.warn "$*"
|
|
printf '%s %s: %s\n' "$(date '+%b %d %H:%M:%S')" "$LOG_TAG" "$*" >> "$LOG_FILE"
|
|
_lc=$(wc -l < "$LOG_FILE" 2>/dev/null || echo 0)
|
|
[ "$_lc" -gt 100 ] && tail -n 60 "$LOG_FILE" > "${LOG_FILE}.tmp" && mv "${LOG_FILE}.tmp" "$LOG_FILE"
|
|
}
|
|
|
|
mkdir -p "$PKG_DIR/var"
|
|
chmod 600 "$KEY" 2>/dev/null
|
|
|
|
# ── Load config ────────────────────────────────────────────────────
|
|
load_config() {
|
|
[ -f "$CONF" ] && . "$CONF" 2>/dev/null
|
|
|
|
: "${DEVICE_ID:=unknown}"
|
|
: "${HUB_HOST:=162.243.83.36}"
|
|
: "${HUB_PORT:=8080}"
|
|
: "${TUNNEL_ENABLE:=1}"
|
|
: "${TAILSCALE_ENABLE:=1}"
|
|
|
|
# Tunnel settings (hub-assigned or defaults)
|
|
: "${TUNNEL_REMOTE_HOST:=162.243.83.36}"
|
|
: "${TUNNEL_REMOTE_USER:=node}"
|
|
: "${TUNNEL_REMOTE_SSH_PORT:=22}"
|
|
: "${TUNNEL_REMOTE_PORT:=0}" # 0 = auto-assign by hub
|
|
: "${TUNNEL_LOCAL_SSH_PORT:=2223}"
|
|
: "${TUNNEL_RETRY_DELAY:=30}"
|
|
|
|
# Tailscale settings
|
|
: "${TAILSCALE_STATEDIR:=/var/packages/Tailscale/var/state}"
|
|
: "${TAILSCALE_SOCKET:=/var/packages/Tailscale/var/run/tailscaled.sock}"
|
|
: "${TAILSCALE_AUTH_KEY:=}"
|
|
: "${TAILSCALE_HOSTNAME:=${DEVICE_ID}}"
|
|
|
|
# Tailscale binary location (varies: SRM has /usr/local/bin, GL has /usr/bin)
|
|
: "${TAILSCALE_BIN:=$(which tailscale 2>/dev/null || echo /usr/local/bin/tailscale)}"
|
|
: "${TAILSCALED_BIN:=$(which tailscaled 2>/dev/null || echo /usr/local/bin/tailscaled)}"
|
|
|
|
# watchdog interval (seconds between forward health checks)
|
|
: "${WATCHDOG_INTERVAL:=60}"
|
|
: "${FORWARD_FAIL_LIMIT:=2}"
|
|
}
|
|
|
|
log "EVENT=DAEMON_START device=${DEVICE_ID}"
|
|
|
|
# ── Start Tailscale ────────────────────────────────────────────────
|
|
start_tailscale() {
|
|
[ "$TAILSCALE_ENABLE" != "1" ] && { log "tailscale: disabled in config"; return 0; }
|
|
[ -z "$TAILSCALE_AUTH_KEY" ] && { log "tailscale: no auth key — skipping"; return 0; }
|
|
|
|
if [ ! -x "$TAILSCALED_BIN" ]; then
|
|
log "tailscale: tailscaled not found at $TAILSCALED_BIN (SRM PATH lacks /usr/local/bin)"
|
|
return 1
|
|
fi
|
|
if [ ! -x "$TAILSCALE_BIN" ]; then
|
|
log "tailscale: binary not found at $TAILSCALE_BIN"
|
|
return 1
|
|
fi
|
|
|
|
mkdir -p "$(dirname "$TAILSCALE_STATEDIR")" "$(dirname "$TAILSCALE_SOCKET")"
|
|
|
|
# Already running?
|
|
if "$TAILSCALE_BIN" --socket="$TAILSCALE_SOCKET" status >/dev/null 2>&1; then
|
|
log "tailscale: already connected — $( "$TAILSCALE_BIN" --socket="$TAILSCALE_SOCKET" ip -4 2>/dev/null || echo no-ip)"
|
|
return 0
|
|
fi
|
|
|
|
# Start tailscaled (userspace mode for SRM compat)
|
|
log "tailscale: starting tailscaled..."
|
|
/usr/bin/setsid "$TAILSCALED_BIN" \
|
|
--statedir="$TAILSCALE_STATEDIR" \
|
|
--tun=userspace-networking \
|
|
--socket="$TAILSCALE_SOCKET" \
|
|
>/dev/null 2>&1 &
|
|
sleep 4
|
|
|
|
# Authenticate
|
|
log "tailscale: authenticating with auth key..."
|
|
"$TAILSCALE_BIN" --socket="$TAILSCALE_SOCKET" up \
|
|
--auth-key "$TAILSCALE_AUTH_KEY" \
|
|
--hostname "${TAILSCALE_HOSTNAME}" \
|
|
--accept-routes=false \
|
|
--accept-dns=false 2>&1 | while read line; do log "tailscale: $line"; done
|
|
|
|
ts_ip=$("$TAILSCALE_BIN" --socket="$TAILSCALE_SOCKET" ip -4 2>/dev/null || echo "unknown")
|
|
log "tailscale: connected — IP=${ts_ip}"
|
|
}
|
|
|
|
# ── Start reverse SSH tunnel ───────────────────────────────────────
|
|
start_tunnel() {
|
|
[ "$TUNNEL_ENABLE" != "1" ] && { log "tunnel: disabled in config"; return 0; }
|
|
[ "$TUNNEL_REMOTE_PORT" = "0" ] && { log "tunnel: no port assigned — run hub registration first"; return 0; }
|
|
|
|
log "tunnel: opening R:0.0.0.0:${TUNNEL_REMOTE_PORT} -> 127.0.0.1:${TUNNEL_LOCAL_SSH_PORT}"
|
|
|
|
ssh \
|
|
-o StrictHostKeyChecking=no \
|
|
-o UserKnownHostsFile=/dev/null \
|
|
-o ServerAliveInterval=15 \
|
|
-o ServerAliveCountMax=3 \
|
|
-o ExitOnForwardFailure=yes \
|
|
-o BatchMode=yes \
|
|
-p "${TUNNEL_REMOTE_SSH_PORT}" \
|
|
-N \
|
|
-R "0.0.0.0:${TUNNEL_REMOTE_PORT}:127.0.0.1:${TUNNEL_LOCAL_SSH_PORT}" \
|
|
-i "$KEY" \
|
|
"${TUNNEL_REMOTE_USER}@${TUNNEL_REMOTE_HOST}" \
|
|
2>"$PKG_DIR/var/ssh_err.tmp" &
|
|
|
|
SSH_PID=$!
|
|
echo "$SSH_PID" > "$PKG_DIR/var/tunnel.pid"
|
|
log "tunnel: SSH PID=${SSH_PID} port=${TUNNEL_REMOTE_PORT}"
|
|
|
|
# Watchdog loop
|
|
_fail_count=0
|
|
while kill -0 "$SSH_PID" 2>/dev/null; do
|
|
sleep "$WATCHDOG_INTERVAL"
|
|
if ! kill -0 "$SSH_PID" 2>/dev/null; then break; fi
|
|
|
|
# Probe forward health
|
|
if timeout 10 ssh \
|
|
-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
|
-o ConnectTimeout=5 -o BatchMode=yes \
|
|
-p "${TUNNEL_REMOTE_SSH_PORT}" -i "$KEY" \
|
|
"${TUNNEL_REMOTE_USER}@${TUNNEL_REMOTE_HOST}" \
|
|
"timeout 3 bash -c 'echo >/dev/tcp/127.0.0.1/${TUNNEL_REMOTE_PORT}' 2>/dev/null" \
|
|
2>/dev/null; then
|
|
[ "$_fail_count" -gt 0 ] && log "tunnel: forward recovered after ${_fail_count} checks"
|
|
_fail_count=0
|
|
else
|
|
_fail_count=$((_fail_count + 1))
|
|
log "tunnel: forward check failed (${_fail_count}/${FORWARD_FAIL_LIMIT})"
|
|
[ "$_fail_count" -ge "$FORWARD_FAIL_LIMIT" ] && break
|
|
fi
|
|
done
|
|
|
|
[ -f "$PKG_DIR/var/tunnel.pid" ] && kill "$(cat "$PKG_DIR/var/tunnel.pid")" 2>/dev/null
|
|
rm -f "$PKG_DIR/var/tunnel.pid"
|
|
log "tunnel: exited — will retry in ${TUNNEL_RETRY_DELAY}s"
|
|
sleep "$TUNNEL_RETRY_DELAY"
|
|
return 1
|
|
}
|
|
|
|
# ── Hub registration — get assigned port ───────────────────────────
|
|
register_with_hub() {
|
|
# Already registered — skip
|
|
if [ "$TUNNEL_REMOTE_PORT" != "0" ] && [ -n "$TUNNEL_REMOTE_PORT" ]; then
|
|
return 0
|
|
fi
|
|
|
|
log "hub: registering device ${DEVICE_ID}..."
|
|
|
|
# --max-time 15: total timeout (connect + response). SRM busybox curl needs this
|
|
# or the daemon hangs forever if the hub accepts the TCP connection but
|
|
# never sends the HTTP response.
|
|
resp=""
|
|
resp=$(curl -s --connect-timeout 10 --max-time 15 \
|
|
"http://${HUB_HOST}:${HUB_PORT}/api/register/${DEVICE_ID}" 2>/dev/null) || true
|
|
|
|
if [ -z "$resp" ]; then
|
|
log "hub: unreachable — will retry next cycle"
|
|
return 1
|
|
fi
|
|
|
|
# Parse JSON response — POSIX-safe, no jq dependency
|
|
port=""
|
|
tskey=""
|
|
port=$(printf '%s' "$resp" | sed -n 's/.*"tunnel_port"[[:space:]]*:[[:space:]]*\([0-9]*\).*/\1/p')
|
|
tskey=$(printf '%s' "$resp" | sed -n 's/.*"tailscale_auth_key"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p')
|
|
|
|
if [ -n "$port" ] && [ "$port" != "0" ]; then
|
|
sed -i "s/^TUNNEL_REMOTE_PORT=.*/TUNNEL_REMOTE_PORT=${port}/" "$CONF" 2>/dev/null
|
|
TUNNEL_REMOTE_PORT="$port"
|
|
log "hub: assigned port ${port}"
|
|
fi
|
|
|
|
if [ -n "$tskey" ] && [ "$tskey" != "$TAILSCALE_AUTH_KEY" ]; then
|
|
sed -i "s|^TAILSCALE_AUTH_KEY=.*|TAILSCALE_AUTH_KEY=${tskey}|" "$CONF" 2>/dev/null
|
|
TAILSCALE_AUTH_KEY="$tskey"
|
|
log "hub: updated Tailscale key"
|
|
fi
|
|
}
|
|
|
|
# ── Main ────────────────────────────────────────────────────────────
|
|
load_config
|
|
echo "$$" > "$DAEMON_PID_FILE"
|
|
|
|
while true; do
|
|
load_config
|
|
|
|
# Register with hub if needed
|
|
register_with_hub 2>/dev/null || true
|
|
|
|
# Start Tailscale in background (stays running)
|
|
start_tailscale 2>/dev/null || true
|
|
|
|
# Start tunnel (blocking — restarts on failure)
|
|
start_tunnel 2>/dev/null || true
|
|
|
|
# start_tunnel is already blocking and self-retrying; no bare wait here — a bare
|
|
# wait blocks forever on the setsid'd tailscaled child and wedges the retry loop.
|
|
done
|