Files
kit-busrouter/kit-connect/bin/connect-daemon.sh
T
kitadmin 879f97c843 fix: x4078 remote recovery + daemon wait-wedge bug + SPK rebuild
- docs/incident-log: mark x4078 recovered remotely via Tailscale (no
  physical access needed); document the recovery steps and root cause
- connect-daemon.sh: remove bare `wait` that wedged the retry loop
  forever on the setsid'd tailscaled child — once the tunnel failed the
  daemon could not self-heal until the package was restarted; now the
  main loop falls through start_tunnel (already blocking+retrying) and
  restarts the whole cycle cleanly
- SPK rebuilt (32839680 bytes) with the daemon fix included
- Also documents the two latent lessons: (1) a field router is only
  inaccessible when ALL three paths fail; (2) always check Tailscale +
  QuickConnect before assuming a truck roll is required

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-07-22 22:23:53 +00:00

216 lines
8.3 KiB
Bash
Executable File

#!/bin/sh
# kit-connect daemon — manages Tailscale + reverse SSH tunnel.
# Config: /etc/kit-connect/connect.conf (fetched from hub on install)
# One SPK, all routers. Hub assigns ports and keys.
PKG="kit-connect"
PKG_DIR="/var/packages/${PKG}/target"
CONF="/etc/${PKG}/connect.conf"
KEY="$PKG_DIR/bin/connect_id_ed25519"
DAEMON_PID_FILE="$PKG_DIR/var/daemon.pid"
LOG_FILE="$PKG_DIR/var/connect.log"
LOG_TAG="kit-connect"
log() {
logger -t "$LOG_TAG" -p local0.warn "$*"
printf '%s %s: %s\n' "$(date '+%b %d %H:%M:%S')" "$LOG_TAG" "$*" >> "$LOG_FILE"
_lc=$(wc -l < "$LOG_FILE" 2>/dev/null || echo 0)
[ "$_lc" -gt 100 ] && tail -n 60 "$LOG_FILE" > "${LOG_FILE}.tmp" && mv "${LOG_FILE}.tmp" "$LOG_FILE"
}
mkdir -p "$PKG_DIR/var"
chmod 600 "$KEY" 2>/dev/null
# ── Load config ────────────────────────────────────────────────────
load_config() {
[ -f "$CONF" ] && . "$CONF" 2>/dev/null
: "${DEVICE_ID:=unknown}"
: "${HUB_HOST:=162.243.83.36}"
: "${HUB_PORT:=8080}"
: "${TUNNEL_ENABLE:=1}"
: "${TAILSCALE_ENABLE:=1}"
# Tunnel settings (hub-assigned or defaults)
: "${TUNNEL_REMOTE_HOST:=162.243.83.36}"
: "${TUNNEL_REMOTE_USER:=node}"
: "${TUNNEL_REMOTE_SSH_PORT:=22}"
: "${TUNNEL_REMOTE_PORT:=0}" # 0 = auto-assign by hub
: "${TUNNEL_LOCAL_SSH_PORT:=2223}"
: "${TUNNEL_RETRY_DELAY:=30}"
# Tailscale settings
: "${TAILSCALE_STATEDIR:=/var/packages/Tailscale/var/state}"
: "${TAILSCALE_SOCKET:=/var/packages/Tailscale/var/run/tailscaled.sock}"
: "${TAILSCALE_AUTH_KEY:=}"
: "${TAILSCALE_HOSTNAME:=${DEVICE_ID}}"
# Tailscale binary location (varies: SRM has /usr/local/bin, GL has /usr/bin)
: "${TAILSCALE_BIN:=$(which tailscale 2>/dev/null || echo /usr/local/bin/tailscale)}"
: "${TAILSCALED_BIN:=$(which tailscaled 2>/dev/null || echo /usr/local/bin/tailscaled)}"
# watchdog interval (seconds between forward health checks)
: "${WATCHDOG_INTERVAL:=60}"
: "${FORWARD_FAIL_LIMIT:=2}"
}
log "EVENT=DAEMON_START device=${DEVICE_ID}"
# ── Start Tailscale ────────────────────────────────────────────────
start_tailscale() {
[ "$TAILSCALE_ENABLE" != "1" ] && { log "tailscale: disabled in config"; return 0; }
[ -z "$TAILSCALE_AUTH_KEY" ] && { log "tailscale: no auth key — skipping"; return 0; }
if [ ! -x "$TAILSCALED_BIN" ]; then
log "tailscale: tailscaled not found at $TAILSCALED_BIN (SRM PATH lacks /usr/local/bin)"
return 1
fi
if [ ! -x "$TAILSCALE_BIN" ]; then
log "tailscale: binary not found at $TAILSCALE_BIN"
return 1
fi
mkdir -p "$(dirname "$TAILSCALE_STATEDIR")" "$(dirname "$TAILSCALE_SOCKET")"
# Already running?
if "$TAILSCALE_BIN" --socket="$TAILSCALE_SOCKET" status >/dev/null 2>&1; then
log "tailscale: already connected — $( "$TAILSCALE_BIN" --socket="$TAILSCALE_SOCKET" ip -4 2>/dev/null || echo no-ip)"
return 0
fi
# Start tailscaled (userspace mode for SRM compat)
log "tailscale: starting tailscaled..."
/usr/bin/setsid "$TAILSCALED_BIN" \
--statedir="$TAILSCALE_STATEDIR" \
--tun=userspace-networking \
--socket="$TAILSCALE_SOCKET" \
>/dev/null 2>&1 &
sleep 4
# Authenticate
log "tailscale: authenticating with auth key..."
"$TAILSCALE_BIN" --socket="$TAILSCALE_SOCKET" up \
--auth-key "$TAILSCALE_AUTH_KEY" \
--hostname "${TAILSCALE_HOSTNAME}" \
--accept-routes=false \
--accept-dns=false 2>&1 | while read line; do log "tailscale: $line"; done
ts_ip=$("$TAILSCALE_BIN" --socket="$TAILSCALE_SOCKET" ip -4 2>/dev/null || echo "unknown")
log "tailscale: connected — IP=${ts_ip}"
}
# ── Start reverse SSH tunnel ───────────────────────────────────────
start_tunnel() {
[ "$TUNNEL_ENABLE" != "1" ] && { log "tunnel: disabled in config"; return 0; }
[ "$TUNNEL_REMOTE_PORT" = "0" ] && { log "tunnel: no port assigned — run hub registration first"; return 0; }
log "tunnel: opening R:0.0.0.0:${TUNNEL_REMOTE_PORT} -> 127.0.0.1:${TUNNEL_LOCAL_SSH_PORT}"
ssh \
-o StrictHostKeyChecking=no \
-o UserKnownHostsFile=/dev/null \
-o ServerAliveInterval=15 \
-o ServerAliveCountMax=3 \
-o ExitOnForwardFailure=yes \
-o BatchMode=yes \
-p "${TUNNEL_REMOTE_SSH_PORT}" \
-N \
-R "0.0.0.0:${TUNNEL_REMOTE_PORT}:127.0.0.1:${TUNNEL_LOCAL_SSH_PORT}" \
-i "$KEY" \
"${TUNNEL_REMOTE_USER}@${TUNNEL_REMOTE_HOST}" \
2>"$PKG_DIR/var/ssh_err.tmp" &
SSH_PID=$!
echo "$SSH_PID" > "$PKG_DIR/var/tunnel.pid"
log "tunnel: SSH PID=${SSH_PID} port=${TUNNEL_REMOTE_PORT}"
# Watchdog loop
_fail_count=0
while kill -0 "$SSH_PID" 2>/dev/null; do
sleep "$WATCHDOG_INTERVAL"
if ! kill -0 "$SSH_PID" 2>/dev/null; then break; fi
# Probe forward health
if timeout 10 ssh \
-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-o ConnectTimeout=5 -o BatchMode=yes \
-p "${TUNNEL_REMOTE_SSH_PORT}" -i "$KEY" \
"${TUNNEL_REMOTE_USER}@${TUNNEL_REMOTE_HOST}" \
"timeout 3 bash -c 'echo >/dev/tcp/127.0.0.1/${TUNNEL_REMOTE_PORT}' 2>/dev/null" \
2>/dev/null; then
[ "$_fail_count" -gt 0 ] && log "tunnel: forward recovered after ${_fail_count} checks"
_fail_count=0
else
_fail_count=$((_fail_count + 1))
log "tunnel: forward check failed (${_fail_count}/${FORWARD_FAIL_LIMIT})"
[ "$_fail_count" -ge "$FORWARD_FAIL_LIMIT" ] && break
fi
done
[ -f "$PKG_DIR/var/tunnel.pid" ] && kill "$(cat "$PKG_DIR/var/tunnel.pid")" 2>/dev/null
rm -f "$PKG_DIR/var/tunnel.pid"
log "tunnel: exited — will retry in ${TUNNEL_RETRY_DELAY}s"
sleep "$TUNNEL_RETRY_DELAY"
return 1
}
# ── Hub registration — get assigned port ───────────────────────────
register_with_hub() {
# Already registered — skip
if [ "$TUNNEL_REMOTE_PORT" != "0" ] && [ -n "$TUNNEL_REMOTE_PORT" ]; then
return 0
fi
log "hub: registering device ${DEVICE_ID}..."
# --max-time 15: total timeout (connect + response). SRM busybox curl needs this
# or the daemon hangs forever if the hub accepts the TCP connection but
# never sends the HTTP response.
resp=""
resp=$(curl -s --connect-timeout 10 --max-time 15 \
"http://${HUB_HOST}:${HUB_PORT}/api/register/${DEVICE_ID}" 2>/dev/null) || true
if [ -z "$resp" ]; then
log "hub: unreachable — will retry next cycle"
return 1
fi
# Parse JSON response — POSIX-safe, no jq dependency
port=""
tskey=""
port=$(printf '%s' "$resp" | sed -n 's/.*"tunnel_port"[[:space:]]*:[[:space:]]*\([0-9]*\).*/\1/p')
tskey=$(printf '%s' "$resp" | sed -n 's/.*"tailscale_auth_key"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p')
if [ -n "$port" ] && [ "$port" != "0" ]; then
sed -i "s/^TUNNEL_REMOTE_PORT=.*/TUNNEL_REMOTE_PORT=${port}/" "$CONF" 2>/dev/null
TUNNEL_REMOTE_PORT="$port"
log "hub: assigned port ${port}"
fi
if [ -n "$tskey" ] && [ "$tskey" != "$TAILSCALE_AUTH_KEY" ]; then
sed -i "s|^TAILSCALE_AUTH_KEY=.*|TAILSCALE_AUTH_KEY=${tskey}|" "$CONF" 2>/dev/null
TAILSCALE_AUTH_KEY="$tskey"
log "hub: updated Tailscale key"
fi
}
# ── Main ────────────────────────────────────────────────────────────
load_config
echo "$$" > "$DAEMON_PID_FILE"
while true; do
load_config
# Register with hub if needed
register_with_hub 2>/dev/null || true
# Start Tailscale in background (stays running)
start_tailscale 2>/dev/null || true
# Start tunnel (blocking — restarts on failure)
start_tunnel 2>/dev/null || true
# start_tunnel is already blocking and self-retrying; no bare wait here — a bare
# wait blocks forever on the setsid'd tailscaled child and wedges the retry loop.
done