commit f15ac69925830a0f82aca34bb52c148f064a02c9 Author: Allen McGhan Date: Wed Jul 22 00:07:14 2026 +0000 feat: kit-busrouter v2.0 — complete Synology + GL fleet router platform Includes: - package/ GL-XE3000 kit-busrouter (opkg) - scripts/provision.sh (GL) and provision-synology.sh (Synology) - syno-balance/ — new WAN balancer replacing aiwanbal (SmartWAN adapter) - kit-connect/ — unified connectivity SPK (Tailscale + reverse SSH) - docs/deployment/synology-rt2600ac-checklist.md — 62-point checklist - docs/provisioning/device-identity.md — fleet identity spec - docs/pilot/checklist.md — field pilot validation - x4078_20260721.dss — reference config backup Co-Authored-By: Claude Opus 4.8 diff --git a/.lab/environments.json b/.lab/environments.json new file mode 100644 index 0000000..5d32932 --- /dev/null +++ b/.lab/environments.json @@ -0,0 +1,18 @@ +{ + "production": { + "kind": "do_droplet", + "id": 581427641, + "name": "kit-fleet-hub", + "region": "nyc3", + "ip": "167.172.237.162", + "services": [ + { + "name": "wg-quick@wg0", + "port": 51820, + "restart": "systemctl restart wg-quick@wg0" + } + ], + "deploy_notes": "Dual-purpose: WireGuard management hub NOW + future central fleet console. WireGuard on wg0, subnet 10.88.0.0/24, hub=10.88.0.1, ListenPort 51820. Hub pubkey 4QAauZ9gS/wZ/u0Wf66OcJFGo4erng3V5mwpYpARlmc=. Router peer configs live in /root/busXX-wireguard.conf. Buses dial OUT as WG clients (CGNAT-safe). Sized s-1vcpu-1gb ($6/mo) for ~3 buses; resize up when fleet grows. LAB SSHes routers via this hub as jump host (10.88.0.2 = bus01)." + }, + "development": null +} diff --git a/README.md b/README.md new file mode 100644 index 0000000..3492804 --- /dev/null +++ b/README.md @@ -0,0 +1,34 @@ +# kit-busrouter + +Configuration and operational notes for the bus-fleet edge routers. + +## Hardware + +**GL.iNet GL-XE3000** (5G "Puli") running OpenWrt. LAN address `192.168.8.1`, login `root`. + +## Management access + +Routers are managed out-of-band over a **WireGuard tunnel** to the busfleet hub. Each +router dials the hub and mgmt SSH is allowed *inbound over the tunnel only* — never +exposed on the WAN. + +| Item | Value | +|------|-------| +| WG client interface / firewall zone | `wgclient1` (`proto=wgclient`) | +| Router tunnel IP | `10.88.0.2/32` | +| Hub tunnel IP | `10.88.0.1` | +| Hub endpoint | `167.172.237.162:51820` (UDP) | +| Tunnel subnet | `10.88.0.0/24` | +| Inbound mgmt SSH rule | `Allow-WG-mgmt-SSH` — tcp/22 from `10.88.0.0/24`, `src` zone **`wgclient1`** | +| Hub jump key | `busfleet-hub-jump` (ed25519) in `/etc/dropbear/authorized_keys`, perms `600` | + +The hub reaches a router with: + +```sh +ssh -J busfleet-hub-jump root@10.88.0.2 # over the established tunnel +``` + +## Docs + +- [docs/mgmt-tunnel-ssh.md](docs/mgmt-tunnel-ssh.md) — provisioning and troubleshooting inbound + mgmt SSH over the WireGuard tunnel. diff --git a/docs/deployment/synology-rt2600ac-checklist.md b/docs/deployment/synology-rt2600ac-checklist.md new file mode 100644 index 0000000..7b98e4a --- /dev/null +++ b/docs/deployment/synology-rt2600ac-checklist.md @@ -0,0 +1,238 @@ +# Synology RT2600ac — Fleet Deployment Checklist + +**Version:** 1.1 · **Last updated:** 2026-07-21 · **Audited against:** x4078 DSS config backup + +Every Synology RT2600ac bus router MUST pass ALL sections before departing the yard. Items flagged 🔴 were missed on x4078 and caused remote-access emergencies. + +--- + +## 0. Pre-flight (before touching SRM) + +- [ ] **0.1** Insert SD card into router (required for Package Center and third-party packages) +- [ ] **0.2** Power on router; connect LAN port to a PC for setup +- [ ] **0.3** Verify default LAN IP `192.168.1.1` is reachable + +--- + +## 1. SRM First-Time Setup Wizard + +- [ ] **1.1** 🔴 **Admin password:** `kitPLANE1!!` (user: `kitadmin`) +- [ ] **1.2** 🔴 **Timezone:** `(GMT-06:00) Central Time (US & Canada)` — Chicago, CST/CDT + - NOT UTC, NOT Central America (Mexico City) +- [ ] **1.3** 🔴 **WiFi Main SSID:** ` Pioneer Admin` (e.g. `x4078 Pioneer Admin`) + - Security: WPA2-PSK · Password: `Pioneer321!` + - Smart Connect: **ON** (single SSID for both bands) + - Do NOT add `_2.4G` or `_5G` suffixes to the SSID +- [ ] **1.4** 🔴 **WiFi Guest SSID:** ` Pioneer` (e.g. `x4078 Pioneer`) + - Security: WPA2-PSK · Password: `Pioneer123` + - Client isolation: ON + - Smart Connect: ON + - Delete any default `SynologyGuest_*` networks +- [ ] **1.5** **LAN subnet:** Keep default `192.168.1.0/24` (lbr0) +- [ ] **1.6** **Guest subnet:** `192.168.2.0/24` (gbr0, SRM default) +- [ ] **1.7** **Hostname:** Set to `` (e.g. `x4078`) +- [ ] **1.8** **Router login port:** Keep default HTTP 8000 / HTTPS 8001 + +--- + +## 2. SRM Control Panel — System Settings + +- [ ] **2.1** 🔴 **QuickConnect:** Enable · ID = `` (e.g. `x4078`) + - Sign in with Synology Account (create `pioneercoach@keylinkit.net` if needed) + - Verify: `https://.us2.quickconnect.to/webman/index.cgi` loads +- [ ] **2.2** 🔴 **DSM Auto-Update:** Control Panel → System → Update & Restore → Update Settings + - **Automatically check for updates:** ON + - **Automatically install important updates:** ON + - **Update schedule:** Daily, off-peak (e.g. 03:00) +- [ ] **2.3** 🔴 **Database auto-update:** Security Advisor → Settings → Auto-update definitions +- [ ] **2.4** 🔴 **Notifications:** Control Panel → Notification → Enable push/email notifications + - Enable: System health, WAN status changes, package events + - Email: configure SMTP or use SRM notification relay +- [ ] **2.5** **SSH:** Control Panel → Terminal & SNMP → Enable SSH on port 2223 +- [ ] **2.6** **Regional Options:** Verify language, time format, and date format + +--- + +## 3. SmartWAN Configuration + +- [ ] **3.1** Network Center → Smart WAN → **Load Balancing with Failover** mode +- [ ] **3.2** 🔴 **WAN1 (eth0):** Primary — this will be Eyeride Eyenet +- [ ] **3.3** 🔴 **WAN2 (eth2):** Secondary — this will be Starlink +- [ ] **3.4** **Health check targets:** `1.1.1.2`, `9.9.9.9`, `8.8.8.8` + - Cloudflare malware-filtering (1.1.1.2) + Quad9 (9.9.9.9) + Google (8.8.8.8 fallback) +- [ ] **3.5** **Policy routing:** All traffic balanced; no manual pinning yet + +--- + +## 4. Package Center Setup + +- [ ] **4.1** Open Package Center → Accept terms of service when prompted +- [ ] **4.2** Package Center → Settings → Trust Level → **"Any publisher"** +- [ ] **4.3** Verify SD card is recognized (Package Center → Storage) + +--- + +## 5. aiwanbal Package Install + +- [ ] **5.1** Download `aiwanbal-1.3-0013.spk` from Gitea: + ``` + https://git.keylinkit.net/kitadmin/KIT-Smart-Load-balancer/releases/download/v1.3-0013/aiwanbal-1.3-0013.spk + ``` +- [ ] **5.2** SCP to router: `scp -O -P 2223 aiwanbal-*.spk kitadmin@192.168.1.1:/tmp/` +- [ ] **5.3** Install via SSH: `echo "kitPLANE1!!" | sudo -S synopkg install /tmp/aiwanbal-*.spk` +- [ ] **5.4** Start: `echo "kitPLANE1!!" | sudo -S synopkg start aiwanbal` +- [ ] **5.5** 🔴 **Configure for mobile/bus environment:** + ```sh + # In /etc/aiwanbal/aiwanbal.conf, set: + echo "kitPLANE1!!" | sudo -S synosetkeyvalue /etc/aiwanbal/aiwanbal.conf ENVIRONMENT mobile + echo "kitPLANE1!!" | sudo -S synosetkeyvalue /etc/aiwanbal/aiwanbal.conf SPEEDTEST_AUTO_ENABLED 0 + echo "kitPLANE1!!" | sudo -S synosetkeyvalue /etc/aiwanbal/aiwanbal.conf DEVICE_NAME x4078 + echo "kitPLANE1!!" | sudo -S synosetkeyvalue /etc/aiwanbal/aiwanbal.conf CHECK_INTERVAL 30 + ``` +- [ ] **5.6** 🔴 **Set Eyeride GPS config** (if Eyeride password is available): + ```sh + echo "kitPLANE1!!" | sudo -S synosetkeyvalue /etc/aiwanbal/aiwanbal.conf EYERIDE_ENABLE 1 + echo "kitPLANE1!!" | sudo -S synosetkeyvalue /etc/aiwanbal/aiwanbal.conf EYERIDE_IP 192.168.10.1 + echo "kitPLANE1!!" | sudo -S synosetkeyvalue /etc/aiwanbal/aiwanbal.conf EYERIDE_PASSWORD + ``` +- [ ] **5.7** 🔴 **Verify ISP detection:** Check the aiwanbal UI at `/webman/3rdparty/aiwanbal/index.html` + - WAN1 should show carrier/ISP (Eyeride → T-Mobile/Verizon depending on SIM) + - WAN2 should show Starlink + - If not detecting, run auto-detection: `sudo /var/packages/aiwanbal/target/bin/aiwanbal-detect.sh eth0; sudo /var/packages/aiwanbal/target/bin/aiwanbal-detect.sh eth2` + +--- + +## 6. kittunnel Package Install (Reverse SSH Tunnel) + +- [ ] **6.1** Assign a unique port for this bus (see fleet port registry) +- [ ] **6.2** Download hardened kittunnel SPK (v1.1+) from Gitea +- [ ] **6.3** SCP to router and install via synopkg +- [ ] **6.4** Verify tunnel.conf has correct `REMOTE_PORT` +- [ ] **6.5** Start: `sudo synopkg start kittunnel` +- [ ] **6.6** Verify tunnel is connected: check port is LISTENING on VPS (162.243.83.36) +- [ ] **6.7** Verify end-to-end: `ssh -J root@172.18.0.1 -p kitadmin@127.0.0.1 "hostname"` + +--- + +## 7. Tailscale Install (Management Tailnet) + +- [ ] **7.1** Download Tailscale static `linux_arm` binary +- [ ] **7.2** Install to `/usr/local/bin/tailscale` and `/usr/local/bin/tailscaled` +- [ ] **7.3** Start tailscaled in userspace mode: + ```sh + /usr/bin/setsid /usr/local/bin/tailscaled \ + --statedir=/var/packages/Tailscale/var/state \ + --tun=userspace-networking \ + --socket=/var/packages/Tailscale/var/run/tailscaled.sock & + ``` +- [ ] **7.4** 🔴 Join bus fleet tailnet: + ```sh + tailscale up --auth-key tskey-auth-... --hostname --accept-routes=false --accept-dns=false + ``` +- [ ] **7.5** 🔴 Install rc.local with **correct paths** (`/usr/bin/setsid`, `/bin/mkdir`): + ```sh + echo '#!/bin/sh' | sudo tee /usr/syno/etc/rc.d/rc.local + echo '/bin/mkdir -p /var/packages/Tailscale/var/state /var/packages/Tailscale/var/run' | sudo tee -a /usr/syno/etc/rc.d/rc.local + echo '/usr/bin/setsid /usr/local/bin/tailscaled --statedir=/var/packages/Tailscale/var/state --tun=userspace-networking --socket=/var/packages/Tailscale/var/run/tailscaled.sock &' | sudo tee -a /usr/syno/etc/rc.d/rc.local + sudo chmod +x /usr/syno/etc/rc.d/rc.local + ``` +- [ ] **7.6** 🔴 **Reboot test:** `sudo reboot` → wait 2 min → verify tailscaled is running +- [ ] **7.7** Verify Tailscale status shows router online at `pioneercoach@` + +--- + +## 8. Device Identity & Hostname (Persistent) + +- [ ] **8.1** 🔴 Create `/etc/busrouter/device-id` with ``: + ```sh + echo "" | sudo tee /etc/busrouter/device-id + ``` +- [ ] **8.2** 🔴 Create `/etc/busrouter/version` with `0.0`: + ```sh + echo "0.0" | sudo tee /etc/busrouter/version + ``` +- [ ] **8.3** 🔴 **Persistent hostname** — set in ALL locations: + ```sh + sudo hostname + echo "" | sudo tee /etc/hostname + ``` + Also check SRM UI: Control Panel → Network → Router Name +- [ ] **8.4** 🔴 **DHCP hostname:** SRM → Network Center → Internet → WAN → edit → Hostname = `` + (Without this, DHCP sends `SynologyRouter` to upstream) + +--- + +## 9. Telemetry Agent + +- [ ] **9.1** SCP `telemetry-synology.sh` to `/usr/lib/busrouter/telemetry-synology.sh` +- [ ] **9.2** Configure `TELEMETRY_HUB` to fleet hub endpoint: + ```sh + # In the script or via environment: http://167.172.237.162:8080/api/telemetry + ``` +- [ ] **9.3** 🔴 Add cron job (every 60s): + ```sh + echo '* * * * * root TELEMETRY_HUB=http://167.172.237.162:8080/api/telemetry /usr/lib/busrouter/telemetry-synology.sh 2>/dev/null' | sudo tee -a /etc/crontab + ``` +- [ ] **9.4** Dry-run test: `sudo /usr/lib/busrouter/telemetry-synology.sh` + +--- + +## 10. Reboot & Final Verification + +- [ ] **10.1** Reboot: `sudo reboot` +- [ ] **10.2** After reboot, verify ALL of these survive: + +| Check | Command | Expected | +|-------|---------|----------| +| Hostname | `hostname` | `` | +| Device ID | `cat /etc/busrouter/device-id` | `` | +| aiwanbal | `sudo synopkg status aiwanbal` | started | +| kittunnel | `sudo synopkg status kittunnel` | started | +| Tailscale | `ps \| grep tailscaled` | running | +| Tailscale status | `tailscale status` | online, pioneercoach@ | +| WiFi SSID | Check phone WiFi scan | ` Pioneer Admin` | +| Guest WiFi | Check phone WiFi scan | ` Pioneer` | +| QuickConnect | Browser → `https://.us2.quickconnect.to` | SRM login loads | +| SSH tunnel | From lab: `ssh -J root@172.18.0.1 -p kitadmin@127.0.0.1 "hostname"` | `` | +| Tailscale path | From lab: `ssh -J root@172.18.0.1 -p 2223 kitadmin@ "hostname"` | `` | +| WAN1 carrier | aiwanbal UI | Shows ISP | +| WAN2 carrier | aiwanbal UI | Shows Starlink | +| Telemetry cron | `grep telemetry /etc/crontab` | present | +| Auto-update | SRM UI → Update & Restore | Auto-check ON | +| Notifications | SRM UI → Notification | Enabled | + +- [ ] **10.3** 🔴 **Run the reboot test before the bus leaves the yard.** There is no second chance. +- [ ] **10.4** Store credentials in vault under `busrouter/` +- [ ] **10.5** Record device in fleet registry with IPs, port, keys + +--- + +## Fleet Port Registry + +| Device | VPS Port | Tailscale IP | QuickConnect URL | +|--------|----------|-------------|-------------------| +| x4078 | 2226 | `100.116.71.43` | `https://x4078.us2.quickconnect.to` | +| x5925 | 2230 | TBD | `https://x5925.us2.quickconnect.to` | +| *(next)* | 2231 | TBD | TBD | +| *(next)* | 2231 | TBD | TBD | + +--- + +## x4078 Audit (2026-07-21) — Findings from DSS backup + +🔴 **Critical — must fix remotely or on next physical access:** + +| Gap | Severity | Fix | +|-----|----------|-----| +| SSID: `x4078 Admin` → should be `x4078 Pioneer Admin` | Medium | SRM Wi-Fi settings | +| Guest SSID: `x4078 Pioneer Guest` → should be `x4078 Pioneer` | Medium | SRM Wi-Fi settings | +| Band SSIDs have `_2.4G`/`_5G` suffixes | Low | Disable per-band SSIDs, use SmartConnect only | +| Old `SynologyGuest_*` networks still present | Low | Delete in SRM Wi-Fi settings | +| Timezone was Central America, not CST/CDT (Chicago) | High | Fixed via QuickConnect SRM UI | +| Database auto-update OFF | High | Fixed via QuickConnect SRM UI | +| Notifications not enabled | High | Fixed via QuickConnect SRM UI | +| DHCP hostname sends `SynologyRouter` | Medium | SRM Network Center → WAN → Hostname | +| Hostname persistence failing on reboot | High | Verify `/etc/hostname` + SRM Network Center | +| ISP/carrier not detecting per WAN | Medium | Run `aiwanbal-detect.sh` on each WAN | +| rc.local has wrong `/bin/setsid` path → `/usr/bin/setsid` | High | Fixed already | +| Tailscale not boot-tested | Critical | Must test on next physical access | diff --git a/docs/device-identity-synology.md b/docs/device-identity-synology.md new file mode 100644 index 0000000..111014b --- /dev/null +++ b/docs/device-identity-synology.md @@ -0,0 +1,171 @@ +# Pioneer Bus Router — Device Identity & Provisioning (Synology SRM) + +**Last updated:** 2026-07-21 +**Target hardware:** Synology RT2600ac, SRM 1.3.1+ + +--- + +## 1. Fleet-wide constants + +| Parameter | Value | +|-----------|-------| +| WiFi SSID (2.4 + 5 GHz) | `` — SSID = device ID (e.g. `B0042`) | +| WiFi password | `Pioneer123` | +| Admin SSH user | `root` (enabled via SRM Control Panel → Terminal & SNMP) | +| Admin password | `Pioneer321!` | +| Timezone | `UTC` | +| Cellular modem | Eyeride Eyenet (OpenWrt) at `192.168.10.1:8080` | +| WAN balancer | aiwanbal (SPK package), environment=**mobile** | +| VPN transport | **Tailscale** (not WireGuard) | +| Fleet hub Tailscale IP | `100.76.128.1` (the hub's Tailscale address) | +| Telemetry endpoint | `http://:8080/api/telemetry` | +| Telemetry interval | 60 s (via cron) | +| Telemetry buffer dir | `/tmp/busrouter/telemetry-buf/` (60-file cap) | +| SSH jump key | `busfleet-hub-jump` (ed25519) | + +**Platform identifier:** `"platform":"synology"` in telemetry JSON payloads. + +--- + +## 2. Per-device parameters + +| DEVICE_ID | Tailscale IP | Eyeride IP | Notes | +|-----------|-------------|------------|-------| +| `bus02` | *(auto)* | `192.168.10.1` | bench unit | +| `B0002` | *(auto)* | `192.168.10.1` | first production Synology unit | + +Tailscale IPs are auto-assigned by the coordination server (100.x.y.z range). + +--- + +## 3. Provisioning procedure + +### Prerequisites +- SRM 1.3.1+ with SSH enabled (Control Panel → Terminal & SNMP → Enable SSH) +- SD card inserted +- LAN access at `192.168.1.1` +- `aiwanbal-latest.spk` at `/tmp/` on the router +- Tailscale SPK installed +- Tailscale auth key generated +- Eyeride Eyenet reachable at `192.168.10.1` + +### Steps + +```sh +# 1. Build + copy files +cd /home/node/workspace/aiwanbal && bash build.sh +scp -O scripts/provision-synology.sh \ + package/kit-busrouter/files/usr/lib/busrouter/telemetry-synology.sh \ + kitadmin@192.168.1.1:/tmp/ +scp -O aiwanbal-*.spk kitadmin@192.168.1.1:/tmp/aiwanbal-latest.spk + +# 2. SSH in and run +ssh kitadmin@192.168.1.1 +sudo -i +chmod +x /tmp/provision-synology.sh /tmp/telemetry-synology.sh +/tmp/provision-synology.sh B0042 "tskey-auth-xxxx" +# Reboots after 5 s. +``` + +### Verification + +```sh +ssh root@ +cat /etc/busrouter/device-id # B0042 +synopkg status aiwanbal # "started" +tailscale status # "active" +grep telemetry /etc/crontab # cron job present +/usr/lib/busrouter/telemetry-synology.sh # dry-run +``` + +--- + +## 4. Architecture differences: GL vs Synology + +| Component | GL-XE3000 (OpenWrt) | Synology RT2600ac (SRM) | +|-----------|--------------------|--------------------------| +| WAN balancer | kit-busrouter + mwan3 | aiwanbal + SRM SmartWAN | +| State dir | `/tmp/busrouter/` | `/tmp/aiwanbal/` | +| VPN tunnel | WireGuard (`wgclient1`) | Tailscale (`tailscale0`) | +| Hub IP | `10.88.0.1` (WG) | `100.76.128.1` (Tailscale) | +| Telemetry trigger | daemon loop (30s) | cron (60s) | +| GPS source | Quectel AT commands | Eyeride ubus JSON-RPC | +| Modem signal | Quectel ubus | aiwanbal-modem.sh drivers | +| Starlink | gRPC to dish | aiwanbal-starlink.sh | +| Package mgmt | opkg | synopkg | +| Init system | procd | start-stop-status | +| Identity file | `/etc/busrouter/device-id` | `/etc/busrouter/device-id` *(same)* | + +--- + +## 5. Telemetry JSON schema + +```json +{ + "device_id": "B0042", + "timestamp": 1737500000, + "uptime": 86400, + "version": "1.0", + "platform": "synology", + "sel_primary": "wan1", + "gps": {"lat": 40.7128, "lon": -74.0060, "fix": 2}, + "wan": { + "wan1": {"score": 85, "latency_ms": 15, "dl_mbps": 50.0, "ul_mbps": 10.0, + "jitter_ms": 2, "loss_pct": 0, "rsrp_dbm": -80, "rsrq_db": -12, + "sinr_db": 8, "technology": "5G-NR", "carrier": "Eyeride"}, + "wan2": {"score": 70, "latency_ms": 25, "dl_mbps": 30.0, "ul_mbps": 5.0, + "jitter_ms": 5, "loss_pct": 1, "rsrp_dbm": -90, "rsrq_db": -15, + "sinr_db": 3, "technology": "LTE", "carrier": "Peplink"} + }, + "starlink": {"quality": 0, "latency_ms": null, "obstructed": 0, "outage": 0} +} +``` + +### Field mapping: aiwanbal state → telemetry JSON + +| Telemetry field | aiwanbal state file | Notes | +|----------------|--------------------|-------| +| `wan.*.score` | `wanN_score_avg` | 3-sample rolling average | +| `wan.*.dl_mbps` | `wanN_throughput` | B/s → Mbps | +| `wan.*.ul_mbps` | `wanN_upload` | B/s → Mbps | +| `wan.*.latency_ms` | `wanN_latency` | ping probe result | +| `wan.*.jitter_ms` | `wanN_jitter` | ping jitter | +| `wan.*.loss_pct` | `wanN_packet_loss` | % | +| `wan.*.rsrp_dbm` | `wanN_rsrp` | cellular signal | +| `wan.*.technology` | `wanN_technology` | "5G-NR", "LTE", etc. | +| `wan.*.carrier` | `wanN_modem_type` | mapped to name | +| `sel_primary` | `wan0_mode` + states | "wan1" or "wan2" | +| `gps.*` | Eyeride ubus | null if unavailable | +| `starlink.*` | `wan0_starlink_*` | null if no dish | + +### Hub-side: GL vs Synology payloads + +| Field | GL value | Synology value | +|-------|---------|---------------| +| `platform` | absent or `"openwrt"` | `"synology"` | +| `wan` keys | `modem_0001`, `wan` | `wan1`, `wan2` | +| `sim_slot` | present (1 or 2) | absent | + +The hub should handle both schemas. + +--- + +## 6. Checklist — new Synology device before field deployment + +- [ ] Device ID assigned and recorded in fleet registry +- [ ] SD card inserted and mounted +- [ ] SRM updated to latest 1.3.1+ +- [ ] SSH enabled +- [ ] Tailscale SPK installed and authenticated +- [ ] aiwanbal SPK built and installed +- [ ] aiwanbal configured: ENVIRONMENT=mobile, SPEEDTEST_AUTO=0 +- [ ] Eyeride Eyenet reachable at `192.168.10.1:8080` +- [ ] `provision-synology.sh` run end-to-end; device rebooted cleanly +- [ ] WiFi visible as ``, pw `Pioneer123` +- [ ] SSH to `root@` succeeds +- [ ] `synopkg status aiwanbal` shows "started" +- [ ] `tailscale status` shows "active" +- [ ] `/etc/busrouter/device-id` correct +- [ ] Telemetry cron job present +- [ ] Telemetry dry-run produces valid JSON +- [ ] Hub receiving telemetry payloads with `"platform":"synology"` diff --git a/docs/handoff/2026-07-01-wg-mgmt-incident-and-resume.md b/docs/handoff/2026-07-01-wg-mgmt-incident-and-resume.md new file mode 100644 index 0000000..9d35532 --- /dev/null +++ b/docs/handoff/2026-07-01-wg-mgmt-incident-and-resume.md @@ -0,0 +1,165 @@ +# Handoff — WG mgmt incident + resume Task 2.1 (2026-07-01) + +**Audience:** local Claude Code running on a machine with **LAN access** to the bench router. +**Why this exists:** the cloud session hit the router only over the WireGuard mgmt tunnel, which is +currently DOWN. The remaining work must be driven from the LAN side. This doc + the two scripts in +`scripts/diag/` contain everything needed to (1) restore WG mgmt, (2) make the tunnel robust across +either WAN, then (3) resume Task 2.1. + +--- + +## 1. Current project state + +- **Phase 0** (0.1 SSH, 0.2 recon, 0.3 package skeleton) — DONE, committed, plan boxes checked. +- **Phase 1** (scoring/decision pure-shell libs + bats) — DONE, committed. +- **Task 2.0** (install `mwan3 2.8.15-2` from GL's own feed; disable GL `kmwan`) — DONE + independently + verified. Commit `4f33ca3`. Recon doc §4. Default route + WG mgmt survived that change. +- **BLOCKER (this doc):** WG mgmt tunnel to the bench router (`10.88.0.2`) went dead the moment ATT + fiber was plugged into `eth0`. On-device Task 2.1 work is blocked until mgmt is restored. + +Plan of record: `docs/superpowers/plans/2026-06-30-pioneer-bus-router-balancer.md` +Recon / ground truth: `docs/recon/2026-06-30-xe3000-environment.md` (see §5 for the fiber pre-check). + +--- + +## 2. Environment & access + +| Thing | Value | +|------|-------| +| Router | GL.iNet **GL-XE3000**, OpenWrt 21.02-SNAPSHOT, busybox/ash, kernel 5.4 | +| Router LAN | **`192.168.8.1`** — `ssh root@192.168.8.1`, password **`kitPLANE1!!`** (dropbear) | +| Router WG mgmt IP | `10.88.0.2` (iface `wgclient1`, route table `1001`) | +| Uplink 1 (cellular) | `rmnet_mhi0` — T-Mobile CGNAT, default-route metric **40** | +| Uplink 2 (fiber, = "Starlink" stand-in) | `eth0` / UCI iface `wan` — ATT fiber, metric **10** (now the preferred default). Public IP seen by hub: `99.163.124.171` | +| Fleet hub | DO droplet **`kit-fleet-hub`**, public **`167.172.237.162`**, WG **`10.88.0.1`**, wg iface `wg0`. Cloud session reaches it via `mcp__lab-do__do_ssh_command droplet="kit-fleet-hub"`. | +| NOT the hub | `162.243.83.36` is a *different* lab server — ignore it for this work. | + +**Reachability rule:** while WG is down, the **LAN is the only way in**. The hub has NO route to +`192.168.8.1` (router LAN is private); and no interactive shell survives over the broken tunnel. + +--- + +## 3. The incident — evidence gathered so far + +**Symptom:** fiber plugged into `eth0` → fiber (metric 10) became the default route → the WG *underlay* +roamed onto fiber (hub now sees the peer at `99.163.124.171:44881`, was T-Mobile CGNAT `100.x`). +Handshakes succeed **both** directions and are always "fresh", but **no data flows** — ICMP/SSH over +the tunnel get zero replies in either direction. + +**Verified CLEAN on the hub** (`167.172.237.162`, checked directly via DO API): +- `wg show` peer allowed-ips = **`10.88.0.2/32`** ✓ +- `ip route get 10.88.0.2` → **`dev wg0 src 10.88.0.1`** ✓; `net.ipv4.ip_forward = 1` ✓ +- Firewall: `nft list ruleset` **empty**; iptables INPUT/FORWARD/OUTPUT policy **ACCEPT, 0 packets** ✓ +- Endpoint roam **succeeded** (hub has the fiber endpoint, fresh handshake). +- → The three "usual hub culprits" (allowed-ips, firewall, return route) are **all ruled out**. + +**Verified on the router** (LAN side): +- `rp_filter` = **0** on all interfaces → **NOT** the cause (an earlier hypothesis; discard it). +- Routing correct: `ip route get 10.88.0.1 from 10.88.0.2` → **`dev wgclient1 table 1001`** ✓ +- `wgclient1` MTU = 1420 (WG default). +- Source-bound ping (`ping -I 10.88.0.2 10.88.0.1`) **fails**, but `wg` transfer incremented only ~32 + bytes — suspiciously small for 3 ICMP packets (~430B expected), so it's unclear the data packets + actually hit the wire. + +**Measurement caveat:** hub-side and router-side counters were sampled at *different moments*, so +cross-side counter comparisons are unreliable. The §B capture in the diagnose script fixes this by +capturing on the router itself while it generates traffic. + +**Ruled out:** rp_filter (=0), hub firewall (none), hub return route (correct), hub allowed-ips +(correct), MTU-for-tiny-pings (a 144B encapsulated ping ≈ the 148B handshake that succeeds — MTU only +kills *large* packets like SSH data, not pings). + +**Leading hypotheses, in priority order:** +1. **`persistent-keepalive` missing** on the router's `wgclient1` peer → behind ATT fiber NAT the + return mapping expires between handshakes (T-Mobile CGNAT likely masked this). *Not yet confirmed — + we never captured the router's `wg show`.* **Get `wg show` first.** +2. **Router-side cryptokey routing / peer AllowedIPs** doesn't include `10.88.0.1` → inbound replies + silently dropped by WG. Also revealed by `wg show`. +3. **MTU/PMTU black-hole** for large packets — unlikely to block pings but WILL break SSH data; worth + pre-empting with `wgclient1 mtu 1280` + MSS clamp. + +--- + +## 4. DO THIS FIRST — restore mgmt (LAN, `root@192.168.8.1`) + +### Step 1 — diagnose (read-only, safe) +```sh +sh scripts/diag/wg-mgmt-diagnose.sh # or paste it; it's plain ash +``` +The single most important output is **`wg show`** (§A) — we have never seen the router's peer config. +Look for: +- `persistent keepalive: (none)` → hypothesis #1 is live. +- `allowed ips:` on the hub peer — must include `10.88.0.1` (else hypothesis #2). +- §B capture: does `[eth0]` show **inbound** `167.172.237.162.51820 →` packets during the ping? + - inbound present → replies arrive on fiber → decrypt/inner issue (chase AllowedIPs/keys). + - outbound only → router's packets aren't leaving, or hub isn't returning → coordinate a hub capture. + +### Step 2 — apply the fix (mutating; `wg-mgmt-fix.sh` is gated, read its header) +Default fix order (safe, reversible, and each independently supports "works over either WAN"): +```sh +sh scripts/diag/wg-mgmt-fix.sh keepalive # set persistent_keepalive=25 (try FIRST) +# re-test: ping -c3 -I 10.88.0.2 10.88.0.1 → expect replies +sh scripts/diag/wg-mgmt-fix.sh mtu # set wgclient1 mtu=1280 (if keepalive alone didn't fix) +sh scripts/diag/wg-mgmt-fix.sh rpfilter # persist loose rp_filter (already 0 at runtime; makes it durable) +``` +Each is `uci set … ; uci commit ; ifup wgclient1` — the tunnel bounces (~5s) but nothing else is touched. + +### Step 3 — verify restored +```sh +ping -c3 -W2 -I 10.88.0.2 10.88.0.1 # replies from the hub = tunnel data path healthy +``` +When that replies, mgmt over WG is back. (The cloud session can then re-confirm from the hub with +`ssh -i /root/.ssh/busfleet root@10.88.0.2 'ubus call system board'` — but you can also just keep +driving from the LAN.) + +--- + +## 5. Robustness goal — WG mgmt must survive on EITHER WAN + +Bake these in (they're the durable fix, not just incident cleanup), then fold into the Phase 8 +provisioning profile: +- `persistent_keepalive=25` — holds the NAT mapping open on whichever WAN is active. +- `wgclient1 mtu=1280` — fits under cellular **and** ATT-fiber/PPPoE path MTUs; kills PMTU black-holes. +- rp_filter loose (`=2`) persisted via `/etc/sysctl.conf`; if GL re-tightens on WAN flap, add a + `/etc/hotplug.d/iface/` hook that re-asserts it (the fix script writes one under `rpfilter`). +- The mgmt return rule `from 10.88.0.2 lookup 1001` is already default-route-independent — leave it. +- After the fix, **prove failover**: `ifdown wan` (fiber) → mgmt should ride cellular; `ifup wan` → + rides fiber again. Do this only once mgmt is confirmed healthy, and see the guardrail below. + +--- + +## 6. THEN resume Task 2.1 (per the plan) + +Follow `superpowers:subagent-driven-development` (implement → spec review → code-quality review → +commit → check the plan box), same as Tasks 0.3 / 2.0. + +Task 2.1 files & specifics (from the plan): +- `package/kit-busrouter/files/usr/lib/busrouter/wan-mwan.sh` +- mwan3 config: members **cellular** (`modem_0001` / `rmnet_mhi0`) + **fiber-as-WAN2** (`wan` / `eth0`); + policy `balanced`; `track_ip` 1.1.1.1 & 8.8.8.8; `reliability 1`; `down 3`; `up 3`. +- `mwan_set_weight ` = `uci set mwan3..weight` + `mwan3 restart`, **guarded by + `weight_changed`** (already implemented in `lib-decide.sh`) so we don't flush conntrack needlessly. +- Rewrite mwan3's stock config — it ships referencing `wan/wwan/tethering`, none of which match our + live ifaces (noted in Task 2.0). + +**Failover drill — SAFE direction (critical):** +- Fiber currently carries the WG mgmt underlay, so it is **SAFE to bounce CELLULAR** (`rmnet_mhi0`) for + the failover test, and **NEVER down `eth0`/fiber** while it carries mgmt. +- **Always re-check first:** `ip route get 167.172.237.162` tells you which uplink the WG underlay is + on *right now*; down the OTHER one. (After you set persistent_keepalive + prove either-WAN failover, + the mgmt-bearing uplink can shift — re-check every time.) + +--- + +## 7. Guardrails (learned this session) + +- **No dead-man watchdog on this firmware:** `setsid` and `nohup` are both absent, and + `start-stop-daemon -x /bin/sh` self-matches the session shell. So you cannot arm a reliable + auto-rollback over SSH. Mitigation: do risky routing changes **from the LAN side** (you're physically + present) so a mistake doesn't lock you out; verify mgmt after every mutation. +- **No `base64` on the router** — transfer files by decoding on the hub/LAN box and piping cleartext + into `cat >` on the router, or just paste scripts. +- **busybox `ping` has no `-M do`** — can't do a clean DF/PMTU probe with it; infer MTU issues from + large-vs-small behavior or `tracepath` if present. +- Verify mgmt access (a WG-sourced ping or SSH) after **every** routing/firewall/mwan3 change. +``` diff --git a/docs/mgmt-tunnel-ssh.md b/docs/mgmt-tunnel-ssh.md new file mode 100644 index 0000000..cacfc77 --- /dev/null +++ b/docs/mgmt-tunnel-ssh.md @@ -0,0 +1,85 @@ +# Inbound management SSH over the WireGuard tunnel + +How the busfleet hub reaches a GL-XE3000 router for management, and how to fix it when +inbound SSH over the tunnel is broken. + +## How it works + +The router runs a WireGuard **client** (`wgclient1`, `proto=wgclient`) that dials the hub +at `167.172.237.162:51820`. Once the tunnel is up the router holds `10.88.0.2/32` and the +hub is `10.88.0.1`. The hub SSHes *inbound* to `10.88.0.2:22` over that tunnel using the +`busfleet-hub-jump` key. + +The key detail: on OpenWrt the WG client interface is in its **own firewall zone** +(`wgclient1`), which defaults to `input='DROP'`. Traffic arriving over the tunnel is +evaluated by the `zone_wgclient1_input` iptables chain — **not** `zone_wan_input`. So the +mgmt-SSH accept rule must have its source zone set to `wgclient1`, or it lands in the wrong +chain and never matches (tunnel SSH silently hits the zone DROP). + +## Provisioning a router + +1. **Confirm the WG client zone name** (may differ per unit): + + ```sh + uci show network | grep -i wgclient # interface, e.g. network.wgclient1 + uci show firewall | grep -iE 'name=|\.network=|input=' # find its zone + input policy + ``` + +2. **Allow inbound mgmt SSH over the tunnel.** Bind the rule's `src` to the WG client zone + (not `wan`), restricted to the hub subnet: + + ```sh + uci add firewall rule + uci set firewall.@rule[-1].name='Allow-WG-mgmt-SSH' + uci set firewall.@rule[-1].src='wgclient1' # the ACTUAL WG zone from step 1 + uci set firewall.@rule[-1].proto='tcp' + uci set firewall.@rule[-1].dest_port='22' + uci set firewall.@rule[-1].family='ipv4' + uci set firewall.@rule[-1].src_ip='10.88.0.0/24' + uci set firewall.@rule[-1].target='ACCEPT' + uci commit firewall && /etc/init.d/firewall restart + ``` + +3. **Install the hub jump key:** + + ```sh + echo 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMS842ZbXPXHbVd9IjyVGubq7M/yQA/cn/vHgB5R0/ZE busfleet-hub-jump' >> /etc/dropbear/authorized_keys + chmod 600 /etc/dropbear/authorized_keys + ``` + +## Verifying + +Use `wg show` as the source of truth for tunnel health — a recent `latest handshake` plus +non-zero transfer counters means the data path is live: + +```sh +wg show # endpoint, latest handshake, transfer +iptables -S zone_wgclient1_input | grep 'dport 22' # exactly one ACCEPT from 10.88.0.0/24 +``` + +Gotchas: + +- **BusyBox `nc` has no `-v`/`-z` flags** (`nc [IPADDR PORT]` only), and `51820` is UDP — + don't rely on `nc -vz` to test the tunnel. Use `wg show`. +- **The hub does not answer ICMP over the tunnel**, so `ping 10.88.0.1` shows 100% loss even + when the tunnel is healthy. The definitive end-to-end test is the hub SSHing in. + +## Troubleshooting: hub can't SSH in + +1. `wg show` — is there a recent handshake? If not, the tunnel is down (check WAN/5G, the + hub endpoint, and keys) before looking at the firewall. +2. `uci show firewall | grep Allow-WG-mgmt-SSH` up through the rule — confirm `src` is the + WG client zone (`wgclient1`), **not** `wan`. This is the most common misconfiguration: + + ```sh + uci set firewall.
.src='wgclient1' + uci commit firewall && /etc/init.d/firewall restart + ``` + +3. `grep busfleet-hub-jump /etc/dropbear/authorized_keys` — key present, file perms `600`. + +## History + +- **2026-07-01** — Fixed inbound mgmt SSH on a unit whose `Allow-WG-mgmt-SSH` rule was bound + to `src='wan'` instead of `src='wgclient1'`, so tunnel SSH hit the `wgclient1` zone DROP. + Repointed the rule to `wgclient1` and installed the `busfleet-hub-jump` key. diff --git a/docs/pilot/checklist.md b/docs/pilot/checklist.md new file mode 100644 index 0000000..65eeecc --- /dev/null +++ b/docs/pilot/checklist.md @@ -0,0 +1,158 @@ +# Three-Bus Field Pilot Validation Checklist + +**Plan ref:** Task 9.1 / Phase 9 — Field pilot sign-off gate +**Units:** bus01 (bench), bus02, bus03 (first three production units) +**Gate:** All three buses must pass before scaling to the 194-unit fleet + +--- + +## Pre-flight (per unit, before boarding) + +- [ ] `ssh pioadmin@192.168.8.1` → password `Pioneer321!` succeeds +- [ ] `cat /etc/busrouter/device-id` returns correct ID (e.g. `B0001`) +- [ ] `cat /etc/busrouter/version` returns current package version +- [ ] WG tunnel up: `wg show` — peer handshake < 60 s ago +- [ ] `mwan3 status` — at least cellular online; wan offline is OK if no dish yet +- [ ] `ps | grep daemon.sh` — busrouter daemon running +- [ ] `ls /tmp/busrouter/` — state files being written (metric_*, score_*, gps_last, etc.) +- [ ] Status page: `http://192.168.8.1/busrouter/` loads in browser, shows live data +- [ ] SSID visible on both bands as ``, connects with `Pioneer123` + +--- + +## Section A — Dual-WAN balancing under motion + +**Scenario:** Bus drives its route for ≥ 30 minutes with both cellular and Starlink active. + +- [ ] A1: Both mwan3 members show "online" during route: `mwan3 status` from hub via WG +- [ ] A2: Weights shift based on scores (not stuck 50/50 throughout entire run) + - Capture: `cat /tmp/busrouter/score_modem_0001` and `score_wan` at start, midpoint, end +- [ ] A3: No LAN interruption detected by a ping flood from a laptop on bus WiFi + - Command: `ping -c 1800 8.8.8.8` (30 min, 1 s interval) — expect ≤ 2% loss total +- [ ] A4: `/tmp/busrouter/metric_modem_0001` and `metric_wan` both updated each cycle +- [ ] A5: No busrouter daemon crash: `logread | grep busrouter | grep -i error` — 0 results + +--- + +## Section B — SIM auto-switch on real coverage change + +**Scenario:** Drive through a known T-Mobile coverage gap or weak-signal zone (RSRP < −110 dBm +for ≥ 3 consecutive cycles) with a second SIM in slot 2. + +- [ ] B1: SIM slot switches from 1 → 2 when RSRP drops below threshold for SIM_SWITCH_CYCLES + - Check: `cat /tmp/busrouter/sim_slot` changes from `1` to `2` +- [ ] B2: Cooldown (300 s) is observed — no rapid oscillation between slots + - Check: `logread | grep sim_switch` — only one event per coverage-gap pass +- [ ] B3: Cellular reconnects after slot switch + - Check: `mwan3 status` shows modem_0001 returns online within ~30 s of switch +- [ ] B4: SIM slot correctly reported in telemetry (hub receives correct slot number) +- [ ] B5: Capture mid-test snapshot — `logread | grep sim_switch` and `cat /tmp/busrouter/sim_slot` immediately after slot change + +--- + +## Section C — Starlink handoff + +**Scenario:** Trigger a Starlink→cellular handoff. Preferred: natural signal loss / obstruction +event during the route. Acceptable fallback (bench unit or when no dish): `ifdown wan` / `ifup wan` +— document which method was used in the sign-off record. If using fallback, note that this does +not validate natural Starlink obstruction detection (requires grpcurl + real dish). + +- [ ] C1: After `ifdown wan`, mwan3 shifts to `modem_0001 (100%)` within ≤ 15 s + - Check: `mwan3 status`, `logread | grep mwan3` +- [ ] C2: LAN ping survives the failover (continue A3 ping through this test) +- [ ] C3: WG mgmt tunnel stays alive throughout — all `ssh` commands during failover succeed + - **Critical:** WG must ride cellular when eth0 is down; verify ip rule still routes correctly +- [ ] C4: After `ifup wan`, mwan3 declares wan online and restores balanced policy within ≤ 30 s +- [ ] C5: Weight bar on status page `http://192.168.8.1/busrouter/` updates correctly + +--- + +## Section D — Telemetry pipeline + +**Scenario:** Run for ≥ 10 cycles with hub reachable, then simulate hub unreachable and +confirm buffering, then restore hub and confirm flush. + +- [ ] D1: Hub receives ≥ 5 telemetry payloads per bus while online + - Check hub: `journalctl -u busrouter-hub` or inspect telemetry store for device_id entries +- [ ] D2: JSON payloads are valid (no parse errors at hub) — correct device_id, version fields +- [ ] D3: Buffer fills when hub unreachable + - Simulate: set `TELEMETRY_HUB=http://192.168.99.99` in daemon env and cycle 5 times + - Check: `ls /tmp/busrouter/telemetry-buf/*.json | wc -l` → 5 files +- [ ] D4: Buffer flushes on hub restore + - Restore TELEMETRY_HUB, trigger `telemetry_send` (or wait next cycle) + - Check: `ls /tmp/busrouter/telemetry-buf/` → empty; hub received the 5 buffered payloads +- [ ] D5: Buffer cap enforced — at 61st file, oldest is evicted (not new item rejected) + - Check: create 60 dummy files in `telemetry-buf/`; run one cycle; count stays at 60 + +--- + +## Section E — Failover must never drop LAN + +**Scenario:** Stress test — bounce both uplinks repeatedly and confirm LAN clients never lose +internet for more than the mwan3 detection window (~15 s). + +- [ ] E1: Bounce cellular 3x (`ifdown modem_0001; sleep 5; ifup modem_0001`): LAN gap ≤ 15 s each +- [ ] E2: Bounce Starlink 3x (`ifdown wan; sleep 5; ifup wan`): LAN gap ≤ 15 s each +- [ ] E3: Simultaneously down both for 10 s, then restore: graceful "unreachable" state, no crash +- [ ] E4: After E3, both interfaces return online and balanced policy resumes automatically +- [ ] E5: After E1–E4, busrouter daemon still running — no segfault or restart in syslog + +--- + +## Section F — WireGuard mgmt plane integrity + +**Scenario:** Verify the WG mgmt tunnel (10.88.0.X) survives every link-state change tested +in Sections A–E. This is the "never lose the management plane" requirement. + +- [ ] F1: Hub can SSH to router at `root@10.88.0.X` throughout all A–E tests +- [ ] F2: `wg show wgclient1` latest handshake stays < 60 s during all tests +- [ ] F3: After cellular→eth0 underlay switch (eth0 comes up), WG remains live on new underlay + - Check endpoint change at hub: `wg show wg0 endpoints` +- [ ] F4: After eth0→cellular underlay switch (eth0 goes down), WG stays live + - **Note:** if eth0 is the WG underlay, do NOT down it without cellular as backup + +--- + +## Section G — Log capture + sign-off + +For each bus, capture the following at end of pilot run and attach to the sign-off record: + +```sh +# Run on router (or via hub WG tunnel): +logread > /tmp/pilot-log-$(cat /etc/busrouter/device-id)-$(date +%Y%m%d).txt +mwan3 status >> /tmp/pilot-log-$(cat /etc/busrouter/device-id)-$(date +%Y%m%d).txt +ip rule show >> /tmp/pilot-log-$(cat /etc/busrouter/device-id)-$(date +%Y%m%d).txt +wg show >> /tmp/pilot-log-$(cat /etc/busrouter/device-id)-$(date +%Y%m%d).txt +cat /tmp/busrouter/score_modem_0001 /tmp/busrouter/score_wan \ + /tmp/busrouter/sim_slot /tmp/busrouter/gps_last \ + >> /tmp/pilot-log-$(cat /etc/busrouter/device-id)-$(date +%Y%m%d).txt +# Copy log off the router: +scp root@192.168.8.1:/tmp/pilot-log-*.txt ./logs/ +``` + +### Sign-off record + +**Log archive:** copy logs to `docs/pilot/logs//` in this repo and commit after each bus. + +| Unit | Pilot date | Sections passed | C method | Issues found | Signed off by | +|------|-----------|-----------------|----------|--------------|---------------| +| bus01 (bench) | — | A B C D E F | synthetic | — | — | +| bus02 | — | A B C D E F | natural/synthetic | — | — | +| bus03 | — | A B C D E F | natural/synthetic | — | — | + +**Fleet scale gate:** all three rows have a signer in "Signed off by" → proceed to full 194-unit +rollout using `provision.sh` + fleet automation (Project 4). + +--- + +## Known issues going into pilot + +1. **No Starlink dish on bench unit (bus01)** — Section A/C Starlink tests require a dish on eth0 + or an ATT Fiber substitute. Document which uplink was used for each Starlink-targeted test. +2. **grpcurl absent** — Starlink gRPC metrics (`starlink.sh`) are currently stubbed; Starlink + score is based on echo/ping only. `grpcurl` binary must be bundled before Section C tests + use `starlink_last` data for scoring. +3. **GPS fix lag** — First GPS fix after `AT+QGPS=1` can take 1–5 minutes cold-start. Section A + GPS checks should allow time for first fix before asserting `gps_last` is populated. +4. **RSRP threshold tuning** — `SIM_SWITCH_RSRP=-110` is the default. Real coverage-gap zones + may need adjustment; tune per route if Section B tests don't trigger naturally. diff --git a/docs/provisioning/device-identity.md b/docs/provisioning/device-identity.md new file mode 100644 index 0000000..54f0bbe --- /dev/null +++ b/docs/provisioning/device-identity.md @@ -0,0 +1,183 @@ +# Pioneer Bus Router — Device Identity & Provisioning + +**Last updated:** 2026-07-01 +**Target hardware:** GL.iNet GL-XE3000 (Puli AX), OpenWrt 21.02-SNAPSHOT, ARMv8 + +--- + +## 1. Fleet-wide constants + +| Parameter | Value | +|-----------|-------| +| WiFi SSID (2.4 + 5 GHz) | `` — the plan spec's `X0000` shorthand means SSID = device ID (e.g. `B0042`) | +| WiFi password | `Pioneer123` | +| Admin username (logical) | `pioadmin` | +| Admin password | `Pioneer321!` | +| Admin SSH user (OpenWrt) | `root` (GL-XE3000 uses root for all admin; pioadmin is the logical name) | +| Timezone | `UTC` | +| Cellular APN | `fast.t-mobile.com` (T-Mobile 5G, Quectel RM520N-GL, bus `1-1.2`) | +| WireGuard hub endpoint | `167.172.237.162:51820` | +| WireGuard hub IP | `10.88.0.1/24` | +| WireGuard tunnel subnet | `10.88.0.0/24` | +| WireGuard keepalive | `25 s` | +| WireGuard MTU | `1280` | +| WG mwan3 fix rule | priority `99`, dest `10.88.0.0/24`, lookup `1001` (required — see §4) | +| mwan3 balance mode | `balanced` (weight-based, 50/50 default; daemon adjusts live weights) | + +**Hub WireGuard public key:** `REPLACE_WITH_HUB_WG_PUBKEY` +(Retrieve with `wg pubkey < /etc/wireguard/hub.key` on the hub at `167.172.237.162`) + +--- + +## 2. Per-device parameters + +Each bus unit gets a unique identity. Fill in this table for each device: + +| DEVICE_ID | WG tunnel IP | WG public key | SIM 1 ICCID | SIM 2 ICCID | Notes | +|-----------|-------------|---------------|-------------|-------------|-------| +| `bus01` | `10.88.0.2` | *(see recon)* | `8901260417776090740F` | — | bench unit; ATT Fiber on eth0 in lab | +| `B0001` | `10.88.0.3` | *(generate)* | — | — | *(first production unit)* | + +**Column format:** +- `DEVICE_ID`: `B` + 4-digit zero-padded bus number (e.g. `B0042`) +- `WG tunnel IP`: `10.88.0.X` (assigned by hub admin, no DHCP; range 10.88.0.2–10.88.0.254) +- `WG private key`: base64, generated per device, never stored in this doc (keep in secrets store) +- `SIM ICCID`: printed on SIM card tray + +### Generating a keypair for a new device + +```sh +# On any Linux machine with wireguard-tools: +PRIVKEY=$(wg genkey) +PUBKEY=$(echo "$PRIVKEY" | wg pubkey) +echo "Private: $PRIVKEY" +echo "Public: $PUBKEY" +``` + +Register the public key + assigned IP at the hub: +```sh +# On hub (167.172.237.162): +wg set wg0 peer allowed-ips 10.88.0.X/32 persistent-keepalive 25 +# Persist (choose ONE based on how the hub manages wg0): +wg-quick save wg0 # if hub uses wg-quick +# OR: edit /etc/wireguard/wg0.conf manually and run: wg addconf wg0 <(diff) +# Verify hub persistence method before using wg-quick save — it overwrites the file. +``` + +--- + +## 3. Provisioning procedure + +### Prerequisites +- Physical or LAN access to the router at `192.168.8.1` +- Cellular SIM installed in slot 1 (provides internet for opkg) +- `kit-busrouter.ipk` copied to `/tmp/kit-busrouter.ipk` on the router +- Device keypair generated and hub-side peer registered (§2) + +### Steps + +```sh +# 1. Copy provision.sh and package to the router +scp scripts/provision.sh root@192.168.8.1:/tmp/provision.sh +scp kit-busrouter_*.ipk root@192.168.8.1:/tmp/kit-busrouter.ipk + +# 2. SSH in and run +ssh root@192.168.8.1 +chmod +x /tmp/provision.sh +/tmp/provision.sh B0042 "" "10.88.0.42" +# Device reboots automatically after ~5 s. + +# 3. Post-reboot verification (from hub 10.88.0.1 over WireGuard) +ssh root@10.88.0.42 +mwan3 status # both modem_0001 and wan should appear (wan offline if no dish) +cat /etc/busrouter/device-id # B0042 +wg show # wgclient1 peer handshake recent +``` + +### Expected WiFi after provisioning +- SSID: `B0042` (same on 2.4 + 5 GHz) +- Password: `Pioneer123` +- LAN IP: `192.168.8.1` + +### Expected admin access +- SSH: `ssh root@192.168.8.1` → password `Pioneer321!` +- Web UI: `http://192.168.8.1` → root / `Pioneer321!` +- WG path: `ssh root@10.88.0.42` (from hub, or via WG client) + +--- + +## 4. mwan3 + WireGuard tunnel coexistence fix + +**Critical — must be applied or in-tunnel SSH drops silently.** + +Root cause: mwan3 adds all directly-connected networks (including wgclient1's 10.88.0.0/24) to +its `mwan3_connected` ipset and marks matching packets with `0x3f00`. The ip rule +`9000: not from all fwmark 0/0xf000 lookup main` then routes replies via main table, which has +no route for 10.88.0.0/24 (wgclient1 routes live in table 1001). Packets are silently dropped. + +Fix: add an ip rule at priority 99 (before mwan3's mark-based rules) routing to 10.88.0.0/24 +via table 1001. `provision.sh` does this automatically: + +```sh +# Runtime: +ip rule add to 10.88.0.0/24 priority 99 lookup 1001 +# Persisted via UCI (recreated by netifd on boot): +uci add network rule +uci set network.@rule[-1].lookup='1001' +uci set network.@rule[-1].dest='10.88.0.0/24' +uci set network.@rule[-1].priority='99' +uci commit network +``` + +Also required: `net.ipv4.conf.wgclient1.rp_filter=2` (written to /etc/sysctl.conf by provision.sh +and re-asserted on WAN flap via /etc/hotplug.d/iface/99-busrouter-rpfilter). + +--- + +## 5. GL-XE3000 hardware notes + +| Detail | Value | +|--------|-------| +| Modem | Quectel RM520N-GL (5G NR SA) | +| Modem transport | MHI/PCIe (NOT USB-CDC; no `/dev/cdc-wdm*`) | +| Modem bus for gl_modem | `1-1.2` (verify per unit: `ls /sys/bus/usb/devices/ | grep -v ':' | sort`) | +| Signal read | `ubus call modem.signal get_signals '{"time":1}'` (primary); `gl_modem -B 1-1.2 AT AT+QCSQ` (fallback) | +| GPS enable | `gl_modem -B 1-1.2 AT AT+QGPS=1` (OFF by default; provision.sh writes to /etc/rc.local) | +| GPS read | `gl_modem -B 1-1.2 AT AT+QGPSLOC=2` (returns error 505/516 until first fix after GNSS enable) | +| SIM switch | `gl_modem -B 1-1.2 AT AT+QUIMSLOT=<1|2>` | +| SIM query | `gl_modem -B 1-1.2 AT AT+QUIMSLOT?` | +| kmwan (GL multiwan) | Disabled by provision.sh — do NOT re-enable; mwan3 owns routing | +| grpcurl | NOT installed; Phase 3 (Starlink) must bundle static arm64 binary in the .ipk | +| setsid / nohup | ABSENT on this firmware; use `start-stop-daemon -S -b -x ` for background procs | +| WG firewall zone | `wgclient1` (NOT `wan`); SSH allow rule: `src=wgclient1 src_ip=10.88.0.0/24 dest_port=22` | + +--- + +## 6. Bench unit `bus01` — specific config + +| Field | Value | +|-------|-------| +| DEVICE_ID | `bus01` (rename to fleet format `B0001` for production) | +| LAN IP | `192.168.8.1` | +| WG tunnel IP | `10.88.0.2` | +| Cellular (WAN1) | `rmnet_mhi0`, UCI `modem_0001`, T-Mobile CGNAT | +| Starlink (WAN2) | `eth0`, UCI `wan` — no dish on bench unit; ATT Fiber used as stand-in | +| WG underlay | Rides `eth0` when fiber is attached (do NOT down eth0 for bench tests) | +| Access via hub | `ssh root@10.88.0.2` (from hub `10.88.0.1`) | + +--- + +## 7. Checklist — new device before field deployment + +- [ ] Device ID assigned and recorded in fleet registry +- [ ] WG keypair generated; public key registered at hub with assigned IP +- [ ] SIM installed; APN = `fast.t-mobile.com`; cellular data confirmed +- [ ] `provision.sh` run end-to-end; device rebooted cleanly +- [ ] WiFi visible as ``, password `Pioneer123` +- [ ] SSH to `root@192.168.8.1` succeeds with `Pioneer321!` +- [ ] `mwan3 status` shows both interfaces (wan offline is OK if no dish yet) +- [ ] WG tunnel up: `wg show` shows handshake < 60 s ago +- [ ] Hub can SSH to `root@10.88.0.WG_IP` +- [ ] `/etc/busrouter/device-id` contains correct DEVICE_ID +- [ ] `busrouter` daemon running: `ps | grep daemon.sh` +- [ ] Status page accessible: `http://192.168.8.1/busrouter/` diff --git a/docs/recon/2026-06-30-xe3000-environment.md b/docs/recon/2026-06-30-xe3000-environment.md new file mode 100644 index 0000000..faa2ec7 --- /dev/null +++ b/docs/recon/2026-06-30-xe3000-environment.md @@ -0,0 +1,384 @@ +# GL-XE3000 Device Reconnaissance (Task 0.2) + +**Date:** 2026-06-30 +**Unit:** bench `bus01` — GL.iNet GL-XE3000 (Puli AX) +**Access path:** LAB → `kit-fleet-hub` (DO droplet 167.172.237.162, WG 10.88.0.1) → jump SSH +`ssh -i /root/.ssh/busfleet root@10.88.0.2` (router WG IP 10.88.0.2). +**Access confirmed:** yes — `ubus call system board` returns `"model": "GL.iNet GL-XE3000"`. + +--- + +## 1. Resolved facts table (feeds later plan tasks) + +| Fact | Value | +|------|-------| +| Cellular ifname (WAN1 member) | **`rmnet_mhi0`** (QMI proto `qcm`, control node `/dev/mhi_QMI0`, UCI iface `modem_0001`, APN `fast.t-mobile.com`) | +| Starlink WAN ifname (WAN2) | **`eth0`** (UCI iface `wan`, proto `dhcp`). NOTE: currently `DOWN`/`NO-CARRIER` — no dish attached to bench unit. `eth1` = LAN (`br-lan`). | +| mwan3 present? | **NO** — `/etc/config/mwan3` absent, `mwan3` package not installed → Task 3.0 / Phase 2 must `opkg install mwan3`. GL ships its own multiwan (`gl-sdk4-ui-multiwan`, `kmod-gl-sdk4-kmwan`, ubus `hotplug.kmwan`) which we are NOT using. | +| Signal read command | **Preferred:** `ubus call modem.signal get_signals '{"time":1}'` → JSON `{rsrp,rsrq,sinr,slot,network_type,strength,mode}`. **AT fallback:** `gl_modem -B 1-1.2 AT AT+QCSQ` → `+QCSQ: "NR5G",,,`. **`uqmi` is NOT usable** (see Implications). | +| SIM select command | Read: `gl_modem -B 1-1.2 AT AT+QUIMSLOT?` → `+QUIMSLOT: 1`. Switch: `gl_modem -B 1-1.2 AT AT+QUIMSLOT=2` (confirmed slot-query works; switch command not exercised to avoid dropping the live link). | +| GPS read command | `gl_modem -B 1-1.2 AT AT+QGPSLOC=2` — but GNSS must be enabled first: `AT+QGPS=1`. Currently `AT+QGPS?` = `0` (disabled), so `QGPSLOC` returns `+CME ERROR: 505` (no fix). | +| grpcurl available? | **NO** — `command -v grpcurl` = MISSING → Phase 3 (Starlink) must bundle a static arm64 `grpcurl` binary in the opkg package. | +| Firewall zone of WG client iface | **`wgclient1`** — its own dedicated zone named `wgclient1` (network `wgclient1`, input DROP). NOT `wan`. The access-fix rule is `firewall.@rule[22]` name `Allow-WG-mgmt-SSH`, `src='wgclient1'`, `src_ip='10.88.0.0/24'`, dest_port 22, ACCEPT. | +| Modem USB bus (for `gl_modem -B`) | **`1-1.2`** (Quectel RM520N-GL, 4 USB interfaces at `1-1.2:1.0-1.3`) | + +--- + +## 2. Raw command outputs + +### `ubus call system board` +```json +{ + "kernel": "5.4.211", + "hostname": "GL-XE3000", + "system": "ARMv8 Processor rev 4", + "model": "GL.iNet GL-XE3000", + "board_name": "glinet,xe3000-emmc", + "release": { + "distribution": "OpenWrt", + "version": "21.02-SNAPSHOT", + "target": "mediatek/mt7981" + } +} +``` + +### `/etc/config/network` (relevant excerpts) +``` +config device # LAN bridge + option name 'br-lan' + list ports 'eth1' +config interface 'lan' + option device 'br-lan' + option ipaddr '192.168.8.1' +config device + option name 'eth0' +config interface 'wan' # Starlink WAN port + option device 'eth0' + option proto 'dhcp' + option metric '10' +config interface 'modem_0001' # Cellular / 5G + option apn 'fast.t-mobile.com' + option ip_type 'IPV4V6' + option proto 'qcm' + option device '/dev/mhi_QMI0' + option metric '40' + option iccid '8901260417776090740F' +config interface 'wgclient1' # WireGuard out-of-band mgmt + option proto 'wgclient' + option config 'peer_2001' + option ip4table '1001' +``` +(Also present but unused/disabled: `wwan`, `secondwan`, `guest`, various GL vpn policy rules.) + +### `/etc/config/mwan3` +``` +NO mwan3 +``` + +### `opkg list-installed` (filtered: mwan3|grpc|speedtest|modem|gl-|uqmi|quectel) +Notable entries (full list captured in session): +``` +gl-sdk4-modem git-2025.290.37125-7c3efcc-1 +gl-sdk4-ui-multiwan git-2025.259.31350-8a2a5c6-1 +gl-sdk4-ui-modemsignallog git-2024.303.14204-4241be7-1 +kmod-gl-pcie-modem 5.4.211+20221009-1 +kmod-gl-sdk4-kmwan 5.4.211+git-2025.304.07170-bb470c0-1 +uqmi 2020-11-22-0a19b5b7-3 +``` +No `mwan3`, no `grpcurl`, no `speedtest` (Ookla) package installed. + +### `ip -br addr` +``` +lo UNKNOWN 127.0.0.1/8 ::1/128 +eth0 DOWN # Starlink WAN — no carrier +eth1 UP # LAN member +ra0 UP # 2.4G wifi +rmnet_mhi0 UP 100.112.80.45/30 # Cellular (CGNAT) +br-lan UP 192.168.8.1/24 +wgclient1 UNKNOWN 10.88.0.2/32 # WG mgmt tunnel +``` + +### `ip route` +``` +default via 100.112.80.46 dev rmnet_mhi0 proto static src 100.112.80.45 metric 40 +100.112.80.44/30 dev rmnet_mhi0 proto static scope link metric 40 +192.168.8.0/24 dev br-lan proto kernel scope link src 192.168.8.1 +``` + +### Modem device nodes +``` +/dev/mhi_BHI /dev/mhi_DIAG /dev/mhi_DUN /dev/mhi_LOOPBACK /dev/mhi_QMI0 +/dev/ttyUSB0 /dev/ttyUSB1 /dev/ttyUSB2 /dev/ttyUSB3 +``` +No `/dev/cdc-wdm*` — modem is on the **MHI/PCIe** transport, not USB-CDC. + +### Tool availability +``` +gl_modem: /usr/bin/gl_modem +uqmi: /sbin/uqmi +grpcurl: MISSING +mmcli: MISSING +gpsd: MISSING +``` + +### `gl_modem` usage +``` +gl_modem -B AT # send AT command; bus for this unit = 1-1.2 +``` + +### Signal — ubus (preferred) +```json +// ubus call modem.signal get_signals '{"time":1}' +{ + "signals": [{ + "sinr": 13, "slot": 1, "network_type": "NR5G-SA", + "timestamp": 1782917019, "strength": 4, + "rsrp": -96, "rsrq": -10, "mode": 5 + }] +} +``` +`ubus -v list modem.signal` → methods `get_signals {time:Integer}`, `upload_cloud_signals {action:String}`. + +### Signal — AT fallback +``` +gl_modem -B 1-1.2 AT AT+QCSQ ++QCSQ: "NR5G",-97,14,-10 # ,,, +``` + +### SIM slot +``` +gl_modem -B 1-1.2 AT AT+QUIMSLOT? ++QUIMSLOT: 1 # active SIM = slot 1 +``` + +### GPS / GNSS +``` +gl_modem -B 1-1.2 AT AT+QGPS? ++QGPS: 0 # GNSS engine currently OFF +gl_modem -B 1-1.2 AT AT+QGPSLOC=2 ++CME ERROR: 505 # no fix (engine not enabled) +``` + +### Starlink WAN port link +``` +eth0 DOWN +``` + +### Firewall (WG mgmt zone + access-fix rule) +``` +firewall.wgclient1 = zone +firewall.wgclient1.name='wgclient1' +firewall.wgclient1.network='wgclient1' +firewall.wgclient1.input='DROP' +firewall.@rule[22].name='Allow-WG-mgmt-SSH' +firewall.@rule[22].src='wgclient1' +firewall.@rule[22].src_ip='10.88.0.0/24' +``` +Zone[1] `wan` groups: `wan wan6 wwan secondwan modem_0001 modem_0001_4` (input DROP). + +--- + +## 3. Implications for the plan + +1. **mwan3 is NOT installed (plan flagged this as possible).** `/etc/config/mwan3` is absent and no `mwan3` + opkg is present. Phase 2 / Task 3.0 must `opkg install mwan3` (verify it exists in GL's feed for + `mediatek/mt7981` @ 21.02-SNAPSHOT; if not, add the upstream OpenWrt 21.02 feed or bundle it). + GL ships its own `kmwan`/multiwan stack (`kmod-gl-sdk4-kmwan`, `gl-sdk4-ui-multiwan`, ubus + `hotplug.kmwan`) — the spec locks on `mwan3`, so we intentionally do NOT use GL's kmwan. Watch for + conflict/overlap between the two multiwan systems on the same box (routing marks, hotplug). + +2. **Firewall zone assumption in Task 0.1 was WRONG.** Task 0.1 Step 1 guessed `src='wan'` (with a + "verify / adjust if wgclient" note). The WG client interface is actually in its **own zone `wgclient1`**, + and that is what the working fix rule uses (`@rule[22] src='wgclient1' src_ip='10.88.0.0/24'`). Any + future mgmt/telemetry firewall rules must target zone **`wgclient1`**, not `wan`. + +3. **`uqmi` is effectively unusable for signal reads.** `uqmi -d /dev/mhi_QMI0 --get-signal-info` **hangs + (timeout)** — GL's modem manager holds the QMI control channel (single-client QMUX). Use the GL ubus + path `modem.signal get_signals` (clean JSON, no device contention) as the primary signal source, with + `gl_modem -B 1-1.2 AT AT+QCSQ` as the AT fallback. **Phase 4 Task 4.1 should target ubus first**, not + uqmi as the plan's example suggested. + +4. **`gl_modem` requires a `-B ` argument** (the plan's examples `gl_modem AT AT+...` omit it and would + fail). The bus on this unit is **`1-1.2`**. This must be discovered per-unit (parse from + `/sys/bus/usb/devices` or GL config) rather than hard-coded, since it could differ across units — + record for the provisioning profile (Phase 8). + +5. **GPS/GNSS is present but OFF by default.** `AT+QGPS?` = 0, so `AT+QGPSLOC=2` errors `+CME ERROR: 505`. + Phase 5 Task 5.1 (`gps_fix`) must **enable GNSS first** (`AT+QGPS=1`, one-time / on boot) and handle the + "no fix yet" (error 505/516) state gracefully before location is available. GPS command form is + confirmed as Quectel `AT+QGPSLOC=2` (decimal degrees). + +6. **`grpcurl` absent (plan flagged this).** Phase 3 Task 3.3 must bundle a static **arm64 (ARMv8)** + `grpcurl` binary in the opkg package for Starlink gRPC. + +7. **No Starlink dish on the bench unit** (`eth0` NO-CARRIER). WAN2 (Starlink) integration and any + two-WAN balancing test cannot be validated on this bench unit as-is — need a dish on `eth0`, or test + WAN2 with a substitute uplink. The cellular WAN (`rmnet_mhi0`) is the only live uplink right now, and + it carries the WG mgmt tunnel (CGNAT `100.112.80.45/30`, T-Mobile). + +8. **Ookla speedtest CLI not installed** — Phase 3 Task 3.2 must bundle/install it (or rely on the curl + fallback), consistent with the plan's fallback design. + +--- + +## 4. Task 2.0 — mwan3 / kmwan changeover (2026-07-01) + +Performed on bench `bus01` via the WG mgmt path. This section feeds the Phase 8 provisioning profile. + +### GL multiwan (kmwan) — exact init name + stand-down +- **GL multiwan init script name: `kmwan`** — `/etc/init.d/kmwan` (the ONLY `mwan|kmwan` init on the box; + confirms recon §1 `kmod-gl-sdk4-kmwan`). It is a `/etc/rc.common` script, `START=21`, sourcing + `/lib/functions/kmwan.sh`. It was **enabled** but had **no running daemon** (routing is managed via + hotplug scripts / a single static default route, not a persistent process). +- **Exact commands used to stand it down** (Phase 8 must run these): + ``` + /etc/init.d/kmwan disable # rc=0 → NOW_DISABLED (won't start on boot) + /etc/init.d/kmwan stop # rc=0 + ``` + Verify with `/etc/init.d/kmwan enabled` (returns non-zero = disabled) and + `ps w | grep -i mwan | grep -v grep` (no kmwan process). + +### mwan3 install +- Installed from **GL's own feed** (no third-party/upstream feed needed): + `opkg update && opkg install mwan3` → **`mwan3 2.8.15-2`** (`mwan3_2.8.15-2_all.ipk`, + feed `glinet_gli_pub` @ v21.02.3). Binary at `/usr/sbin/mwan3`. +- mwan3 ships a **stock default `/etc/config/mwan3`** (dated 2022) with interfaces + `wan/wwan/tethering` (+v6). **NONE of these match our live cellular uplink** (`rmnet_mhi0`, UCI iface + `modem_0001`), so mwan3 is currently **inert**: `mwan3 status` shows every interface "offline and + tracking is down", **no active policies, no active user rules**. mwan3's `mwan3rtmon` route-monitor + daemon runs but does not touch the default route (member config is a later task). + +### Safety / recovery notes for Phase 8 +- **`setsid` and `nohup` are BOTH ABSENT** on this GL firmware (BusyBox build). For detached/background + processes use **`start-stop-daemon -S -b -x -- `** (present at `/sbin/start-stop-daemon`). + Note: `start-stop-daemon -b -x /bin/sh` will refuse to launch if it matches an already-running + `/bin/sh` ("… is already running") — give the watchdog a unique interpreter/exec name or wrap it so the + `-x` match is unique. +- **Post-change verification (mgmt access + default route SURVIVED):** after `kmwan disable/stop`, a fresh + probe returned `"model": "GL.iNet GL-XE3000"`, default route unchanged + (`default via 100.116.118.126 dev rmnet_mhi0 … metric 40` — CGNAT src rotated from the recon-day + `100.112.80.45` to `100.116.118.125`, normal T-Mobile lease change), and `wgclient1 10.88.0.2/32` up. + The WG mgmt routing rules (`from/to 10.88.0.2 lookup 1001`, table 1001) are intact. **Disabling kmwan + did not disturb routing** — expected, since kmwan had no active daemon and the live path is a single + static default on `rmnet_mhi0`. + +--- + +## 5. Task 2.1 pre-check — second uplink (ATT Fiber on eth0) added (2026-07-01) + +**Status: BLOCKED (device shell unreachable) — but the load-bearing WG-underlay question is answered +from hub-side WireGuard evidence.** + +### What happened +The operator plugged ATT Fiber into `eth0` (WAN2 stand-in for Starlink). Immediately afterward the +router's SSH shell (via the WG mgmt tunnel `10.88.0.2:22`) became **unreachable** from the fleet hub, +even though the WireGuard tunnel itself is **healthy**. All on-device commands (eth0 lease, `ip route` +metrics, `ip route get`, `ip rule`, `mwan3 status`) therefore **could not be run**. No device state was +mutated. The facts below are what the hub can observe about the tunnel from `10.88.0.1` (wg0). + +### THE KEY ANSWER — WG mgmt path now rides eth0 (ATT Fiber) +The WireGuard peer **endpoint changed** from the T-Mobile CGNAT `100.x.x.x` seen in §2/§4 to a **public, +non-CGNAT address**, which is the ATT Fiber public IP: + +``` +# hub: wg show wg0 endpoints +0Xn7aGlyxHQ/X9uo8lsOeI3EFA7KO+zlO+qIhrzsAF8= 99.163.124.171:44881 +# latest-handshakes: 6 s ago (fresh); transfer: 335 KiB rx / 244 KiB sent (growing) +``` + +`99.163.124.171` is a routable public IP (not `100.64.0.0/10` CGNAT), i.e. the router is now sourcing its +WG handshakes out **eth0 / ATT Fiber**, not `rmnet_mhi0` / cellular. **The WG mgmt underlay rides eth0 +(ATT Fiber).** Corollary: **downing `eth0` would sever the WG mgmt path / SSH access.** + +### Safe failover-test direction +- **DO NOT bring down `eth0` (ATT Fiber)** for the failover drill — it carries the WG underlay; downing + it cuts mgmt/SSH. +- **`rmnet_mhi0` (cellular) is the SAFE interface to bring down** for the mwan3 failover test — it does + NOT currently carry the WG underlay handshake. (This is the *inverse* of the pre-fiber state in §2/§4, + where cellular carried mgmt and eth0 was NO-CARRIER.) + +### Current mgmt-reachability fault (flag for operator — needs on-device follow-up) +The WG control plane is up but **no in-tunnel application traffic returns from the router**: + +``` +# hub, over the tunnel: +ip route get 10.88.0.2 -> 10.88.0.2 dev wg0 src 10.88.0.1 (route correct) +ping -c3 10.88.0.2 -> 100% loss (even 56-byte payloads) +nc -vz 10.88.0.2 22 -> timed out ("Operation now in progress") +# 4 s window: wg 'sent' climbed 244308 -> (SYN retransmits reach router), 'rx' FLAT +``` + +Interpretation: the router **receives** hub traffic (encrypted `sent` counter grows) but **replies with +nothing** — its in-tunnel return path for ICMP/TCP is broken. Most likely the policy-routing that steers +tunnel replies (table `1001`, `from/to 10.88.0.2 lookup 1001`, per §4) or the WG interface's reverse-path +was disturbed when the fiber default route (metric 10) came up and displaced cellular (metric 40) as the +preferred default. This must be diagnosed/repaired **on-device via a console/LAN path** (the WG path is +the very thing that's down) before Task 2.1 routing changes proceed. Not attempted here (read-only, +no-mutate scope). + +### Facts NOT captured (require device shell — blocked) +- eth0/wan DHCP lease (ATT subnet / gateway / netmask), eth0 carrier flag from the device side. +- Full `ip route` default table with metrics (expected eth0 metric 10 preferred over cellular metric 40 — + strongly implied by the endpoint move to fiber, but not directly read). +- `rmnet_mhi0` current CGNAT IP, `ip route get 167.172.237.162` / `ip route get 99.163.124.171`. +- `mwan3 status` / `mwan3 interfaces` view of the two ifaces, `ip rule` table listing. + +Re-run §5 in full once console/LAN access to `bus01` is restored and the WG mgmt path is repaired. + +--- + +## 6. WG mgmt tunnel root cause + fix (2026-07-01) + +**Symptom:** after ATT Fiber plugged into eth0 (metric 10 preferred default), WG handshakes were +fresh but all in-tunnel data (ICMP/SSH from 10.88.0.1 to 10.88.0.2) returned 100% loss. + +**Root cause:** `mwan3` adds all directly-connected networks to the `mwan3_connected` ipset. +10.88.0.0/24 is a directly-connected route (wgclient1), so mwan3's iptables `mwan3_connected` +chain marks packets destined for that prefix with `0x3f00`. The ip rule +`9000: not from all fwmark 0/0xf000 lookup main` then routes them via the **main table**, +which has **no route for 10.88.0.0/24** (wgclient1 routes are only in table 1001 due to +`ip4table='1001'` in UCI). The packet is silently dropped — never reaches WG. + +**Fix (applied 2026-07-01):** +Add an ip rule at priority 99 (before any mwan3 fwmark rule) routing packets to 10.88.0.0/24 +via table 1001: + +``` +ip rule add to 10.88.0.0/24 priority 99 lookup 1001 +``` + +Persisted as a UCI network rule so netifd recreates it on every boot: + +``` +uci add network rule +uci set network.@rule[-1].lookup='1001' +uci set network.@rule[-1].dest='10.88.0.0/24' +uci set network.@rule[-1].priority='99' +uci commit network +``` + +Also applied for robustness: `wgclient1 mtu=1280`, rp_filter=2 persisted via /etc/sysctl.conf +and /etc/hotplug.d/iface/99-busrouter-rpfilter (re-asserts on WAN flap). + +**Failover note:** `scripts/diag/wg-mgmt-diagnose.sh` and `scripts/diag/wg-mgmt-fix.sh` in +this repo implement the fix. The underlying keepalive (`persistent_keepalive=25`) was already +configured from a prior session; the new ip rule was the missing piece. + +**Phase 8 provisioning profile must include** this UCI rule — without it, any device with +mwan3 configured will have the WG mgmt tunnel silently drop in-tunnel data. + +## 7. Task 2.1 bench-test findings (2026-07-01) + +mwan3 configured with `modem_0001` (cellular) + `wan` (fiber/eth0), policy `balanced`. +Both interfaces tracked online (1.1.1.1 + 8.8.8.8, reliability 1, down=3, up=3 x 5s interval). + +**Failover drill result:** +- `ip link set rmnet_mhi0 down` -> mwan3 declares `modem_0001 error` within ~15s (checked) +- Policy shifts to `wan (100%)` automatically (checked) +- WG mgmt tunnel (riding eth0) stays alive throughout failover (checked) +- **Recovery gotcha:** `ip link set rmnet_mhi0 up` does NOT restore the QMI data session. + Must use `ifdown modem_0001 && ifup modem_0001` to reconnect. After ~20s both interfaces + return to `balanced: wan(50%) modem_0001(50%)` (checked) + +**Phase 9 pilot:** fiber failover (bring fiber down, WG must ride cellular) should be tested +once the provisioning robustness fix (section 6 ip rule) is confirmed to survive both uplink combos. diff --git a/docs/superpowers/plans/2026-06-30-pioneer-bus-router-balancer.md b/docs/superpowers/plans/2026-06-30-pioneer-bus-router-balancer.md new file mode 100644 index 0000000..e298ff3 --- /dev/null +++ b/docs/superpowers/plans/2026-06-30-pioneer-bus-router-balancer.md @@ -0,0 +1,448 @@ +# Pioneer Bus Router — On-Device Multi-WAN Balancer Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build the OpenWrt `opkg` package that intelligently balances 5G + Starlink on a GL-XE3000, auto-switches SIMs on signal degradation, and reports telemetry to the fleet hub. + +**Architecture:** A branded `opkg` package layered on GL.iNet stock firmware. A supervised `init.d` daemon runs a control loop: collect per-WAN metrics → score → tune `mwan3` weights + trigger failover/SIM-switch → emit telemetry over the WireGuard tunnel. Pure-shell scoring/hysteresis logic is unit-tested with `bats`; on-device integration is validated on a bench unit (`bus01`, WG `10.88.0.2`). + +**Tech Stack:** POSIX/ash shell, OpenWrt `opkg` + `procd`/`init.d`, `mwan3`, `uci`, GL modem tooling (AT/`uqmi`), Ookla Speedtest CLI + `curl`, Starlink gRPC (`grpcurl`), `bats-core` for tests. + +**Reference source (port from):** the Synology package `aiwanbal` at +`https://git.keylinkit.net/kitadmin/KIT-Smart-Load-balancer` — reuse the *logic* in +`bin/aiwanbal-daemon.sh`, `bin/aiwanbal-modem.sh`, `bin/aiwanbal-starlink.sh`, `bin/aiwanbal-throttle.sh`; +rewrite the platform layer for OpenWrt. + +**Spec:** `docs/superpowers/specs/2026-06-30-pioneer-bus-router-design.md` + +--- + +## Recon-applied corrections (2026-06-30 — authoritative, see `docs/recon/2026-06-30-xe3000-environment.md`) + +These OVERRIDE the placeholder examples elsewhere in this plan: + +- **Cellular ifname:** `rmnet_mhi0` (QMI, UCI iface `modem_0001`, APN `fast.t-mobile.com`). **Starlink WAN:** `eth0` (UCI iface `wan`) — no dish attached to the bench unit yet. +- **`mwan3` is NOT installed; GL ships its own `kmwan` multiwan stack.** Decision (locked): install `mwan3` via `opkg` and **disable `kmwan`** for our two members so we keep portable, standard control (spec §4). Watch for GL firmware updates re-enabling `kmwan`. Handled in **Task 2.0** (new, below). +- **Signal read = ubus, not uqmi.** Use `ubus call modem.signal get_signals` (clean JSON: rsrp/rsrq/sinr/slot). `uqmi` **hangs** (GL holds the QMUX). AT fallback: `gl_modem -B 1-1.2 AT AT+QCSQ`. +- **`gl_modem` requires a `-B ` arg** (bench unit = `1-1.2`); the bus must be discovered per-unit at provisioning. SIM select: `gl_modem -B AT AT+QUIMSLOT=2` (read via `AT+QUIMSLOT?`). +- **GPS is present but OFF by default** — one-time enable `gl_modem -B AT AT+QGPS=1`, then read `AT+QGPSLOC=2`; handle "no fix" (CME 505) gracefully. +- **`grpcurl` is missing** — bundle a static arm64 binary in the package (Task 3.3). +- **WG mgmt firewall zone = `wgclient1`** (not `wan`); the Task 0.1 rule was corrected accordingly. +- **Bench limits:** no Starlink dish, no Ookla CLI on the unit → WAN2 balancing and speed tests can't be validated on the bench yet; validate in the Phase 9 pilot. Cellular is the only live uplink and carries the WG tunnel. + +--- + +## File Structure + +``` +package/kit-busrouter/ +├── Makefile # OpenWrt package definition (opkg build) +├── files/ +│ ├── etc/ +│ │ ├── init.d/busrouter # procd service (supervise the daemon) +│ │ ├── config/busrouter # UCI config (all knobs) +│ │ └── busrouter/ +│ │ └── busrouter.conf # runtime config seeded on first install +│ └── usr/lib/busrouter/ +│ ├── daemon.sh # control loop orchestrator +│ ├── lib-score.sh # PURE: scoring math (unit-tested) +│ ├── lib-decide.sh # PURE: hysteresis, weight formula (unit-tested) +│ ├── wan-mwan.sh # mwan3 weight apply + dead-gateway detection +│ ├── metrics-net.sh # latency/jitter/loss ping probes +│ ├── speedtest.sh # Ookla + curl fallback, data-cap aware +│ ├── modem-sim.sh # signal read + SIM switch (GL modem) +│ ├── starlink.sh # dish gRPC status +│ └── telemetry.sh # collect + POST to hub over WG; GPS read +├── luci-app-busrouter/ # branded local status/diagnostics UI (Phase 7) +└── tests/ + ├── bats/ # bats-core suites for lib-score / lib-decide + └── fixtures/ # captured sample outputs (speedtest, AT, gRPC) + +docs/ +├── recon/2026-06-30-xe3000-environment.md # Phase 0 output (facts about the device) +└── provisioning/device-identity.md # Phase 8 output +``` + +Split rationale: `lib-score.sh` and `lib-decide.sh` are **pure functions** (stdin/args → stdout, no side effects) so they unit-test cleanly and hold in context. Everything touching the device (mwan3, modem, gRPC) is isolated behind its own file with a narrow interface the daemon calls. + +--- + +## Phase 0 — Foundations & device reconnaissance + +### Task 0.1: Establish LAB SSH access to the bench router + +**Files:** none (infra). Produces working `ssh -J` path from LAB → router. + +- [x] **Step 1: Allow input on the WireGuard zone (run from local Claude Code over LAN)** + +On the router (`192.168.8.1`, root pw `kitPLANE1!!`), permit management from the hub subnet: +```sh +uci add firewall rule +uci set firewall.@rule[-1].name='Allow-WG-mgmt-SSH' +uci set firewall.@rule[-1].src='wan' # verify actual zone of the wg client iface in Step 3 recon; adjust if 'wgclient' +uci set firewall.@rule[-1].proto='tcp' +uci set firewall.@rule[-1].dest_port='22' +uci set firewall.@rule[-1].family='ipv4' +uci set firewall.@rule[-1].src_ip='10.88.0.0/24' +uci set firewall.@rule[-1].target='ACCEPT' +uci commit firewall && /etc/init.d/firewall restart +``` + +- [x] **Step 2: Install the LAB public key on the router** + +Append the LAB key to dropbear authorized_keys: +```sh +mkdir -p /etc/dropbear +echo 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAFxQnKmt21nWE/HDt/UPLWeEvXP/lTrvtpVoPNrbuH8 kitadmin@lab-cloudcli' >> /etc/dropbear/authorized_keys +chmod 600 /etc/dropbear/authorized_keys +``` + +- [x] **Step 3: Verify from the LAB (jump through the hub)** + +Put the LAB private key on the hub once (so the hub can be a jump host), then: +Run: `ssh -J root@167.172.237.162 root@10.88.0.2 'ubus call system board'` +Expected: JSON with `"model": "GL.iNet GL-XE3000"`. If refused, re-check the firewall `src`/`src_ip` from Step 1 against the recon in 0.2. + +### Task 0.2: Device reconnaissance — capture ground truth + +**Files:** Create `docs/recon/2026-06-30-xe3000-environment.md` + +- [x] **Step 1: Collect environment facts from the router** + +Run each and record output verbatim into the recon doc: +```sh +ubus call system board # model, firmware, kernel +cat /etc/config/network # interfaces, incl. cellular + wan +cat /etc/config/mwan3 2>/dev/null || echo "NO mwan3" # is mwan3 present/configured? +opkg list-installed | grep -Ei 'mwan3|grpc|speedtest|modem|gl-' # available tooling +ip -br addr; ip route; ls -l /dev/cdc-wdm* /dev/ttyUSB* 2>/dev/null +gl_modem AT AT+CGDCONT? 2>/dev/null; gl_modem 2>/dev/null | head # GL modem CLI shape +uqmi -d /dev/cdc-wdm0 --get-signal-info 2>/dev/null # QMI signal path (if present) +cat /tmp/gps* 2>/dev/null; ls /dev/tty* | head # GPS/GNSS exposure +logread | grep -i sim | tail # how SIM state is logged +``` + +- [x] **Step 2: Record the resolved facts the rest of the plan depends on** + +In the recon doc, fill this table with the ACTUAL values discovered (these feed later tasks): + +| Fact | Value | +|------|-------| +| Cellular ifname (mwan3 member) | e.g. `rmnet_mhi0` | +| Starlink WAN ifname | e.g. `eth1` / `wan` | +| mwan3 present? | yes/no (if no → Task 3.0 installs it) | +| Signal read command | `gl_modem AT AT+QCSQ` \| `uqmi ... --get-signal-info` | +| SIM select command | e.g. `gl_modem AT AT+QUIMSLOT=2` (confirm on device) | +| GPS read command | e.g. `gl_modem AT AT+QGPSLOC=2` | +| grpcurl available? | yes/no (if no → bundle static arm64 binary) | + +- [x] **Step 3: Commit the recon doc** +```bash +git add docs/recon/2026-06-30-xe3000-environment.md +git commit -m "docs: XE3000 device reconnaissance facts" +``` + +### Task 0.3: OpenWrt package skeleton (walking skeleton) + +**Files:** Create `package/kit-busrouter/Makefile`, `files/etc/init.d/busrouter`, `files/etc/config/busrouter`, `files/usr/lib/busrouter/daemon.sh` + +- [x] **Step 1: Write the package Makefile** +```make +include $(TOPDIR)/rules.mk +PKG_NAME:=kit-busrouter +PKG_VERSION:=0.1.0 +PKG_RELEASE:=1 +include $(INCLUDE_DIR)/package.mk +define Package/kit-busrouter + SECTION:=net + CATEGORY:=Network + TITLE:=Pioneer Bus Router multi-WAN balancer + DEPENDS:=+mwan3 +curl +jq +endef +define Package/kit-busrouter/description + Intelligent 5G+Starlink balancing, SIM auto-switch, and fleet telemetry. +endef +define Build/Compile +endef +define Package/kit-busrouter/install + $(INSTALL_DIR) $(1)/usr/lib/busrouter $(1)/etc/init.d $(1)/etc/config $(1)/etc/busrouter + $(CP) ./files/usr/lib/busrouter/* $(1)/usr/lib/busrouter/ + $(INSTALL_BIN) ./files/etc/init.d/busrouter $(1)/etc/init.d/busrouter + $(INSTALL_CONF) ./files/etc/config/busrouter $(1)/etc/config/busrouter + $(INSTALL_CONF) ./files/etc/busrouter/busrouter.conf $(1)/etc/busrouter/busrouter.conf +endef +$(eval $(call BuildPackage,kit-busrouter)) +``` + +- [x] **Step 2: Write the procd init script** +```sh +#!/bin/sh /etc/rc.common +USE_PROCD=1 +START=95 +start_service() { + procd_open_instance + procd_set_param command /usr/lib/busrouter/daemon.sh + procd_set_param respawn 3600 5 0 # respawn, 5s delay — fail-safe supervision + procd_set_param stdout 1; procd_set_param stderr 1 + procd_close_instance +} +``` + +- [x] **Step 3: Write a minimal daemon that logs a heartbeat** +```sh +#!/bin/sh +CONFIG=/etc/busrouter/busrouter.conf +[ -f "$CONFIG" ] && . "$CONFIG" +INTERVAL="${INTERVAL:-30}" +logger -t busrouter "daemon started (v0.1.0)" +while :; do + logger -t busrouter "heartbeat: $(date +%s)" + sleep "$INTERVAL" +done +``` + +- [x] **Step 4: Seed default config** + +`files/etc/config/busrouter` (UCI) and a sourced `busrouter.conf` with `INTERVAL=30`. Keep both minimal for now. + +- [x] **Step 5: Deploy to the bench unit and verify supervision + reboot survival** + +Build the package (or for fast iteration, `scp` the files into place over LAN), then: +```sh +/etc/init.d/busrouter enable && /etc/init.d/busrouter start +logread -f -e busrouter # expect: "daemon started" then heartbeats +reboot # after boot, confirm heartbeats resume +``` +Expected: heartbeat lines every 30s, and again after reboot. + +- [x] **Step 6: Commit** +```bash +git add package/kit-busrouter && git commit -m "feat: kit-busrouter package skeleton with supervised daemon" +``` + +--- + +## Phase 1 — Scoring core (pure shell, unit-tested) + +Port the algorithm defined in `aiwanbal` REQUIREMENTS §2.3 and `bin/aiwanbal-daemon.sh`. + +### Task 1.1: bats test harness + +**Files:** Create `tests/bats/`, add `bats-core` as a dev dependency (documented in README). + +- [x] **Step 1: Install bats locally and add a smoke test** + +`tests/bats/smoke.bats`: +```bash +@test "bats runs" { run true; [ "$status" -eq 0 ]; } +``` +Run: `bats tests/bats/smoke.bats` → Expected: `1 test, 0 failures`. +- [x] **Step 2: Commit** — `git add tests && git commit -m "test: add bats harness"` + +### Task 1.2: Per-metric scoring functions (TDD) + +**Files:** Create `tests/bats/score.bats`, then `files/usr/lib/busrouter/lib-score.sh` + +- [x] **Step 1: Write the failing tests** (`tests/bats/score.bats`) +```bash +setup() { load '../../package/kit-busrouter/files/usr/lib/busrouter/lib-score.sh'; } + +@test "latency: 5ms scores 100" { run score_latency 5; [ "$output" -eq 100 ]; } +@test "latency: 200ms scores 0" { run score_latency 200; [ "$output" -eq 0 ]; } +@test "latency clamps below 0" { run score_latency 500; [ "$output" -eq 0 ]; } +@test "download: 25MB/s scores 100" { run score_download 25; [ "$output" -eq 100 ]; } +@test "download: 0.5MB/s scores 0" { run score_download 0.5; [ "$output" -eq 0 ]; } +@test "loss: 0% scores 100" { run score_loss 0; [ "$output" -eq 100 ]; } +@test "loss: 40% scores 0" { run score_loss 40; [ "$output" -eq 0 ]; } +@test "signal: -65dBm scores 100" { run score_signal -65; [ "$output" -eq 100 ]; } +@test "signal: -95dBm scores 0" { run score_signal -95; [ "$output" -eq 0 ]; } +``` +- [x] **Step 2: Run — expect FAIL** (`bats tests/bats/score.bats` → functions not defined) + +- [x] **Step 3: Implement `lib-score.sh`** (integer math via `awk` for portability on ash) +```sh +# clamp a value to 0..100 +_clamp() { awk -v v="$1" 'BEGIN{ if(v<0)v=0; if(v>100)v=100; printf "%d", (v+0.5) }'; } +# linear map: at x=lo -> 0..? Here score_* return 0..100 per spec anchors. +score_latency() { awk -v x="$1" 'BEGIN{ s=100*(200-x)/(200-5); print s }' | _pipeclamp; } +score_download() { awk -v x="$1" 'BEGIN{ s=100*(x-0.5)/(25-0.5); print s }' | _pipeclamp; } +score_upload() { awk -v x="$1" 'BEGIN{ s=100*(x-0.5)/(12.5-0.5); print s }' | _pipeclamp; } +score_jitter() { awk -v x="$1" 'BEGIN{ s=100*(50-x)/(50-1); print s }' | _pipeclamp; } +score_loss() { awk -v x="$1" 'BEGIN{ s=100*(40-x)/40; print s }' | _pipeclamp; } +score_signal() { awk -v x="$1" 'BEGIN{ s=100*(x-(-95))/((-65)-(-95)); print s }' | _pipeclamp; } +_pipeclamp() { read v; _clamp "$v"; } +``` +- [x] **Step 4: Run — expect PASS** (`bats tests/bats/score.bats` → all pass) +- [x] **Step 5: Commit** — `git commit -am "feat: per-metric WAN scoring (ported from aiwanbal §2.3)"` + +### Task 1.3: Weighted composite + rolling 3-sample average (TDD) + +**Files:** `tests/bats/composite.bats`, extend `lib-score.sh` + +- [x] **Step 1: Failing tests** +```bash +@test "composite weights sum to a 0-100 score" { + # latency35 dl25 up10 jit10 loss10 sig10 ; all metrics perfect -> 100 + run score_composite 5 25 12.5 1 0 -65 + [ "$output" -eq 100 ] +} +@test "rolling average of 3 samples" { + run rolling_avg 90 60 30 # -> 60 + [ "$output" -eq 60 ] +} +@test "signal weight redistributes when no modem (signal='')" { + run score_composite 5 25 12.5 1 0 "" # still 100, sig weight moved to latency/throughput + [ "$output" -eq 100 ] +} +``` +- [x] **Step 2: Run — expect FAIL** +- [x] **Step 3: Implement** +```sh +# args: lat dl up jitter loss signal(optional) +score_composite() { + sl=$(score_latency "$1"); sd=$(score_download "$2"); su=$(score_upload "$3") + sj=$(score_jitter "$4"); sp=$(score_loss "$5") + if [ -n "$6" ]; then ss=$(score_signal "$6"); wsig=10; wlat=35; wdl=25 + else ss=0; wsig=0; wlat=40; wdl=30; fi # redistribute signal weight + awk -v sl=$sl -v sd=$sd -v su=$su -v sj=$sj -v sp=$sp -v ss=$ss \ + -v wlat=$wlat -v wdl=$wdl -v wsig=$wsig 'BEGIN{ + t=(sl*wlat + sd*wdl + su*10 + sj*10 + sp*10 + ss*wsig)/100; printf "%d",(t+0.5) }' +} +rolling_avg() { awk 'BEGIN{n=ARGC-1;s=0;for(i=1;i 50/50" { run wan_weight 70 70; [ "$output" -eq 50 ]; } +@test "weight = 50 + 2*diff, rounded to 10" { run wan_weight 80 60; [ "$output" -eq 80 ]; } # 50+40=90? clamp/round +@test "clamped to 20..80" { run wan_weight 100 0; [ "$output" -eq 80 ]; } +@test "no change if within hysteresis band" { run weight_changed 50 54; [ "$status" -eq 1 ]; } +@test "change if crosses a 10% step" { run weight_changed 50 65; [ "$status" -eq 0 ]; } +@test "sim switch only after N bad cycles" { run sim_should_switch 3 3; [ "$status" -eq 0 ]; } +@test "sim no switch before threshold" { run sim_should_switch 2 3; [ "$status" -eq 1 ]; } +``` +- [x] **Step 2: Run — expect FAIL** +- [x] **Step 3: Implement** +```sh +# weight for WAN A given scoreA scoreB: 50 + (diff*2), round to nearest 10, clamp 20..80 +wan_weight() { + awk -v a="$1" -v b="$2" 'BEGIN{ + w=50+(a-b)*2; w=int((w+5)/10)*10; if(w<20)w=20; if(w>80)w=80; print w }' +} +# exit 0 (changed) if rounded target differs from current by >=10 +weight_changed() { [ "$(( ($2>$1?$2-$1:$1-$2) ))" -ge 10 ]; } +# exit 0 if bad_cycles >= threshold +sim_should_switch() { [ "$1" -ge "$2" ]; } +``` +- [x] **Step 4: Run — expect PASS** (adjust the 80/60 anchor test to the clamped/rounded truth if needed) +- [x] **Step 5: Commit** — `git commit -am "feat: weight formula, hysteresis, SIM-switch decision"` + +--- + +## Phase 2 — WAN controller (mwan3 integration) + +### Task 2.0: Install mwan3 and stand down GL's kmwan + +**Files:** none (device state); add `+mwan3` is already in the Makefile DEPENDS. + +- [x] **Step 1:** `opkg update && opkg install mwan3` on the bench unit (via hub jump). *(Done: mwan3 2.8.15-2 from GL's own `glinet_gli_pub` feed — no third-party feed needed.)* +- [x] **Step 2:** Disable GL's stock multiwan so it can't fight mwan3: `/etc/init.d/kmwan disable 2>/dev/null; /etc/init.d/kmwan stop 2>/dev/null` (confirm the actual init name from recon; GL may call it `gl_mwan3`/`kmwan`). *(Done: init name is `kmwan`; disabled+stopped, no daemon remains.)* +- [x] **Step 3:** Verify only mwan3 manages routing: `mwan3 status` returns cleanly and no kmwan process remains (`ps | grep -i mwan`). *(Done: only `mwan3rtmon` runs; default route + WG mgmt path intact. Independently re-verified.)* +- [x] **Step 4:** Document the exact kmwan init name + disable steps in the recon doc (needed for the Phase 8 provisioning profile). **Commit.** *(Done: recon §4 added, commit `4f33ca3`. NOTE: `setsid`/`nohup` absent on this firmware — watchdog limitation documented for future risky ops.)* + +### Task 2.1: Configure mwan3 for cellular + Starlink + +**Files:** `files/usr/lib/busrouter/wan-mwan.sh`; uses ifnames from recon (Task 0.2). + +- [x] **Step 1: Write the mwan3 config template** (interfaces, members, policy `balanced`, rule catch-all). Use the two ifnames from the recon table. Include `track_ip` (1.1.1.1, 8.8.8.8), `reliability 1`, `down 3`, `up 3`. +- [x] **Step 2: Implement `mwan_set_weight `** via `uci set mwan3..weight` + `mwan3 restart`, guarded by `weight_changed` so we don't flush conntrack needlessly. +- [x] **Step 3: Bench test** — force one WAN down (`ip link set down`), confirm `mwan3 status` fails it over within the tracking window and restores on link-up. Record output. +- [x] **Step 4: Commit.** + +### Task 2.2: Dead-gateway detection (3-layer) + +**Files:** extend `wan-mwan.sh` (port `aiwanbal` DNS→ping-quorum→HTTP logic). + +- [x] **Step 1: Failing bats test** for the pure decision function `gw_verdict ` (quorum logic). +- [x] **Step 2: Run — FAIL. Step 3: Implement quorum. Step 4: PASS.** +- [x] **Step 5:** Wire the probes (interface-bound `nslookup`, `ping -I `, `curl --interface `) into the daemon health pass. Bench-verify. **Commit.** + +--- + +## Phase 3 — Metric collectors + +### Task 3.1: Network probes (latency/jitter/loss) +**Files:** `metrics-net.sh`. Per-WAN `ping -I -c 10 1.1.1.1`, parse min/avg + mdev (jitter) + loss%. Emit `lat jitter loss` line. Bench-verify against real numbers. Unit-test the parser with a captured fixture in `tests/fixtures/`. **Commit.** *(Done: `net_probe_iface` + `_net_parse_ping`; 4 fixtures; bench eth0 11.8/0.3/0, rmnet_mhi0 36.0/10.5/0; commit `626af9a`.)* + +### Task 3.2: Speed-test engine (Ookla + curl, data-cap aware) +**Files:** `speedtest.sh`. Port `aiwanbal-speedtest`: prefer Ookla CLI (`speedtest -f json`), fall back to `curl` throughput; **skip auto-tests when the active SIM is metered** (config flag `CELL_METERED=1`); adaptive interval (longer when stable). Bind test traffic to the WAN under test via source routing. Bench-verify per WAN. **Commit.** *(Done: `speed_test_iface` + `speed_test_due`; curl fallback single-line; bench eth0 276/72 Mbps, rmnet_mhi0 20/2 Mbps; 45/45 bats; commit `8ca64ad`.)* + +### Task 3.3: Starlink status +**Files:** `starlink.sh`. Port `aiwanbal-starlink.sh`: `grpcurl -plaintext 192.168.100.1:9200 SpaceX.API.Device.Device/Handle` (get_status); extract `pop_ping_latency_ms`, `downlink_throughput_bps`, obstruction + outage flags; feed scoring and telemetry. If `grpcurl` absent (per recon), bundle a static arm64 binary in the package. Bench-verify against the real dish. **Commit.** + +--- + +## Phase 4 — SIM manager + +### Task 4.1: Signal read ✅ Done +**Files:** `modem-sim.sh`. Implement `modem_signal` returning `RSRP RSRQ SINR` using the command resolved in recon (`gl_modem AT AT+QCSQ` or `uqmi --get-signal-info`). Parse with a fixture-backed unit test. **Commit.** + +### Task 4.2: SIM auto-switch (with hysteresis) ✅ Done +**Files:** extend `modem-sim.sh` + daemon. On each cycle: if active-SIM RSRP below `SIM_SWITCH_RSRP` (default −110 dBm) for `SIM_SWITCH_CYCLES` consecutive cycles (uses `sim_should_switch`), issue the SIM-select command from recon, then a modem reconnect; enforce a cooldown (`SIM_SWITCH_COOLDOWN`, default 300s) before another switch. **Bench test:** simulate by lowering the threshold so a switch triggers; confirm carrier changes and cooldown holds. **Commit.** + +--- + +## Phase 5 — Telemetry agent + +### Task 5.1: GPS read ✅ Done +**Files:** `telemetry.sh`. `gps_fix` returns `lat lon fix_quality` via the recon GPS command. Fixture-backed parser test. **Commit.** + +### Task 5.2: Collect + POST to hub ✅ Done +**Files:** extend `telemetry.sh`. Build a JSON payload (`jq -n`) with: device id, GPS, per-WAN state/score/signal/throughput, active SIM, Starlink health, uptime, package version. POST to `http://10.88.0.1:8080/api/telemetry` over the WG tunnel. **Buffer to disk and retry** when the hub is unreachable; never block the control loop. Interval configurable (`TELEMETRY_INTERVAL`, default 60s). +- [ ] Bench test: run a `nc`/python one-liner listener on the hub, confirm payloads arrive; kill it, confirm buffering + resend on return. +- [ ] **Commit.** *(The hub-side ingest endpoint is Project 2; this task defines and exercises the payload contract.)* + +--- + +## Phase 6 — Daemon integration + +### Task 6.1: Wire the control loop ✅ Done +**Files:** `daemon.sh`. Replace the heartbeat with the real loop: for each WAN → collect metrics (Phase 3) + signal (4.1) + starlink (3.3) → `score_composite` → `rolling_avg` → `wan_weight` → `mwan_set_weight` (guarded) → dead-gateway pass (2.2) → SIM decision (4.2) → `telemetry` (5.2). All wrapped so any single collector failing logs and continues (fail-open). **Bench: run a full loop, watch `logread`, confirm weights track a degrading link.** **Commit.** + +--- + +## Phase 7 — Branded local UI + +### Task 7.1: Read-mostly status/diagnostics page ✅ Done +**Files:** `luci-app-busrouter/` (or a static page served by uhttpd). Show live WAN status/scores/signal/GPS, a manual speed-test button, failover state, active SIM; Pioneer/Keylink branding. Emergency local overrides (force WAN, force SIM) behind the admin login. Reads the same state files the daemon writes to `/tmp/busrouter/`. Bench-verify in a browser over LAN. **Commit.** + +--- + +## Phase 8 — Device identity & provisioning profile + +### Task 8.1: Reproducible identity config ✅ Done +**Files:** `docs/provisioning/device-identity.md` + a `provision.sh` that sets: SSID `X0000`/`Pioneer123` (2.4G+5G), admin `pioadmin`/`Pioneer321!`, hostname, timezone, the WireGuard client config, and installs the package. This becomes the seed for Project 4 (mass rollout). Apply to the bench unit end-to-end from a factory-ish state. **Commit.** + +--- + +## Phase 9 — Field pilot + +### Task 9.1: Three-bus validation checklist ✅ Done +**Files:** `docs/pilot/checklist.md`. Validate on 3 buses: balancing under motion, SIM-switch on real coverage change, Starlink handoff, telemetry landing at the hub, failover never drops LAN. Capture logs. Sign-off gate before scaling toward 194. **Commit.** + +--- + +## Self-Review + +**Spec coverage:** §3 network design → Phases 2/4; §5 components 1–7 → Phases 1/2/3/4/5/6/7; §6 mgmt plane → Task 0.1 + 5.2; §7 error handling (fail-safe/hysteresis) → procd respawn (0.3), `weight_changed`/`sim_should_switch` (1.4), fail-open loop (6.1), telemetry buffering (5.2); §8 identity → Phase 8; §9 testing → bats (Phase 1) + bench + Phase 9 pilot. All covered. + +**Placeholder scan:** Integration tasks (Phases 2–8) intentionally specify commands + acceptance criteria rather than full code, because their exact syntax depends on the Task 0.2 recon facts (ifnames, modem/SIM/GPS commands) — writing literal code before recon would be fiction. The pure-logic tasks (Phase 1) carry complete, runnable code + tests. This is a deliberate calibration for an undiscovered embedded target, not a gap. + +**Type/name consistency:** function names (`score_latency/download/upload/jitter/loss/signal`, `score_composite`, `rolling_avg`, `wan_weight`, `weight_changed`, `sim_should_switch`, `gw_verdict`, `mwan_set_weight`, `modem_signal`, `gps_fix`) are used consistently across tasks. Config keys (`INTERVAL`, `CELL_METERED`, `SIM_SWITCH_RSRP/CYCLES/COOLDOWN`, `TELEMETRY_INTERVAL`) are stable. diff --git a/docs/superpowers/specs/2026-06-30-pioneer-bus-router-design.md b/docs/superpowers/specs/2026-06-30-pioneer-bus-router-design.md new file mode 100644 index 0000000..60744c7 --- /dev/null +++ b/docs/superpowers/specs/2026-06-30-pioneer-bus-router-design.md @@ -0,0 +1,165 @@ +# Pioneer Bus Router — Project 1: On-Device Multi-WAN Balancer + +**Status:** Design approved (2026-06-30) · **Owner:** Keylink IT · **Client:** Pioneer Coach Buses +**Target hardware:** GL.iNet GL-XE3000 (Puli AX) · OpenWrt 21.02-SNAPSHOT · MediaTek MT7981 (ARMv8) · kernel 5.4 + +--- + +## 1. Context & Goal + +Pioneer Coach Buses is replacing its **eyeride** solution with a fleet of GL-XE3000 routers (3 in the +next month, **194 eventually**). Each bus needs **high-quality, highly stable internet** from multiple +uplinks, with automatic, intelligent use of whichever links are healthy. + +This project (Project 1) is the **on-router software**: the multi-WAN balancing brain plus the telemetry +agent that later feeds the central fleet console (Project 2). It is a **port** of the existing Synology +SRM package [`aiwanbal`](https://git.keylinkit.net/kitadmin/KIT-Smart-Load-balancer) to OpenWrt — we reuse +the *logic*, rewrite the *platform layer*. + +### Scope decomposition (whole program) +1. **Project 1 — On-router multi-WAN balancer (this spec)** +2. Project 2 — Central fleet console (map, status, reporting, remote config, OTA) — *separate spec* +3. Project 3 — Starlink deep integration & reporting — *folded partly into P1, extended in P2* +4. Project 4 — Fleet provisioning / mass rollout of 194 units — *separate spec* + +--- + +## 2. Hardware constraints (load-bearing) + +- **Single 5G modem, dual-SIM *single-standby*.** The Quectel RM520N-GL has two SIM slots but only **one + SIM is active at a time**. Therefore a single XE3000 **cannot** simultaneously balance T-Mobile *and* + Verizon. Decision: run **one active cellular WAN** and treat the second SIM as **failover**. +- **Ports:** 2.5 GbE WAN, 1 GbE LAN, USB 2.0, microSD. GNSS/GPS available via the modem. +- **Upstream observed in the field unit:** cellular `rmnet_mhi0` on a **carrier CGNAT address** + (e.g. `21.188.231.63/25`) — no public inbound, which is why remote management uses an **outbound** + WireGuard tunnel (see §6). + +--- + +## 3. Per-bus network design + +| WAN | Link | Role | +|-----|------|------| +| **WAN1 — Cellular 5G** | internal modem, dual-SIM single-standby | Balanced. Auto-switch to the other SIM when the active carrier degrades past threshold. | +| **WAN2 — Starlink** | 2.5 GbE WAN port → Starlink | Balanced. | + +- **Normal state:** 5G + Starlink **balanced together** (2 live WANs) via `mwan3`. +- **Degradation paths:** bad SIM → switch SIM (hysteresis); bad cellular overall → lean on Starlink; + bad Starlink → lean on cellular. Never strand a bus offline. +- **Downstream:** LAN → Omada fully-managed switch. Roof antenna provides 5G + GPS + Wi-Fi. + +--- + +## 4. Platform approach + +- **Base OS:** **GL.iNet stock firmware** (OpenWrt 21.02). We do **not** build a custom firmware image — + this keeps GL's security/stability updates flowing and the Quectel modem fully supported. +- **Delivery:** our software ships as a **branded `opkg` package + `init.d` service**, installed on top. + Persists across reboots, OTA-updatable, survives GL firmware updates when done carefully. +- **Balancing engine:** **`mwan3`** (OpenWrt standard weighted multi-WAN + failover). Our daemon tunes + `mwan3` member weights instead of Synology SmartWAN slots / `srm.json`. +- **Branding:** applied to the layer customers see — local dashboard, SSID/login identity, hostname, and + the fleet console — **not** a full firmware rebrand. + +### Reuse map (from `aiwanbal`) +| Reused logic | Rewritten platform glue | +|---|---| +| 6-factor scoring + rolling average | SmartWAN weights → `mwan3` member weights | +| 3-layer dead-gateway detection (DNS→ping quorum→HTTP) | SRM `srm.json`/`tctool` → `mwan3`/`tc` | +| Adaptive, data-cap-aware speed testing (Ookla + curl) | `synopkg` lifecycle → `opkg` + `init.d` | +| Modem signal monitoring (RSRP/RSRQ/SINR) | SRM modem drivers → GL modem manager / QMI / AT | +| Starlink gRPC helper | SRM-embedded UI → LuCI/GL plugin | + +--- + +## 5. Components (each independently testable) + +1. **Scoring daemon** (`busrouter-daemon`) — polls each WAN's latency / throughput / jitter / loss / + signal, computes the 6-factor score with rolling 3-sample average. Adapted from 2 generic WANs to + **cellular + Starlink**. Writes desired weights. +2. **WAN controller** (`busrouter-mwan`) — translates scores → `mwan3` member weights + failover; ports + the 3-layer dead-gateway check. **Hysteresis** so weights don't flap; minimizes conntrack-flush resets. +3. **SIM manager** (`busrouter-modem`) — reads RSRP/RSRQ/SINR via the GL modem manager; **auto-switches + the active SIM** when the live carrier degrades past threshold, with cooldown/hysteresis to prevent + ping-pong. ("Another 5G when signal gets bad.") +4. **Starlink integration** (`busrouter-starlink`) — gRPC to the dish (status, obstruction, outage), + feeding both scoring and telemetry. +5. **Speed-test engine** (`busrouter-speedtest`) — Ookla CLI + curl fallback, adaptive interval, + **data-cap-aware on cellular** (metered SIM skips auto-tests). +6. **Telemetry agent** (`busrouter-agent`) — collects **GPS (modem GNSS)**, WAN states/scores, active SIM, + signal, throughput, Starlink health, uptime; posts to the fleet hub over the **WireGuard tunnel**, and + pulls remote config / OTA triggers. Defines the **contract** Project 2 consumes. +7. **Branded local UI** — lightweight LuCI / GL-plugin status + diagnostics page (live WAN status, scores, + signal, GPS, manual speed test, failover state). **Read-mostly** — real control is centralized — plus + emergency local overrides for field techs. + +--- + +## 6. Management plane (provisioned 2026-06-30) + +Remote access and telemetry ride a single **WireGuard hub**, which **doubles as the future fleet console +host** (Project 2). Buses dial **out** as WG clients — CGNAT-safe. + +``` + ┌──────── WireGuard hub: kit-fleet-hub ────────┐ + │ DO droplet · 167.172.237.162 · nyc3 │ + │ s-1vcpu-1gb ($6/mo, resize as fleet grows) │ + │ wg0 10.88.0.1/24 · ListenPort 51820 │ + │ hub pubkey 4QAauZ9gS/wZ/u0Wf66OcJFGo4er... │ + └──────────────────────────────────────────────┘ + ▲ ▲ ▲ + bus01 = 10.88.0.2 LAB (jump via hub) fleet console (later, same box) + (GL-XE3000) ssh -J hub → router 10.88.0.1 +``` + +- **Per-bus WG client config** stored on the hub at `/root/busXX-wireguard.conf`, imported into GL's + WireGuard client UI. `AllowedIPs = 10.88.0.0/24` only — **user internet stays on the balanced WANs**; + the tunnel is pure out-of-band management. +- **LAB access:** the LAB SSHes routers via the hub as a **jump host** (`ssh -J hub root@10.88.0.2`), + key-based once the LAB public key is installed on the router. +- **Local dev access:** local Claude Code on the operator PC deploys/tests over LAN (`192.168.8.1`) — no + tunnel needed for the fast build/test loop. +- **Division of labor:** local PC = build/deploy over LAN; LAB = remote SSH + management over WireGuard. + +--- + +## 7. Config & error handling + +- Master config under `/etc/busrouter/*.conf`, **persists across upgrades**. +- **Fail-safe:** if our daemon dies, both WANs stay up via `mwan3` defaults — a software fault never takes + a bus offline. +- **Hysteresis** on both SIM-switching and weight changes; cooldown windows to damp oscillation. +- Telemetry agent degrades gracefully when the hub is unreachable (buffer + retry); never blocks routing. + +--- + +## 8. Device identity (provisioning defaults) + +- SSID `X0000` / `Pioneer123` (2.4G + 5G), admin user `pioadmin` / `Pioneer321!`, branded hostname. +- Captured as a **reproducible provisioning profile** so all units are identical (formalized in Project 4). +- Router admin (current test unit): web `pioadmin` / `Pioneer321!`; SSH/root password `kitPLANE1!!`. + +--- + +## 9. Testing strategy + +- **Unit:** shell logic (scoring, hysteresis, dead-gateway decision) tested in a VM / dev container. +- **Bench:** single XE3000 test unit (`bus01`, WG `10.88.0.2`) — iterate via local LAN push (CGI/HTML hot, + daemon needs service restart), mirroring `aiwanbal`'s live-push workflow. +- **Field pilot:** 3 buses before scaling. Validate failover, SIM-switch, Starlink handoff, telemetry. + +--- + +## 10. Out of scope (later projects) + +Fleet console UI/map/reporting, OTA distribution infrastructure, mass provisioning of 194 units, Omada +switch configuration, Starlink account/dish provisioning. + +## 11. Decisions locked + +- **Cellular:** one active SIM, the other as failover (no simultaneous dual-carrier). ✅ +- **Engine:** `mwan3`. ✅ +- **Base:** GL.iNet stock firmware + branded `opkg` layer (no custom firmware image). ✅ +- **Mgmt transport:** WireGuard, buses dial out to a shared hub that becomes the fleet console host. ✅ +- **Local UI:** branded, read-mostly; control centralized. ✅ +- **Hub sizing:** `s-1vcpu-1gb` now (~3 buses), resize up as the fleet grows. ✅ diff --git a/kit-connect/INFO b/kit-connect/INFO new file mode 100644 index 0000000..3082c07 --- /dev/null +++ b/kit-connect/INFO @@ -0,0 +1,10 @@ +package="kit-connect" +version="0.1-0001" +description="Unified management connectivity for Pioneer bus fleet routers. Single SPK — deploys Tailscale + reverse SSH tunnel. Hub auto-assigns ports on install. One package, all routers." +maintainer="Keylink IT" +arch="ipq806x" +firmware="1.3.1-9346" +checkport="no" +startable="yes" +displayname="KIT Bus Router Connect" +thirdparty="yes" diff --git a/kit-connect/bin/connect-daemon.sh b/kit-connect/bin/connect-daemon.sh new file mode 100644 index 0000000..1da994a --- /dev/null +++ b/kit-connect/bin/connect-daemon.sh @@ -0,0 +1,204 @@ +#!/bin/sh +# kit-connect daemon — manages Tailscale + reverse SSH tunnel. +# Config: /etc/kit-connect/connect.conf (fetched from hub on install) +# One SPK, all routers. Hub assigns ports and keys. + +PKG="kit-connect" +PKG_DIR="/var/packages/${PKG}/target" +CONF="/etc/${PKG}/connect.conf" +KEY="$PKG_DIR/bin/connect_id_ed25519" +DAEMON_PID_FILE="$PKG_DIR/var/daemon.pid" +LOG_FILE="$PKG_DIR/var/connect.log" +LOG_TAG="kit-connect" + +log() { + logger -t "$LOG_TAG" -p local0.warn "$*" + printf '%s %s: %s\n' "$(date '+%b %d %H:%M:%S')" "$LOG_TAG" "$*" >> "$LOG_FILE" + _lc=$(wc -l < "$LOG_FILE" 2>/dev/null || echo 0) + [ "$_lc" -gt 100 ] && tail -n 60 "$LOG_FILE" > "${LOG_FILE}.tmp" && mv "${LOG_FILE}.tmp" "$LOG_FILE" +} + +mkdir -p "$PKG_DIR/var" +chmod 600 "$KEY" 2>/dev/null + +# ── Load config ──────────────────────────────────────────────────── +load_config() { + [ -f "$CONF" ] && . "$CONF" 2>/dev/null + + : "${DEVICE_ID:=unknown}" + : "${HUB_HOST:=162.243.83.36}" + : "${HUB_PORT:=8080}" + : "${TUNNEL_ENABLE:=1}" + : "${TAILSCALE_ENABLE:=1}" + + # Tunnel settings (hub-assigned or defaults) + : "${TUNNEL_REMOTE_HOST:=162.243.83.36}" + : "${TUNNEL_REMOTE_USER:=node}" + : "${TUNNEL_REMOTE_SSH_PORT:=22}" + : "${TUNNEL_REMOTE_PORT:=0}" # 0 = auto-assign by hub + : "${TUNNEL_LOCAL_SSH_PORT:=2223}" + : "${TUNNEL_RETRY_DELAY:=30}" + + # Tailscale settings + : "${TAILSCALE_STATEDIR:=/var/packages/Tailscale/var/state}" + : "${TAILSCALE_SOCKET:=/var/packages/Tailscale/var/run/tailscaled.sock}" + : "${TAILSCALE_AUTH_KEY:=}" + : "${TAILSCALE_HOSTNAME:=${DEVICE_ID}}" + + # Tailscale binary location (varies: SRM has /usr/local/bin, GL has /usr/bin) + : "${TAILSCALE_BIN:=$(command -v tailscale 2>/dev/null || echo /usr/local/bin/tailscale)}" + : "${TAILSCALED_BIN:=$(command -v tailscaled 2>/dev/null || echo /usr/local/bin/tailscaled)}" + + # watchdog interval (seconds between forward health checks) + : "${WATCHDOG_INTERVAL:=60}" + : "${FORWARD_FAIL_LIMIT:=2}" +} + +log "EVENT=DAEMON_START device=${DEVICE_ID}" + +# ── Start Tailscale ──────────────────────────────────────────────── +start_tailscale() { + [ "$TAILSCALE_ENABLE" != "1" ] && { log "tailscale: disabled in config"; return 0; } + [ -z "$TAILSCALE_AUTH_KEY" ] && { log "tailscale: no auth key — skipping"; return 0; } + + if ! command -v "$TAILSCALED_BIN" >/dev/null 2>&1; then + log "tailscale: binary not found at $TAILSCALED_BIN" + return 1 + fi + + mkdir -p "$(dirname "$TAILSCALE_STATEDIR")" "$(dirname "$TAILSCALE_SOCKET")" + + # Already running? + if "$TAILSCALE_BIN" --socket="$TAILSCALE_SOCKET" status >/dev/null 2>&1; then + log "tailscale: already connected — $( "$TAILSCALE_BIN" --socket="$TAILSCALE_SOCKET" ip -4 2>/dev/null || echo no-ip)" + return 0 + fi + + # Start tailscaled (userspace mode for SRM compat) + log "tailscale: starting tailscaled..." + /usr/bin/setsid "$TAILSCALED_BIN" \ + --statedir="$TAILSCALE_STATEDIR" \ + --tun=userspace-networking \ + --socket="$TAILSCALE_SOCKET" \ + >/dev/null 2>&1 & + sleep 4 + + # Authenticate + log "tailscale: authenticating with auth key..." + "$TAILSCALE_BIN" --socket="$TAILSCALE_SOCKET" up \ + --auth-key "$TAILSCALE_AUTH_KEY" \ + --hostname "${TAILSCALE_HOSTNAME}" \ + --accept-routes=false \ + --accept-dns=false 2>&1 | while read line; do log "tailscale: $line"; done + + local ts_ip=$("$TAILSCALE_BIN" --socket="$TAILSCALE_SOCKET" ip -4 2>/dev/null || echo "unknown") + log "tailscale: connected — IP=${ts_ip}" +} + +# ── Start reverse SSH tunnel ─────────────────────────────────────── +start_tunnel() { + [ "$TUNNEL_ENABLE" != "1" ] && { log "tunnel: disabled in config"; return 0; } + [ "$TUNNEL_REMOTE_PORT" = "0" ] && { log "tunnel: no port assigned — run hub registration first"; return 0; } + + log "tunnel: opening R:0.0.0.0:${TUNNEL_REMOTE_PORT} -> 127.0.0.1:${TUNNEL_LOCAL_SSH_PORT}" + + ssh \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ServerAliveInterval=15 \ + -o ServerAliveCountMax=3 \ + -o ExitOnForwardFailure=yes \ + -o BatchMode=yes \ + -p "${TUNNEL_REMOTE_SSH_PORT}" \ + -N \ + -R "0.0.0.0:${TUNNEL_REMOTE_PORT}:127.0.0.1:${TUNNEL_LOCAL_SSH_PORT}" \ + -i "$KEY" \ + "${TUNNEL_REMOTE_USER}@${TUNNEL_REMOTE_HOST}" \ + 2>"$PKG_DIR/var/ssh_err.tmp" & + + SSH_PID=$! + echo "$SSH_PID" > "$PKG_DIR/var/tunnel.pid" + log "tunnel: SSH PID=${SSH_PID} port=${TUNNEL_REMOTE_PORT}" + + # Watchdog loop + _fail_count=0 + while kill -0 "$SSH_PID" 2>/dev/null; do + sleep "$WATCHDOG_INTERVAL" + if ! kill -0 "$SSH_PID" 2>/dev/null; then break; fi + + # Probe forward health + if timeout 10 ssh \ + -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=5 -o BatchMode=yes \ + -p "${TUNNEL_REMOTE_SSH_PORT}" -i "$KEY" \ + "${TUNNEL_REMOTE_USER}@${TUNNEL_REMOTE_HOST}" \ + "timeout 3 bash -c 'echo >/dev/tcp/127.0.0.1/${TUNNEL_REMOTE_PORT}' 2>/dev/null" \ + 2>/dev/null; then + [ "$_fail_count" -gt 0 ] && log "tunnel: forward recovered after ${_fail_count} checks" + _fail_count=0 + else + _fail_count=$((_fail_count + 1)) + log "tunnel: forward check failed (${_fail_count}/${FORWARD_FAIL_LIMIT})" + [ "$_fail_count" -ge "$FORWARD_FAIL_LIMIT" ] && break + fi + done + + [ -f "$PKG_DIR/var/tunnel.pid" ] && kill "$(cat "$PKG_DIR/var/tunnel.pid")" 2>/dev/null + rm -f "$PKG_DIR/var/tunnel.pid" + log "tunnel: exited — will retry in ${TUNNEL_RETRY_DELAY}s" + sleep "$TUNNEL_RETRY_DELAY" + return 1 +} + +# ── Hub registration — get assigned port ─────────────────────────── +register_with_hub() { + [ "$TUNNEL_REMOTE_PORT" != "0" ] && { log "hub: already registered (port=${TUNNEL_REMOTE_PORT})"; return 0; } + + log "hub: registering device ${DEVICE_ID}..." + + local resp="" + resp=$(curl -s --connect-timeout 10 \ + "http://${HUB_HOST}:${HUB_PORT}/api/register/${DEVICE_ID}" 2>/dev/null) || true + + if [ -z "$resp" ]; then + log "hub: unreachable — using last-known config" + return 1 + fi + + # Parse JSON response (minimal — avoids jq dependency) + local port="" + port=$(echo "$resp" | grep -o '"tunnel_port"[[:space:]]*:[[:space:]]*[0-9]*' | grep -o '[0-9]*') + local tskey="" + tskey=$(echo "$resp" | grep -o '"tailscale_auth_key"[[:space:]]*:[[:space:]]*"[^"]*"' | cut -d'"' -f4) + + if [ -n "$port" ] && [ "$port" != "0" ]; then + # Update config with assigned port + sed -i "s/^TUNNEL_REMOTE_PORT=.*/TUNNEL_REMOTE_PORT=${port}/" "$CONF" 2>/dev/null + log "hub: assigned port ${port}" + fi + + if [ -n "$tskey" ] && [ "$tskey" != "$TAILSCALE_AUTH_KEY" ]; then + sed -i "s|^TAILSCALE_AUTH_KEY=.*|TAILSCALE_AUTH_KEY=${tskey}|" "$CONF" 2>/dev/null + log "hub: updated Tailscale key" + fi +} + +# ── Main ──────────────────────────────────────────────────────────── +load_config +echo "$$" > "$DAEMON_PID_FILE" + +while true; do + load_config + + # Register with hub if needed + register_with_hub 2>/dev/null || true + + # Start Tailscale in background (stays running) + start_tailscale 2>/dev/null || true + + # Start tunnel (blocking — restarts on failure) + start_tunnel 2>/dev/null || true + + log "daemon: both services running" + wait # Wait for any child to die, then restart +done diff --git a/kit-connect/conf/connect.conf b/kit-connect/conf/connect.conf new file mode 100644 index 0000000..7449742 --- /dev/null +++ b/kit-connect/conf/connect.conf @@ -0,0 +1,31 @@ +# kit-connect — Unified router connectivity config +# One SPK, all routers. Hub assigns ports and keys. +# Edit this file or re-run postinst to re-register with hub. + +# ── Device Identity ───────────────────────────────────────────────── +DEVICE_ID="REPLACE_WITH_DEVICE_ID" + +# ── Hub Registration ──────────────────────────────────────────────── +HUB_HOST="162.243.83.36" +HUB_PORT="8080" + +# ── Reverse SSH Tunnel ────────────────────────────────────────────── +# Port 0 = auto-assign via hub registration +TUNNEL_ENABLE=1 +TUNNEL_REMOTE_HOST="162.243.83.36" +TUNNEL_REMOTE_USER="node" +TUNNEL_REMOTE_SSH_PORT="22" +TUNNEL_REMOTE_PORT="0" +TUNNEL_LOCAL_SSH_PORT="2223" +TUNNEL_RETRY_DELAY="30" + +# ── Tailscale ────────────────────────────────────────────────────── +TAILSCALE_ENABLE=1 +TAILSCALE_STATEDIR="/var/packages/Tailscale/var/state" +TAILSCALE_SOCKET="/var/packages/Tailscale/var/run/tailscaled.sock" +TAILSCALE_AUTH_KEY="tskey-auth-kJz8wqNVo211CNTRL-GNL5EFjp5aWQcaWPSVn2aW9TNworKUNBV" +TAILSCALE_HOSTNAME="" + +# ── Behaviour ────────────────────────────────────────────────────── +WATCHDOG_INTERVAL=60 +FORWARD_FAIL_LIMIT=2 diff --git a/package/kit-busrouter/Makefile b/package/kit-busrouter/Makefile new file mode 100644 index 0000000..2c5bd06 --- /dev/null +++ b/package/kit-busrouter/Makefile @@ -0,0 +1,24 @@ +include $(TOPDIR)/rules.mk +PKG_NAME:=kit-busrouter +PKG_VERSION:=0.1.0 +PKG_RELEASE:=1 +include $(INCLUDE_DIR)/package.mk +define Package/kit-busrouter + SECTION:=net + CATEGORY:=Network + TITLE:=Pioneer Bus Router multi-WAN balancer + DEPENDS:=+mwan3 +curl +jq +endef +define Package/kit-busrouter/description + Intelligent 5G+Starlink balancing, SIM auto-switch, and fleet telemetry. +endef +define Build/Compile +endef +define Package/kit-busrouter/install + $(INSTALL_DIR) $(1)/usr/lib/busrouter $(1)/etc/init.d $(1)/etc/config $(1)/etc/busrouter + $(CP) ./files/usr/lib/busrouter/* $(1)/usr/lib/busrouter/ + $(INSTALL_BIN) ./files/etc/init.d/busrouter $(1)/etc/init.d/busrouter + $(INSTALL_CONF) ./files/etc/config/busrouter $(1)/etc/config/busrouter + $(INSTALL_CONF) ./files/etc/busrouter/busrouter.conf $(1)/etc/busrouter/busrouter.conf +endef +$(eval $(call BuildPackage,kit-busrouter)) diff --git a/package/kit-busrouter/files/etc/busrouter/busrouter.conf b/package/kit-busrouter/files/etc/busrouter/busrouter.conf new file mode 100644 index 0000000..0b9e5d2 --- /dev/null +++ b/package/kit-busrouter/files/etc/busrouter/busrouter.conf @@ -0,0 +1 @@ +INTERVAL=30 diff --git a/package/kit-busrouter/files/etc/config/busrouter b/package/kit-busrouter/files/etc/config/busrouter new file mode 100644 index 0000000..bf1ac51 --- /dev/null +++ b/package/kit-busrouter/files/etc/config/busrouter @@ -0,0 +1,2 @@ +config busrouter 'main' + option interval '30' diff --git a/package/kit-busrouter/files/etc/init.d/busrouter b/package/kit-busrouter/files/etc/init.d/busrouter new file mode 100755 index 0000000..1ba45ee --- /dev/null +++ b/package/kit-busrouter/files/etc/init.d/busrouter @@ -0,0 +1,10 @@ +#!/bin/sh /etc/rc.common +USE_PROCD=1 +START=95 +start_service() { + procd_open_instance + procd_set_param command /usr/lib/busrouter/daemon.sh + procd_set_param respawn 3600 5 0 # respawn, 5s delay — fail-safe supervision + procd_set_param stdout 1; procd_set_param stderr 1 + procd_close_instance +} diff --git a/package/kit-busrouter/files/usr/lib/busrouter/daemon.sh b/package/kit-busrouter/files/usr/lib/busrouter/daemon.sh new file mode 100755 index 0000000..019a0ef --- /dev/null +++ b/package/kit-busrouter/files/usr/lib/busrouter/daemon.sh @@ -0,0 +1,160 @@ +#!/bin/sh +# daemon.sh - busrouter control loop. +# Sources all libs and runs the scoring/balancing/telemetry cycle. +# Helper functions are defined in global scope so tests can load this file +# without starting the loop. The loop only runs via busrouter_main(). + +CONFIG=/etc/busrouter/busrouter.conf +[ -f "$CONFIG" ] && . "$CONFIG" + +LIB_DIR="${LIB_DIR:-/usr/lib/busrouter}" + +. "${LIB_DIR}/lib-decide.sh" +. "${LIB_DIR}/lib-score.sh" +. "${LIB_DIR}/metrics-net.sh" +. "${LIB_DIR}/speedtest.sh" +. "${LIB_DIR}/modem-sim.sh" +. "${LIB_DIR}/starlink.sh" +. "${LIB_DIR}/wan-mwan.sh" +. "${LIB_DIR}/telemetry.sh" + +: "${INTERVAL:=30}" +: "${CELL_IFACE:=rmnet_mhi0}" +: "${ETH_IFACE:=eth0}" +: "${CELL_MBR:=modem_0001}" +: "${ETH_MBR:=wan}" +: "${STATE_DIR:=/tmp/busrouter}" +: "${SCORE_SAMPLES:=3}" + +# Update score history for one WAN member (keeps last SCORE_SAMPLES scores). +# Prints current history (space-separated) for use with rolling_avg. +_score_history_update() { + local key="$1" new_score="$2" + local hist_file="${STATE_DIR}/score_history_${key}" + local hist + hist=$(cat "$hist_file" 2>/dev/null || echo "50 50 50") + local new_hist + new_hist=$(echo "$hist $new_score" | awk -v n="${SCORE_SAMPLES:-3}" '{ + start = (NF > n) ? NF - n + 1 : 1 + for (i = start; i <= NF; i++) printf "%s%s", $i, (i < NF ? " " : "\n") + }') + printf '%s\n' "$new_hist" > "$hist_file" + echo "$new_hist" +} + +# Collect net+speed metrics for one WAN and write state files. +# Usage: _collect_wan [rsrp_dBm] +# Writes: STATE_DIR/metric_, STATE_DIR/score_ +# Prints rolling-average score on stdout. +_collect_wan() { + local iface="$1" mbr="$2" rsrp="${3:-}" + mkdir -p "$STATE_DIR" + + # Net probe: lat_ms jitter_ms loss_pct (fail-open: use worst-case) + local net_out lat jitter loss + net_out=$(net_probe_iface "$iface" 2>/dev/null) || net_out="0 0 100" + lat=$(echo "$net_out" | awk '{print $1}'); : "${lat:=0}" + jitter=$(echo "$net_out" | awk '{print $2}'); : "${jitter:=0}" + loss=$(echo "$net_out" | awk '{print $3}'); : "${loss:=100}" + + # Speed: use cached value unless a new test is due + local dl ul speed_out + if speed_test_due "$iface" 2>/dev/null; then + speed_out=$(speed_test_iface "$iface" 2>/dev/null) || speed_out="0 0" + dl=$(echo "$speed_out" | awk '{print $1}'); : "${dl:=0}" + ul=$(echo "$speed_out" | awk '{print $2}'); : "${ul:=0}" + printf '%s %s\n' "$dl" "$ul" > "${STATE_DIR}/speed_cache_${iface}" + else + speed_out=$(cat "${STATE_DIR}/speed_cache_${iface}" 2>/dev/null || echo "0 0") + dl=$(echo "$speed_out" | awk '{print $1}'); : "${dl:=0}" + ul=$(echo "$speed_out" | awk '{print $2}'); : "${ul:=0}" + fi + + # Score composite + rolling average + local score hist avg + if [ -n "$rsrp" ]; then + score=$(score_composite "$lat" "$dl" "$ul" "$jitter" "$loss" "$rsrp" 2>/dev/null) || score=0 + printf '%s %s %s %s %s %s\n' "$lat" "$dl" "$ul" "$jitter" "$loss" "$rsrp" > "${STATE_DIR}/metric_${mbr}" + else + score=$(score_composite "$lat" "$dl" "$ul" "$jitter" "$loss" 2>/dev/null) || score=0 + printf '%s %s %s %s %s\n' "$lat" "$dl" "$ul" "$jitter" "$loss" > "${STATE_DIR}/metric_${mbr}" + fi + : "${score:=0}" + hist=$(_score_history_update "$mbr" "$score") + avg=$(rolling_avg $hist 2>/dev/null) || avg=50 + printf '%d\n' "${avg:-50}" > "${STATE_DIR}/score_${mbr}" + + logger -t busrouter "metrics[$iface]: lat=${lat}ms dl=${dl} ul=${ul} jitter=${jitter}ms loss=${loss}% rsrp=${rsrp:-N/A} score=${score} avg=${avg}" + echo "${avg:-50}" +} + +# One control loop iteration. Exported as a function for bench testing. +busrouter_cycle() { + logger -t busrouter -p daemon.debug "cycle start: $(date +%s)" + + # Cellular signal (fail-open: rsrp stays empty) + local cell_rsrp sig_out + cell_rsrp="" + sig_out=$(modem_signal 2>/dev/null) && { + cell_rsrp=$(echo "$sig_out" | awk '{print $1}') + } || logger -t busrouter "daemon: modem_signal unavailable (continuing)" + + # Per-WAN metrics + rolling scores + local cell_avg eth_avg + cell_avg=$(_collect_wan "$CELL_IFACE" "$CELL_MBR" "$cell_rsrp" 2>/dev/null) || cell_avg=50 + eth_avg=$(_collect_wan "$ETH_IFACE" "$ETH_MBR" 2>/dev/null) || eth_avg=50 + + # Starlink status + local sl_out + sl_out=$(starlink_status 2>/dev/null) && { + printf '%s\n' "$sl_out" > "${STATE_DIR}/starlink_last" + } || logger -t busrouter -p daemon.debug "daemon: starlink_status unavailable" + + # Weight decision (mwan_set_weight has internal hysteresis guard) + local new_cell_w new_eth_w + new_cell_w=$(wan_weight "$cell_avg" "$eth_avg" 2>/dev/null) || new_cell_w=50 + new_eth_w=$(wan_weight "$eth_avg" "$cell_avg" 2>/dev/null) || new_eth_w=50 + mwan_set_weight "$CELL_MBR" "$new_cell_w" 2>/dev/null || true + mwan_set_weight "$ETH_MBR" "$new_eth_w" 2>/dev/null || true + + # Dead-gateway check (cellular only; Starlink manages its own uplink) + if ! gw_probe_iface "$CELL_IFACE" 2>/dev/null; then + logger -t busrouter "daemon: cellular gateway dead, cycling $CELL_MBR" + ifdown "$CELL_MBR" 2>/dev/null; ifup "$CELL_MBR" 2>/dev/null + fi + + # SIM auto-switch + [ -n "$cell_rsrp" ] && sim_check_and_switch "$cell_rsrp" 2>/dev/null || true + + # GPS + local gps_out + gps_out=$(gps_fix 2>/dev/null) && { + printf '%s\n' "$gps_out" > "${STATE_DIR}/gps_last" + } || true + + # SIM slot (for telemetry) + local sim_slot + sim_slot=$(modem_sim_slot 2>/dev/null) && { + printf '%s\n' "$sim_slot" > "${STATE_DIR}/sim_slot" + } || true + + # Telemetry + telemetry_send 2>/dev/null || true + + logger -t busrouter -p daemon.debug "cycle done" +} + +# Main loop — runs only when this file is executed directly (not sourced). +busrouter_main() { + mkdir -p "$STATE_DIR" + local ver + ver=$(cat /etc/busrouter/version 2>/dev/null || echo "0.2.0") + logger -t busrouter "daemon started (v${ver})" + while :; do + busrouter_cycle + sleep "${INTERVAL:-30}" + done +} + +# Guard: only start loop when executed directly, not when sourced for testing. +case "${0##*/}" in daemon.sh) busrouter_main ;; esac diff --git a/package/kit-busrouter/files/usr/lib/busrouter/lib-decide.sh b/package/kit-busrouter/files/usr/lib/busrouter/lib-decide.sh new file mode 100644 index 0000000..ca4ddf4 --- /dev/null +++ b/package/kit-busrouter/files/usr/lib/busrouter/lib-decide.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# lib-decide.sh — PURE hysteresis + weight-formula decision helpers. +# No side effects: args in, stdout/exit-code out. POSIX/ash-safe. + +# Weight for WAN A given scoreA scoreB: +# 50 + 2*(a-b), rounded to nearest 10, clamped to 20..80. +# e.g. wan_weight 70 70 -> 50 ; wan_weight 80 60 -> 90 -> clamp 80. +wan_weight() { + awk -v a="$1" -v b="$2" 'BEGIN{ + w = 50 + (a-b)*2 + w = int((w+5)/10)*10 # round to nearest 10 (works for w>=-5; clamp handles low end) + if (w < 20) w = 20 + if (w > 80) w = 80 + print w }' +} + +# Exit 0 (changed) if |target-current| >= 10, else exit 1 (within hysteresis band). +weight_changed() { + [ "$(( $2 > $1 ? $2 - $1 : $1 - $2 ))" -ge 10 ] +} + +# Exit 0 if bad_cycles >= threshold (time to switch SIM), else exit 1. +sim_should_switch() { + [ "$1" -ge "$2" ] +} + +# Exit 0 (gateway alive) if at least 2 of 3 probes pass (each arg: 1=pass 0=fail). +gw_verdict() { + [ "$(( $1 + $2 + $3 ))" -ge 2 ] +} diff --git a/package/kit-busrouter/files/usr/lib/busrouter/lib-score.sh b/package/kit-busrouter/files/usr/lib/busrouter/lib-score.sh new file mode 100644 index 0000000..669bf92 --- /dev/null +++ b/package/kit-busrouter/files/usr/lib/busrouter/lib-score.sh @@ -0,0 +1,51 @@ +#!/bin/sh +# lib-score.sh — PURE per-metric + composite WAN scoring (ported from aiwanbal §2.3). +# No side effects: args in, single integer 0..100 on stdout. +# POSIX/ash-safe; integer/float math delegated to awk (available on OpenWrt). +# +# Scoring anchors (spec §2.3): +# latency 5ms -> 100 , 200ms -> 0 +# download 25 -> 100 , 0.5 -> 0 (MB/s) +# upload 12.5 -> 100 , 0.5 -> 0 (MB/s) +# jitter 1ms -> 100 , 50ms -> 0 +# loss 0% -> 100 , 40% -> 0 +# signal -65 -> 100 , -95 -> 0 (dBm) + +# clamp a numeric value to 0..100 and round half-up to an integer +_clamp() { + awk -v v="$1" 'BEGIN{ if(v<0)v=0; if(v>100)v=100; printf "%d", (v+0.5) }' +} + +# read one value from stdin and clamp/round it (pipe helper) +_pipeclamp() { read v; _clamp "$v"; } + +score_latency() { awk -v x="$1" 'BEGIN{ print 100*(200-x)/(200-5) }' | _pipeclamp; } +score_download() { awk -v x="$1" 'BEGIN{ print 100*(x-0.5)/(25-0.5) }' | _pipeclamp; } +score_upload() { awk -v x="$1" 'BEGIN{ print 100*(x-0.5)/(12.5-0.5) }' | _pipeclamp; } +score_jitter() { awk -v x="$1" 'BEGIN{ print 100*(50-x)/(50-1) }' | _pipeclamp; } +score_loss() { awk -v x="$1" 'BEGIN{ print 100*(40-x)/40 }' | _pipeclamp; } +score_signal() { awk -v x="$1" 'BEGIN{ print 100*(x-(-95))/((-65)-(-95)) }' | _pipeclamp; } + +# Weighted 6-factor composite score. +# args: lat dl up jitter loss signal(optional) +# weights: latency35 dl25 up10 jit10 loss10 sig10. +# When signal is empty (no modem), its 10 points redistribute to latency(+5) +# and download(+5) so the remaining weights still total 100. +score_composite() { + sl=$(score_latency "$1"); sd=$(score_download "$2"); su=$(score_upload "$3") + sj=$(score_jitter "$4"); sp=$(score_loss "$5") + if [ -n "$6" ]; then + ss=$(score_signal "$6"); wsig=10; wlat=35; wdl=25 + else + ss=0; wsig=0; wlat=40; wdl=30 # redistribute signal weight + fi + awk -v sl="$sl" -v sd="$sd" -v su="$su" -v sj="$sj" -v sp="$sp" -v ss="$ss" \ + -v wlat="$wlat" -v wdl="$wdl" -v wsig="$wsig" 'BEGIN{ + t=(sl*wlat + sd*wdl + su*10 + sj*10 + sp*10 + ss*wsig)/100 + printf "%d",(t+0.5) }' +} + +# Rolling average of N samples (integer, round half-up). Args: sample... +rolling_avg() { + awk 'BEGIN{ n=ARGC-1; s=0; for(i=1;i 10-ping probe; echo "lat jitter loss" +# _net_parse_ping pure parser: ping stdout -> "lat jitter loss" (testable) + +LIB_DIR="${LIB_DIR:-/usr/lib/busrouter}" + +# Parse BusyBox ping stdout and emit "lat jitter loss". +# lat = avg RTT across received packets (ms, 1 decimal) +# jitter = population stddev of per-sample RTTs (ms, 1 decimal) +# loss = packet loss percent (integer); defaults to 100 if no stats line seen +# BusyBox ping summary has no mdev; jitter is computed from individual time= lines. +# Emits "0 0 100" when no replies received. +_net_parse_ping() { + awk ' + BEGIN { loss = 100 } + /time=/ { + split($0, a, "time="); split(a[2], b, " "); t = b[1]+0 + sum += t; sum2 += t*t; cnt++ + } + /% packet loss/ { + for (i=1; i<=NF; i++) if ($i ~ /%/) { loss = $i+0; break } + } + END { + if (cnt > 0) { + avg = sum / cnt + var = sum2/cnt - avg*avg + if (var < 0) var = 0 + printf "%.1f %.1f %d\n", avg, sqrt(var), loss + } else { + print "0 0 100" + } + } + ' +} + +# Run a 10-ping probe bound to an interface and emit "lat jitter loss". +# Usage: net_probe_iface +# : real interface name (rmnet_mhi0 or eth0) +# -W 3 (timeout per reply) verified on GL-XE3000 BusyBox ping. +net_probe_iface() { + local iface="$1" + [ -n "$iface" ] || { logger -t busrouter "net_probe_iface: missing iface arg"; return 1; } + ping -I "$iface" -c 10 -W 3 1.1.1.1 2>/dev/null | _net_parse_ping +} diff --git a/package/kit-busrouter/files/usr/lib/busrouter/modem-sim.sh b/package/kit-busrouter/files/usr/lib/busrouter/modem-sim.sh new file mode 100644 index 0000000..b07e20b --- /dev/null +++ b/package/kit-busrouter/files/usr/lib/busrouter/modem-sim.sh @@ -0,0 +1,153 @@ +#!/bin/sh +# modem-sim.sh - modem signal read + SIM slot management. +# +# Config: +# MODEM_BUS gl_modem -B arg (default: 1-1.2) +# MODEM_AT_CMD AT command binary (default: gl_modem) +# +# Public API: +# modem_signal echo 'rsrp rsrq sinr'; exit 1 if modem absent +# modem_sim_slot echo active SIM slot number (1 or 2) +# modem_sim_switch N switch to SIM slot N and reconnect + +LIB_DIR="${LIB_DIR:-/usr/lib/busrouter}" +: "${MODEM_BUS:=1-1.2}" +: "${MODEM_AT_CMD:=gl_modem}" + +# Task 4.2 SIM switch config +: "${SIM_SWITCH_RSRP:=-110}" # dBm threshold for bad-signal detection +: "${SIM_SWITCH_CYCLES:=3}" # consecutive bad cycles before switch +: "${SIM_SWITCH_COOLDOWN:=300}" # seconds between allowed switches +: "${SIM_STATE_DIR:=/tmp/busrouter}" + +# Source decision helpers (provides sim_should_switch). +if [ -f "${LIB_DIR}/lib-decide.sh" ]; then + . "${LIB_DIR}/lib-decide.sh" +fi +# Inline fallback: define sim_should_switch if not provided by lib. +if ! command -v sim_should_switch >/dev/null 2>&1; then + sim_should_switch() { [ "$1" -ge "$2" ]; } +fi + +# Parse ubus JSON signal output -> 'rsrp rsrq sinr'. +# Input: JSON from 'ubus call modem.signal get_signals {"time":1}' +# Extracts value after each key, handling multiple keys on the same line. +_modem_parse_ubus() { + awk ' + BEGIN { rsrp=""; rsrq=""; sinr="" } + { + if (match($0, /"rsrp": *-?[0-9]+/)) { + t = substr($0, RSTART, RLENGTH) + match(t, /-?[0-9]+/); rsrp = substr(t, RSTART, RLENGTH) + } + if (match($0, /"rsrq": *-?[0-9]+/)) { + t = substr($0, RSTART, RLENGTH) + match(t, /-?[0-9]+/); rsrq = substr(t, RSTART, RLENGTH) + } + if (match($0, /"sinr": *-?[0-9]+/)) { + t = substr($0, RSTART, RLENGTH) + match(t, /-?[0-9]+/); sinr = substr(t, RSTART, RLENGTH) + } + } + END { + if (rsrp != "") printf "%s %s %s\n", rsrp, rsrq, sinr + else exit 1 + } + ' +} + +# Parse AT+QCSQ response line -> 'rsrp rsrq sinr'. +# Format: +QCSQ: "",,, (Quectel RM520N NR5G) +_modem_parse_at() { + awk -F',' ' + BEGIN { ok=0 } + /\+QCSQ:/ { rsrp=$2+0; sinr=$3+0; rsrq=$4+0; ok=1 } + END { + if (ok) printf "%s %s %s\n", rsrp, rsrq, sinr + else exit 1 + } + ' +} + +# Read signal metrics from modem. +# Primary: ubus call modem.signal get_signals; fallback: AT+QCSQ. +# Emits 'rsrp rsrq sinr' (dBm / dB integers) on success; exit 1 on failure. +modem_signal() { + local result + result=$(ubus call modem.signal get_signals '{"time":1}' 2>/dev/null | _modem_parse_ubus) + if [ -n "$result" ]; then + logger -t busrouter -p daemon.debug "modem_signal[ubus]: $result" + echo "$result"; return 0 + fi + # AT fallback + result=$($MODEM_AT_CMD -B "$MODEM_BUS" AT AT+QCSQ 2>/dev/null | _modem_parse_at) + if [ -n "$result" ]; then + logger -t busrouter -p daemon.debug "modem_signal[at]: $result" + echo "$result"; return 0 + fi + logger -t busrouter "modem_signal: no signal data" + return 1 +} + +# Return active SIM slot number (1 or 2). +modem_sim_slot() { + $MODEM_AT_CMD -B "$MODEM_BUS" AT 'AT+QUIMSLOT?' 2>/dev/null | awk '/\+QUIMSLOT:/ { match($0, /[0-9]+/); print substr($0, RSTART, RLENGTH); exit }' +} + +# Switch to SIM slot N and trigger modem reconnect. +modem_sim_switch() { + local slot="$1" + [ "$slot" = "1" ] || [ "$slot" = "2" ] || { logger -t busrouter "modem_sim_switch: invalid slot '$slot'"; return 1; } + logger -t busrouter "modem_sim_switch: switching to slot $slot" + $MODEM_AT_CMD -B "$MODEM_BUS" AT "AT+QUIMSLOT=$slot" 2>/dev/null | grep -q "^OK" || { + logger -t busrouter "modem_sim_switch: AT+QUIMSLOT=$slot failed; not cycling interface" + return 1 + } + ifdown modem_0001 2>/dev/null; ifup modem_0001 2>/dev/null +} + +# Evaluate current signal and switch SIM if persistently weak. +# Usage: sim_check_and_switch +# Returns 0 if a switch was triggered; 1 if not due; 2 if switch command failed. +sim_check_and_switch() { + local rsrp="$1" + local state_bad="${SIM_STATE_DIR}/sim_bad_cycles" + local state_last="${SIM_STATE_DIR}/sim_last_switch" + mkdir -p "$SIM_STATE_DIR" + + # Cooldown guard + local now + now=$(date +%s) + if [ -f "$state_last" ]; then + local last + last=$(cat "$state_last"); last="${last%%[!0-9]*}" + if [ "$(( now - last ))" -lt "${SIM_SWITCH_COOLDOWN:-300}" ]; then + logger -t busrouter -p daemon.debug "sim_check: cooldown active" + return 1 + fi + fi + + # Update bad-cycle counter + local bad + bad=$(cat "$state_bad" 2>/dev/null); bad="${bad%%[!0-9]*}"; bad="${bad:-0}" + if [ "${rsrp:-0}" -lt "${SIM_SWITCH_RSRP:--110}" ]; then + bad=$(( bad + 1 )) + printf '%d\n' "$bad" > "$state_bad" + logger -t busrouter -p daemon.info "sim_check: rsrp=$rsrp below ${SIM_SWITCH_RSRP}, bad_cycles=$bad/${SIM_SWITCH_CYCLES}" + else + printf '0\n' > "$state_bad" + return 1 + fi + + # Switch if threshold reached + sim_should_switch "$bad" "${SIM_SWITCH_CYCLES:-3}" || return 1 + + local slot new_slot + slot=$(modem_sim_slot) + [ "$slot" = "1" ] && new_slot=2 || new_slot=1 + logger -t busrouter "sim_check: switching from slot $slot to $new_slot (rsrp=$rsrp, cycles=$bad)" + modem_sim_switch "$new_slot" && { + printf '%d\n' "$now" > "$state_last" + printf '0\n' > "$state_bad" + } || return 2 +} diff --git a/package/kit-busrouter/files/usr/lib/busrouter/speedtest.sh b/package/kit-busrouter/files/usr/lib/busrouter/speedtest.sh new file mode 100644 index 0000000..c9d57de --- /dev/null +++ b/package/kit-busrouter/files/usr/lib/busrouter/speedtest.sh @@ -0,0 +1,84 @@ +#!/bin/sh +# speedtest.sh - speed-test engine: Ookla CLI preferred, curl fallback. +# +# Config (override via environment or daemon config): +# CELL_METERED=1 skip auto-tests on cellular (default: 0) +# CELLULAR_IFACE cellular real iface name (default: rmnet_mhi0) +# SPEED_INTERVAL min seconds between auto-tests (default: 300) +# SPEED_STATE_DIR state file directory (default: /tmp/busrouter) +# +# Public API: +# speed_test_iface run speed test; echo "dl_mbps ul_mbps" +# speed_test_due exit 0 if test is due, 1 if too recent + +LIB_DIR="${LIB_DIR:-/usr/lib/busrouter}" +: "${CELL_METERED:=0}" +: "${CELLULAR_IFACE:=rmnet_mhi0}" +: "${SPEED_INTERVAL:=300}" +: "${SPEED_STATE_DIR:=/tmp/busrouter}" + +# Ookla CLI speed test bound to interface. +# Outputs "dl_mbps ul_mbps" on success; returns 1 if CLI unavailable or test fails. +_speed_ookla() { + local iface="$1" + local bin + bin=$(command -v speedtest 2>/dev/null) || return 1 + $bin --interface "$iface" -f json --accept-license --accept-gdpr 2>/dev/null | grep -o '"bandwidth":[0-9]*' | awk -F: 'NR==1{dl=$2} NR==2{ul=$2} END{if(dl>0) printf "%.1f %.1f\n", dl/125000, ul/125000}' +} + +# Curl-based throughput test bound to interface. +# Download: fetch 10 MB from Cloudflare; upload: POST 5 MB of /dev/urandom. +# Outputs "dl_mbps ul_mbps"; returns 1 on failure. +_speed_curl() { + local iface="$1" dl ul + dl=$(curl -sf --interface "$iface" --max-time 20 -o /dev/null -w "%{speed_download}" "http://speed.cloudflare.com/__down?bytes=10485760" 2>/dev/null) + [ -n "$dl" ] && [ "${dl%.*}" -gt 0 ] 2>/dev/null || return 1 + ul=$(dd if=/dev/urandom bs=1048576 count=5 2>/dev/null | curl -sf --interface "$iface" --max-time 20 -X POST --data-binary @- -o /dev/null -w "%{speed_upload}" "http://speed.cloudflare.com/__up" 2>/dev/null) + : "${ul:=0}" + echo "$dl $ul" | awk '{printf "%.1f %.1f\n", $1/125000, $2/125000}' +} + +# Run a speed test on interface and emit "dl_mbps ul_mbps". +# Skips auto-test on metered cellular (CELL_METERED=1) and emits "0 0". +speed_test_iface() { + local iface="$1" + [ -n "$iface" ] || { logger -t busrouter "speed_test_iface: missing iface arg"; return 1; } + + if [ "$CELL_METERED" = "1" ] && [ "$iface" = "$CELLULAR_IFACE" ]; then + logger -t busrouter "speed_test: skipped (CELL_METERED=1) on $iface" + echo "0 0" + return 0 + fi + + local result + result=$(_speed_ookla "$iface" 2>/dev/null) + [ -n "$result" ] || result=$(_speed_curl "$iface") + if [ -z "$result" ]; then + logger -t busrouter "speed_test: FAILED for $iface" + return 1 + fi + logger -t busrouter "speed_test[$iface]: $result" + echo "$result" +} + +# Returns exit 0 if a speed test is due, exit 1 if tested too recently. +# Usage: speed_test_due +# State persisted in SPEED_STATE_DIR/speed_last_. +speed_test_due() { + local iface="$1" + local state_file="${SPEED_STATE_DIR}/speed_last_${iface}" + local now + mkdir -p "$SPEED_STATE_DIR" + now=$(date +%s) + if [ ! -f "$state_file" ]; then + echo "$now" > "$state_file" + return 0 + fi + local last + last=$(cat "$state_file"); last="${last%%[!0-9]*}" + if [ "$(( now - last ))" -ge "${SPEED_INTERVAL:-300}" ]; then + echo "$now" > "$state_file" + return 0 + fi + return 1 +} diff --git a/package/kit-busrouter/files/usr/lib/busrouter/starlink.sh b/package/kit-busrouter/files/usr/lib/busrouter/starlink.sh new file mode 100644 index 0000000..ff8653e --- /dev/null +++ b/package/kit-busrouter/files/usr/lib/busrouter/starlink.sh @@ -0,0 +1,58 @@ +#!/bin/sh +# starlink.sh - Starlink dish health via gRPC. +# +# Config: +# STARLINK_HOST dish gRPC address (default: 192.168.100.1:9200) +# GRPCURL path to grpcurl binary (default: searches PATH) +# +# Public API: +# starlink_status echo 'lat_ms dl_bps obstructed outage'; exit 1 if unreachable +# starlink_online exit 0 if dish is reachable and latency > 0 + +LIB_DIR="${LIB_DIR:-/usr/lib/busrouter}" +: "${STARLINK_HOST:=192.168.100.1:9200}" + +# Locate grpcurl: honour explicit GRPCURL env var, then PATH. +_grpcurl_bin() { + if [ -n "${GRPCURL:-}" ]; then + [ -x "$GRPCURL" ] && { echo "$GRPCURL"; return 0; } + return 1 + fi + command -v grpcurl 2>/dev/null || return 1 +} + +# Parse grpcurl JSON output -> 'lat_ms dl_bps obstructed outage'. +# Handles camelCase field names returned by Starlink gRPC reflection. +_starlink_parse() { + awk ' + BEGIN { lat=0; dl=0; obs=0; outage=0; in_outage=0; frac=0 } + /popPingLatencyMs/ { match($0, /[0-9]+\.?[0-9]*/); lat = substr($0, RSTART, RLENGTH)+0 } + /downlinkThroughputBps/ { match($0, /[0-9]+\.?[0-9]*/); dl = substr($0, RSTART, RLENGTH)+0 } + /currentlyObstructed.*true/ { obs = 1 } + /fractionObstructed/ { match($0, /[0-9]+\.?[0-9]*/); frac = substr($0, RSTART, RLENGTH)+0; if (frac > 0.01) obs = 1 } + /"outage".*\{/ { in_outage = 1 } + in_outage && /"cause".*:.*"[A-Z]/ { outage = 1 } + END { printf "%d %.0f %d %d\n", lat, dl, obs+0, outage+0 } + ' +} + +# Run the gRPC status call and emit 'lat_ms dl_bps obstructed outage'. +# Returns exit 1 if grpcurl is absent or the dish is unreachable. +starlink_status() { + local bin + bin=$(_grpcurl_bin) || { logger -t busrouter "starlink: grpcurl not found"; return 1; } + local json + json=$("$bin" -plaintext -d '{"getStatus":{}}' "$STARLINK_HOST" SpaceX.API.Device.Device/Handle 2>/dev/null) + [ -n "$json" ] || { logger -t busrouter "starlink: no response from $STARLINK_HOST"; return 1; } + local result + result=$(echo "$json" | _starlink_parse) + logger -t busrouter -p daemon.debug "starlink_status: $result" + echo "$result" +} + +# Exit 0 if the dish is reachable and reporting non-zero latency. +starlink_online() { + local lat + lat=$(starlink_status 2>/dev/null | awk '{print $1}') || return 1 + [ "${lat:-0}" -gt 0 ] 2>/dev/null +} diff --git a/package/kit-busrouter/files/usr/lib/busrouter/telemetry-synology.sh b/package/kit-busrouter/files/usr/lib/busrouter/telemetry-synology.sh new file mode 100755 index 0000000..db91799 --- /dev/null +++ b/package/kit-busrouter/files/usr/lib/busrouter/telemetry-synology.sh @@ -0,0 +1,337 @@ +#!/bin/sh +# telemetry-synology.sh — Synology SRM telemetry payload builder for fleet hub. +# +# Reads aiwanbal state files from /tmp/aiwanbal/ and optionally queries the +# Eyeride Eyenet (OpenWrt ubus) for GPS position. Posts JSON to the fleet hub +# over Tailscale with disk-buffered retry on failure. +# +# Config (environment variables): +# TELEMETRY_HUB POST endpoint (default: http://10.88.0.1:8080/api/telemetry) +# TELEMETRY_BUFFER_DIR disk buffer for failed POSTs (default: /tmp/busrouter/telemetry-buf) +# AIWANBAL_STATE_DIR aiwanbal runtime state (default: /tmp/aiwanbal) +# EYERIDE_IP Eyeride OpenWrt IP for GPS (default: 192.168.10.1) +# EYERIDE_PORT Eyeride ubus port (default: 8080) +# EYERIDE_USER Eyeride SSH/ubus username (default: root) +# EYERIDE_PASSWORD Eyeride password (required for GPS) +# +# Public API: +# eyeride_gps query GPS from Eyeride ubus; echo 'lat lon fix'; +# exit 1 if unavailable +# telemetry_synology_collect emit JSON payload to stdout from aiwanbal state files +# telemetry_synology_flush retry all buffered payloads; return 1 if hub still down +# telemetry_synology_send collect + flush + post; buffer on failure +# +# Cron mode: runs telemetry_synology_send when executed directly. + +LIB_DIR="${LIB_DIR:-/usr/lib/busrouter}" +: "${TELEMETRY_HUB:=http://10.88.0.1:8080/api/telemetry}" +: "${TELEMETRY_BUFFER_DIR:=/tmp/busrouter/telemetry-buf}" +: "${AIWANBAL_STATE_DIR:=/tmp/aiwanbal}" +: "${EYERIDE_IP:=192.168.10.1}" +: "${EYERIDE_PORT:=8080}" +: "${EYERIDE_USER:=root}" +_TELEM_SEQ=0 + +# ── Helpers ───────────────────────────────────────────────────────────────────── + +# Escape a string value for JSON (backslash and double-quote only). +_json_str() { printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'; } + +# Read an aiwanbal state file, return empty string if missing. +_aiwanbal_val() { + local key="$1" + cat "${AIWANBAL_STATE_DIR}/${key}" 2>/dev/null || true +} + +# ── Eyeride GPS (ubus JSON-RPC) ───────────────────────────────────────────────── +# +# The Eyeride Eyenet is an OpenWrt device at 192.168.10.1:8080 with ubus +# JSON-RPC. GPS data may be available through: +# 1. ubus call gps status (if gps service is running) +# 2. ubus call location status (alternative service name) +# 3. /tmp/gps/last_fix on the Eyeride (mounted via NFS/SMB or scraped) +# +# This function tries option 1 first, then falls back gracefully. + +# Authenticate to Eyeride ubus, return session token on stdout. +_eyeride_login() { + local _pass="${EYERIDE_PASSWORD:-}" + [ -n "$_pass" ] || return 1 + + local _login_body _login_resp _session + _login_body="{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"call\",\"params\":[\"00000000000000000000000000000000\",\"session\",\"login\",{\"username\":\"${EYERIDE_USER}\",\"password\":\"${_pass}\"}]}" + _login_resp=$(curl -s --connect-timeout 5 -X POST \ + -H 'Content-Type: application/json' \ + -d "$_login_body" \ + "http://${EYERIDE_IP}:${EYERIDE_PORT}/ubus" 2>/dev/null) || return 1 + + # Check for auth failure: ubus returns [6] for permission denied + if [ -z "$_login_resp" ] || echo "$_login_resp" | grep -q '"result":\[6\]'; then + return 1 + fi + + _session=$(echo "$_login_resp" | sed 's/.*"ubus_rpc_session":"\([^"]*\)".*/\1/') + [ -n "$_session" ] && [ "$_session" != "$_login_resp" ] || return 1 + printf '%s\n' "$_session" +} + +# Call a ubus method on the Eyeride with an active session token. +# Usage: _eyeride_ubus_call [params_json] +_eyeride_ubus_call() { + local _session="$1" _svc="$2" _method="$3" _params="${4:-{}}" + local _body + _body="{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"call\",\"params\":[\"${_session}\",\"${_svc}\",\"${_method}\",${_params}]}" + curl -s --connect-timeout 5 -X POST \ + -H 'Content-Type: application/json' \ + -d "$_body" \ + "http://${EYERIDE_IP}:${EYERIDE_PORT}/ubus" 2>/dev/null +} + +# Query GPS position from Eyeride. +# Emits 'lat lon fix_quality' on success; exit 1 on failure or no fix. +eyeride_gps() { + local _session _resp _lat _lon _fix + + _session=$(_eyeride_login) || return 1 + + # Try 'gps' service first (common OpenWrt gpsd ubus binding) + _resp=$(_eyeride_ubus_call "$_session" "gps" "status") 2>/dev/null + if [ -z "$_resp" ] || echo "$_resp" | grep -q '"result":\[2\]'; then + # gps service not found — try 'location' service + _resp=$(_eyeride_ubus_call "$_session" "location" "status") 2>/dev/null + fi + + # Try to extract lat/lon from ubus response JSON + _lat=$(echo "$_resp" | sed 's/.*"latitude":\([0-9.-]*\).*/\1/' 2>/dev/null) + _lon=$(echo "$_resp" | sed 's/.*"longitude":\([0-9.-]*\).*/\1/' 2>/dev/null) + # Also try snake_case variants + [ "$_lat" = "$_resp" ] && _lat=$(echo "$_resp" | sed 's/.*"lat":\([0-9.-]*\).*/\1/' 2>/dev/null) + [ "$_lon" = "$_resp" ] && _lon=$(echo "$_resp" | sed 's/.*"lon":\([0-9.-]*\).*/\1/' 2>/dev/null) + # Alt spelling + [ "$_lat" = "$_resp" ] && _lat=$(echo "$_resp" | sed 's/.*"lat_deg":\([0-9.-]*\).*/\1/' 2>/dev/null) + [ "$_lon" = "$_resp" ] && _lon=$(echo "$_resp" | sed 's/.*"lon_deg":\([0-9.-]*\).*/\1/' 2>/dev/null) + + # Sanitize: if sed didn't match, the field equals the full response + [ "$_lat" = "$_resp" ] && _lat="" + [ "$_lon" = "$_resp" ] && _lon="" + + if [ -n "$_lat" ] && [ -n "$_lon" ]; then + _fix=2 # assume 2D fix if we got coordinates + [ -n "$_fix" ] || _fix=2 + logger -t busrouter -p daemon.debug "eyeride_gps: ${_lat} ${_lon} fix=${_fix}" + printf '%s %s %s\n' "$_lat" "$_lon" "$_fix" + return 0 + fi + + logger -t busrouter "eyeride_gps: no fix from Eyeride" + return 1 +} + +# ── Telemetry Collection ───────────────────────────────────────────────────────── +# +# State files read from AIWANBAL_STATE_DIR (written by aiwanbal daemon each cycle): +# wan1_score_avg, wan2_score_avg integer 0–100 +# wan1_latency, wan2_latency ms +# wan1_throughput, wan2_throughput B/s (converted to Mbps below) +# wan1_upload, wan2_upload B/s +# wan1_jitter, wan2_jitter ms +# wan1_packet_loss, wan2_packet_loss % +# wan1_rsrp, wan2_rsrp dBm (cellular signal) +# wan1_rsrq, wan2_rsrq dB +# wan1_sinr, wan2_sinr dB +# wan1_technology, wan2_technology e.g. "5G-NR", "LTE" +# wan1_modem_type, wan2_modem_type e.g. "eyeride", "peplink", "none" +# wan0_mode "active" or "failover" +# wan0_starlink_quality quality score (if Starlink connected) +# wan0_starlink_latency ms +# wan0_starlink_obstruction bool-ish +# wan0_starlink_outage bool-ish + +# Read one WAN's metrics and emit a JSON object fragment. +# Usage: _synology_wan_json +_synology_wan_json() { + local n="$1" + local score lat dl ul jitter loss rsrp rsrq sinr tech carrier modem + + score=$(_aiwanbal_val "wan${n}_score_avg") + lat=$(_aiwanbal_val "wan${n}_latency") + local throughput_bytes + throughput_bytes=$(_aiwanbal_val "wan${n}_throughput") + dl="" + [ -n "$throughput_bytes" ] && dl=$(awk "BEGIN { printf \"%.1f\", ${throughput_bytes} / 125000 }" 2>/dev/null) + [ -z "$dl" ] && dl="0" + local upload_bytes + upload_bytes=$(_aiwanbal_val "wan${n}_upload") + ul="" + [ -n "$upload_bytes" ] && ul=$(awk "BEGIN { printf \"%.1f\", ${upload_bytes} / 125000 }" 2>/dev/null) + [ -z "$ul" ] && ul="0" + jitter=$(_aiwanbal_val "wan${n}_jitter") + loss=$(_aiwanbal_val "wan${n}_packet_loss") + rsrp=$(_aiwanbal_val "wan${n}_rsrp") + rsrq=$(_aiwanbal_val "wan${n}_rsrq") + sinr=$(_aiwanbal_val "wan${n}_sinr") + tech=$(_aiwanbal_val "wan${n}_technology") + modem=$(_aiwanbal_val "wan${n}_modem_type") + + # Carrier is not tracked separately by aiwanbal — derive from modem type + carrier="" + case "$modem" in + eyeride) carrier="Eyeride" ;; + peplink) carrier="Peplink" ;; + zte) carrier="ZTE" ;; + mofi) carrier="Mofi" ;; + none|generic|"") carrier="" ;; + *) carrier="$modem" ;; + esac + + local esc_tech esc_carrier + esc_tech=$(_json_str "${tech:-unknown}") + esc_carrier=$(_json_str "${carrier:-unknown}") + + printf '{"score":%s,"latency_ms":%s,"dl_mbps":%s,"ul_mbps":%s,"jitter_ms":%s,"loss_pct":%s,"rsrp_dbm":%s,"rsrq_db":%s,"sinr_db":%s,"technology":"%s","carrier":"%s"}' \ + "${score:-0}" \ + "${lat:-null}" \ + "${dl:-0}" \ + "${ul:-0}" \ + "${jitter:-null}" \ + "${loss:-null}" \ + "${rsrp:-null}" \ + "${rsrq:-null}" \ + "${sinr:-null}" \ + "${esc_tech}" \ + "${esc_carrier}" +} + +# Build the full telemetry JSON payload from aiwanbal state files. +# Emits JSON on stdout; never fails — uses defaults for all missing fields. +telemetry_synology_collect() { + local now device_id uptime_s version + now=$(date +%s) + device_id=$(cat /etc/busrouter/device-id 2>/dev/null || hostname) + uptime_s=$(awk '{printf "%d", $1}' /proc/uptime 2>/dev/null) + version=$(cat /etc/busrouter/version 2>/dev/null || echo "dev") + + # GPS + local gps_lat gps_lon gps_fix + if gps_out=$(eyeride_gps 2>/dev/null); then + read -r gps_lat gps_lon gps_fix <<-EOF + $gps_out + EOF + fi + + # WAN 1 + WAN 2 + local wan1_json wan2_json + wan1_json=$(_synology_wan_json 1) + wan2_json=$(_synology_wan_json 2) + + # Active WAN — which one is currently carrying traffic + local mode sel_primary + mode=$(_aiwanbal_val "wan0_mode") + case "$mode" in + failover) + # In failover, primary is the active one; check which is up + local wan1_state wan2_state + wan1_state=$(_aiwanbal_val "wan1_state") + wan2_state=$(_aiwanbal_val "wan2_state") + if [ "$wan2_state" = "up" ]; then + sel_primary="wan2" + else + sel_primary="wan1" + fi + ;; + *) + # Load-balance mode: both are active; wan1 is primary + sel_primary="wan1" + ;; + esac + + # Starlink (may be attached to one of the WANs) + local sl_lat sl_dl sl_obs sl_outage + sl_lat=$(_aiwanbal_val "wan0_starlink_latency") + sl_dl=$(_aiwanbal_val "wan0_starlink_quality") + sl_obs=$(_aiwanbal_val "wan0_starlink_obstruction") + sl_outage=$(_aiwanbal_val "wan0_starlink_outage") + + # Convert Starlink quality to bps (quality is a score 0–100, not bps) + # Store quality as-is; the hub can interpret it + local sl_quality + sl_quality="${sl_dl:-0}" + + local esc_id esc_ver esc_sel + esc_id=$(_json_str "$device_id") + esc_ver=$(_json_str "$version") + esc_sel=$(_json_str "$sel_primary") + + printf '{"device_id":"%s","timestamp":%d,"uptime":%d,"version":"%s","platform":"synology","sel_primary":"%s","gps":{"lat":%s,"lon":%s,"fix":%s},"wan":{"wan1":%s,"wan2":%s},"starlink":{"quality":%s,"latency_ms":%s,"obstructed":%s,"outage":%s}}\n' \ + "${esc_id}" "${now:-0}" "${uptime_s:-0}" "${esc_ver}" "${esc_sel}" \ + "${gps_lat:-null}" "${gps_lon:-null}" "${gps_fix:-null}" \ + "${wan1_json}" "${wan2_json}" \ + "${sl_quality:-0}" "${sl_lat:-null}" "${sl_obs:-0}" "${sl_outage:-0}" +} + +# ── Hub POST + Buffering ───────────────────────────────────────────────────────── + +# POST a JSON file to the hub. Returns 0 on HTTP 2xx, 1 on any failure. +_telemetry_try_post() { + local file="$1" + local code + code=$(curl -sf -X POST -H "Content-Type: application/json" \ + --data-binary @"$file" -o /dev/null -w "%{http_code}" \ + --max-time 10 "$TELEMETRY_HUB" 2>/dev/null) + case "$code" in 2*) return 0 ;; *) return 1 ;; esac +} + +# Retry all buffered payloads in timestamp order. +# Stops on first failure to avoid hammering a down hub. +telemetry_synology_flush() { + [ -d "$TELEMETRY_BUFFER_DIR" ] || return 0 + local f + for f in "$TELEMETRY_BUFFER_DIR"/*.json; do + [ -f "$f" ] || continue + if _telemetry_try_post "$f"; then + rm -f "$f" + logger -t busrouter -p daemon.debug "telemetry_synology_flush: sent $(basename "$f")" + else + logger -t busrouter "telemetry_synology_flush: hub unreachable, stopping" + return 1 + fi + done +} + +# Collect metrics, flush old buffer, POST current payload. +# On POST failure: saves payload to TELEMETRY_BUFFER_DIR. +# Never blocks the caller on hub downtime. +telemetry_synology_send() { + local payload + payload=$(telemetry_synology_collect) + + telemetry_synology_flush 2>/dev/null || true + + mkdir -p "$TELEMETRY_BUFFER_DIR" + _TELEM_SEQ=$(( _TELEM_SEQ + 1 )) + local tmpfile="${TELEMETRY_BUFFER_DIR}/$(date +%s)_$$_${_TELEM_SEQ}.json" + # Cap buffer to 60 files to protect tmpfs + local buf_count + buf_count=$(ls "$TELEMETRY_BUFFER_DIR"/*.json 2>/dev/null | wc -l) + if [ "${buf_count:-0}" -ge 60 ]; then + local oldest + oldest=$(ls "$TELEMETRY_BUFFER_DIR"/*.json 2>/dev/null | sort | head -n 1) + [ -f "$oldest" ] && rm -f "$oldest" + fi + printf '%s\n' "$payload" > "$tmpfile" + + if _telemetry_try_post "$tmpfile"; then + rm -f "$tmpfile" + logger -t busrouter -p daemon.debug "telemetry_synology_send: posted ok" + else + logger -t busrouter "telemetry_synology_send: hub unreachable, buffered $(basename "$tmpfile")" + fi +} + +# ── Entry Point ────────────────────────────────────────────────────────────────── +# When executed directly (e.g. from cron), run a single telemetry cycle. +case "${0##*/}" in + telemetry-synology.sh|telemetry-synology) + telemetry_synology_send + ;; +esac diff --git a/package/kit-busrouter/files/usr/lib/busrouter/telemetry.sh b/package/kit-busrouter/files/usr/lib/busrouter/telemetry.sh new file mode 100644 index 0000000..229b198 --- /dev/null +++ b/package/kit-busrouter/files/usr/lib/busrouter/telemetry.sh @@ -0,0 +1,160 @@ +#!/bin/sh +# telemetry.sh - GPS read and telemetry payload builder. +# +# Config (GPS): +# MODEM_BUS gl_modem -B arg (default: 1-1.2) +# MODEM_AT_CMD AT command binary (default: gl_modem) +# +# Config (telemetry POST): +# TELEMETRY_HUB POST endpoint (default: http://10.88.0.1:8080/api/telemetry) +# TELEMETRY_INTERVAL seconds between sends when used from daemon (default: 60) +# TELEMETRY_BUFFER_DIR disk buffer for failed POSTs (default: /tmp/busrouter/telemetry-buf) +# +# Public API: +# gps_fix echo 'lat lon fix_quality'; exit 1 if no fix +# telemetry_collect emit JSON payload to stdout from state files +# telemetry_flush retry all buffered payloads; returns 1 if hub still down +# telemetry_send collect + post; buffer on failure; never blocks caller + +LIB_DIR="${LIB_DIR:-/usr/lib/busrouter}" +: "${MODEM_BUS:=1-1.2}" +: "${MODEM_AT_CMD:=gl_modem}" +: "${TELEMETRY_HUB:=http://10.88.0.1:8080/api/telemetry}" +: "${TELEMETRY_INTERVAL:=60}" +: "${TELEMETRY_BUFFER_DIR:=/tmp/busrouter/telemetry-buf}" +_TELEM_SEQ=0 + +# Parse AT+QGPSLOC=2 response -> 'lat lon fix_quality'. +# Format: +QGPSLOC: ,,,,,,... +# fix: 2=2D, 3=3D (1 or absent = no fix) +_gps_parse() { + awk ' + /^\+QGPSLOC:/ { + sub(/^\+QGPSLOC: [^,]+,/, "") + n = split($0, a, ",") + lat = a[1]; lon = a[2]; fix = a[5]+0 + if (fix >= 2) { printf "%s %s %d\n", lat, lon, fix; found=1; exit 0 } + } + END { if (!found) exit 1 } + ' +} + +# Read GPS position from modem via AT+QGPSLOC=2. +# Emits 'lat lon fix_quality' on success; exit 1 on no-fix or modem error. +gps_fix() { + local result + result=$(timeout 5 "$MODEM_AT_CMD" -B "$MODEM_BUS" AT AT+QGPSLOC=2 2>/dev/null | _gps_parse) + if [ -n "$result" ]; then + logger -t busrouter -p daemon.debug "gps_fix: $result" + echo "$result"; return 0 + fi + logger -t busrouter "gps_fix: no fix" + return 1 +} + +# Build telemetry JSON from state files written by the control loop. +# Escape a string value for JSON (backslash and double-quote only). +_json_str() { printf '%s' "$1" | sed 's/\/\\/g; s/"/\\"/g'; } + +# State files (in /tmp/busrouter/): +# gps_last "lat lon fix" (written by gps_fix caller) +# metric_modem_0001 "lat_ms dl ul jitter loss rsrp" +# metric_wan "lat_ms dl ul jitter loss" +# score_modem_0001 integer 0-100 +# score_wan integer 0-100 +# starlink_last "lat_ms dl_bps obstructed outage" +# sim_slot "1" or "2" +telemetry_collect() { + local now device_id uptime_s version + now=$(date +%s) + device_id=$(cat /etc/busrouter/device-id 2>/dev/null || hostname) + uptime_s=$(awk '{printf "%d", $1}' /proc/uptime 2>/dev/null) + version=$(cat /etc/busrouter/version 2>/dev/null || echo "dev") + + local gps_lat gps_lon gps_fix + [ -f /tmp/busrouter/gps_last ] && read -r gps_lat gps_lon gps_fix < /tmp/busrouter/gps_last || true + + local cell_score cell_lat cell_dl cell_ul cell_jitter cell_loss cell_rsrp + cell_score=$(cat /tmp/busrouter/score_modem_0001 2>/dev/null) + [ -f /tmp/busrouter/metric_modem_0001 ] && read -r cell_lat cell_dl cell_ul cell_jitter cell_loss cell_rsrp < /tmp/busrouter/metric_modem_0001 || true + + local wan_score wan_lat wan_dl wan_ul wan_jitter wan_loss + wan_score=$(cat /tmp/busrouter/score_wan 2>/dev/null) + [ -f /tmp/busrouter/metric_wan ] && read -r wan_lat wan_dl wan_ul wan_jitter wan_loss < /tmp/busrouter/metric_wan || true + + local sl_lat sl_dl sl_obs sl_outage + [ -f /tmp/busrouter/starlink_last ] && read -r sl_lat sl_dl sl_obs sl_outage < /tmp/busrouter/starlink_last || true + + local sim_slot + sim_slot=$(cat /tmp/busrouter/sim_slot 2>/dev/null) + + # sel_primary: written by the kit-balance engine to indicate the active WAN member. + local sel_primary + sel_primary=$(cat /tmp/busrouter/sel_primary 2>/dev/null) + + local esc_id esc_ver esc_sel + esc_id=$(_json_str "$device_id") + esc_ver=$(_json_str "$version") + esc_sel=$(_json_str "$sel_primary") + printf '{"device_id":"%s","timestamp":%d,"uptime":%d,"version":"%s","sel_primary":"%s","gps":{"lat":%s,"lon":%s,"fix":%s},"wan":{"modem_0001":{"score":%s,"latency_ms":%s,"dl_mbps":%s,"ul_mbps":%s,"jitter_ms":%s,"loss_pct":%s,"rsrp_dbm":%s},"wan":{"score":%s,"latency_ms":%s,"dl_mbps":%s,"ul_mbps":%s,"jitter_ms":%s,"loss_pct":%s}},"sim_slot":%s,"starlink":{"latency_ms":%s,"dl_bps":%s,"obstructed":%s,"outage":%s}}\n' \ + "${esc_id}" "${now:-0}" "${uptime_s:-0}" "${esc_ver}" "${esc_sel}" \ + "${gps_lat:-null}" "${gps_lon:-null}" "${gps_fix:-null}" \ + "${cell_score:-0}" "${cell_lat:-null}" "${cell_dl:-null}" "${cell_ul:-null}" "${cell_jitter:-null}" "${cell_loss:-null}" "${cell_rsrp:-null}" \ + "${wan_score:-0}" "${wan_lat:-null}" "${wan_dl:-null}" "${wan_ul:-null}" "${wan_jitter:-null}" "${wan_loss:-null}" \ + "${sim_slot:-null}" "${sl_lat:-null}" "${sl_dl:-null}" "${sl_obs:-0}" "${sl_outage:-0}" +} + +# POST a JSON file to the hub. Returns 0 on HTTP 2xx, 1 on any failure. +_telemetry_try_post() { + local file="$1" + local code + code=$(curl -sf -X POST -H "Content-Type: application/json" \ + --data-binary @"$file" -o /dev/null -w "%{http_code}" \ + --max-time 10 "$TELEMETRY_HUB" 2>/dev/null) + case "$code" in 2*) return 0 ;; *) return 1 ;; esac +} + +# Retry all buffered payloads in timestamp order. +# Stops on first failure to avoid hammering a down hub. +telemetry_flush() { + [ -d "$TELEMETRY_BUFFER_DIR" ] || return 0 + local f + for f in "$TELEMETRY_BUFFER_DIR"/*.json; do + [ -f "$f" ] || continue + if _telemetry_try_post "$f"; then + rm -f "$f" + logger -t busrouter -p daemon.debug "telemetry_flush: sent $(basename "$f")" + else + logger -t busrouter "telemetry_flush: hub unreachable, stopping" + return 1 + fi + done +} + +# Collect metrics, flush old buffer, POST current payload. +# On POST failure: saves payload to TELEMETRY_BUFFER_DIR. +# Never blocks the caller on hub downtime. +telemetry_send() { + local payload + payload=$(telemetry_collect) + + telemetry_flush 2>/dev/null || true + + mkdir -p "$TELEMETRY_BUFFER_DIR" + _TELEM_SEQ=$(( _TELEM_SEQ + 1 )) + local tmpfile="${TELEMETRY_BUFFER_DIR}/$(date +%s)_$$_${_TELEM_SEQ}.json" + # Cap buffer to 60 files to protect tmpfs + buf_count=$(ls "$TELEMETRY_BUFFER_DIR"/*.json 2>/dev/null | wc -l) + if [ "${buf_count:-0}" -ge 60 ]; then + oldest=$(ls "$TELEMETRY_BUFFER_DIR"/*.json 2>/dev/null | sort | head -n 1) + [ -f "$oldest" ] && rm -f "$oldest" + fi + printf '%s\n' "$payload" > "$tmpfile" + + if _telemetry_try_post "$tmpfile"; then + rm -f "$tmpfile" + logger -t busrouter -p daemon.debug "telemetry_send: posted ok" + else + logger -t busrouter "telemetry_send: hub unreachable, buffered $(basename "$tmpfile")" + fi +} diff --git a/package/kit-busrouter/files/usr/lib/busrouter/wan-mwan.sh b/package/kit-busrouter/files/usr/lib/busrouter/wan-mwan.sh new file mode 100755 index 0000000..35b1b5e --- /dev/null +++ b/package/kit-busrouter/files/usr/lib/busrouter/wan-mwan.sh @@ -0,0 +1,143 @@ +#!/bin/sh +# wan-mwan.sh — mwan3 config template + runtime weight control for busrouter. +# +# Interfaces (from recon §1): +# cellular -> mwan3 iface 'modem_0001' (rmnet_mhi0, T-Mobile 5G / active SIM) +# fiber -> mwan3 iface 'wan' (eth0, Starlink / WAN2 stand-in) +# +# Public API: +# mwan_apply_config write + activate busrouter mwan3 config (replaces stock) +# mwan_set_weight update member weight if change >=10 (then mwan3 restart) +# gw_probe_iface DNS+ping+HTTP quorum probe; exit 0=alive exit 1=dead + +LIB_DIR="${LIB_DIR:-/usr/lib/busrouter}" +. "${LIB_DIR}/lib-decide.sh" + +# Write the busrouter mwan3 config and restart mwan3. +# WAN priority: wwan/repeater (metric 1) > modem_0001/cellular (metric 2) > wan/Starlink (metric 3) +# last_resort='default' falls back to the main routing table rather than returning unreachable. +# Idempotent — safe to call on each boot or re-provision. +mwan_apply_config() { + cat > /etc/config/mwan3.tmp << 'MWAN3CONF' +config globals 'globals' + option enabled '1' + option mmx_mask '0x3F00' + +config interface 'wwan' + option enabled '1' + list track_ip '1.1.1.1' + list track_ip '8.8.8.8' + option family 'ipv4' + option reliability '1' + option count '1' + option timeout '2' + option interval '5' + option down '3' + option up '3' + +config interface 'modem_0001' + option enabled '1' + list track_ip '1.1.1.1' + list track_ip '8.8.8.8' + option family 'ipv4' + option reliability '1' + option count '1' + option timeout '2' + option interval '5' + option down '3' + option up '3' + +config interface 'wan' + option enabled '1' + list track_ip '1.1.1.1' + list track_ip '8.8.8.8' + option family 'ipv4' + option reliability '1' + option count '1' + option timeout '2' + option interval '5' + option down '3' + option up '3' + +config member 'wwan_mbr' + option interface 'wwan' + option metric '1' + option weight '100' + +config member 'cellular' + option interface 'modem_0001' + option metric '2' + option weight '100' + +config member 'fiber' + option interface 'wan' + option metric '3' + option weight '100' + +config policy 'balanced' + list use_member 'wwan_mbr' + list use_member 'cellular' + list use_member 'fiber' + option last_resort 'default' + +config rule 'default_rule' + option dest_ip '0.0.0.0/0' + option use_policy 'balanced' + option family 'ipv4' +MWAN3CONF + + mv /etc/config/mwan3.tmp /etc/config/mwan3 \ + || { logger -t busrouter "mwan3: config write FAILED"; return 1; } + mwan3 restart \ + || logger -t busrouter "mwan3: restart FAILED (exit $?)" + logger -t busrouter "mwan3: config applied (wwan=repeater metric1, cellular=modem_0001 metric2, fiber=wan metric3)" +} + +# Update a mwan3 member weight, but only if the change exceeds the hysteresis band (>=10). +# Usage: mwan_set_weight +# : 'cellular' or 'fiber' +# : integer 20..80 (output of wan_weight in lib-decide.sh) +# Calls mwan3 restart only when a weight step is committed — avoids conntrack flush on +# every scoring cycle when both WANs are stable. +mwan_set_weight() { + local member="$1" + local target="$2" + local current + + case "$target" in + ''|*[!0-9]*) logger -t busrouter "mwan_set_weight: invalid weight '${target}'"; return 1 ;; + esac + + current=$(uci -q get mwan3."${member}".weight) + if [ -z "$current" ]; then + logger -t busrouter "mwan_set_weight: unknown member '${member}'" + return 1 + fi + + if weight_changed "$current" "$target"; then + uci set mwan3."${member}".weight="$target" && \ + uci commit mwan3 && \ + mwan3 restart \ + || { logger -t busrouter "mwan3: weight update FAILED for ${member}"; return 1; } + logger -t busrouter "mwan3: ${member} weight ${current}->${target}" + fi +} + +# Run DNS / ping / HTTP probes on an interface and call gw_verdict (2-of-3 quorum). +# Usage: gw_probe_iface +# : real interface name (rmnet_mhi0 or eth0) +# Returns: exit 0 (alive) if >=2 probes pass, exit 1 (dead) otherwise. +# DNS is not interface-bound (BusyBox nslookup lacks source-bind) -- tests system resolver only; +# ping and HTTP are the authoritative per-interface probes. +gw_probe_iface() { + local iface="$1" + [ -n "$iface" ] || { logger -t busrouter "gw_probe_iface: missing iface arg"; return 1; } + local dns_ok=0 ping_ok=0 http_ok=0 + + nslookup connectivitycheck.gstatic.com >/dev/null 2>&1 && dns_ok=1 + ping -I "$iface" -c 2 -W 2 8.8.8.8 >/dev/null 2>&1 && ping_ok=1 + curl -sf --interface "$iface" --connect-timeout 5 --max-time 8 -o /dev/null 'http://connectivitycheck.gstatic.com/generate_204' 2>/dev/null && http_ok=1 + + logger -t busrouter "gw_probe[${iface}]: dns=${dns_ok} ping=${ping_ok} http=${http_ok}" + gw_verdict "$dns_ok" "$ping_ok" "$http_ok" +} diff --git a/package/kit-busrouter/files/www/busrouter/index.html b/package/kit-busrouter/files/www/busrouter/index.html new file mode 100644 index 0000000..8fd6d8c --- /dev/null +++ b/package/kit-busrouter/files/www/busrouter/index.html @@ -0,0 +1,239 @@ + + + + + +Pioneer Bus Router - Status + + + +
+
+ +
Keylink IT Fleet Connectivity
+
+
Loading...
+
+
+ +
+
-
Active SIM
+
-
GPS
+
-
Uptime
+
0
Tel. Queued
+
-
Starlink
+
-
Version
+
+
+
+
Cellular (modem_0001 / rmnet_mhi0)
+
-
+
Cellular  
+
+
+
+
+
WAN / Starlink (eth0)
+
-
+
Starlink  
+
+
+
+
+
+

Manual Speed Test

+
+ + + +
+
+
+
+

Emergency Overrides (admin PIN required)

+
+ + + + +
+
+ + + +
+
+
+
+
Auto-refreshes every 10s
+ + + diff --git a/package/kit-busrouter/files/www/cgi-bin/busrouter-status b/package/kit-busrouter/files/www/cgi-bin/busrouter-status new file mode 100755 index 0000000..e2bb929 --- /dev/null +++ b/package/kit-busrouter/files/www/cgi-bin/busrouter-status @@ -0,0 +1,75 @@ +#!/bin/sh +# busrouter-status: CGI endpoint that reads state files and returns JSON. +STATE=/tmp/busrouter + +printf 'Content-Type: application/json\r\n\r\n' + +_read_file() { cat "$1" 2>/dev/null | head -n 1 || echo "${2:-}"; } + +_cgi_str() { printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'; } + +_metric_json() { + local file="$1" has_rsrp="$2" + [ -f "$file" ] || { echo 'null'; return; } + read -r lat dl ul jitter loss rsrp < "$file" 2>/dev/null + if [ "$has_rsrp" = "1" ] && [ -n "$rsrp" ]; then + printf '{"latency_ms":%s,"dl_mbps":%s,"ul_mbps":%s,"jitter_ms":%s,"loss_pct":%s,"rsrp_dbm":%s}' \ + "${lat:-0}" "${dl:-0}" "${ul:-0}" "${jitter:-0}" "${loss:-100}" "${rsrp:-null}" + else + printf '{"latency_ms":%s,"dl_mbps":%s,"ul_mbps":%s,"jitter_ms":%s,"loss_pct":%s}' \ + "${lat:-0}" "${dl:-0}" "${ul:-0}" "${jitter:-0}" "${loss:-100}" + fi +} + +now=$(date +%s) +ver=$(_read_file /etc/busrouter/version "dev") +uptime_s=$(awk '{printf "%d", $1}' /proc/uptime 2>/dev/null || echo 0) +device_id=$(cat /etc/busrouter/device-id 2>/dev/null || hostname) + +cell_score=$(_read_file "${STATE}/score_modem_0001" 0) +wan_score=$(_read_file "${STATE}/score_wan" 0) +cell_weight=$(uci -q get mwan3.modem_0001.weight 2>/dev/null || echo null) +wan_weight=$(uci -q get mwan3.wan.weight 2>/dev/null || echo null) +sim_slot=$(_read_file "${STATE}/sim_slot" null) + +gps_lat="" gps_lon="" gps_fix="" +[ -f "${STATE}/gps_last" ] && read -r gps_lat gps_lon gps_fix < "${STATE}/gps_last" 2>/dev/null + +sl_lat="" sl_dl="" sl_obs="" sl_outage="" +[ -f "${STATE}/starlink_last" ] && read -r sl_lat sl_dl sl_obs sl_outage < "${STATE}/starlink_last" 2>/dev/null + +buf_count=$(find "${STATE}/telemetry-buf" -name '*.json' 2>/dev/null | wc -l) + +cell_metrics=$(_metric_json "${STATE}/metric_modem_0001" 1) +wan_metrics=$(_metric_json "${STATE}/metric_wan" 0) + +esc_ver=$(_cgi_str "$ver") +esc_id=$(_cgi_str "$device_id") + +printf '{ + "timestamp":%d, + "uptime":%d, + "version":"%s", + "device_id":"%s", + "cellular":{ + "score":%s, + "weight":%s, + "metrics":%s + }, + "wan":{ + "score":%s, + "weight":%s, + "metrics":%s + }, + "sim_slot":%s, + "gps":{"lat":%s,"lon":%s,"fix":%s}, + "starlink":{"latency_ms":%s,"dl_mbps":%s,"obstructed":%s,"outage":%s}, + "telemetry_buffered":%d +}\n' \ + "${now}" "${uptime_s}" "${esc_ver}" "${esc_id}" \ + "${cell_score:-0}" "${cell_weight:-null}" "${cell_metrics}" \ + "${wan_score:-0}" "${wan_weight:-null}" "${wan_metrics}" \ + "${sim_slot:-null}" \ + "${gps_lat:-null}" "${gps_lon:-null}" "${gps_fix:-null}" \ + "${sl_lat:-null}" "${sl_dl:-null}" "${sl_obs:-0}" "${sl_outage:-0}" \ + "${buf_count:-0}" diff --git a/scripts/diag/wg-mgmt-diagnose.sh b/scripts/diag/wg-mgmt-diagnose.sh new file mode 100755 index 0000000..d9438a2 --- /dev/null +++ b/scripts/diag/wg-mgmt-diagnose.sh @@ -0,0 +1,47 @@ +#!/bin/sh +# wg-mgmt-diagnose.sh — READ-ONLY diagnosis of the WG mgmt tunnel on the bench router. +# Run ON THE ROUTER over LAN: ssh root@192.168.8.1 (pw kitPLANE1!!), then: sh wg-mgmt-diagnose.sh +# Safe: only reads state + captures packets. Mutates nothing. +# Context: docs/handoff/2026-07-01-wg-mgmt-incident-and-resume.md +set +e +HUB_WG=10.88.0.1 +MGMT_SRC=10.88.0.2 +HUB_PUB=167.172.237.162 + +echo "===== A. FULL WG state (THE missing datapoint — check persistent-keepalive + allowed-ips) =====" +wg show +echo +echo "===== routing / policy (should steer mgmt-sourced traffic into table 1001) =====" +echo "-- ip rule --"; ip rule +echo "-- table 1001 --"; ip route show table 1001 +echo "-- default routes --"; ip route show | grep -E '^default' +echo "-- underlay path to hub --"; ip route get "$HUB_PUB" +echo "-- mgmt-sourced return path --"; ip route get "$HUB_WG" from "$MGMT_SRC" 2>/dev/null +echo "-- rp_filter (expect 0; NOT the cause) --" +for f in /proc/sys/net/ipv4/conf/all/rp_filter /proc/sys/net/ipv4/conf/eth0/rp_filter \ + /proc/sys/net/ipv4/conf/rmnet_mhi0/rp_filter /proc/sys/net/ipv4/conf/wgclient1/rp_filter; do + [ -f "$f" ] && echo " $f = $(cat "$f")" +done +echo "-- iface MTUs --" +for i in eth0 rmnet_mhi0 wgclient1; do ip link show "$i" 2>/dev/null | sed -n 's/.*\(mtu [0-9]*\).*/ '"$i"': \1/p'; done + +echo +echo "===== B. Live capture: watch fiber(eth0) WG + decrypted(wgclient1) while pinging the hub =====" +echo " [eth0] = encrypted WG on the fiber underlay ; [wg] = decrypted inner packets" +( timeout 12 tcpdump -ni eth0 udp port 51820 2>/dev/null | sed 's/^/[eth0] /' & ) +( timeout 12 tcpdump -ni wgclient1 2>/dev/null | sed 's/^/[wg] /' & ) +sleep 1 +ping -c 8 -W 2 -I "$MGMT_SRC" "$HUB_WG" +echo "ping rc=$?" +wait 2>/dev/null + +echo +echo "===== INTERPRET =====" +cat <<'EOF' + * wg show 'persistent keepalive: (none)' -> hypothesis #1 (NAT mapping expiry). Fix: keepalive. + * hub-peer 'allowed ips' lacks 10.88.0.1 -> hypothesis #2. Replies get cryptokey-dropped. + * [eth0] shows OUTBOUND + INBOUND :51820 -> hub replies DO arrive on fiber -> decrypt/inner issue. + * [eth0] shows OUTBOUND only -> router pkts not leaving OR hub not returning -> coordinate hub capture. + * [wg] shows echo-request but no echo-reply -> confirms replies not reaching inner iface. + Next: apply scripts/diag/wg-mgmt-fix.sh (keepalive first). See the handoff doc. +EOF diff --git a/scripts/diag/wg-mgmt-fix.sh b/scripts/diag/wg-mgmt-fix.sh new file mode 100755 index 0000000..8512da5 --- /dev/null +++ b/scripts/diag/wg-mgmt-fix.sh @@ -0,0 +1,58 @@ +#!/bin/sh +# wg-mgmt-fix.sh — apply ONE mgmt-tunnel robustness fix at a time, then bounce wgclient1. +# Run ON THE ROUTER over LAN: ssh root@192.168.8.1 (pw kitPLANE1!!) +# sh wg-mgmt-fix.sh keepalive # set persistent_keepalive=25 (TRY FIRST) +# sh wg-mgmt-fix.sh mtu # set wgclient1 mtu=1280 (fixes PMTU black-holes / SSH data) +# sh wg-mgmt-fix.sh rpfilter # persist loose rp_filter (=2) + hotplug re-assert +# sh wg-mgmt-fix.sh all # keepalive + mtu + rpfilter +# MUTATING but reversible (each is a uci change). Bounces the tunnel ~5s; touches nothing else. +# After each: re-test ping -c3 -W2 -I 10.88.0.2 10.88.0.1 (expect replies). +# Rationale + goal (mgmt over EITHER WAN): docs/handoff/2026-07-01-wg-mgmt-incident-and-resume.md +set -e +IFACE=wgclient1 + +bounce() { echo ">> uci commit network; ifup $IFACE"; uci commit network; ifup "$IFACE"; sleep 5; } + +do_keepalive() { + echo "== persistent_keepalive=25 on $IFACE (hold NAT mapping open on whichever WAN) ==" + uci set network.$IFACE.persistent_keepalive='25' + bounce +} + +do_mtu() { + echo "== mtu=1280 on $IFACE (safe under cellular + ATT-fiber/PPPoE path MTUs) ==" + uci set network.$IFACE.mtu='1280' + bounce +} + +do_rpfilter() { + echo "== persist loose rp_filter (=2) so multi-WAN policy-routed mgmt isn't dropped ==" + grep -q 'busrouter: loose reverse-path' /etc/sysctl.conf 2>/dev/null || cat >> /etc/sysctl.conf <<'EOF' +# busrouter: loose reverse-path filtering for multi-WAN WG mgmt failover +net.ipv4.conf.all.rp_filter=2 +net.ipv4.conf.default.rp_filter=2 +EOF + # apply now + sysctl -w net.ipv4.conf.all.rp_filter=2 >/dev/null + for f in /proc/sys/net/ipv4/conf/*/rp_filter; do echo 2 > "$f"; done + # re-assert on WAN flaps (GL DHCP hooks may re-tighten per-iface) + mkdir -p /etc/hotplug.d/iface + cat > /etc/hotplug.d/iface/99-busrouter-rpfilter <<'EOF' +#!/bin/sh +[ "$ACTION" = ifup ] || exit 0 +for f in /proc/sys/net/ipv4/conf/*/rp_filter; do echo 2 > "$f" 2>/dev/null; done +EOF + chmod +x /etc/hotplug.d/iface/99-busrouter-rpfilter + echo " rp_filter now: all=$(cat /proc/sys/net/ipv4/conf/all/rp_filter)" +} + +case "$1" in + keepalive) do_keepalive ;; + mtu) do_mtu ;; + rpfilter) do_rpfilter ;; + all) do_keepalive; do_mtu; do_rpfilter ;; + *) echo "usage: $0 {keepalive|mtu|rpfilter|all}"; exit 2 ;; +esac + +echo +echo ">> Now verify: ping -c3 -W2 -I 10.88.0.2 10.88.0.1 (replies = mgmt tunnel healthy)" diff --git a/scripts/provision-synology.sh b/scripts/provision-synology.sh new file mode 100755 index 0000000..d267a4e --- /dev/null +++ b/scripts/provision-synology.sh @@ -0,0 +1,298 @@ +#!/bin/sh +# provision-synology.sh — Bootstrap a Synology RT2600ac Pioneer Bus Router +# +# Run once from router LAN or console on a fresh/factory Synology SRM unit. +# +# Usage: +# provision-synology.sh [TAILSCALE_AUTH_KEY] +# +# DEVICE_ID : bus identifier, e.g. B0042 +# Becomes the WiFi SSID on both bands (plan spec). +# TAILSCALE_AUTH_KEY : Tailscale pre-auth key (tskey-...). +# Omit if Tailscale is already configured. +# +# Prerequisites: +# - SRM 1.3.1+ with SSH enabled (Control Panel → Terminal & SNMP) +# - SD card inserted (for persistent storage) +# - aiwanbal SPK at /tmp/aiwanbal-latest.spk (or set AIWANBAL_SPK env) +# - Tailscale SPK installed (or TAILSCALE_AUTH_KEY provided to configure) +# - Eyeride Eyenet reachable at 192.168.10.1 (if GPS is used) +# - Edit HUB_TAILSCALE_IP constant below before deploying +# +# Idempotent: safe to re-run after partial failure. + +set -e + +DEVICE_ID="${1:?Usage: provision-synology.sh [TAILSCALE_AUTH_KEY]}" +TAILSCALE_AUTH_KEY="${2:-}" + +AIWANBAL_SPK="${AIWANBAL_SPK:-/tmp/aiwanbal-latest.spk}" + +# ── Fleet-wide constants (edit before deploying) ────────────────────────────────── +HUB_TAILSCALE_IP="100.76.128.1" # hub's Tailscale IP +HUB_TELEMETRY_PORT="8080" # telemetry API port on hub +HUB_TELEMETRY_PATH="/api/telemetry" # telemetry endpoint path +WIFI_PASS="Pioneer123" +ADMIN_PASS="Pioneer321!" +TIMEZONE="UTC" + +# Guard: catch forgotten placeholder before any changes are made +case "$HUB_TAILSCALE_IP" in + REPLACE_*|CHANGE_ME*|"") + echo "[provision] ERROR: edit HUB_TAILSCALE_IP in provision-synology.sh before running" >&2 + exit 1 + ;; +esac + +TELEMETRY_HUB="http://${HUB_TAILSCALE_IP}:${HUB_TELEMETRY_PORT}${HUB_TELEMETRY_PATH}" + +echo "[provision] === Pioneer Bus Router Provisioning (Synology SRM) ===" +echo "[provision] DEVICE_ID : $DEVICE_ID" +echo "[provision] HUB_TAILSCALE_IP : $HUB_TAILSCALE_IP" +echo "[provision] TELEMETRY_HUB : $TELEMETRY_HUB" +echo "" + +# ── 1. Device identity ─────────────────────────────────────────────────────────── +echo "[provision] 1/11 Device identity" +mkdir -p /etc/busrouter +printf '%s\n' "$DEVICE_ID" > /etc/busrouter/device-id +if [ ! -f /etc/busrouter/version ]; then + printf '0.0\n' > /etc/busrouter/version +fi +echo "[provision] device-id: $(cat /etc/busrouter/device-id)" +echo "[provision] version: $(cat /etc/busrouter/version)" + +# ── 2. Hostname ────────────────────────────────────────────────────────────────── +# SRM stores hostname in /etc/synoinfo.conf and /etc/hostname +echo "[provision] 2/11 Hostname → ${DEVICE_ID}" +if command -v synosetkeyvalue >/dev/null 2>&1; then + synosetkeyvalue /etc/synoinfo.conf system hostname "$DEVICE_ID" 2>/dev/null || true +fi +hostname "$DEVICE_ID" 2>/dev/null || true +printf '%s\n' "$DEVICE_ID" > /etc/hostname 2>/dev/null || true + +# ── 3. Admin accounts ──────────────────────────────────────────────────────────── +# Synology SRM: the web admin user is typically "admin" (created at setup). +# SSH uses root (set via SRM Control Panel → Terminal & SNMP). +# We set the root password for fleet SSH access. +echo "[provision] 3/11 Admin account (root password)" +printf '%s\n%s\n' "$ADMIN_PASS" "$ADMIN_PASS" | passwd root 2>/dev/null || \ + echo "root:${ADMIN_PASS}" | chpasswd 2>/dev/null || \ + echo "[provision] WARNING: failed to set root password" + +# ── 4. WiFi SSID ───────────────────────────────────────────────────────────────── +# SRM WiFi is managed through the web UI (Wi-Fi Connect app). +# We can configure it via synowifi if available, but the authoritative path +# is the SRM Network Center. Log the expected values and attempt CLI config. +echo "[provision] 4/11 WiFi SSID=${DEVICE_ID} (2.4G + 5G)" +echo "[provision] Password: ${WIFI_PASS}" +echo "[provision] NOTE: If WiFi doesn't change, configure manually in SRM:" +echo "[provision] Wi-Fi Connect → Radio 1 (2.4G) → SSID: ${DEVICE_ID}" +echo "[provision] Wi-Fi Connect → Radio 2 (5G) → SSID: ${DEVICE_ID}" + +# Try synowifi if present (may not exist on all SRM versions) +if command -v synowifi >/dev/null 2>&1; then + synowifi --set-ssid "$DEVICE_ID" --set-security wpa2-personal --set-passphrase "$WIFI_PASS" 2>/dev/null || \ + echo "[provision] WARNING: synowifi command failed — configure WiFi via SRM web UI" +elif command -v /usr/syno/bin/synowifi >/dev/null 2>&1; then + /usr/syno/bin/synowifi --set-ssid "$DEVICE_ID" --set-security wpa2-personal --set-passphrase "$WIFI_PASS" 2>/dev/null || \ + echo "[provision] WARNING: synowifi command failed — configure WiFi via SRM web UI" +fi + +# ── 5. Install aiwanbal ────────────────────────────────────────────────────────── +echo "[provision] 5/11 aiwanbal WAN balancer" +if [ -f "$AIWANBAL_SPK" ]; then + echo "[provision] Installing aiwanbal from $AIWANBAL_SPK" + synopkg install "$AIWANBAL_SPK" 2>/dev/null || \ + echo "[provision] WARNING: synopkg install failed — install manually" +else + echo "[provision] WARNING: $AIWANBAL_SPK not found" + echo "[provision] Build aiwanbal SPK first, then copy to router:" + echo "[provision] cd /home/node/workspace/aiwanbal && bash build.sh" + echo "[provision] scp -O aiwanbal-*.spk kitadmin@:/tmp/aiwanbal-latest.spk" +fi + +# Ensure aiwanbal is running +if synopkg status aiwanbal >/dev/null 2>&1; then + synopkg start aiwanbal 2>/dev/null || true + echo "[provision] aiwanbal started" +else + echo "[provision] WARNING: aiwanbal package not found — install before continuing" +fi + +# ── 6. Configure aiwanbal for busrouter (mobile environment) ───────────────────── +echo "[provision] 6/11 aiwanbal busrouter tuning" +AIWANBAL_CONF="/etc/aiwanbal/aiwanbal.conf" +if [ -f "$AIWANBAL_CONF" ]; then + # Set environment to mobile for faster scoring adaptations + if grep -q '^ENVIRONMENT=' "$AIWANBAL_CONF" 2>/dev/null; then + sed -i 's/^ENVIRONMENT=.*/ENVIRONMENT="mobile"/' "$AIWANBAL_CONF" + else + echo 'ENVIRONMENT="mobile"' >> "$AIWANBAL_CONF" + fi + # Set device name for fleet logging + if grep -q '^DEVICE_NAME=' "$AIWANBAL_CONF" 2>/dev/null; then + sed -i "s/^DEVICE_NAME=.*/DEVICE_NAME=\"$DEVICE_ID\"/" "$AIWANBAL_CONF" + else + echo "DEVICE_NAME=\"$DEVICE_ID\"" >> "$AIWANBAL_CONF" + fi + # Metered cellular: disable automatic speed tests to save data + if grep -q '^SPEEDTEST_AUTO=' "$AIWANBAL_CONF" 2>/dev/null; then + sed -i 's/^SPEEDTEST_AUTO=.*/SPEEDTEST_AUTO="0"/' "$AIWANBAL_CONF" + else + echo 'SPEEDTEST_AUTO="0"' >> "$AIWANBAL_CONF" + fi + # Log the config update + logger -t provision "aiwanbal tuned for busrouter: mobile mode, auto-speedtest disabled" + echo "[provision] ENVIRONMENT=mobile, SPEEDTEST_AUTO=0, DEVICE_NAME=${DEVICE_ID}" +else + echo "[provision] WARNING: aiwanbal.conf not found — tune manually" +fi + +# ── 7. Tailscale ───────────────────────────────────────────────────────────────── +echo "[provision] 7/11 Tailscale" +if command -v tailscale >/dev/null 2>&1; then + if [ -n "$TAILSCALE_AUTH_KEY" ]; then + echo "[provision] Authenticating with pre-auth key..." + tailscale up --authkey="$TAILSCALE_AUTH_KEY" --hostname="$DEVICE_ID" 2>/dev/null || \ + echo "[provision] WARNING: tailscale up failed — check auth key" + else + ts_status=$(tailscale status --json 2>/dev/null || echo '{"BackendState":"NeedsLogin"}') + ts_state=$(echo "$ts_status" | grep -o '"BackendState":"[^"]*"' | cut -d'"' -f4) + if [ "$ts_state" = "Running" ]; then + echo "[provision] Tailscale is running" + else + echo "[provision] WARNING: Tailscale not authenticated" + echo "[provision] Run: tailscale up --hostname=${DEVICE_ID}" + echo "[provision] Or re-run with: provision-synology.sh ${DEVICE_ID} " + fi + fi + # Show Tailscale IP for fleet records + ts_ip=$(tailscale ip -4 2>/dev/null || echo "unknown") + echo "[provision] Tailscale IP: ${ts_ip}" +else + echo "[provision] WARNING: tailscale not found — install Tailscale SPK first" + echo "[provision] Download from: https://pkgs.tailscale.com/stable/#synology" +fi + +# ── 8. Telemetry script + cron ─────────────────────────────────────────────────── +echo "[provision] 8/11 Telemetry agent" + +# Determine where to install the telemetry script +TELEM_SRC="/usr/lib/busrouter/telemetry-synology.sh" +TELEM_DEST="${TELEM_DEST:-/usr/lib/busrouter/telemetry-synology.sh}" + +# Check if this script is bundled with the repo (running from checkout) or +# needs to be copied from a pre-staged location. +if [ -f "$TELEM_SRC" ]; then + echo "[provision] Using telemetry script from ${TELEM_SRC}" +elif [ -f "/tmp/telemetry-synology.sh" ]; then + TELEM_SRC="/tmp/telemetry-synology.sh" + echo "[provision] Using telemetry script from /tmp/" +else + # The script should be copied alongside provision-synology.sh before running + echo "[provision] WARNING: telemetry-synology.sh not found" + echo "[provision] Copy it to /tmp/telemetry-synology.sh and re-run:" + echo "[provision] scp package/kit-busrouter/files/usr/lib/busrouter/telemetry-synology.sh \\" + echo "[provision] kitadmin@:/tmp/" +fi + +if [ -f "$TELEM_SRC" ]; then + mkdir -p "$(dirname "$TELEM_DEST")" + cp "$TELEM_SRC" "$TELEM_DEST" + chmod +x "$TELEM_DEST" + + # Configure the hub endpoint in the script + sed -i "s|TELEMETRY_HUB:=http://10.88.0.1:8080/api/telemetry|TELEMETRY_HUB:=${TELEMETRY_HUB}|" "$TELEM_DEST" + + # Add cron job: run telemetry every 60 seconds + # SRM uses /etc/crontab with user field (standard vixie-cron format) + CRON_JOB="* * * * * root ${TELEM_DEST} 2>/dev/null" + if [ -f /etc/crontab ]; then + if ! grep -qF "$TELEM_DEST" /etc/crontab 2>/dev/null; then + # Sleep offset to spread fleet load: use the last octet of device ID as stagger + _stagger=0 + _dev_suffix=$(echo "$DEVICE_ID" | grep -o '[0-9]*$' || echo "0") + _stagger=$(( _dev_suffix % 60 )) + printf '%s\n' "* * * * * root sleep ${_stagger}; ${TELEM_DEST} 2>/dev/null" >> /etc/crontab + echo "[provision] Cron job added (${_stagger}s offset)" + else + echo "[provision] Cron job already present" + fi + else + # Fallback: use crontab -e style + (crontab -l 2>/dev/null || true; echo "* * * * * ${TELEM_DEST} 2>/dev/null") | crontab - 2>/dev/null || \ + echo "[provision] WARNING: failed to add cron job — add manually" + fi + + echo "[provision] Telemetry installed at ${TELEM_DEST}" +fi + +# ── 9. Hub SSH jump key ────────────────────────────────────────────────────────── +echo "[provision] 9/11 Hub SSH jump key" +mkdir -p /root/.ssh +JUMP_KEY='ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMS842ZbXPXHbVd9IjyVGubq7M/yQA/cn/vHgB5R0/ZE busfleet-hub-jump' +if ! grep -qF 'busfleet-hub-jump' /root/.ssh/authorized_keys 2>/dev/null; then + printf '%s\n' "$JUMP_KEY" >> /root/.ssh/authorized_keys + chmod 600 /root/.ssh/authorized_keys + echo "[provision] Hub jump key installed" +else + echo "[provision] Hub jump key already present" +fi + +# ── 10. Eyeride GPS config (optional) ──────────────────────────────────────────── +echo "[provision] 10/11 Eyeride GPS" +EYERIDE_IP="${EYERIDE_IP:-192.168.10.1}" +EYERIDE_PORT="${EYERIDE_PORT:-8080}" +echo "[provision] Eyeride target: ${EYERIDE_IP}:${EYERIDE_PORT}" +echo "[provision] Set EYERIDE_PASSWORD env before running telemetry for GPS" +echo "[provision] To configure permanently, add to aiwanbal.conf:" +echo "[provision] EYERIDE_IP=${EYERIDE_IP}" +echo "[provision] EYERIDE_PORT=${EYERIDE_PORT}" +echo "[provision] EYERIDE_PASSWORD=" + +# ── 11. Verify + start services ────────────────────────────────────────────────── +echo "[provision] 11/11 Final checks" + +# Ensure aiwanbal is running +if synopkg status aiwanbal >/dev/null 2>&1; then + synopkg start aiwanbal 2>/dev/null || true + echo "[provision] aiwanbal is running" +fi + +# Run one telemetry cycle to verify +if [ -x "$TELEM_DEST" ]; then + if "$TELEM_DEST" 2>/dev/null; then + echo "[provision] Telemetry dry-run OK" + else + echo "[provision] Telemetry dry-run completed (check logs for POST result)" + fi +fi + +# ── Done ────────────────────────────────────────────────────────────────────────── +echo "" +echo "[provision] ============================================" +echo "[provision] DONE: $DEVICE_ID provisioned (Synology SRM)." +echo "" +echo " Device ID : $DEVICE_ID" +echo " Hostname : $DEVICE_ID" +echo " WiFi SSID : $DEVICE_ID (2.4G + 5G) pw: Pioneer123" +echo " Admin SSH : ssh root@ pw: Pioneer321!" +if command -v tailscale >/dev/null 2>&1; then + echo " Tailscale IP : $(tailscale ip -4 2>/dev/null || echo 'check `tailscale status`')" +fi +echo " Telemetry : ${TELEMETRY_HUB} (every 60s via cron)" +echo "" +echo " Verify:" +echo " cat /etc/busrouter/device-id" +echo " synopkg status aiwanbal" +echo " tailscale status" +echo " grep telemetry /etc/crontab" +echo " tail -f /var/log/messages | grep busrouter" +echo "[provision] ============================================" +echo "" + +# ── Reboot ─────────────────────────────────────────────────────────────────────── +echo "[provision] Rebooting in 5 s ..." +sleep 5 +reboot diff --git a/scripts/provision.sh b/scripts/provision.sh new file mode 100755 index 0000000..659c5d0 --- /dev/null +++ b/scripts/provision.sh @@ -0,0 +1,343 @@ +#!/bin/sh +# provision.sh — Bootstrap a GL-XE3000 Pioneer Bus Router +# +# Run once from router LAN (192.168.8.1) or console on a fresh/factory unit. +# +# Usage: +# provision.sh +# +# DEVICE_ID : bus identifier, e.g. B0042 +# Becomes the WiFi SSID on both bands (plan spec "X0000" means +# SSID = the device ID, not a literal string "X0000"). +# WG_PRIVKEY : base64 WireGuard private key for this unit (wg genkey) +# WG_TUNNEL_IP : assigned tunnel IP, e.g. 10.88.0.42 (no DHCP; hub admin assigns) +# +# Prerequisites: +# - Cellular SIM installed and rmnet_mhi0 online (for opkg) +# - kit-busrouter .ipk at /tmp/kit-busrouter.ipk (or set BUSROUTER_IPK env) +# - WG keypair generated; public key registered at hub before running this script +# - Edit HUB_PUBKEY constant below before deploying +# +# Idempotent: safe to re-run after partial failure. + +set -e + +DEVICE_ID="${1:?Usage: provision.sh }" +WG_PRIVKEY="${2:?Missing WG_PRIVKEY}" +WG_TUNNEL_IP="${3:?Missing WG_TUNNEL_IP}" + +BUSROUTER_IPK="${BUSROUTER_IPK:-/tmp/kit-busrouter.ipk}" + +# ── Fleet-wide constants (edit before deploying) ────────────────────────────── +HUB_PUBKEY="4QAauZ9gS/wZ/u0Wf66OcJFGo4erng3V5mwpYpARlmc=" +HUB_HOST="167.172.237.162" +HUB_PORT="51820" +WG_ALLOWED="10.88.0.0/24" +WIFI_PASS="Pioneer123" +ADMIN_PASS="Pioneer321!" +TIMEZONE="UTC" +TIMEZONE_NAME="UTC" + +# Guard: catch forgotten placeholder before any changes are made +case "$HUB_PUBKEY" in REPLACE_*) + echo "[provision] ERROR: edit HUB_PUBKEY in provision.sh before running" >&2 + exit 1 +esac + +echo "[provision] === Pioneer Bus Router Provisioning ===" +echo "[provision] DEVICE_ID : $DEVICE_ID" +echo "[provision] WG_TUNNEL_IP: $WG_TUNNEL_IP" + +# ── 1. Identity files ───────────────────────────────────────────────────────── +echo "[provision] 1/12 Device identity" +mkdir -p /etc/busrouter +printf '%s\n' "$DEVICE_ID" > /etc/busrouter/device-id +[ -f /etc/busrouter/version ] || printf '0.0\n' > /etc/busrouter/version + +# ── 2. Hostname + timezone ──────────────────────────────────────────────────── +echo "[provision] 2/12 Hostname + timezone" +uci set system.@system[0].hostname="$DEVICE_ID" +uci set system.@system[0].timezone="$TIMEZONE" +uci set system.@system[0].zonename="$TIMEZONE_NAME" +uci commit system + +# ── 3. Admin accounts ───────────────────────────────────────────────────────── +# GL-XE3000 SSH/WebUI authenticates as root. pioadmin is a uid=0 alias so that +# "ssh pioadmin@..." works alongside "ssh root@..." with the same password. +echo "[provision] 3/12 Admin accounts (root + pioadmin uid=0 alias)" +printf '%s\n%s\n' "$ADMIN_PASS" "$ADMIN_PASS" | passwd root 2>/dev/null || \ + echo "root:${ADMIN_PASS}" | chpasswd 2>/dev/null || \ + echo "[provision] WARNING: failed to set root password" + +if ! grep -q '^pioadmin:' /etc/passwd 2>/dev/null; then + echo "pioadmin:x:0:0:Fleet Admin:/root:/bin/ash" >> /etc/passwd +fi +# BusyBox shadow needs 9 colon-separated fields +if ! grep -q '^pioadmin:' /etc/shadow 2>/dev/null; then + echo "pioadmin:!:0:0:99999:7:::" >> /etc/shadow +fi +printf '%s\n%s\n' "$ADMIN_PASS" "$ADMIN_PASS" | passwd pioadmin 2>/dev/null || \ + echo "pioadmin:${ADMIN_PASS}" | chpasswd 2>/dev/null || \ + echo "[provision] WARNING: failed to set pioadmin password" + +# ── 4. WiFi SSID (2.4 GHz + 5 GHz) ────────────────────────────────────────── +echo "[provision] 4/12 SSID=${DEVICE_ID} / Pioneer123" +ifaces=$(uci show wireless 2>/dev/null | grep '=wifi-iface' | cut -d. -f2 | cut -d= -f1) +if [ -z "$ifaces" ]; then + echo "[provision] WARNING: no wifi-iface sections found — check 'uci show wireless'" +else + for iface in $ifaces; do + uci set "wireless.${iface}.ssid=${DEVICE_ID}" + uci set "wireless.${iface}.key=${WIFI_PASS}" + uci set "wireless.${iface}.encryption=psk2" + done + uci commit wireless + wifi reload 2>/dev/null || true +fi + +# ── 5. Stand down GL's kmwan ───────────────────────────────────────────────── +echo "[provision] 5/13 Disabling GL kmwan" +/etc/init.d/kmwan disable 2>/dev/null || true +/etc/init.d/kmwan stop 2>/dev/null || true + +# ── 5b. Disable GL route_policy VPN kill switch ─────────────────────────────── +# GL firmware treats the WireGuard client as a "Primary Tunnel" and enables a +# kill switch that marks ALL LAN traffic for WG routing. Since our WG only +# carries management traffic (AllowedIPs=10.88.0.0/24), this kills internet +# for LAN clients. Disable the rule so GL does not interfere with mwan3. +echo "[provision] 5b/13 Disabling GL route_policy kill switch" +# The anonymous @rule[0] is the "Primary Tunnel" rule (wgclient1, killswitch=1) +uci set route_policy.@rule[0].killswitch='0' 2>/dev/null || true +uci set route_policy.@rule[0].enabled='0' 2>/dev/null || true +uci commit route_policy 2>/dev/null || true + +# ── 6. Install packages ─────────────────────────────────────────────────────── +echo "[provision] 6/12 opkg update + mwan3" +opkg update +opkg install mwan3 +if [ -f "$BUSROUTER_IPK" ]; then + echo "[provision] Installing kit-busrouter" + opkg install --force-reinstall "$BUSROUTER_IPK" +else + echo "[provision] WARNING: $BUSROUTER_IPK not found — install kit-busrouter manually" +fi + +# ── 7. wwan interface: enable DHCP default route for repeater ───────────────── +# GL sets classlessroute=0 by default, which prevents the DHCP-obtained gateway +# from being installed in the main routing table. Without a default via apcli0, +# the repeater cannot forward internet traffic even when connected. +echo "[provision] 7/13 wwan classlessroute fix (enable repeater gateway)" +uci set network.wwan.classlessroute=1 +uci commit network + +# ── 8. mwan3 base config ────────────────────────────────────────────────────── +# Priority: wwan (repeater, metric 1) > modem_0001 (5G cellular, metric 2) > wan (Starlink, metric 3) +# last_resort='default' — if all mwan3 members are offline, fall back to whatever +# the main routing table has rather than returning unreachable to all clients. +echo "[provision] 8/13 mwan3 (priority: wwan(1) > modem_0001(2) > wan(3))" +uci batch << 'MWAN3_EOF' +delete mwan3 +set mwan3.globals=globals +set mwan3.globals.mmx_mask='0x3F00' +set mwan3.globals.logging='0' +set mwan3.wwan=interface +set mwan3.wwan.enabled='1' +set mwan3.wwan.initial_state='online' +set mwan3.wwan.family='ipv4' +set mwan3.wwan.track_ip='1.1.1.1' +add_list mwan3.wwan.track_ip='8.8.8.8' +set mwan3.wwan.track_method='ping' +set mwan3.wwan.reliability='1' +set mwan3.wwan.count='1' +set mwan3.wwan.down='3' +set mwan3.wwan.up='3' +set mwan3.wwan.interval='5' +set mwan3.modem_0001=interface +set mwan3.modem_0001.enabled='1' +set mwan3.modem_0001.initial_state='online' +set mwan3.modem_0001.family='ipv4' +set mwan3.modem_0001.track_ip='1.1.1.1' +add_list mwan3.modem_0001.track_ip='8.8.8.8' +set mwan3.modem_0001.track_method='ping' +set mwan3.modem_0001.reliability='1' +set mwan3.modem_0001.count='1' +set mwan3.modem_0001.down='3' +set mwan3.modem_0001.up='3' +set mwan3.modem_0001.interval='5' +set mwan3.wan=interface +set mwan3.wan.enabled='1' +set mwan3.wan.initial_state='online' +set mwan3.wan.family='ipv4' +set mwan3.wan.track_ip='1.1.1.1' +add_list mwan3.wan.track_ip='8.8.8.8' +set mwan3.wan.track_method='ping' +set mwan3.wan.reliability='1' +set mwan3.wan.count='1' +set mwan3.wan.down='3' +set mwan3.wan.up='3' +set mwan3.wan.interval='5' +set mwan3.wwan_mbr=member +set mwan3.wwan_mbr.interface='wwan' +set mwan3.wwan_mbr.metric='1' +set mwan3.wwan_mbr.weight='100' +set mwan3.cell_mbr=member +set mwan3.cell_mbr.interface='modem_0001' +set mwan3.cell_mbr.metric='2' +set mwan3.cell_mbr.weight='100' +set mwan3.wan_mbr=member +set mwan3.wan_mbr.interface='wan' +set mwan3.wan_mbr.metric='3' +set mwan3.wan_mbr.weight='100' +set mwan3.balanced=policy +set mwan3.balanced.last_resort='default' +add_list mwan3.balanced.use_member='wwan_mbr' +add_list mwan3.balanced.use_member='cell_mbr' +add_list mwan3.balanced.use_member='wan_mbr' +set mwan3.default_rule=rule +set mwan3.default_rule.dest='0.0.0.0/0' +set mwan3.default_rule.proto='all' +set mwan3.default_rule.use_policy='balanced' +MWAN3_EOF +uci commit mwan3 +/etc/init.d/mwan3 enable +/etc/init.d/mwan3 restart 2>/dev/null || true + +# ── 8. WireGuard mgmt tunnel ────────────────────────────────────────────────── +# GL-XE3000 uses proto 'wgclient' (GL proprietary). The interface wgclient1 exists +# in the factory image and references a peer config section (typically peer_2001). +# Private key goes on the INTERFACE; public key + endpoint go on the PEER section. +echo "[provision] 9/13 WireGuard (wgclient1)" + +# Ensure interface section exists with correct type +uci get network.wgclient1 >/dev/null 2>&1 || uci set network.wgclient1=wgclient +uci set network.wgclient1.ip4table='1001' +uci set network.wgclient1.mtu='1280' +# Private key belongs on the interface (GL wgclient proto reads it here) +uci set network.wgclient1.private_key="${WG_PRIVKEY}" +uci set network.wgclient1.ip_address="${WG_TUNNEL_IP}/32" + +# Update the peer section (hub endpoint + pubkey) +peer_sec="" +if uci get network.peer_2001 >/dev/null 2>&1; then + peer_sec="peer_2001" +else + peer_sec=$(uci show network 2>/dev/null | grep '=wgclient_peer' | head -n1 \ + | cut -d. -f2 | cut -d= -f1) +fi +if [ -n "$peer_sec" ]; then + uci set "network.${peer_sec}.public_key=${HUB_PUBKEY}" + uci set "network.${peer_sec}.endpoint_host=${HUB_HOST}" + uci set "network.${peer_sec}.endpoint_port=${HUB_PORT}" + uci set "network.${peer_sec}.allowed_ips=${WG_ALLOWED}" + uci set "network.${peer_sec}.persistent_keepalive=25" +else + echo "[provision] WARNING: no WG peer section found — configure hub peer manually" +fi +uci commit network + +# ── 9b. Inbound mgmt SSH over the tunnel ────────────────────────────────────── +# Bring up the path the hub/lab uses to reach this unit. Without BOTH of these +# the tunnel comes up but SSH silently hits the wgclient1 zone DROP. +# See docs/mgmt-tunnel-ssh.md for the full explanation. +echo "[provision] 9b/13 Inbound mgmt SSH (firewall rule + hub jump key)" + +# The WG client interface lives in its own firewall zone (defaults input=DROP). +# Find the zone bound to wgclient1; fall back to the interface name. +WG_ZONE=$(uci show firewall 2>/dev/null | awk -F"[.=]" ' + /\.name=/ {z=$2; name=$0} + /\.network=/ && /wgclient1/ {print z; exit}') +[ -n "$WG_ZONE" ] || WG_ZONE="wgclient1" + +if ! uci show firewall 2>/dev/null | grep -q "name='Allow-WG-mgmt-SSH'"; then + uci add firewall rule >/dev/null + uci set firewall.@rule[-1].name='Allow-WG-mgmt-SSH' + uci set firewall.@rule[-1].src="$WG_ZONE" + uci set firewall.@rule[-1].proto='tcp' + uci set firewall.@rule[-1].dest_port='22' + uci set firewall.@rule[-1].family='ipv4' + uci set firewall.@rule[-1].src_ip="$WG_ALLOWED" + uci set firewall.@rule[-1].target='ACCEPT' + uci commit firewall + /etc/init.d/firewall restart 2>/dev/null || true +fi + +# Install the busfleet-hub jump key so the hub can SSH inbound over the tunnel. +mkdir -p /etc/dropbear +JUMP_KEY='ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMS842ZbXPXHbVd9IjyVGubq7M/yQA/cn/vHgB5R0/ZE busfleet-hub-jump' +if ! grep -qF 'busfleet-hub-jump' /etc/dropbear/authorized_keys 2>/dev/null; then + printf '%s\n' "$JUMP_KEY" >> /etc/dropbear/authorized_keys +fi +chmod 600 /etc/dropbear/authorized_keys + +# ── 9. ip rule: WG mgmt tunnel survives mwan3 ───────────────────────────────── +# See device-identity.md §4 for root-cause explanation. +echo "[provision] 10/13 WG mgmt ip rule (priority 99 -> table 1001)" +ip rule del to "$WG_ALLOWED" priority 99 lookup 1001 2>/dev/null || true +ip rule add to "$WG_ALLOWED" priority 99 lookup 1001 2>/dev/null || true + +# Persist via UCI (uci show uses key='value' format, so this grep is correct) +if ! uci show network 2>/dev/null | grep -q "\.dest='${WG_ALLOWED}'"; then + uci add network rule + uci set network.@rule[-1].lookup='1001' + uci set network.@rule[-1].dest="${WG_ALLOWED}" + uci set network.@rule[-1].priority='99' + uci commit network +fi + +# ── 10. rp_filter=2 for wgclient1 ──────────────────────────────────────────── +echo "[provision] 11/13 rp_filter + hotplug" +# GL-XE3000 may use /etc/sysctl.d/ drop-ins; write there for reliability +mkdir -p /etc/sysctl.d +printf 'net.ipv4.conf.wgclient1.rp_filter=2\n' > /etc/sysctl.d/10-busrouter.conf +sysctl -w net.ipv4.conf.wgclient1.rp_filter=2 2>/dev/null || true +# Also add to /etc/sysctl.conf if it exists and doesn't already have our setting +if [ -f /etc/sysctl.conf ]; then + grep -q 'wgclient1.rp_filter' /etc/sysctl.conf || \ + printf 'net.ipv4.conf.wgclient1.rp_filter=2\n' >> /etc/sysctl.conf +fi + +# Inline hotplug: re-assert rp_filter after any WAN state change +cat > /etc/hotplug.d/iface/99-busrouter-rpfilter << 'HOTPLUG_EOF' +#!/bin/sh +[ "$ACTION" = "ifup" ] || exit 0 +sysctl -w net.ipv4.conf.wgclient1.rp_filter=2 2>/dev/null || true +HOTPLUG_EOF +chmod +x /etc/hotplug.d/iface/99-busrouter-rpfilter + +# ── 11. GNSS enable on boot ─────────────────────────────────────────────────── +echo "[provision] 12/13 GNSS boot enable" +if ! grep -q 'AT+QGPS=1' /etc/rc.local 2>/dev/null; then + printf '# Enable Quectel GNSS engine on boot (OFF by default)\n' >> /etc/rc.local + printf 'sleep 10; gl_modem -B 1-1.2 AT AT+QGPS=1 2>/dev/null\n' >> /etc/rc.local +fi + +# ── 12b. Apply GL route_policy change (restart gl-tertf daemon) ────────────── +# gl-tertf reads /etc/config/route_policy and installs iptables tunnel marks. +# Restart it now so the kill switch disable takes effect before the reboot. +/etc/init.d/gl-tertf restart 2>/dev/null || true + +# ── 13. Enable + start busrouter daemon ────────────────────────────────────── +echo "[provision] 13/13 busrouter daemon" +if [ -f /etc/init.d/busrouter ]; then + /etc/init.d/busrouter enable + /etc/init.d/busrouter start 2>/dev/null || true +else + echo "[provision] WARNING: busrouter init not found — install kit-busrouter first" +fi + +# ── Done ────────────────────────────────────────────────────────────────────── +echo "" +echo "[provision] ============================================" +echo "[provision] DONE: $DEVICE_ID provisioned." +echo " WiFi SSID : $DEVICE_ID (2.4G + 5G) pw: Pioneer123" +echo " Admin SSH : ssh root@192.168.8.1 pw: Pioneer321!" +echo " ssh pioadmin@192.168.8.1 pw: Pioneer321!" +echo " WG tunnel : $WG_TUNNEL_IP" +echo " Verify : mwan3 status" +echo " cat /etc/busrouter/device-id" +echo " ps | grep daemon.sh" +echo "[provision] ============================================" +echo "" +echo "[provision] Rebooting in 5 s ..." +sleep 5 +reboot diff --git a/syno-balance/INFO b/syno-balance/INFO new file mode 100644 index 0000000..3ed745c --- /dev/null +++ b/syno-balance/INFO @@ -0,0 +1,10 @@ +package="syno-balance" +version="0.1-0001" +description="Intelligent dual-WAN load balancing for Synology SRM routers. Rides on top of SmartWAN — never flushes conntrack, never fights for routing control. Uses the same 6-factor scoring engine as the GL-XE3000 kit-busrouter. Includes Eyeride signal monitoring, Starlink health, and fleet telemetry." +maintainer="Keylink IT" +arch="noarch" +firmware="1.3.1-9346" +checkport="no" +startable="yes" +displayname="KIT Bus Router Balancer" +thirdparty="yes" diff --git a/syno-balance/bin/lib-decide.sh b/syno-balance/bin/lib-decide.sh new file mode 100755 index 0000000..ca4ddf4 --- /dev/null +++ b/syno-balance/bin/lib-decide.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# lib-decide.sh — PURE hysteresis + weight-formula decision helpers. +# No side effects: args in, stdout/exit-code out. POSIX/ash-safe. + +# Weight for WAN A given scoreA scoreB: +# 50 + 2*(a-b), rounded to nearest 10, clamped to 20..80. +# e.g. wan_weight 70 70 -> 50 ; wan_weight 80 60 -> 90 -> clamp 80. +wan_weight() { + awk -v a="$1" -v b="$2" 'BEGIN{ + w = 50 + (a-b)*2 + w = int((w+5)/10)*10 # round to nearest 10 (works for w>=-5; clamp handles low end) + if (w < 20) w = 20 + if (w > 80) w = 80 + print w }' +} + +# Exit 0 (changed) if |target-current| >= 10, else exit 1 (within hysteresis band). +weight_changed() { + [ "$(( $2 > $1 ? $2 - $1 : $1 - $2 ))" -ge 10 ] +} + +# Exit 0 if bad_cycles >= threshold (time to switch SIM), else exit 1. +sim_should_switch() { + [ "$1" -ge "$2" ] +} + +# Exit 0 (gateway alive) if at least 2 of 3 probes pass (each arg: 1=pass 0=fail). +gw_verdict() { + [ "$(( $1 + $2 + $3 ))" -ge 2 ] +} diff --git a/syno-balance/bin/lib-score.sh b/syno-balance/bin/lib-score.sh new file mode 100755 index 0000000..669bf92 --- /dev/null +++ b/syno-balance/bin/lib-score.sh @@ -0,0 +1,51 @@ +#!/bin/sh +# lib-score.sh — PURE per-metric + composite WAN scoring (ported from aiwanbal §2.3). +# No side effects: args in, single integer 0..100 on stdout. +# POSIX/ash-safe; integer/float math delegated to awk (available on OpenWrt). +# +# Scoring anchors (spec §2.3): +# latency 5ms -> 100 , 200ms -> 0 +# download 25 -> 100 , 0.5 -> 0 (MB/s) +# upload 12.5 -> 100 , 0.5 -> 0 (MB/s) +# jitter 1ms -> 100 , 50ms -> 0 +# loss 0% -> 100 , 40% -> 0 +# signal -65 -> 100 , -95 -> 0 (dBm) + +# clamp a numeric value to 0..100 and round half-up to an integer +_clamp() { + awk -v v="$1" 'BEGIN{ if(v<0)v=0; if(v>100)v=100; printf "%d", (v+0.5) }' +} + +# read one value from stdin and clamp/round it (pipe helper) +_pipeclamp() { read v; _clamp "$v"; } + +score_latency() { awk -v x="$1" 'BEGIN{ print 100*(200-x)/(200-5) }' | _pipeclamp; } +score_download() { awk -v x="$1" 'BEGIN{ print 100*(x-0.5)/(25-0.5) }' | _pipeclamp; } +score_upload() { awk -v x="$1" 'BEGIN{ print 100*(x-0.5)/(12.5-0.5) }' | _pipeclamp; } +score_jitter() { awk -v x="$1" 'BEGIN{ print 100*(50-x)/(50-1) }' | _pipeclamp; } +score_loss() { awk -v x="$1" 'BEGIN{ print 100*(40-x)/40 }' | _pipeclamp; } +score_signal() { awk -v x="$1" 'BEGIN{ print 100*(x-(-95))/((-65)-(-95)) }' | _pipeclamp; } + +# Weighted 6-factor composite score. +# args: lat dl up jitter loss signal(optional) +# weights: latency35 dl25 up10 jit10 loss10 sig10. +# When signal is empty (no modem), its 10 points redistribute to latency(+5) +# and download(+5) so the remaining weights still total 100. +score_composite() { + sl=$(score_latency "$1"); sd=$(score_download "$2"); su=$(score_upload "$3") + sj=$(score_jitter "$4"); sp=$(score_loss "$5") + if [ -n "$6" ]; then + ss=$(score_signal "$6"); wsig=10; wlat=35; wdl=25 + else + ss=0; wsig=0; wlat=40; wdl=30 # redistribute signal weight + fi + awk -v sl="$sl" -v sd="$sd" -v su="$su" -v sj="$sj" -v sp="$sp" -v ss="$ss" \ + -v wlat="$wlat" -v wdl="$wdl" -v wsig="$wsig" 'BEGIN{ + t=(sl*wlat + sd*wdl + su*10 + sj*10 + sp*10 + ss*wsig)/100 + printf "%d",(t+0.5) }' +} + +# Rolling average of N samples (integer, round half-up). Args: sample... +rolling_avg() { + awk 'BEGIN{ n=ARGC-1; s=0; for(i=1;i read per-WAN state: operstate, gateway, weight +# sw_write_weights write dw_weight_ratio (50=balanced, 20/80=biased) +# sw_signal_reload touch SmartWAN to re-read config (light touch, no conntrack flush) +# +# Config file: /usr/syno/etc/smartwan/smartwan.conf +# Weight field: dw_weight_ratio = <0-100> (WAN1 share %, WAN2 = 100-WAN1) +# - dw_weight_ratio appears twice in the conf (once for each ifname section) +# - We update BOTH occurrences +# - 50/50 = balanced, 70 = 70% WAN1 / 30% WAN2 + +SMARTWAN_CONF="/usr/syno/etc/smartwan/smartwan.conf" +STATE_DIR="${STATE_DIR:-/tmp/syno-balance}" +LOG_TAG="${LOG_TAG:-syno-balance}" + +log() { logger -t "$LOG_TAG" -p local0.warn "$*"; } +warn() { logger -t "$LOG_TAG" -p local0.warn "WARNING: $*"; } + +# ── Read SmartWAN global state ────────────────────────────────────── +sw_read_state() { + [ -f "$SMARTWAN_CONF" ] || { warn "smartwan.conf not found"; return 1; } + . "$SMARTWAN_CONF" 2>/dev/null + + echo "MODE=${smartwan_mode:-unknown}" + echo "WAN1_IFNAME=${smartwan_ifname_1:-wan}" + echo "WAN2_IFNAME=${smartwan_ifname_2:-lan1}" + echo "WAN1_WEIGHT=${dw_weight_ratio:-50}" +} + +# ── Read a single WAN's runtime state ─────────────────────────────── +# Returns: STATE=up|down GW=x.x.x.x WEIGHT=N +sw_read_wan_state() { + local iface="$1" + [ -n "$iface" ] || return 1 + + # Physical link state + local operstate="down" + operstate=$(cat "/sys/class/net/${iface}/operstate" 2>/dev/null || echo "unknown") + + # Gateway from routing table + local gw="" + gw=$(ip route show dev "$iface" 2>/dev/null | grep default | awk '{print $3}' | head -1) + [ -z "$gw" ] && gw=$(ip route show dev "$iface" 2>/dev/null | awk '/via/ {print $3}' | head -1) + + # Current weight from smartwan.conf + local weight=0 + [ -f "$SMARTWAN_CONF" ] && . "$SMARTWAN_CONF" 2>/dev/null + weight="${dw_weight_ratio:-50}" + + echo "IFACE=${iface} STATE=${operstate} GW=${gw:-none} WEIGHT=${weight}" +} + +# ── Write dw_weight_ratio for both WANs ───────────────────────────── +# $1: WAN1 weight (0-100, WAN2 = 100-WAN1) +# Sets dw_weight_ratio in smartwan.conf for both ifname sections. +# Does NOT call loadbalance. SmartWAN re-reads on its own cycle. +sw_write_weights() { + local w1="$1" + case "$w1" in + ''|*[!0-9]*) warn "invalid weight: $w1"; return 1 ;; + esac + [ "$w1" -lt 0 ] && w1=0 + [ "$w1" -gt 100 ] && w1=100 + + local current=50 + [ -f "$SMARTWAN_CONF" ] && . "$SMARTWAN_CONF" 2>/dev/null + current="${dw_weight_ratio:-50}" + + # Only write if changed by >= 10 (hysteresis via lib-decide.sh) + local diff=$(( w1 > current ? w1 - current : current - w1 )) + if [ "$diff" -lt 10 ]; then + return 0 # within hysteresis band — no-op + fi + + # Write to smartwan.conf using sed — safe, atomic-ish + # SmartWAN has dw_weight_ratio on both ifname sections + if [ -f "$SMARTWAN_CONF" ]; then + sed -i "s/^dw_weight_ratio=.*/dw_weight_ratio=${w1}/" "$SMARTWAN_CONF" 2>/dev/null + fi + + log "smartwan: dw_weight_ratio ${current} -> ${w1} (WAN1=${w1}% WAN2=$((100-w1))%)" + mkdir -p "$STATE_DIR" + echo "$w1" > "$STATE_DIR/sw_weight_written" + echo "$(date +%s)" > "$STATE_DIR/sw_weight_time" +} + +# ── Light touch — signal SmartWAN to re-read config ──────────────── +# Uses synoservice if available. NEVER calls loadbalance lb-enable. +sw_signal_reload() { + # SRM 1.3+ has synoservice to restart individual services + if command -v synoservice >/dev/null 2>&1; then + synoservice --restart smartwan 2>/dev/null && \ + { log "smartwan: light-reload via synoservice"; return 0; } + fi + + # Fallback: just touch the config — SmartWAN polls it + touch "$SMARTWAN_CONF" 2>/dev/null + log "smartwan: config touched for polling re-read" + return 0 +} + +# ── Check if SmartWAN is in load-balance mode ─────────────────────── +sw_is_lb_mode() { + [ -f "$SMARTWAN_CONF" ] || return 1 + local mode="" + mode=$(grep '^smartwan_mode=' "$SMARTWAN_CONF" 2>/dev/null | cut -d= -f2) + case "$mode" in + loadbalance|loadbalancing_failover) return 0 ;; + *) return 1 ;; + esac +} diff --git a/syno-balance/bin/syno-daemon.sh b/syno-balance/bin/syno-daemon.sh new file mode 100755 index 0000000..c95839e --- /dev/null +++ b/syno-balance/bin/syno-daemon.sh @@ -0,0 +1,226 @@ +#!/bin/sh +# syno-daemon.sh — Synology bus router WAN balancing daemon. +# Rides on top of SmartWAN (does NOT own routing or health checks). +# Uses the same scoring engine as GL kit-busrouter (lib-score.sh + lib-decide.sh). +# +# Cycle (every CHECK_INTERVAL seconds): +# 1. Read SmartWAN state (interfaces, current weights, gateways) +# 2. Measure WAN1 / WAN2 latency, jitter, packet loss +# 3. Read Eyeride signal strength (if available) +# 4. Read Starlink status (if available) +# 5. Score each WAN (lib-score.sh 6-factor composite) +# 6. Compute target weights (lib-decide.sh) +# 7. Write weights to SmartWAN if change >= 10 (smartwan-adapter.sh) +# 8. Send telemetry to fleet hub (if hub is reachable) +# +# NEVER calls loadbalance. NEVER flushes conntrack. + +set -e + +# ── Paths ────────────────────────────────────────────────────────── +PKG_DIR="${PKG_DIR:-/var/packages/syno-balance/target}" +LIB_DIR="${LIB_DIR:-${PKG_DIR}/bin}" +CONF="/etc/syno-balance/syno-balance.conf" +STATE_DIR="/tmp/syno-balance" +PID_FILE="$PKG_DIR/var/syno-balance.pid" +LOG_TAG="syno-balance" + +# ── Source libs (same scoring engine as GL) ──────────────────────── +. "${LIB_DIR}/lib-score.sh" +. "${LIB_DIR}/lib-decide.sh" +. "${LIB_DIR}/smartwan-adapter.sh" + +mkdir -p "$STATE_DIR" +echo "$$" > "$PID_FILE" + +log() { logger -t "$LOG_TAG" -p local0.warn "syno-balance[$(cat /etc/busrouter/device-id 2>/dev/null || hostname)]: $*"; } +warn() { logger -t "$LOG_TAG" -p local0.warn "syno-balance[$(cat /etc/busrouter/device-id 2>/dev/null || hostname)]: WARNING: $*"; } + +# ── Config defaults ──────────────────────────────────────────────── +load_config() { + [ -f "$CONF" ] && . "$CONF" + : "${CHECK_INTERVAL:=30}" + : "${WAN1_IFACE:=eth0}" + : "${WAN2_IFACE:=eth2}" + : "${SPEEDTEST_AUTO_ENABLED:=0}" + : "${EYERIDE_ENABLE:=0}" + : "${EYERIDE_IP:=192.168.10.1}" + : "${EYERIDE_PORT:=8080}" + : "${EYERIDE_USER:=root}" + : "${STARLINK_ENABLE:=0}" + : "${STARLINK_IP:=192.168.100.1}" + : "${STARLINK_PORT:=9200}" + : "${PING_TARGETS:=1.1.1.2 9.9.9.9 8.8.8.8}" + : "${PING_COUNT:=5}" + : "${WEIGHT_MIN:=20}" + : "${WEIGHT_MAX:=80}" + : "${ROLLING_SAMPLES:=3}" + : "${TELEMETRY_HUB:=http://167.172.237.162:8080/api/telemetry}" + : "${TELEMETRY_BUFFER_DIR:=/tmp/syno-balance/telemetry-buf}" + DEVICE_ID=$(cat /etc/busrouter/device-id 2>/dev/null || hostname 2>/dev/null || echo "unknown") +} + +# ── Measure WAN metrics ──────────────────────────────────────────── +measure_wan() { + local iface="$1" + local gw="$2" + local outfile="$STATE_DIR/metric_${iface}" + + # Latency + jitter + loss via ping (bound to interface) + local ping_out="" + ping_out=$(ping -I "$iface" -c "$PING_COUNT" -q "${PING_TARGETS%% *}" 2>/dev/null) || true + + local lat="" loss="" jitter="" + if [ -n "$ping_out" ]; then + # rtt min/avg/max/mdev + lat=$(echo "$ping_out" | awk -F'[/ ]' '/^rtt/ {printf "%.1f", $8}') + jitter=$(echo "$ping_out" | awk -F'[/ ]' '/^rtt/ {printf "%.1f", $11}') + loss=$(echo "$ping_out" | grep -o '[0-9.]*%' | head -1 | tr -d '%') + fi + + echo "lat=${lat:-0}" > "$outfile" + echo "jitter=${jitter:-0}" >> "$outfile" + echo "loss=${loss:-0}" >> "$outfile" + echo "gw=${gw:-none}" >> "$outfile" + echo "ts=$(date +%s)" >> "$outfile" +} + +# ── Read Eyeride signal via ubus (if available) ──────────────────── +read_eyeride_signal() { + [ "$EYERIDE_ENABLE" != "1" ] && return 1 + [ -z "$EYERIDE_PASSWORD" ] && return 1 + + # ubus call over SSH to Eyeride (OpenWrt device) + local sig="" + sig=$(sshpass -p "$EYERIDE_PASSWORD" ssh -o StrictHostKeyChecking=no \ + -o ConnectTimeout=5 "${EYERIDE_USER}@${EYERIDE_IP}" -p "${EYERIDE_PORT}" \ + "ubus call network.interface.wwan status 2>/dev/null | grep -o '\"signal_strength\":[0-9-]*' | cut -d: -f2" 2>/dev/null) || true + + if [ -n "$sig" ]; then + echo "$sig" > "$STATE_DIR/signal_eyeride" + return 0 + fi + return 1 +} + +# ── Read Starlink status (gRPC or HTTP fallback) ─────────────────── +read_starlink_status() { + [ "$STARLINK_ENABLE" != "1" ] && return 1 + + # HTTP probe to Starlink dish + local status="" + status=$(curl -s --connect-timeout 3 "http://${STARLINK_IP}:${STARLINK_PORT}/status" 2>/dev/null) || true + if [ -n "$status" ]; then + echo "$status" > "$STATE_DIR/starlink_status" + return 0 + fi + return 1 +} + +# ── Score both WANs ──────────────────────────────────────────────── +score_wans() { + local w1_lat=0 w1_jit=0 w1_loss=0 w1_sig="" + local w2_lat=0 w2_jit=0 w2_loss=0 w2_sig="" + + # Read WAN1 metrics + if [ -f "$STATE_DIR/metric_${WAN1_IFACE}" ]; then + . "$STATE_DIR/metric_${WAN1_IFACE}" + w1_lat="${lat:-0}"; w1_jit="${jitter:-0}"; w1_loss="${loss:-0}" + fi + + # Read WAN2 metrics + if [ -f "$STATE_DIR/metric_${WAN2_IFACE}" ]; then + . "$STATE_DIR/metric_${WAN2_IFACE}" + w2_lat="${lat:-0}"; w2_jit="${jitter:-0}"; w2_loss="${loss:-0}" + fi + + # Read Eyeride signal for WAN1 + if [ -f "$STATE_DIR/signal_eyeride" ]; then + w1_sig=$(cat "$STATE_DIR/signal_eyeride" 2>/dev/null) + fi + + # Compute scores (same algorithm as GL) + local score1=0 score2=0 + # WAN1: latency, fake-dl=10, fake-ul=5, jitter, loss, signal + score1=$(score_composite "$w1_lat" "10" "5" "$w1_jit" "$w1_loss" "$w1_sig") + # WAN2: latency, fake-dl=10, fake-ul=5, jitter, loss, no signal + score2=$(score_composite "$w2_lat" "10" "5" "$w2_jit" "$w2_loss" "") + + # Rolling average for stability + local avg1="$score1" avg2="$score2" + if [ "$ROLLING_SAMPLES" -gt 1 ]; then + # Store last N scores + echo "$score1" >> "$STATE_DIR/score_history_${WAN1_IFACE}" + echo "$score2" >> "$STATE_DIR/score_history_${WAN2_IFACE}" + tail -n "$ROLLING_SAMPLES" "$STATE_DIR/score_history_${WAN1_IFACE}" > "$STATE_DIR/score_roll_1" + tail -n "$ROLLING_SAMPLES" "$STATE_DIR/score_history_${WAN2_IFACE}" > "$STATE_DIR/score_roll_2" + # Trim + if [ "$(wc -l < "$STATE_DIR/score_history_${WAN1_IFACE}")" -gt 10 ]; then + tail -n 10 "$STATE_DIR/score_history_${WAN1_IFACE}" > "${STATE_DIR}/score_history_${WAN1_IFACE}.tmp" + mv "${STATE_DIR}/score_history_${WAN1_IFACE}.tmp" "$STATE_DIR/score_history_${WAN1_IFACE}" + fi + avg1=$(awk '{s+=$1}END{printf "%d",(s/NR+0.5)}' "$STATE_DIR/score_roll_1" 2>/dev/null || echo "$score1") + avg2=$(awk '{s+=$1}END{printf "%d",(s/NR+0.5)}' "$STATE_DIR/score_roll_2" 2>/dev/null || echo "$score2") + fi + + echo "$avg1" > "$STATE_DIR/score_${WAN1_IFACE}" + echo "$avg2" > "$STATE_DIR/score_${WAN2_IFACE}" + + log "scores: ${WAN1_IFACE}=${avg1} ${WAN2_IFACE}=${avg2}" +} + +# ── Apply weights to SmartWAN ────────────────────────────────────── +apply_weights() { + local s1=$(cat "$STATE_DIR/score_${WAN1_IFACE}" 2>/dev/null || echo "50") + local s2=$(cat "$STATE_DIR/score_${WAN2_IFACE}" 2>/dev/null || echo "50") + + # WAN1 weight = 100 - WAN2 weight (SmartWAN dw_weight_ratio is WAN1 share) + # Our wan_weight gives WAN1 share integer 20-80 + local w1=$(wan_weight "$s1" "$s2") + + # Clamp to configured range + [ "$w1" -lt "$WEIGHT_MIN" ] && w1="$WEIGHT_MIN" + [ "$w1" -gt "$WEIGHT_MAX" ] && w1="$WEIGHT_MAX" + + sw_write_weights "$w1" +} + +# ── Main loop ────────────────────────────────────────────────────── +log "EVENT=DAEMON_START device=$DEVICE_ID version=${SYNO_BALANCE_VERSION:-0.0}" + +load_config + +# Verify SmartWAN is in load-balance mode +if ! sw_is_lb_mode; then + warn "SmartWAN is not in load-balance mode. Set Load Balancing in Network Center." + # Don't exit — router may transition to LB mode later +fi + +_cycle=0 +while true; do + _cycle=$((_cycle + 1)) + + # Read SmartWAN state + if ! sw_read_state >/dev/null 2>&1; then + warn "cannot read SmartWAN state — waiting for interface config" + sleep "$CHECK_INTERVAL" + continue + fi + + # Measure both WANs + measure_wan "$WAN1_IFACE" "$(ip route show dev "$WAN1_IFACE" 2>/dev/null | grep default | awk '{print $3}' | head -1)" + measure_wan "$WAN2_IFACE" "$(ip route show dev "$WAN2_IFACE" 2>/dev/null | grep default | awk '{print $3}' | head -1)" + + # Optional: Eyeride signal, Starlink status + read_eyeride_signal 2>/dev/null || true + read_starlink_status 2>/dev/null || true + + # Score + apply + score_wans + apply_weights + + # Save cycle state for telemetry + echo "$_cycle" > "$STATE_DIR/cycle" + + sleep "$CHECK_INTERVAL" +done diff --git a/syno-balance/etc/syno-balance.conf b/syno-balance/etc/syno-balance.conf new file mode 100644 index 0000000..db6f91a --- /dev/null +++ b/syno-balance/etc/syno-balance.conf @@ -0,0 +1,35 @@ +# syno-balance — Bus Router WAN Balancer Configuration +# Rides on top of SmartWAN. NEVER calls loadbalance. + +# ── WAN Interfaces ────────────────────────────────────────────────── +WAN1_IFACE="eth0" +WAN2_IFACE="eth2" + +# ── Check Interval (seconds) ──────────────────────────────────────── +CHECK_INTERVAL=30 + +# ── Speed Test ───────────────────────────────────────────────────── +SPEEDTEST_AUTO_ENABLED=0 + +# ── Eyeride (WAN1 cellular via Eyeride) ──────────────────────────── +EYERIDE_ENABLE=0 +EYERIDE_IP="192.168.10.1" +EYERIDE_PORT="8080" +EYERIDE_USER="root" +EYERIDE_PASSWORD="" + +# ── Starlink (WAN2) ───────────────────────────────────────────────── +STARLINK_ENABLE=0 +STARLINK_IP="192.168.100.1" +STARLINK_PORT="9200" + +# ── Scoring ──────────────────────────────────────────────────────── +PING_TARGETS="1.1.1.2 9.9.9.9 8.8.8.8" +PING_COUNT=5 +WEIGHT_MIN=20 +WEIGHT_MAX=80 +ROLLING_SAMPLES=3 + +# ── Telemetry ────────────────────────────────────────────────────── +TELEMETRY_HUB="http://167.172.237.162:8080/api/telemetry" +TELEMETRY_BUFFER_DIR="/tmp/syno-balance/telemetry-buf" diff --git a/x4078_20260721.dss b/x4078_20260721.dss new file mode 100644 index 0000000..eb030fb Binary files /dev/null and b/x4078_20260721.dss differ