feat(ns-api): add live trace troubleshooting API

packages/ns-api/Makefile

@@ -23,14 +23,17 @@ define Package/ns-api
 	URL:=https://github.com/NethServer/nethsecurity-controller/
 	DEPENDS:= \
 		+adblock \
+		+conntrack \
 		+coreutils-date \
 		+coreutils-stty \
+		+ip-full \
 		+python3-idna \
 		+python3-nethsec \
 		+python3-openssl \
 		+python3-requests \
 		+python3-semver \
 		+python3-urllib \
+		+tcpdump \
 		+whiptail \
 		+sshpass \
 		+wireguard-tools
@@ -179,6 +182,8 @@ define Package/ns-api/install
 	$(INSTALL_DATA) ./files/ns.ha.json $(1)/usr/share/rpcd/acl.d/
 	$(INSTALL_BIN) ./files/ns.flows $(1)/usr/libexec/rpcd/
 	$(INSTALL_DATA) ./files/ns.flows.json $(1)/usr/share/rpcd/acl.d/
+	$(INSTALL_BIN) ./files/ns.livetrace $(1)/usr/libexec/rpcd/
+	$(INSTALL_DATA) ./files/ns.livetrace.json $(1)/usr/share/rpcd/acl.d/
 	$(INSTALL_DIR) $(1)/lib/upgrade/keep.d
 	$(INSTALL_CONF) files/ns-api.keep $(1)/lib/upgrade/keep.d/ns-api
 	$(INSTALL_CONF) files/msmtp.keep $(1)/lib/upgrade/keep.d/msmtp
@@ -187,6 +192,7 @@ define Package/ns-api/install
 	$(INSTALL_BIN) ./files/load-kernel-modules $(1)/usr/sbin/load-kernel-modules
 	$(INSTALL_BIN) ./files/setup $(1)/usr/sbin/setup
 	$(INSTALL_BIN) ./files/firewall-apply-default-logging $(1)/usr/sbin/firewall-apply-default-logging
+	$(INSTALL_BIN) ./files/ns-livetrace.py $(1)/usr/libexec/ns-livetrace
 	$(INSTALL_DIR) $(1)/etc/config
 	$(INSTALL_CONF) ./files/config/ns-api $(1)/etc/config/ns-api
 	$(INSTALL_CONF) ./files/config/ns-wizard $(1)/etc/config/ns-wizard

packages/ns-api/README.md

@@ -7991,6 +7991,219 @@ Example response:
 }
 ```
 
+## ns.livetrace
+
+Trace the live traffic of one client to find which layer blocks it.
+One session at a time. Results stay readable after the session ends.
+
+### start
+
+Start a session. A running session is stopped first.
+
+Parameters:
+
+- `ip`: client IP address
+- `duration`: seconds, from 10 to 600, default 120
+- `port`: optional, destination port of new connections, from 1 to 65535
+- `protocol`: optional, `tcp` or `udp`
+
+Missing, `null`, `0` and `""` mean not set. `port` and `protocol` do not filter DNS queries.
+
+```bash
+api-cli ns.livetrace start --data '{"ip": "10.10.10.50", "duration": 120, "port": 443, "protocol": "tcp"}'
+```
+
+Example response:
+
+```json
+{
+   "message": "success"
+}
+```
+
+Errors:
+
+- validation: `invalid_ip_address` (also for scoped, IPv4-mapped, multicast, loopback and unspecified addresses),
+  `invalid_duration`, `invalid_port`, `invalid_protocol`
+- `collector_not_started`: the collector did not start, the reason is in `failure` of `status`
+- `session_busy`: the running session did not stop within 5 seconds
+
+### stop
+
+Stop the running session:
+
+```bash
+api-cli ns.livetrace stop
+```
+
+Example response:
+
+```json
+{
+   "message": "success"
+}
+```
+
+### status
+
+Return the session state:
+
+```bash
+api-cli ns.livetrace status
+```
+
+Example response:
+
+```json
+{
+   "running": true,
+   "ip": "10.10.10.50",
+   "duration": 120,
+   "port": 0,
+   "protocol": "",
+   "remaining": 87,
+   "blocked": [
+      {
+         "protocol": "tcp",
+         "source": "10.10.10.50",
+         "destination": "1.0.0.1",
+         "destination_port": "80",
+         "layer": "threat_shield_ip",
+         "table": "banIP",
+         "chain": "_outbound",
+         "rule": "blocklist.v4",
+         "in_interface": "eth2",
+         "out_interface": "eth1",
+         "count": 5,
+         "first_seen": 1793706147,
+         "last_seen": 1793706152
+      }
+   ],
+   "dns": [
+      {
+         "name": "doubleclick.com",
+         "type": "A",
+         "result": "NXDOMAIN",
+         "filtered": true,
+         "count": 1,
+         "last_seen": 1793706213
+      }
+   ],
+   "connections": [
+      {
+         "source": "10.10.10.50",
+         "destination": "1.1.1.1",
+         "protocol": "tcp",
+         "source_port": "39494",
+         "destination_port": "80",
+         "source_stats": {
+            "packets": 6,
+            "bytes": 391
+         },
+         "destination_stats": {
+            "packets": 4,
+            "bytes": 602
+         },
+         "id": "4272941068",
+         "timeout": "115",
+         "state": "TIME_WAIT",
+         "labels": [
+            "netify-analyzed"
+         ],
+         "in_interface": "eth2",
+         "out_interface": "eth1",
+         "rule": "Accept lan to wan forwarding",
+         "application": "wikimedia",
+         "dpi_protocol": "DNS",
+         "host": "auth.wikimedia.org"
+      }
+   ],
+   "allowed": [],
+   "failure": "",
+   "truncated": false
+}
+```
+
+Example `allowed` when the session is not running:
+
+```json
+[
+   {
+      "protocol": "tcp",
+      "source": "10.10.10.50",
+      "destination": "1.1.1.1",
+      "destination_port": "443",
+      "in_interface": "eth2",
+      "out_interface": "eth1",
+      "rule": "Accept lan to wan forwarding",
+      "count": 3,
+      "source_ports": ["51234", "51236", "51240"]
+   }
+]
+```
+
+Fields:
+
+- `port`, `protocol`: session filters, `0` and `""` when not set
+- `failure`: error message of the collector, empty if none
+- `truncated`: true if something was skipped: new connections over the rate limit, more than 200 `blocked` or
+  `dns` entries, more than 1000 flows (oldest dropped from `allowed`)
+
+`blocked`: new connections dropped or rejected.
+
+- `layer`: `firewall`, `dpi`, `threat_shield_ip`, `ips`, `dns_filter`, `mac_binding`, `routing` or the nftables table name
+- `rule`: fw4 rule name, matching set name (e.g. banIP list) or nftables rule text
+- `count`: number of distinct connections
+- `application`, `host`: only with layer `dpi`, read from ns-flows, empty if the flow is unknown
+- layer `routing` has empty `table` and `chain`, `rule` is one of:
+  - `no_route`: no route to the destination, or packet refused by reverse path filtering;
+    multicast and TTL 1 packets are not reported
+  - `private_to_wan`: a private destination (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, 100.64.0.0/10, fc00::/7)
+    left through the device of the default route: missing route or tunnel down; the connection is also in `connections`
+
+`dns`: queries sent by the client and their replies.
+
+- `result`: response code (`NOERROR`, `NXDOMAIN`, ...), empty if no reply was seen
+- `filtered`: true if adblock is enabled and the name, or a parent domain, is in its list;
+  the most specific allowlist or blocklist entry wins
+
+`connections`: conntrack entries of the client, same format as `ns.conntrack list`. Filled only while the session is running.
+
+- `rule`: fw4 rule that accepted the connection, only if the connection was traced
+- `in_interface`, `out_interface`: from the trace, or from a route lookup if the connection was not traced
+- `application`, `dpi_protocol`, `host`: read from ns-flows, if the flow is classified
+
+`allowed`: traced allowed connections of the last session, without traffic counters. Filled only when the session is not running.
+
+- grouped by `protocol`, `source`, `destination`, `destination_port`, `in_interface`, `out_interface` and `rule`
+- `count`: number of traced connections, `source_ports`: their source ports
+- sorted by `count`, descending
+
+Limitations:
+
+- only the first packet of each new connection and DPI-blocked packets are traced
+- rate limit: 50 packets per second per direction, burst 100; needs `net.netfilter.nf_conntrack_acct=1` (OpenWrt default)
+- Snort (IPS) drops are not shown: a connection dropped by Snort is listed as `allowed`
+- a DPI block shows up after the flow is classified: the first packets pass
+- chains with priority lower than -199 (e.g. the raw table) are not visible
+- DNS filter detection covers adblock with dnsmasq only, not FlashStart
+- `filtered` is also set when the query goes to an external resolver, and for adblock SafeSearch domains
+  (rewritten, not blocked)
+- DNS over TCP and DNS over HTTPS are not captured
+- with `port` or `protocol` set, ICMP is not traced
+- connections opened before the session have no `rule`; the route lookup of their interfaces ignores
+  policy routing (multi-WAN) and port forwards
+- a wrong route is detected only when a private destination leaves through the device of the default route
+- port forwards: trace the public source IP with the public port; tracing the internal server shows no `rule`
+  and no `out_interface`; connections show the public destination address and port
+- after a ruleset reload (firewall, banIP, DPI) `nft monitor trace` is restarted: packets traced before the restart
+  have an empty `rule` and may show the inner chain (e.g. `reject_to_wan`), connections started during the
+  restart are not listed
+- if the traced IP is the remote side of a masqueraded connection, a DPI-blocked reply shows the WAN address and
+  the client port as destination and is also listed in `allowed`: trace the client instead
+- if the collector is killed with SIGKILL, `nft monitor`, `tcpdump` and the `inet ns_livetrace` table stay
+  until `stop` or the next `start`
+
 ## ns.objects
 
 Manage domain sets and host sets.