From 5dae17fb7362a9e8d7f95df52efa10c365e25224 Mon Sep 17 00:00:00 2001 From: Keith Solomon Date: Sun, 8 Mar 2026 18:53:37 -0500 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8feature:=20Enhance=20scanning=20capabi?= =?UTF-8?q?lities=20with=20Docker=20insights=20and=20concurrent=20processi?= =?UTF-8?q?ng?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 24 +++++ app/config.py | 12 +++ app/main.py | 43 +++++--- app/scanner.py | 256 ++++++++++++++++++++++++++++++++++++++++----- docker-compose.yml | 14 +++ requirements.txt | 1 + 6 files changed, 309 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 83e3fa2..fedfbea 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,9 @@ Results are persisted in SQLite for change tracking (new/updated/missing devices - Right-top: selected machine details - Right-bottom: collapsible port records with headers/banners - Background scan execution +- Live scan progress with ETA - SQLite persistence for historical tracking +- Concurrent host scanning for faster runs ## Tech Stack @@ -40,12 +42,19 @@ Database file is stored at `./data/nettrak.db` via a bind mount. Environment variables: - `NETTRAK_DB_PATH` (default: `/data/nettrak.db`) - `NETTRAK_SUBNET` (default: `192.168.2.0/24`) +- `NETTRAK_TOP_PORTS` (default: `100`) +- `NETTRAK_SCAN_WORKERS` (default: `12`) +- `NETTRAK_PORT_PROBE_TIMEOUT` (default: `0.4`) +- `NETTRAK_ENABLE_OS_DETECTION` (default: `0`) +- `NETTRAK_ENABLE_DOCKER_INSIGHTS` (default: `0`) +- `NETTRAK_DOCKER_HOST_IP` (optional, used when Docker publishes on `0.0.0.0`) In Compose, these are already set. ## LAN Scanning Notes - LAN host discovery can be limited in bridged container networking. +- MAC addresses are best-effort in bridged mode; for most reliable MAC/ARP discovery, run in host networking and keep `NET_RAW`/`NET_ADMIN` capabilities. - For best results on Linux hosts, enable host networking in `docker-compose.yml`: ```yaml @@ -54,6 +63,21 @@ network_mode: host - Some `nmap` OS detection capabilities may require elevated privileges. The app automatically falls back if OS detection fails. +## Docker Container Port Awareness + +NetTrak can optionally annotate host ports that are published by Docker containers on the scan host. + +To enable: +- set `NETTRAK_ENABLE_DOCKER_INSIGHTS=1` +- mount the Docker socket: + +```yaml +volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro +``` + +If your Docker bindings are `0.0.0.0`, set `NETTRAK_DOCKER_HOST_IP` to the host LAN IP so mappings can be attributed correctly. + ## API Endpoints - `GET /api/health` diff --git a/app/config.py b/app/config.py index 5f59b4e..4770df5 100644 --- a/app/config.py +++ b/app/config.py @@ -3,3 +3,15 @@ import os DB_PATH = os.getenv("NETTRAK_DB_PATH", "/data/nettrak.db") DEFAULT_SUBNET = os.getenv("NETTRAK_SUBNET", "192.168.2.0/24") SCAN_TIMEOUT_SECONDS = int(os.getenv("NETTRAK_SCAN_TIMEOUT", "1800")) +SCAN_TOP_PORTS = int(os.getenv("NETTRAK_TOP_PORTS", "100")) +SCAN_WORKERS = int(os.getenv("NETTRAK_SCAN_WORKERS", "12")) +PORT_PROBE_TIMEOUT_SECONDS = float(os.getenv("NETTRAK_PORT_PROBE_TIMEOUT", "0.4")) +ENABLE_OS_DETECTION = os.getenv("NETTRAK_ENABLE_OS_DETECTION", "0").lower() in {"1", "true", "yes", "on"} +ENABLE_DOCKER_INSIGHTS = os.getenv("NETTRAK_ENABLE_DOCKER_INSIGHTS", "0").lower() in { + "1", + "true", + "yes", + "on", +} +DOCKER_SOCKET = os.getenv("NETTRAK_DOCKER_SOCKET", "unix:///var/run/docker.sock") +DOCKER_HOST_IP = os.getenv("NETTRAK_DOCKER_HOST_IP") diff --git a/app/main.py b/app/main.py index 3d7a6cd..b58062c 100644 --- a/app/main.py +++ b/app/main.py @@ -1,5 +1,6 @@ from __future__ import annotations +from concurrent.futures import ThreadPoolExecutor, as_completed import threading from typing import Any @@ -7,9 +8,9 @@ from fastapi import FastAPI, HTTPException from fastapi.responses import FileResponse from fastapi.staticfiles import StaticFiles -from .config import DEFAULT_SUBNET +from .config import DEFAULT_SUBNET, SCAN_WORKERS from .db import init_db -from .scanner import discover_hosts, scan_host +from .scanner import HostResult, discover_docker_ports, discover_hosts, merge_docker_ports, scan_host from .service import ( complete_scan, create_scan, @@ -77,23 +78,31 @@ def run_scan(subnet: str | None = None) -> dict[str, Any]: try: discovered = discover_hosts(subnet) scan_state.set_total_hosts(len(discovered)) + docker_ports_by_ip = discover_docker_ports({h["ip"] for h in discovered}) - for idx, host in enumerate(discovered, start=1): - scan_state.set_current_host(host["ip"]) - detailed = scan_host(host["ip"]) - if not detailed: - scan_state.update_progress(idx, host_count) - continue - host_count += 1 - if not detailed.hostname and host.get("hostname"): - detailed.hostname = host["hostname"] - if not detailed.mac and host.get("mac"): - detailed.mac = host["mac"] - if not detailed.vendor and host.get("vendor"): - detailed.vendor = host["vendor"] + processed_count = 0 + max_workers = min(max(1, SCAN_WORKERS), max(1, len(discovered))) + with ThreadPoolExecutor(max_workers=max_workers) as pool: + future_map = {pool.submit(scan_host, host["ip"], host): host for host in discovered} - upsert_host(scan_id, detailed) - scan_state.update_progress(idx, host_count) + for future in as_completed(future_map): + seed = future_map[future] + scan_state.set_current_host(seed["ip"]) + processed_count += 1 + try: + detailed = future.result() + except Exception: + detailed = HostResult( + ip=seed["ip"], + hostname=seed.get("hostname"), + mac=seed.get("mac"), + vendor=seed.get("vendor"), + ) + + detailed = merge_docker_ports(detailed, docker_ports_by_ip.get(detailed.ip, [])) + upsert_host(scan_id, detailed) + host_count += 1 + scan_state.update_progress(processed_count, host_count) mark_missing_devices(scan_id) complete_scan(scan_id, "completed", host_count) diff --git a/app/scanner.py b/app/scanner.py index d274957..ae77707 100644 --- a/app/scanner.py +++ b/app/scanner.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re import socket import ssl import subprocess @@ -7,6 +8,20 @@ import xml.etree.ElementTree as ET from dataclasses import dataclass, field from typing import Any +from .config import ( + DOCKER_HOST_IP, + DOCKER_SOCKET, + ENABLE_DOCKER_INSIGHTS, + ENABLE_OS_DETECTION, + PORT_PROBE_TIMEOUT_SECONDS, + SCAN_TOP_PORTS, +) + +try: + import docker +except Exception: # pragma: no cover - optional dependency/runtime + docker = None + HTTP_PORTS = {80, 81, 443, 8000, 8080, 8081, 8443, 8888} @@ -46,9 +61,64 @@ def run_nmap(args: list[str]) -> ET.Element: return ET.fromstring(proc.stdout) +def _run_cmd(args: list[str]) -> str: + try: + proc = subprocess.run(args, capture_output=True, text=True, check=False) + except OSError: + return "" + if proc.returncode != 0: + return "" + return proc.stdout + + +def _ip_neigh_table() -> dict[str, dict[str, str | None]]: + output = _run_cmd(["ip", "neigh", "show"]) + if not output: + return {} + + table: dict[str, dict[str, str | None]] = {} + for line in output.splitlines(): + parts = line.strip().split() + if len(parts) < 5: + continue + ip = parts[0] + mac = None + if "lladdr" in parts: + idx = parts.index("lladdr") + if idx + 1 < len(parts): + mac = parts[idx + 1] + if mac and re.fullmatch(r"[0-9a-fA-F:]{17}", mac): + table[ip] = {"mac": mac.upper(), "vendor": None} + return table + + +def _arp_table() -> dict[str, dict[str, str | None]]: + output = _run_cmd(["arp", "-an"]) + if not output: + return {} + + table: dict[str, dict[str, str | None]] = {} + for line in output.splitlines(): + match = re.search(r"\((\d+\.\d+\.\d+\.\d+)\)\s+at\s+([0-9a-fA-F:]{17})", line) + if not match: + continue + ip, mac = match.group(1), match.group(2).upper() + table[ip] = {"mac": mac, "vendor": None} + return table + + +def arp_neighbors() -> dict[str, dict[str, str | None]]: + table = _ip_neigh_table() + if table: + return table + return _arp_table() + + def discover_hosts(subnet: str) -> list[dict[str, Any]]: - root = run_nmap(["-sn", subnet]) + root = run_nmap(["-sn", "-n", subnet]) hosts: list[dict[str, Any]] = [] + arp_cache = arp_neighbors() + for host in root.findall("host"): status = host.find("status") if status is None or status.attrib.get("state") != "up": @@ -69,8 +139,15 @@ def discover_hosts(subnet: str) -> list[dict[str, Any]]: first = names.find("hostname") if first is not None: hostname = first.attrib.get("name") - if ip: - hosts.append({"ip": ip, "hostname": hostname, "mac": mac, "vendor": vendor}) + + if not ip: + continue + + if not mac and ip in arp_cache: + mac = arp_cache[ip]["mac"] + vendor = vendor or arp_cache[ip]["vendor"] + + hosts.append({"ip": ip, "hostname": hostname, "mac": mac, "vendor": vendor}) return hosts @@ -146,9 +223,9 @@ def parse_detailed_host(xml_root: ET.Element) -> HostResult | None: return result -def probe_port(ip: str, port: int, timeout: float = 1.5) -> tuple[str | None, dict[str, str]]: +def probe_port(ip: str, port: int, timeout: float = PORT_PROBE_TIMEOUT_SECONDS) -> tuple[str | None, dict[str, str]]: if port not in HTTP_PORTS: - return grab_banner(ip, port, timeout), {} + return None, {} try: if port in {443, 8443}: @@ -160,7 +237,7 @@ def probe_port(ip: str, port: int, timeout: float = 1.5) -> tuple[str | None, di with socket.create_connection((ip, port), timeout=timeout) as sock: return http_probe(sock, ip) except Exception: - return grab_banner(ip, port, timeout), {} + return None, {} def http_probe(sock: socket.socket, host: str) -> tuple[str | None, dict[str, str]]: @@ -186,25 +263,156 @@ def http_probe(sock: socket.socket, host: str) -> tuple[str | None, dict[str, st return status, headers -def grab_banner(ip: str, port: int, timeout: float) -> str | None: - try: - with socket.create_connection((ip, port), timeout=timeout) as sock: - sock.settimeout(timeout) - data = sock.recv(1024) - if not data: - return None - return data.decode("utf-8", errors="ignore").strip()[:300] - except Exception: - return None +def scan_host(ip: str, seed_host: dict[str, Any] | None = None) -> HostResult: + base_args = [ + "-Pn", + "-n", + "--open", + "-sV", + "--version-light", + "--top-ports", + str(max(SCAN_TOP_PORTS, 1)), + "-T4", + "--max-retries", + "1", + "--host-timeout", + "45s", + ip, + ] + result: HostResult | None = None + if ENABLE_OS_DETECTION: + try: + root = run_nmap([*base_args, "-O", "--osscan-guess"]) + result = parse_detailed_host(root) + except Exception: + result = None -def scan_host(ip: str) -> HostResult | None: - # `-O` may fail without extra privileges, so we gracefully retry without it. - base_args = ["-Pn", "-sV", "--top-ports", "200", ip] - try: - root = run_nmap([*base_args, "-O", "--osscan-guess"]) - return parse_detailed_host(root) - except Exception: + if result is None: root = run_nmap(base_args) - return parse_detailed_host(root) + result = parse_detailed_host(root) + if result is None: + result = HostResult(ip=ip) + + if seed_host: + if not result.hostname and seed_host.get("hostname"): + result.hostname = seed_host["hostname"] + if not result.mac and seed_host.get("mac"): + result.mac = seed_host["mac"] + if not result.vendor and seed_host.get("vendor"): + result.vendor = seed_host["vendor"] + + if not result.mac: + cached = arp_neighbors().get(ip) + if cached: + result.mac = cached.get("mac") + result.vendor = result.vendor or cached.get("vendor") + + return result + + +def _local_ipv4s() -> set[str]: + local_ips: set[str] = {"127.0.0.1"} + try: + addrs = socket.getaddrinfo(socket.gethostname(), None, socket.AF_INET) + for addr in addrs: + local_ips.add(addr[4][0]) + except Exception: + pass + + output = _run_cmd(["ip", "-4", "addr", "show"]) + for match in re.findall(r"inet\s+(\d+\.\d+\.\d+\.\d+)/\d+", output): + local_ips.add(match) + return local_ips + + +def discover_docker_ports(target_ips: set[str]) -> dict[str, list[PortResult]]: + if not ENABLE_DOCKER_INSIGHTS or docker is None or not target_ips: + return {} + + local_ips = _local_ipv4s() + if DOCKER_HOST_IP: + local_ips.add(DOCKER_HOST_IP) + + try: + client = docker.DockerClient(base_url=DOCKER_SOCKET) + containers = client.containers.list() + except Exception: + return {} + + ports_by_ip: dict[str, list[PortResult]] = {} + seen: set[tuple[str, int, str]] = set() + + for container in containers: + try: + attrs = container.attrs + except Exception: + continue + port_map = attrs.get("NetworkSettings", {}).get("Ports", {}) or {} + image = attrs.get("Config", {}).get("Image") + name = (attrs.get("Name") or "").lstrip("/") or container.name + + for container_port_proto, bindings in port_map.items(): + if not bindings: + continue + container_port, protocol = container_port_proto.split("/", 1) + for binding in bindings: + host_port_raw = binding.get("HostPort") + host_ip = binding.get("HostIp") or "" + if not host_port_raw: + continue + try: + host_port = int(host_port_raw) + except ValueError: + continue + + candidate_ips: set[str] = set() + if host_ip and host_ip not in {"0.0.0.0", "::"}: + candidate_ips.add(host_ip) + else: + explicit = (DOCKER_HOST_IP or "").strip() + if explicit: + candidate_ips.add(explicit) + else: + candidate_ips |= (local_ips & target_ips) + + for ip in candidate_ips: + if ip not in target_ips: + continue + key = (ip, host_port, protocol) + if key in seen: + continue + seen.add(key) + ports_by_ip.setdefault(ip, []).append( + PortResult( + port=host_port, + protocol=protocol, + state="open", + service="docker-published", + product=image, + extra_info=f"container={name}, internal={container_port_proto}", + ) + ) + return ports_by_ip + + +def merge_docker_ports(host: HostResult, docker_ports: list[PortResult]) -> HostResult: + if not docker_ports: + return host + + by_key = {(p.port, p.protocol): p for p in host.ports} + for dp in docker_ports: + key = (dp.port, dp.protocol) + existing = by_key.get(key) + if existing: + note = dp.extra_info or "docker-published" + existing.extra_info = f"{existing.extra_info}; {note}" if existing.extra_info else note + if not existing.product: + existing.product = dp.product + if not existing.service: + existing.service = dp.service + continue + host.ports.append(dp) + by_key[key] = dp + return host diff --git a/docker-compose.yml b/docker-compose.yml index e7fbb70..c8227e2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,13 +7,27 @@ services: environment: - NETTRAK_DB_PATH=/data/nettrak.db - NETTRAK_SUBNET=192.168.2.0/24 + - NETTRAK_TOP_PORTS=100 + - NETTRAK_SCAN_WORKERS=12 + - NETTRAK_PORT_PROBE_TIMEOUT=0.4 + - NETTRAK_ENABLE_OS_DETECTION=0 + - NETTRAK_ENABLE_DOCKER_INSIGHTS=0 + # Set this if Docker published ports are bound to 0.0.0.0 and host IP cannot be inferred. + # - NETTRAK_DOCKER_HOST_IP=192.168.2.10 # For best host discovery on Linux, you can switch to host mode. # If you do that, remove the `ports` section and ensure APP_PORT is free. # network_mode: host + # Helps nmap discover MAC addresses/OS details in containerized runs. + cap_add: + - NET_ADMIN + - NET_RAW + ports: - "1337:1337" volumes: - ./data:/data + # Optional: mount Docker socket to include container-published host ports in results. + # - /var/run/docker.sock:/var/run/docker.sock:ro diff --git a/requirements.txt b/requirements.txt index babedd0..74e915e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ fastapi==0.116.1 uvicorn[standard]==0.35.0 +docker==7.1.0