From 3bccb79238b042f636a9b177c1f8627b70811824 Mon Sep 17 00:00:00 2001 From: Keith Solomon Date: Tue, 29 Sep 2026 06:46:40 -0500 Subject: [PATCH] feat: Add scripts --- README-plex-state.md | 123 ++++++++++++ README-tree-rebuild.md | 182 +++++++++++++++++ capture-tree.py | 204 +++++++++++++++++++ plex-db-verify | 10 + plex-deep-scan | 20 ++ plex-plugin-scan | 27 +++ plex-recover.sh | 4 + plex-scan.sh | 4 + plex-state.py | 443 +++++++++++++++++++++++++++++++++++++++++ restore-tree.py | 206 +++++++++++++++++++ 10 files changed, 1223 insertions(+) create mode 100644 README-plex-state.md create mode 100644 README-tree-rebuild.md create mode 100755 capture-tree.py create mode 100644 plex-db-verify create mode 100644 plex-deep-scan create mode 100644 plex-plugin-scan create mode 100755 plex-recover.sh create mode 100755 plex-scan.sh create mode 100755 plex-state.py create mode 100755 restore-tree.py diff --git a/README-plex-state.md b/README-plex-state.md new file mode 100644 index 0000000..1884187 --- /dev/null +++ b/README-plex-state.md @@ -0,0 +1,123 @@ +# Selective Plex state salvage — Banshee + +**Purpose:** Recover Plex identity, settings, library databases, scheduled database backups and small customizations **without** crawling Plex's `Metadata`, `Media`, `Cache`, `Logs`, transcoding, `.nv` or `xcode` trees. + +This tool is specific to the Banshee recovery incident. It is not a general-purpose ext4 recovery program: both the damaged LV and known-dead block range are hard-coded in `plex-state.py`. + +## What we know + +- Damaged source LV: `/dev/mapper/plex--new-plex--new`, ext4, 4,096-byte filesystem blocks. +- Confirmed blank replacement-PV extent in **filesystem block numbers**: **3,418,636,288 through 4,395,389,951**, inclusive. +- Recovery drive: `/mnt/recovery` (currently exFAT; physically separate from damaged LV). +- Original Plex state directory: `/plexmediaserver/Library/Application Support/Plex Media Server`. +- The important Plex directories and their contents were located using `debugfs -c` because the damaged ext4 filesystem cannot be mounted. +- `Preferences.xml` (inode 249299786, 1,416 bytes, physical block 2,379,907,133) and `.LocalAdminToken` (inode 249299774, 42 bytes, block 33,822,208) both lie **outside** the dead block range. They still need extracting; knowing their extents does not itself create a backup. +- The inspected `Plug-in Support/Databases` directory contained two live databases and **four dated pairs** from 2026-08-20, -23, -26 and -29: roughly **2.94 GiB** for all 10 database files combined. The directory listing alone **does not establish** which of these files survived; scan them before promising a restore. + +## Package + +- `plex-state.py`: shared scanner and extractor. +- `plex-scan.sh`: select files by explicit allowlist and inspect their full ext4 block mapping. **No file data is copied.** +- `plex-recover.sh`: extract only files classified `SAFE` in that profile's scan manifest. + +These names won't collide with the generic `scan.sh` / `recover.sh` already in your tools directory. + +## The two profiles + +**`core` (default)** includes: + +- `Preferences.xml` and `.LocalAdminToken` at the Plex Media Server root; +- live library and blobs databases, matching scheduled dated backups, and any present `-wal` / `-shm` companions; +- XML files directly beneath `Plug-in Support/Preferences` (including the observed Hama/Audnexus preferences). + +**`extras` (optional)** recursively includes **only**: + +- `Plug-ins/Hama.bundle`; +- `Plug-ins/Audnexus.bundle`; +- `Scanners/Series` (your custom scanner location). + +Neither profile includes Plex's giant `Metadata`, `Media`, `Cache`, `Plug-in Support/Caches`, diagnostics, crash reports or `.nv` / `xcode` directories. **Plugin-specific `Plug-in Support/Data` is deliberately excluded**: top-level inspection found many agent directories, including Hama and Audnexus, but we haven't inspected their children to distinguish durable custom settings from caches. If you need those directories too, inspect them first rather than blindly copying everything. + +## Before scanning + +Plex must remain stopped; don't start it against the incomplete/empty media tree. Confirm the recovery drive is mounted and that the source LV is still kernel read-only: + +```bash +findmnt /mnt/recovery +blockdev --getro /dev/mapper/plex--new-plex--new # must print 1 +``` + +The scripts independently enforce these checks and refuse to run otherwise. No software installation is required beyond existing Python 3 and `debugfs`/`blockdev`/`findmnt`. + +## Scan, inspect, recover + +Unzip this package into your existing `plex-tools` directory. Then: + +```bash +chmod +x plex-*.sh plex-state.py +./plex-scan.sh # defaults to 'core' +cat /mnt/recovery/plex-state-core-scan-summary.txt +``` + +**Review the summary and optionally upload it for assessment before extracting anything.** It lists the individual database files and their extent classifications. In particular, determine whether a complete *matching pair* of the library DB and library blobs DB survives, ideally using the same scheduled-backup date if the live files aren't both usable. + +If the scan results justify extraction: + +```bash +./plex-recover.sh # core +cat /mnt/recovery/plex-state-core-recovery-summary.txt +``` + +Optional plugin and scanner pass: + +```bash +./plex-scan.sh extras +cat /mnt/recovery/plex-state-extras-scan-summary.txt +./plex-recover.sh extras # only after reviewing the extras scan +``` + +Each profile maintains an independent manifest, scan summary, extraction log and recovery summary under `/mnt/recovery`. The extracted files keep their source directory structure under `/mnt/recovery/plexmediaserver/...` so they're easy to identify for later restoration. A small temporary directory lives under `/mnt/recovery/.plex-state-tmp`. + +Recoveries may be restarted. Existing destination files **of the same size** are skipped, but this is *not* an integrity check of an existing copy; check your earlier logs/hashes or delete the untrusted destination copy before rerunning it. + +## How the strict block checks work + +The original generic media salvage script classified a file SAFE if its *observed* data extents missed the dead disk. That's useful for media triage, but insufficiently conservative for Plex databases: missing extent-tree blocks could hide additional portions of the file. + +This scanner also: + +1. Rejects files whose extent-tree metadata (`ETB`) is on the dead disk **even if some data extents are visible**. +2. Requires the data extents to map **every logical file block**, in order, without overlaps or holes, with matching logical/physical lengths. +3. Rejects unrecognized extent formats instead of assuming they're safe. +4. Rechecks each SAFE inode and its mapping immediately before recovery. +5. Requires the extracted byte count to match the inode size and computes a SHA-256 digest of each newly extracted file. Those digests go in the profile's `*-recovery.jsonl` log. + +**A matching size, SHA-256 hash of the recovered copy, and surviving extents do not prove that a SQLite database is internally consistent.** Perform offline Plex-compatible database integrity checks before restoring the databases. Legitimately sparse files may be classified UNKNOWN by these conservative checks. + +A scan may also warn about inaccessible/missing directories. Review the warnings: an unreadable directory cannot be traversed, and even a readable directory listing may be incomplete if filesystem metadata was badly damaged. + +## Restoring Plex after rebuilding + +1. Finish media recovery and rebuild the array. Confirm the *new* `/storage` is mounted and all required bind mounts/shares are configured. +2. **Do not start Plex yet.** Copy the salvaged state from the recovery drive to its original location on the new filesystem, retaining complete paths. Consider restoring the August 29 **matched** database pair if the live pair is incomplete or fails validation. Never mix the two dates arbitrarily. +3. The exFAT Passport does **not** preserve original POSIX file ownership and permissions. The inspected Plex tree was largely UID:GID `998:998`, but verify the UID/GID used by your rebuilt Plex service before setting ownership. Restrict `Preferences.xml`, `.LocalAdminToken` and database file access to the Plex service account. +4. **Validate the database before launching Plex** using a supported Plex SQLite workflow. If a `-wal` file is present, keep it with its *matching* live database during validation; `-shm` is transient and shouldn't be blindly restored. Dated database backups should be validated as corresponding pairs. +5. Keep automatic trash emptying disabled and avoid scanning libraries until the actual media directories and their expected paths are present, or Plex may forget missing library items. +6. Keep the extracted state and all manifests/logs untouched until Plex starts successfully, recognizes its libraries and preserves expected accounts/settings/watch history. + +### Token security + +Both `Preferences.xml` and `.LocalAdminToken` may contain credentials or authentication material. This recovery target currently uses **exFAT**, so `chmod`/`chown` protections on the Passport don't behave like ext4. Keep the recovery drive physically secure, avoid uploading these two files or pasting their contents into chat, and move them to appropriately permissioned storage as soon as practical. The scan logs only metadata and paths, never file contents. + +### If the original geometry is different next time + +**Do not reuse these numeric block boundaries on another LV.** Re-derive them from the actual `dmsetup table` mapping for the missing PV and convert 512-byte sectors to ext4 filesystem blocks. In this incident, the dead segment began at LV sector `27,349,090,304` and occupied `7,814,029,312` sectors. With 4 KiB filesystem blocks, 8 sectors/block: + +```text +first missing filesystem block = 27,349,090,304 / 8 = 3,418,636,288 +missing filesystem block count = 7,814,029,312 / 8 = 976,753,664 +last missing filesystem block = 3,418,636,288 + 976,753,664 - 1 + = 4,395,389,951 +``` + +Your broader LVM recovery README documents how to identify the missing segment and create a replacement PV without writing over surviving data. Never run `mkfs`, `e2fsck -y` or a destructive `vgcfgrestore` on the old array before completing desired salvage. diff --git a/README-tree-rebuild.md b/README-tree-rebuild.md new file mode 100644 index 0000000..1cc7aa6 --- /dev/null +++ b/README-tree-rebuild.md @@ -0,0 +1,182 @@ +# Rebuilding the empty directory tree after LVM recovery + +These scripts are **independent** of the file-salvage scripts. Their job is to +save the names (and, when available, numeric UID/GID/mode) of directories +**before you wipe the damaged array**, then recreate those empty directories +on the new `/storage` filesystem. + +## Before rebuilding: export the surviving tree + +The current damaged ext4 volume cannot mount normally. `capture-tree.py` uses +`debugfs -c` instead, just as the file-salvage scripts do. It enumerates +**directory metadata only**. It does not read or recover the file contents. + +Keep the damaged LV read-only. Check: + +```bash +blockdev --getro /dev/mapper/plex--new-plex--new # must print 1 +findmnt /mnt/recovery # Passport must be mounted +``` + +Capture all *reachable* top-level directories and their descendants: + +```bash +sudo python3 capture-tree.py \ + --source / \ + --include /Backups \ + --output /mnt/recovery/plex-directory-tree.jsonl +``` + +`--include /Backups` adds that root manually because, in this incident, its +inode metadata is zeroed out. **It will not restore unknown children beneath +it.** Repeat `--include` for any additional known directory path you need to +exist later, including the share roots that have missing metadata. + +To scan only selected trees (less time, but incomplete): + +```bash +sudo python3 capture-tree.py \ + --source /Media \ + --source /Warez \ + --include /Backups \ + --output /mnt/recovery/plex-directory-tree.jsonl +``` + +This writes three files **to the Passport**: + +- `plex-directory-tree.jsonl` — directory paths with owner/mode metadata (when + available); preferred input for the restore script. +- `plex-directory-tree.txt` — one absolute filesystem path per line, easy to + inspect or edit, and also valid restore input (without ownership/modes). +- `plex-directory-tree.warnings.txt` — unreadable directories and ambiguous + entries. **Read this** before deciding the capture is complete. + +You can manually append known missing directory paths to the `.txt` file, or +re-run the capture with additional `--include` arguments. Save the resulting +three files somewhere safe besides the array you're about to rebuild. + +**Important:** If a directory's inode or parent directory block was lost, no +metadata-only tool can reconstruct the unknown descendants. Keep a list of +missing Samba share roots and any especially important subdirectories to add +manually. If a directory entry's type is destroyed, `capture-tree.py` reports +it as ambiguous in the warnings file; it does *not* invent a directory. + +## Alternative: use an existing full-path folder list + +`restore-tree.py` also accepts a plain UTF-8 text file with **one full directory +path per line**, for example: + +```text +/Media +/Media/Videos +/Media/Videos/Television +/Media/Videos/Television/Bob's Burgers +/Media/Videos/Movies +/Warez +/Warez/Books +/Backups +``` + +Folder paths may contain spaces and apostrophes; don't quote individual lines +in the text file. Empty directory parents are created automatically. An +indented/box-drawing `tree` diagram is **not** valid input; use `tree -dfi +--noreport` or `find ... -type d` output that prints a full path on each line. + +If a working source had been mounted under `/storage`, a text export might +contain `/storage/Media/...` rather than `/Media/...`; see `--strip-prefix` below. + +## After rebuilding: validate, dry-run, then apply + +**Do not run `restore-tree.py --apply` until the newly rebuilt filesystem is +mounted at `/storage`.** The script refuses to apply if `/storage` is just an +unmounted directory on the OS disk. It also refuses symlinks/non-directories +along the destination path. + +Verify that the *correct* filesystem is mounted: + +```bash +findmnt /storage +lsblk -f +``` + +Dry run using the captured JSONL: + +```bash +python3 restore-tree.py /mnt/recovery/plex-directory-tree.jsonl \ + --target /storage +``` + +If it looks correct, create the empty directories: + +```bash +sudo python3 restore-tree.py /mnt/recovery/plex-directory-tree.jsonl \ + --target /storage --apply +``` + +**Optional:** Restore original numeric owners and permission modes where the +capture has valid metadata. Check that the same service accounts/UIDs exist on +the rebuilt host first. Files on the exFAT Passport do not themselves preserve +Unix owner/mode metadata; the JSONL does. + +```bash +sudo python3 restore-tree.py /mnt/recovery/plex-directory-tree.jsonl \ + --target /storage --apply --restore-metadata +``` + +Use `--expect-source /dev/mapper/` for an extra safety check if +you know the exact `findmnt` SOURCE string for the rebuilt array. + +For an existing text list whose paths include the old mountpoint: + +```bash +python3 restore-tree.py old-tree.txt --target /storage \ + --strip-prefix /storage + +sudo python3 restore-tree.py old-tree.txt --target /storage \ + --strip-prefix /storage --apply +``` + +Re-running the restore is supported. Existing directories are retained. +`--restore-metadata` **does** change mode and owner of existing directories +listed in the snapshot, so use it deliberately. + +## What this does **not** restore + +- File contents, missing/unreachable directory names, symlinks, or hardlinks. +- Samba definitions (`smb.conf` or included shares), user accounts, ACLs, and + extended attributes. Those must already exist or be separately restored. +- Docker bind-mount configuration and application databases. Creating the + directory is not sufficient if a container also needs its files or metadata. + +After restoration, verify the expected share root directories exist and have +appropriate permissions before starting services. For Samba, `testparm` can +validate the existing share configuration; directory creation alone cannot +recreate the share definitions. + +## Recovery-incident particulars (Banshee) + +- Damaged logical volume: `/dev/mapper/plex--new-plex--new`. +- Filesystem: ext4, 4,096-byte blocks. +- Previously calculated missing data-block range: `3418636288–4395389951`. + (The directory-tree scripts do **not** use this range; they simply traverse + readable directory metadata. See the main salvage README for dead-block + calculations.) +- `/Backups` root entry survives, but inode `496369665` is zeroed and cannot + be traversed. Use `--include /Backups` plus any known child directories. +- `/mnt/recovery` is the removable Passport; export onto that or another + device, **not** onto the array being rebuilt. + +**Checkpoint:** Copy all three capture files off the recovery system and +review the warnings and share roots *before* formatting or recreating the LV. +Also make sure your existing Samba configuration and share definitions are +backed up independently. Useful commands before the rebuild: + +```bash +testparm -s # inspect configured Samba shares and paths +cp -a /etc/samba /mnt/recovery/samba-config-pre-rebuild +``` + +If the new storage will be served by containers instead, independently save +their Compose files and bind-mount declarations. If the operating-system disk +isn't being rebuilt, those configurations may remain intact, but it's worth +having copies before changing the storage layout. diff --git a/capture-tree.py b/capture-tree.py new file mode 100755 index 0000000..cb21c3b --- /dev/null +++ b/capture-tree.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Export surviving ext4 directories from an UNMOUNTED damaged LV using debugfs. + +Does not read file contents or write to the source. Unreachable trees must be +supplemented manually; no software can infer names below missing dir metadata. +""" + +import argparse +import json +import os +import re +import stat +import subprocess +import sys +from pathlib import Path, PurePosixPath + +DEFAULT_LV = "/dev/mapper/plex--new-plex--new" +DEFAULT_OUTPUT = "/mnt/recovery/plex-directory-tree.jsonl" +INODE_RE = re.compile(r"\bInode:\s*(\d+)\s+Type:\s*([^\s]+)") +MODE_RE = re.compile(r"\bMode:\s*([0-7]+)") +OWNER_RE = re.compile(r"\bUser:\s*(\d+)\s+Group:\s*(\d+)") + + +def fail(message): + sys.exit(f"ERROR: {message}") + + +def normalize_path(value): + if not value.startswith("/"): + fail(f"Source directories must be absolute filesystem paths: {value!r}") + # Avoid guessing about special path syntax; '/' is meaningful within debugfs. + parts = value.split("/") + if any(part in (".", "..") for part in parts): + fail(f"Source paths must not contain '.' or '..': {value!r}") + return "/" + "/".join(p for p in parts if p) + + +def quote_debugfs(value): + return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"' + + +def debugfs(lv, command): + env = os.environ.copy() + env["DEBUGFS_PAGER"] = "cat" + return subprocess.run( + ["debugfs", "-c", "-R", command, lv], + text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, + errors="replace", + ) + + +def get_inode(lv, path): + r = debugfs(lv, f"stat {quote_debugfs(path)}") + match = INODE_RE.search(r.stdout) + if not match or match.group(2) != "directory": + return None, r + mode_match = MODE_RE.search(r.stdout) + owner_match = OWNER_RE.search(r.stdout) + return { + "inode": int(match.group(1)), + "mode": int(mode_match.group(1), 8) if mode_match else None, + "uid": int(owner_match.group(1)) if owner_match else None, + "gid": int(owner_match.group(2)) if owner_match else None, + }, r + + +def list_dir(lv, inode): + r = debugfs(lv, f"ls -p <{inode}>") + entries = [] + bad_lines = [] + for raw in r.stdout.splitlines(): + if not raw.startswith("/"): + continue + # debugfs ls -p: /inode/mode/uid/gid/name/size/ + # Linux names cannot contain a slash. The last empty field is expected. + fields = raw.split("/") + if len(fields) < 7: + bad_lines.append(raw) + continue + try: + ino = int(fields[1]) + mode = int(fields[2], 8) + uid = int(fields[3]) + gid = int(fields[4]) + except ValueError: + bad_lines.append(raw) + continue + name = fields[5] + if name in (".", "..") or ino == 0: + continue + entries.append((ino, mode, uid, gid, name)) + # debugfs always writes a version banner to stderr; strip that banner. + stderr = "\n".join( + line for line in r.stderr.splitlines() + if not line.startswith("debugfs ") + ).strip() + return entries, stderr, bad_lines + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", action="append", default=[], + help="Existing source directory to traverse; repeatable (default: /)") + parser.add_argument("--include", action="append", default=[], + help="Add an expected directory even if its inode is unreadable; repeatable") + parser.add_argument("--lv", default=DEFAULT_LV) + parser.add_argument("--output", default=DEFAULT_OUTPUT, + help="Write JSONL plus a .txt folder-path list and .warnings.txt") + args = parser.parse_args() + + if os.geteuid() != 0: + fail("Run capture as root so debugfs can access the block device.") + ro = subprocess.run(["blockdev", "--getro", args.lv], capture_output=True, text=True) + if ro.returncode or ro.stdout.strip() != "1": + fail("Source LV must be kernel read-only. Set it read-only before capture.") + + output = Path(args.output).absolute() + if output.suffix != ".jsonl": + fail("--output must end with .jsonl (two companion files are generated).") + if str(output).startswith("/mnt/recovery/") and not os.path.ismount("/mnt/recovery"): + fail("/mnt/recovery isn't mounted. Refusing to write the snapshot to an unmounted recovery drive.") + output.parent.mkdir(parents=True, exist_ok=True) + # A temporary JSONL is renamed only when the traversal is complete. + tmp = Path(str(output) + ".incomplete") + txt_path = output.with_suffix(".txt") + warning_path = output.with_suffix(".warnings.txt") + sources = [normalize_path(s) for s in (args.source or ["/"])] + additions = [normalize_path(s) for s in args.include] + records = {} + warnings = [] + visited = set() + pending = [] + + def add_record(path, inode=None, mode=None, uid=None, gid=None, origin="debugfs"): + if path == "/": + return + if path not in records: + records[path] = { + "path": path, "inode": inode, "mode": mode, "uid": uid, + "gid": gid, "origin": origin, + } + elif origin == "debugfs": + records[path].update( + inode=inode, mode=mode, uid=uid, gid=gid, origin=origin, + ) + + for path in additions: + add_record(path, origin="manual") + + for path in sources: + info, r = get_inode(args.lv, path) + if info is None: + warnings.append(f"UNREADABLE_SOURCE {path}: {r.stderr.strip()} {r.stdout.strip()}") + add_record(path, origin="unreadable-source") + continue + add_record(path, **info) + pending.append((info["inode"], path)) + + while pending: + inode, current_path = pending.pop() + if inode in visited: + continue + visited.add(inode) + entries, stderr, bad_lines = list_dir(args.lv, inode) + if stderr: + warnings.append(f"DIR_READ_WARNING {current_path} (inode {inode}): {stderr}") + for raw in bad_lines: + warnings.append(f"UNPARSEABLE_ENTRY {current_path}: {raw}") + for child_ino, mode, uid, gid, name in entries: + path = current_path.rstrip("/") + "/" + name + if stat.S_ISDIR(mode): + add_record(path, inode=child_ino, mode=stat.S_IMODE(mode), + uid=uid, gid=gid) + pending.append((child_ino, path)) + elif stat.S_IFMT(mode) == 0: + warnings.append( + f"AMBIGUOUS_ENTRY {path} (inode {child_ino}): bad/missing type; " + "might be a lost directory. Manually include if required." + ) + if len(visited) % 200 == 0: + print(f"Visited {len(visited):,} directories; found {len(records):,} paths", flush=True) + + with tmp.open("w", encoding="utf-8") as f: + f.write(json.dumps({"_meta": { + "format": "plex-directory-tree-v1", "lv": args.lv, + "sources": sources, "manual_includes": additions, + "warnings": len(warnings), + "note": "Only reachable ext4 metadata is exported; check warnings and missing share roots." + }}, ensure_ascii=False) + "\n") + for path in sorted(records): + f.write(json.dumps(records[path], ensure_ascii=False) + "\n") + f.flush() + os.fsync(f.fileno()) + tmp.replace(output) + txt_path.write_text("".join(p + "\n" for p in sorted(records)), encoding="utf-8") + warning_path.write_text("\n".join(warnings) + ("\n" if warnings else ""), encoding="utf-8") + print(f"Exported {len(records):,} directories ({len(visited):,} visited)") + print(f"Manifest: {output}\nPaths: {txt_path}\nWarnings: {warning_path}") + if warnings: + print(f"WARNING: {len(warnings)} warnings / ambiguous entries. Review before rebuilding.") + + +if __name__ == "__main__": + main() diff --git a/plex-db-verify b/plex-db-verify new file mode 100644 index 0000000..099ed45 --- /dev/null +++ b/plex-db-verify @@ -0,0 +1,10 @@ +DB_DIR="/mnt/recovery/plexmediaserver/Library/Application Support/Plex Media Server/Plug-in Support/Databases" +SQLITE="/usr/lib/plexmediaserver/Plex SQLite" + +for db in \ + "$DB_DIR/com.plexapp.plugins.library.db-2026-08-20" \ + "$DB_DIR/com.plexapp.plugins.library.blobs.db-2026-08-20" +do + echo "Checking $(basename "$db")" + "$SQLITE" -readonly "$db" "PRAGMA integrity_check;" +done diff --git a/plex-deep-scan b/plex-deep-scan new file mode 100644 index 0000000..c7dad96 --- /dev/null +++ b/plex-deep-scan @@ -0,0 +1,20 @@ +LV='/dev/mapper/plex--new-plex--new' +BASE='/plexmediaserver/Library/Application Support/Plex Media Server' + +{ + for path in \ + '/plexmediaserver/Library' \ + '/plexmediaserver/Library/Application Support' \ + "$BASE" \ + "$BASE/Plug-in Support" \ + "$BASE/Plug-in Support/Databases" \ + "$BASE/Plug-in Support/Preferences" \ + "$BASE/Plug-ins" + do + printf '\n========== %s ==========\n' "$path" + + DEBUGFS_PAGER=cat debugfs -c \ + -R "ls -l \"$path\"" \ + "$LV" 2>&1 + done +} | tee /mnt/recovery/plex-deep-layout.txt diff --git a/plex-plugin-scan b/plex-plugin-scan new file mode 100644 index 0000000..a425bb5 --- /dev/null +++ b/plex-plugin-scan @@ -0,0 +1,27 @@ +LV='/dev/mapper/plex--new-plex--new' +BASE='/plexmediaserver/Library/Application Support/Plex Media Server' + +{ + for path in \ + "$BASE/Plug-in Support/Data" \ + "$BASE/Plug-in Support/Metadata Combination" \ + "$BASE/Scanners" + do + printf '\n========== %s ==========\n' "$path" + + DEBUGFS_PAGER=cat debugfs -c \ + -R "ls -l \"$path\"" \ + "$LV" 2>&1 + done + + for inode in 249299786 249299774 + do + printf '\n========== INODE %s ==========\n' "$inode" + + DEBUGFS_PAGER=cat debugfs -c \ + -R "stat <$inode>" \ + "$LV" 2>&1 + done + +} | tee /mnt/recovery/plex-final-inspection.txt + diff --git a/plex-recover.sh b/plex-recover.sh new file mode 100755 index 0000000..761ecae --- /dev/null +++ b/plex-recover.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +exec python3 "$SCRIPT_DIR/plex-state.py" recover "${1:-core}" diff --git a/plex-scan.sh b/plex-scan.sh new file mode 100755 index 0000000..5c55983 --- /dev/null +++ b/plex-scan.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +exec python3 "$SCRIPT_DIR/plex-state.py" scan "${1:-core}" diff --git a/plex-state.py b/plex-state.py new file mode 100755 index 0000000..7171422 --- /dev/null +++ b/plex-state.py @@ -0,0 +1,443 @@ +#!/usr/bin/env python3 +"""Strict, selective, read-only Plex state salvage from Banshee's damaged ext4 LV.""" +import argparse +import datetime as dt +import hashlib +import json +import math +import os +import re +import shutil +import stat +import subprocess +import sys +from pathlib import Path + +LV = '/dev/mapper/plex--new-plex--new' +DEST_MOUNT = Path('/mnt/recovery') +BASE = '/plexmediaserver/Library/Application Support/Plex Media Server' +DEAD_FIRST = 3418636288 +DEAD_LAST = 4395389951 +BLOCK_SIZE = 4096 +RESERVE = 2 * 1024**3 +VERSION = 1 + +ROOT_FILES = ['Preferences.xml', '.LocalAdminToken'] +DATABASE_DIR = f'{BASE}/Plug-in Support/Databases' +PREFERENCES_DIR = f'{BASE}/Plug-in Support/Preferences' +EXTRA_TREES = [ + f'{BASE}/Plug-ins/Hama.bundle', + f'{BASE}/Plug-ins/Audnexus.bundle', + f'{BASE}/Scanners/Series', +] +DB_NAME = re.compile(r'^com\.plexapp\.plugins\.library(?:\.blobs)?\.db(?:-(?:[0-9]{4}-[0-9]{2}-[0-9]{2}|wal|shm))?$') +ETB = re.compile(r'\(ETB([0-9]+)\):([0-9]+)') +EXTENT = re.compile(r'\(([0-9]+)(?:-([0-9]+))?\):([0-9]+)(?:-([0-9]+))?') +INODE = re.compile(r'\bInode:\s*([0-9]+)\s+Type:\s*([^\s]+)') +SIZE = re.compile(r'\bSize:\s*([0-9]+)') +OWNER = re.compile(r'\bUser:\s*([0-9]+)\s+Group:\s*([0-9]+)') +MODE = re.compile(r'\bMode:\s*([0-7]{3,4})') + +def human(n): + units = ('B', 'KiB', 'MiB', 'GiB', 'TiB') + v = float(n) + for unit in units: + if v < 1024 or unit == units[-1]: + return f'{v:.2f} {unit}' + v /= 1024 + +def die(message): + raise SystemExit(f'ERROR: {message}') + +def run(cmd): + return subprocess.run(cmd, capture_output=True, text=True, errors='replace') + +def dbg(command): + env = os.environ.copy() + env['DEBUGFS_PAGER'] = 'cat' + return subprocess.run( + ['debugfs', '-c', '-R', command, LV], + capture_output=True, text=True, errors='replace', env=env, + ) + +def quote(path): + return '"' + path.replace('\\', '\\\\').replace('"', '\\"') + '"' + +def warnings(result): + return '\n'.join( + line for line in result.stderr.splitlines() + if line.strip() and not re.match(r'^debugfs [0-9]', line.strip()) + ).strip() + +def config(profile): + prefix = f'plex-state-{profile}' + return { + 'profile': profile, + 'manifest': DEST_MOUNT / f'{prefix}-manifest.jsonl', + 'summary': DEST_MOUNT / f'{prefix}-scan-summary.txt', + 'recover_log': DEST_MOUNT / f'{prefix}-recovery.jsonl', + 'recover_summary': DEST_MOUNT / f'{prefix}-recovery-summary.txt', + 'tmp': DEST_MOUNT / '.plex-state-tmp', + } + +def preflight(cfg, need_manifest=False): + if os.geteuid() != 0: + die('Run as root; the source block device requires read access.') + for binary in ('debugfs', 'blockdev', 'findmnt'): + if shutil.which(binary) is None: + die(f'Required program not installed: {binary}') + if not Path(LV).exists(): + die(f'LV missing: {LV}') + ro = run(['blockdev', '--getro', LV]) + if ro.returncode != 0 or ro.stdout.strip() != '1': + die(f'{LV} is not kernel read-only; refusing to proceed') + m = run(['findmnt', '-n', '-o', 'TARGET,SOURCE,FSTYPE', '-T', str(DEST_MOUNT)]) + if m.returncode != 0 or not m.stdout.strip(): + die(f'No mounted recovery filesystem detected at {DEST_MOUNT}') + parts = m.stdout.split() + if len(parts) < 3 or parts[0] != str(DEST_MOUNT): + die(f'{DEST_MOUNT} must itself be the mounted recovery filesystem: {m.stdout.strip()}') + if LV in m.stdout or 'plex--new' in m.stdout: + die('Recovery destination appears to be the damaged LV') + if not os.access(DEST_MOUNT, os.W_OK): + die(f'{DEST_MOUNT} is not writable') + if need_manifest and not cfg['manifest'].exists(): + die(f'Missing {cfg["manifest"]}. Scan {cfg["profile"]} first.') + print(f'Source: {LV} (read-only)') + print(f'Destination: {m.stdout.strip()}') + print(f'Dead physical filesystem blocks: {DEAD_FIRST}-{DEAD_LAST}') + if parts[2].lower() == 'exfat': + print('NOTICE: destination is exFAT. Tokens and database files are plaintext here;') + print(' Linux ownership/permissions cannot be preserved on this filesystem.') + print() + +def get_stat(inode_or_path): + spec = f'<{inode_or_path}>' if isinstance(inode_or_path, int) else quote(inode_or_path) + r = dbg(f'stat {spec}') + match = INODE.search(r.stdout) + if not match: + return None, (warnings(r) or (r.stdout + '\n' + r.stderr).strip()) + owner = OWNER.search(r.stdout) + mode = MODE.search(r.stdout) + size = SIZE.search(r.stdout) + info = { + 'inode': int(match.group(1)), + 'type': match.group(2), + 'size': int(size.group(1)) if size else None, + 'uid': int(owner.group(1)) if owner else None, + 'gid': int(owner.group(2)) if owner else None, + 'mode': mode.group(1) if mode else None, + 'text': r.stdout, + } + return info, warnings(r) + +def get_dir_entries(path): + info, issue = get_stat(path) + if not info or info['type'] != 'directory': + return None, f'Cannot open directory: {path} | {issue or info}' + r = dbg(f'ls -p <{info["inode"]}>') + issue = warnings(r) + # Do not silently accept a partial directory listing if debugfs reported errors. + if issue: + return None, f'Error listing {path} (inode {info["inode"]}): {issue}' + entries = [] + for line in r.stdout.splitlines(): + if not line.startswith('/'): + continue + bits = line.split('/') + if len(bits) < 7: + return None, f'Unparseable directory entry in {path}: {line!r}' + try: + inode, mode = int(bits[1]), int(bits[2], 8) + except ValueError: + return None, f'Unparseable inode/mode in {path}: {line!r}' + name = bits[5] + if not name or name in ('.', '..') or inode == 0: + continue + entries.append((name, inode, mode)) + return entries, None + +def inspect_file(path, inode): + info, issue = get_stat(inode) + rec = {'path': path, 'inode': inode} + if not info or info['type'] != 'regular' or info['size'] is None: + return {**rec, 'status': 'UNKNOWN', 'reason': f'Inode unavailable: {issue}'} + rec.update({key: info[key] for key in ('size', 'uid', 'gid', 'mode')}) + if issue: + return {**rec, 'status': 'UNKNOWN', 'reason': f'debugfs reported an issue: {issue}'} + if info['size'] == 0: + return {**rec, 'status': 'SAFE', 'reason': 'empty file', 'extents': []} + if 'EXTENTS:' not in info['text']: + return {**rec, 'status': 'UNKNOWN', 'reason': 'No EXTENTS section (legacy block maps not supported)'} + + data = info['text'].split('EXTENTS:', 1)[1] + etbs = [{'level': int(l), 'block': int(b)} for l, b in ETB.findall(data)] + extents = [] + for m in EXTENT.finditer(data): + ll = int(m.group(1)); lh = int(m.group(2) or m.group(1)) + pl = int(m.group(3)); ph = int(m.group(4) or m.group(3)) + extents.append({'logical_start': ll, 'logical_end': lh, 'physical_start': pl, 'physical_end': ph}) + rec.update({'extents': extents, 'etbs': etbs}) + + dead_etbs = [x for x in etbs if DEAD_FIRST <= x['block'] <= DEAD_LAST] + if dead_etbs: + return {**rec, 'status': 'ETB_LOST', 'reason': 'Required extent-tree metadata is on dead PV'} + if not extents: + return {**rec, 'status': 'ETB_SURVIVES' if etbs else 'UNKNOWN', + 'reason': 'No usable data extents resolved'} + + if any(x['physical_start'] <= DEAD_LAST and x['physical_end'] >= DEAD_FIRST for x in extents): + return {**rec, 'status': 'LOST', 'reason': 'Data extents overlap dead PV'} + + # Strict check: a file can look SAFE if debugfs returned only the surviving + # *subset* of extents. Demand complete, non-overlapping, physically plausible + # block coverage with no holes; intentionally reject sparse/uninitialized files. + needed = math.ceil(info['size'] / BLOCK_SIZE) + cursor = 0 + for x in sorted(extents, key=lambda row: row['logical_start']): + if x['logical_start'] != cursor: + return {**rec, 'status': 'UNKNOWN', 'reason': + f'Incomplete/overlapping logical extent coverage at file block {cursor}'} + logical_count = x['logical_end'] - x['logical_start'] + 1 + physical_count = x['physical_end'] - x['physical_start'] + 1 + if logical_count != physical_count or logical_count <= 0 or x['physical_start'] == 0: + return {**rec, 'status': 'UNKNOWN', 'reason': 'Extent length mismatch / invalid physical extent'} + cursor = x['logical_end'] + 1 + if cursor != needed: + return {**rec, 'status': 'UNKNOWN', 'reason': + f'Extent coverage ends at {cursor} blocks; expected {needed}'} + + # Reject unexpected extent syntax (e.g. uninitialized extent annotations) + # rather than treating any recognized extents as a complete mapping. + unparsed = ETB.sub('', data) + unparsed = EXTENT.sub('', unparsed) + unparsed = re.sub(r'[\s,]+', '', unparsed) + if unparsed: + return {**rec, 'status': 'UNKNOWN', 'reason': f'Unexpected EXTENTS text: {unparsed[:100]}'} + return {**rec, 'status': 'SAFE', 'reason': 'Complete extent coverage outside dead PV'} + +def scan(profile): + cfg = config(profile) + preflight(cfg) + selected = {} + issues = [] + skipped = 0 + visited_dirs = set() + + def select_file(path, inode=None): + nonlocal skipped + if path in selected: + return + if inode is None: + info, issue = get_stat(path) + if not info or info['type'] != 'regular': + issues.append(f'MISSING/UNREADABLE FILE {path}: {issue or info}') + return + inode = info['inode'] + selected[path] = inspect_file(path, inode) + rec = selected[path] + if rec['status'] != 'SAFE': + print(f'{rec["status"]:13} {path}: {rec.get("reason", "")}') + + def select_directory(directory, filter_func=lambda _name, _mode: True, recursive=False): + nonlocal skipped + stack = [directory] + while stack: + current = stack.pop() + if current in visited_dirs: + continue + visited_dirs.add(current) + entries, error = get_dir_entries(current) + if error: + issues.append(error) + print('WARNING:', error) + continue + for name, inode, mode in entries: + path = current + '/' + name + if stat.S_ISDIR(mode): + if recursive and filter_func(name, mode): + stack.append(path) + elif stat.S_ISREG(mode): + if filter_func(name, mode): + select_file(path, inode) + else: + skipped += 1 + else: + issues.append(f'IGNORED nonregular entry: {path} mode={oct(mode)}') + + if profile == 'core': + for filename in ROOT_FILES: + select_file(f'{BASE}/{filename}') + select_directory(DATABASE_DIR, lambda name, _mode: bool(DB_NAME.fullmatch(name))) + select_directory(PREFERENCES_DIR, lambda name, _mode: name.endswith('.xml')) + elif profile == 'extras': + for directory in EXTRA_TREES: + select_directory(directory, recursive=True) + else: + die(f'Unknown profile {profile}') + + counts = {} + sizes = {} + for rec in selected.values(): + status = rec['status'] + counts[status] = counts.get(status, 0) + 1 + sizes[status] = sizes.get(status, 0) + (rec.get('size') or 0) + + meta = { + 'version': VERSION, 'profile': profile, 'base': BASE, + 'lv': LV, 'dead_first': DEAD_FIRST, 'dead_last': DEAD_LAST, + 'block_size': BLOCK_SIZE, 'selected_count': len(selected), + 'created_at': dt.datetime.now(dt.timezone.utc).isoformat(), + 'issues': issues, + } + new_path = Path(str(cfg['manifest']) + '.new') + with new_path.open('w', encoding='utf-8') as f: + f.write(json.dumps({'_meta': meta}, ensure_ascii=False) + '\n') + for rec in sorted(selected.values(), key=lambda item: item['path']): + f.write(json.dumps(rec, ensure_ascii=False) + '\n') + new_path.replace(cfg['manifest']) + + lines = [ + f'PLEX SELECTIVE SCAN: {profile}', + '=' * 35, + f'Source: {BASE}', + f'Selected regular files: {len(selected)}', + f'Skipped by allowlist: {skipped}', + f'Scanned directories: {len(visited_dirs)}', + ] + for status in ('SAFE', 'LOST', 'ETB_LOST', 'ETB_SURVIVES', 'UNKNOWN'): + lines.append(f'{status:14} {counts.get(status, 0):>4} files {human(sizes.get(status, 0)):>12}') + lines += [ + '', + f'Available recovery space NOW: {human(shutil.disk_usage(DEST_MOUNT).free)}', + f'Issues/warnings: {len(issues)}', + ] + lines += [' ' + issue for issue in issues] + + if profile == 'core': + lines += ['', 'DATABASE CANDIDATES (extent classification; NOT SQLite integrity):'] + for rec in sorted(selected.values(), key=lambda r: r['path']): + if rec['path'].startswith(DATABASE_DIR + '/'): + lines.append(f' {rec["status"]:12} {Path(rec["path"]).name}') + live = [rec for rec in selected.values() if + rec['path'].startswith(DATABASE_DIR + '/') and + Path(rec['path']).name in ('com.plexapp.plugins.library.db', 'com.plexapp.plugins.library.blobs.db')] + if len(live) < 2: + lines.append(' WARNING: One or both live database files absent/unreadable') + + lines += ['', f'Manifest: {cfg["manifest"]}', + 'CAUTION: SAFE is based on block mapping; validate recovered databases separately.'] + report = '\n'.join(lines) + '\n' + cfg['summary'].write_text(report, encoding='utf-8') + print('\n' + report) + if issues: + print('WARNING: scan had missing/inaccessible entries. Review warning lines; do not assume completeness.') + +def recover(profile): + cfg = config(profile) + preflight(cfg, need_manifest=True) + with cfg['manifest'].open(encoding='utf-8') as f: + parsed = [json.loads(line) for line in f if line.strip()] + if not parsed or '_meta' not in parsed[0]: + die('No manifest metadata header') + meta = parsed[0]['_meta'] + if any((meta.get(k) != expected for k, expected in ( + ('profile', profile), ('lv', LV), ('dead_first', DEAD_FIRST), + ('dead_last', DEAD_LAST), ('block_size', BLOCK_SIZE), + ('version', VERSION), + ))): + die('Manifest geometry/profile/version does not match this script') + entries = [r for r in parsed[1:] if r.get('status') == 'SAFE'] + print(f'{len(entries)} SAFE files selected, {human(sum(r["size"] for r in entries))} total') + if meta.get('issues'): + print('WARNING: scan had missing/unreadable paths; review scan summary.') + cfg['tmp'].mkdir(exist_ok=True) + stats = {'RECOVERED': 0, 'ALREADY_PRESENT_UNVERIFIED': 0, 'ERROR': 0, 'NO_SPACE': 0} + bytes_new = 0 + mode = 'a' if cfg['recover_log'].exists() else 'w' + with cfg['recover_log'].open(mode, encoding='utf-8') as logfile: + for idx, rec in enumerate(entries, 1): + src = rec['path'] + size = rec['size'] + destination = DEST_MOUNT / src.lstrip('/') + label = None; detail = '' + digest = None + # Recheck the live inode's mapping and byte size before copying. + check = inspect_file(src, rec['inode']) + if check['status'] != 'SAFE' or check.get('size') != size or check.get('extents') != rec.get('extents'): + label = 'ERROR'; detail = 'Source inode extent map changed or fails strict safety recheck' + elif destination.exists(): + if destination.is_file() and destination.stat().st_size == size: + label = 'ALREADY_PRESENT_UNVERIFIED'; detail = 'Already present with matching size; not rewritten' + else: + label = 'ERROR'; detail = 'Destination exists with different size or is not a file; refusing overwrite' + elif shutil.disk_usage(DEST_MOUNT).free < size + RESERVE: + label = 'NO_SPACE'; detail = 'Need file size plus 2 GiB free-space reserve' + else: + tmp = cfg['tmp'] / f'{profile}-{rec["inode"]}.part' + if tmp.exists(): + tmp.unlink() + result = dbg(f'dump <{rec["inode"]}> {tmp}') + issue = warnings(result) + if issue: + label = 'ERROR'; detail = f'debugfs reported: {issue[:400]}' + elif not tmp.exists() or tmp.stat().st_size != size: + label = 'ERROR'; detail = 'Missing output file or byte-size mismatch' + else: + digest_obj = hashlib.sha256() + with tmp.open('rb') as source: + for chunk in iter(lambda: source.read(4 * 1024 * 1024), b''): + digest_obj.update(chunk) + digest = digest_obj.hexdigest() + destination.parent.mkdir(parents=True, exist_ok=True) + try: + tmp.replace(destination) + label = 'RECOVERED'; bytes_new += size + except OSError as err: + label = 'ERROR'; detail = f'Rename failed: {err}' + if label == 'ERROR' and tmp.exists(): + try: + tmp.unlink() + except OSError: + pass + if label in ('ERROR', 'NO_SPACE'): + print(f'{label:10} {src}: {detail}') + else: + print(f'{label:26} [{idx}/{len(entries)}] {src}') + stats[label] += 1 + logfile.write(json.dumps({ + 'status': label, 'path': src, 'size': size, 'inode': rec['inode'], + 'sha256': digest, 'detail': detail, + }, ensure_ascii=False) + '\n') + logfile.flush() + if label == 'NO_SPACE': + break + + lines = [ + f'PLEX SELECTIVE RECOVERY: {profile}', '=' * 35, + f'Newly recovered: {stats["RECOVERED"]}, {human(bytes_new)}', + f'Already present (size-only): {stats["ALREADY_PRESENT_UNVERIFIED"]}', + f'Errors: {stats["ERROR"]}', f'Space stops: {stats["NO_SPACE"]}', + f'Remaining free: {human(shutil.disk_usage(DEST_MOUNT).free)}', + f'Manifest: {cfg["manifest"]}', + f'Hash log: {cfg["recover_log"]}', + 'Hashes verify extracted copies, NOT correctness of Plex databases.', + ] + report = '\n'.join(lines) + '\n' + cfg['recover_summary'].write_text(report, encoding='utf-8') + print('\n' + report) + return 1 if stats['ERROR'] or stats['NO_SPACE'] else 0 + +def main(): + os.umask(0o077) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('action', choices=('scan', 'recover')) + parser.add_argument('profile', choices=('core', 'extras'), nargs='?', default='core') + args = parser.parse_args() + if args.action == 'scan': + scan(args.profile) + else: + sys.exit(recover(args.profile)) + +if __name__ == '__main__': + main() diff --git a/restore-tree.py b/restore-tree.py new file mode 100755 index 0000000..646a660 --- /dev/null +++ b/restore-tree.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Dry-run or rebuild empty folder tree on an explicitly mounted new filesystem.""" + +import argparse +import json +import os +import stat +import subprocess +import sys +from pathlib import Path + + +def fail(msg): + sys.exit(f"ERROR: {msg}") + + +def normalize_source_path(raw, strip_prefix=None): + if not isinstance(raw, str) or not raw.startswith("/") or "\x00" in raw: + fail(f"Invalid source directory path: {raw!r}") + # Never silently normalize parent traversals or hidden whitespace. + if any(part in (".", "..") for part in raw.split("/")): + fail(f"Refusing dangerous path: {raw!r}") + path = "/" + "/".join(part for part in raw.split("/") if part) + if strip_prefix: + if path == strip_prefix: + return "/" + if path.startswith(strip_prefix + "/"): + path = path[len(strip_prefix):] + else: + fail(f"Path does not start with --strip-prefix {strip_prefix}: {path}") + return path + + +def load_rows(input_path, strip_prefix): + rows = {} + is_jsonl = input_path.suffix.lower() == ".jsonl" + with input_path.open(encoding="utf-8-sig") as f: + for no, raw in enumerate(f, 1): + line = raw.rstrip("\r\n") + if not line or (not is_jsonl and line.startswith("#")): + continue + if is_jsonl: + try: + record = json.loads(line) + except json.JSONDecodeError as e: + fail(f"Invalid JSONL at line {no}: {e}") + if "_meta" in record: + continue + path = record.get("path") + else: + # Accept output from capture-tree.py, find -type d, and tree -dfi + # --noreport. Do not accept the diagram/box-drawing variant. + if line.startswith(("│", "├", "└", "|", "`")): + fail(f"Line {no} looks like an indented tree diagram, not full paths") + record = {"path": line} + path = line + path = normalize_source_path(path, strip_prefix) + if path == "/": + continue # Never chmod/chown the target mount's root. + if path in rows: + old = rows[path] + if old.get("mode") is None and record.get("mode") is not None: + rows[path] = record + else: + rows[path] = record + return rows + + +def verify_mount(target, expected_source): + if not target.is_absolute() or str(target) == "/" or target.is_symlink(): + fail("--target must be an absolute, non-symlink mountpoint other than /.") + if not target.is_dir() or not os.path.ismount(target): + fail(f"{target} is NOT itself a mounted filesystem. Nothing was created.") + if os.statvfs(target).f_flag & os.ST_RDONLY: + fail(f"{target} is mounted read-only.") + r = subprocess.run(["findmnt", "-n", "-o", "SOURCE,FSTYPE", "--mountpoint", str(target)], + capture_output=True, text=True) + if r.returncode != 0 or not r.stdout.strip(): + fail(f"Could not confirm mount source for {target}") + mount_source, fs_type = r.stdout.strip().split(None, 1) + print(f"Verified mount: {target} <- {mount_source} ({fs_type})") + if expected_source and mount_source != expected_source: + fail(f"Mount source is {mount_source}, expected {expected_source}.") + + +def guarded_directory(target, rel, do_apply): + """Create each component, refusing existing symlinks and non-directories.""" + here = target + created = 0 + for component in rel.parts: + if component in ("", "."): + continue + if component == "..": + fail("Unexpected .. while creating directories") + here = here / component + try: + st = here.lstat() + except FileNotFoundError: + if do_apply: + here.mkdir(mode=0o755) + created += 1 + continue + if stat.S_ISLNK(st.st_mode): + fail(f"Refusing to follow symlink: {here}") + if not stat.S_ISDIR(st.st_mode): + fail(f"Path exists but is not a directory: {here}") + return created + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("tree", type=Path, + help="capture-tree.py JSONL or one-full-path-per-line .txt") + parser.add_argument("--target", type=Path, default=Path("/storage"), + help="Mountpoint on newly rebuilt array (default: /storage)") + parser.add_argument("--strip-prefix", default=None, + help="Strip the OLD mountpoint from text entries, e.g. /storage") + parser.add_argument("--apply", action="store_true", + help="Actually create the folders. Default: dry run, no changes") + parser.add_argument("--restore-metadata", action="store_true", + help="With --apply, use captured numeric UID/GID and mode where known") + parser.add_argument("--expect-source", default=None, + help="With --apply, require findmnt to report this exact mount SOURCE") + args = parser.parse_args() + + if args.restore_metadata and not args.apply: + fail("--restore-metadata requires --apply") + if not args.tree.is_file(): + fail(f"Input file not found: {args.tree}") + if args.strip_prefix: + args.strip_prefix = normalize_source_path(args.strip_prefix) + if args.strip_prefix == "/": + fail("--strip-prefix / is redundant; omit the option") + target = args.target.absolute() + if target.is_symlink() or str(target) == "/": + fail("Refusing a symlink target or filesystem root /.") + if args.apply: + if os.geteuid() != 0: + fail("Run as root when applying directory creation.") + verify_mount(target, args.expect_source) + + rows = load_rows(args.tree, args.strip_prefix) + if not rows: + fail("No directory paths found in input. Nothing to do.") + + # Parents of input paths are created automatically; source folder names are + # treated as filesystem-relative, NOT as host paths. + ordered = sorted(rows, key=lambda path: (path.count("/"), path)) + counts = {"input": len(ordered), "created": 0, "existing": 0, "metadata": 0} + print("MODE:", "APPLY" if args.apply else "DRY RUN") + print("Target:", target) + print("Input directories:", len(ordered)) + print("Top-level directories:", ", ".join(sorted({p.split("/")[1] for p in ordered}))) + print("Sample paths:") + for path in ordered[:12]: + print(f" {target / path.lstrip('/')}") + if len(ordered) > 12: + print(f" ... and {len(ordered)-12:,} more") + if not args.apply: + print("\nDRY RUN ONLY: no directories created. Review your tree, then add --apply.") + return + + # Apply phase: create top-down, with explicit symlink checks. + failures = [] + for path in ordered: + rel = Path(path.lstrip("/")) + dest = target / rel + existed = dest.is_dir() and not dest.is_symlink() + try: + guarded_directory(target, rel, do_apply=True) + except (OSError, SystemExit) as e: + failures.append(f"MKDIR {path}: {e}") + print(f"FAILED {path}: {e}", file=sys.stderr) + continue + if existed: + counts["existing"] += 1 + else: + counts["created"] += 1 + + if args.restore_metadata: + # Work bottom-up so restrictive permissions can't block child creation. + for path in reversed(ordered): + info = rows[path] + dest = target / path.lstrip("/") + if not dest.is_dir() or dest.is_symlink(): + continue + uid, gid, mode = (info.get("uid"), info.get("gid"), info.get("mode")) + if uid is None or gid is None or mode is None: + continue + try: + os.chown(dest, int(uid), int(gid), follow_symlinks=False) + os.chmod(dest, int(mode)) # symlinks explicitly rejected above + counts["metadata"] += 1 + except OSError as e: + failures.append(f"METADATA {path}: {e}") + print(f"METADATA FAILED {path}: {e}", file=sys.stderr) + + print(f"\nInput: {counts['input']:,}; newly created: {counts['created']:,}; " + f"already present: {counts['existing']:,}; metadata restored: {counts['metadata']:,}; " + f"failures: {len(failures):,}") + if failures: + sys.exit(1) + + +if __name__ == "__main__": + main()