diff --git a/assets/files/gitea/forge-alert.sh b/assets/files/gitea/forge-alert.sh new file mode 100644 index 0000000..fcc407d --- /dev/null +++ b/assets/files/gitea/forge-alert.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +level="${1:-INFO}" # e.g., INFO, WARN, ERROR +event="${2:-general}" # e.g., backup, restore-test +message="${3:-}" + +# Telegram config via environment +BOT_TOKEN="${TELEGRAM_BOT_TOKEN:-}" +CHAT_ID="${TELEGRAM_CHAT_ID:-}" + +# Log to syslog +logger -t soloforge "[$level][$event] $message" + +# Send to Telegram if configured +if [[ -n "$BOT_TOKEN" && -n "$CHAT_ID" ]]; then + # Keep the message simple to avoid escaping headaches + safe_msg=$(echo "$message" | tr '"' "'" ) + + curl -sS -X POST \ + "https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" \ + -d "chat_id=${CHAT_ID}" \ + --data-urlencode "text=[SoloForge][$level][$event] $safe_msg" \ + >/dev/null 2>&1 || true +fi diff --git a/assets/files/gitea/forge-b2-backup.sh b/assets/files/gitea/forge-b2-backup.sh index 11d69b2..cec868b 100644 --- a/assets/files/gitea/forge-b2-backup.sh +++ b/assets/files/gitea/forge-b2-backup.sh @@ -23,8 +23,8 @@ mv "$GITEA_DIR/$dump_file" "$BACKUP_DIR/$dump_file" echo "[backup] Dump created at $BACKUP_DIR/$dump_file" # Optional: Upload to Backblaze B2 via rclone -# Make sure you configured a remote named 'b2' -rclone copy "$BACKUP_DIR/$dump_file" b2:soloforge-backups +# Make sure you configured a remote named 'B2' +rclone copy "$BACKUP_DIR/$dump_file" B2:soloforge-backups echo "[backup] Uploaded $dump_file to Backblaze B2" echo "[backup] All done." diff --git a/assets/files/gitea/forge-restore-test.sh b/assets/files/gitea/forge-restore-test.sh new file mode 100644 index 0000000..fbb5cd5 --- /dev/null +++ b/assets/files/gitea/forge-restore-test.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail + +BACKUP_DIR="/gitea/backups" +TMP_BASE="/tmp/soloforge-restore-test" +CONTAINER_IMAGE="gitea/gitea:latest" # for future deeper tests if desired + +mkdir -p "$TMP_BASE" + +latest="$(ls -1t "$BACKUP_DIR"/gitea-dump-*.zip 2>/dev/null | head -n1 || true)" + +if [[ -z "$latest" ]]; then + echo "[restore-test] No backup files found in $BACKUP_DIR" + forge-alert.sh "ERROR" "restore-test" "No backup files found in $BACKUP_DIR" + + exit 1 +fi + +echo "[restore-test] Using latest backup: $latest" + +# Basic ZIP integrity test +echo "[restore-test] Running unzip -t ..." +if ! unzip -t "$latest" >/dev/null 2>&1; then + echo "[restore-test] unzip -t failed for $latest" + forge-alert.sh "ERROR" "restore-test" "Backup failed unzip test: $(basename "$latest")" + + exit 1 +fi + +# Optional: extraction smoke test +tmpdir="$(mktemp -d "$TMP_BASE/restore-test-XXXXXX")" +echo "[restore-test] Extracting into $tmpdir ..." +if ! unzip -qq "$latest" -d "$tmpdir"; then + echo "[restore-test] unzip extract failed for $latest" + forge-alert.sh "ERROR" "restore-test" "Backup failed extraction: $(basename "$latest")" + rm -rf "$tmpdir" + + exit 1 +fi + +# Clean up +rm -rf "$tmpdir" + +echo "[restore-test] Backup passed unzip + extraction checks." +forge-alert.sh "INFO" "restore-test" "Backup $(basename "$latest") passed restore tests." diff --git a/assets/files/gitea/forge b/assets/files/gitea/forge.sh similarity index 89% rename from assets/files/gitea/forge rename to assets/files/gitea/forge.sh index 94d3105..ee3f49b 100644 --- a/assets/files/gitea/forge +++ b/assets/files/gitea/forge.sh @@ -48,6 +48,7 @@ Usage: forge gitea-logs Tail logs from Gitea container forge runner-logs Tail logs from Actions runner container forge backup Run a Gitea dump and move it into BACKUP_DIR + forge restore-test Run backup restore sanity checks (unzip + extract) forge restart-gitea Restart Gitea stack forge restart-runner Restart Actions runner stack forge runner-reset Re-register runner with current labels (destroys .runner) @@ -95,26 +96,26 @@ cmd_backup() { dest="$BACKUP_DIR/gitea-dump-$ts.zip" echo "Running Gitea dump inside container: $GITEA_CONTAINER_NAME" - # Gitea dump will write gitea-dump-*.zip under /data which is mounted to $GITEA_DIR/gitea - docker exec -u 1000 "$GITEA_CONTAINER_NAME" gitea dump -c /data/gitea/conf/app.ini --file /data/gitea-dump-$ts.zip + if ! docker exec -u 1000 "$GITEA_CONTAINER_NAME" sh -lc \ + "gitea dump -c /data/gitea/conf/app.ini --file /data/gitea-dump-$ts.zip"; then + echo "Error: gitea dump failed" >&2 + forge-alert.sh "ERROR" "backup" "gitea dump failed" - echo "Locating latest dump..." - local dump - dump=$(ls -1t "$GITEA_DIR"/gitea/gitea-dump-*.zip 2>/dev/null | head -n1 || true) - - if [[ -z "$dump" ]]; then - echo "Error: no gitea-dump-*.zip found under $GITEA_DIR/gitea" >&2 exit 1 fi - local base - base=$(basename "$dump") - local dest="$BACKUP_DIR/$base" + if [[ ! -f "$dump" ]]; then + echo "Error: expected dump file not found: $dump" >&2 + forge-alert.sh "ERROR" "backup" "dump file missing: $dump" + + exit 1 + fi echo "Moving $dump -> $dest" mv "$dump" "$dest" echo "Backup complete: $dest" + forge-alert.sh "INFO" "backup" "Backup complete: $(basename "$dest")" } cmd_restart_gitea() { @@ -205,6 +206,9 @@ case "$cmd" in backup) cmd_backup ;; + restore-test) + /usr/local/bin/forge-restore-test.sh + ;; restart-gitea) cmd_restart_gitea ;; diff --git a/content/soloforge-docs.md b/content/soloforge-docs.md index dfe59a1..6bce60f 100644 --- a/content/soloforge-docs.md +++ b/content/soloforge-docs.md @@ -215,9 +215,11 @@ If the runner needs re-registration, follow section 4.3. --- -## 10. `forge` CLI Tool +## 10. CLI Toolkit -A custom CLI tool [`forge`](/assets/files/gitea/forge) exists to help manage common tasks: +### [`forge`](/assets/files/gitea/forge.sh) Tool + +A custom CLI tool to help manage common tasks: | Command | Description | | --------------------------- | ---------------------------------------------------------- | @@ -226,11 +228,24 @@ A custom CLI tool [`forge`](/assets/files/gitea/forge) exists to help manage com | `forge gitea-logs` | Tail logs from Gitea container | | `forge runner-logs` | Tail logs from Actions runner container | | `forge backup` | Run a Gitea dump and move it into BACKUP_DIR | +| `forge restore-test` | Run backup restore sanity checks (unzip + extract) | | `forge restart-gitea` | Restart Gitea stack | | `forge restart-runner` | Restart Actions runner stack | | `forge runner-reset` | Re-register runner with current labels (destroys .runner) | | `forge diag` | Quick diagnostic summary | +### [`forge-alert`](/assets/files/gitea/forge-alert.sh) Tool + +A companion script that serves as a basic reusable alert sender, capable of logging to syslog and sending notifications via Telegram. Telegram bot token and chat ID must be set in environment variables by editing `/etc/environment` in a root/`sudo` shell. + +### [`forge-b2-backup`](/assets/files/gitea/forge-b2-backup.sh) Tool + +A backup script that uploads Gitea dumps to Backblaze B2 cloud storage. Utilizes `rclone`, so make sure to configure an appropriate remote (`B2` by default)before use. Alerts via `forge-alert.sh` for backup status. + +### [`forge-restore-test`](/assets/files/gitea/forge-restore-test.sh) Tool + +A restore test script that performs basic integrity checks on the latest Gitea dump file, including unzip testing and extraction smoke test. Alerts via `forge-alert.sh` if any issues are detected. + --- ## TL;DR Cheat Sheet