add production readiness and recovery assets

scripts/backup_sqlite.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+set -eu
+umask 077
+DB_PATH=${1:-${PROSPECT_API_DB:-${DATA_DIR:-/data}/prospects.db}}
+BACKUP_DIR=${2:-${BACKUP_DIR:-/var/backups/prospect-platform}}
+RETENTION=${3:-${BACKUP_RETENTION:-30}}
+case "$RETENTION" in ''|*[!0-9]*) echo 'retention must be a non-negative integer' >&2; exit 2;; esac
+case "$BACKUP_DIR" in /*) ;; *) echo 'backup directory must be an absolute path' >&2; exit 2;; esac
+case "$DB_PATH" in *.env|*secret*|*credentials*) echo 'refusing secret-like database path' >&2; exit 2;; esac
+mkdir -p -- "$BACKUP_DIR"
+python3 - "$DB_PATH" "$BACKUP_DIR" "$RETENTION" <<'PY'
+import hashlib, os, sqlite3, sys, tempfile, time
+from pathlib import Path
+src, out_dir, retention = Path(sys.argv[1]).expanduser(), Path(sys.argv[2]).expanduser(), int(sys.argv[3])
+if not src.is_file() or not src.is_absolute(): raise SystemExit('database must be an existing absolute regular file')
+out_dir = out_dir.resolve()
+if src.resolve() == out_dir: raise SystemExit('backup directory must differ from database')
+out_dir.mkdir(parents=True, exist_ok=True)
+name = f"{os.environ.get('BACKUP_PREFIX', 'prospects')}-{time.strftime('%Y%m%dT%H%M%SZ', time.gmtime())}-{os.getpid()}"
+final = out_dir / (name + '.db')
+fd, temp_name = tempfile.mkstemp(prefix='.backup-', suffix='.tmp', dir=out_dir)
+os.close(fd)
+try:
+    with sqlite3.connect(f'file:{src}?mode=ro', uri=True) as source, sqlite3.connect(temp_name) as target:
+        source.backup(target)
+        target.execute('PRAGMA integrity_check')
+        target.commit()
+    with open(temp_name, 'rb') as handle:
+        os.fsync(handle.fileno())
+    os.chmod(temp_name, 0o600); os.replace(temp_name, final)
+    digest = hashlib.sha256(final.read_bytes()).hexdigest()
+    sidecar = Path(str(final) + '.sha256')
+    sidecar.write_text(f'{digest}  {final.name}\n', encoding='ascii'); os.chmod(sidecar, 0o600)
+    candidates = sorted([*out_dir.glob('prospects-*.db'), *out_dir.glob('pre-restore-*.db')], key=lambda p: p.stat().st_mtime, reverse=True)
+    for old in candidates[retention:]:
+        old.unlink(missing_ok=True); Path(str(old)+'.sha256').unlink(missing_ok=True)
+    print(final)
+finally:
+    Path(temp_name).unlink(missing_ok=True)
+PY

scripts/healthcheck.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+set -eu
+URL=${HEALTHCHECK_URL:-http://127.0.0.1:8000/api/v1/health/ready}
+TIMEOUT=${HEALTHCHECK_TIMEOUT:-5}
+python3 - "$URL" "$TIMEOUT" <<'PY'
+import json, sys, urllib.request
+url, timeout = sys.argv[1], float(sys.argv[2])
+try:
+    with urllib.request.urlopen(url, timeout=timeout) as response:
+        payload=json.loads(response.read())
+        if response.status != 200 or payload.get('status') != 'ok' or payload.get('ready') is not True:
+            raise RuntimeError('service is not ready')
+        if payload.get('outreach_enabled') is not False: raise RuntimeError('outreach safety check failed')
+except Exception as exc:
+    print(f'healthcheck failed: {type(exc).__name__}', file=sys.stderr)
+    raise SystemExit(1)
+print('ok')
+PY

scripts/restore_sqlite.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+set -eu
+umask 077
+if [ "${3:-}" != "--confirm-restore" ]; then
+  echo 'Refusing restore: pass --confirm-restore explicitly.' >&2
+  exit 2
+fi
+SOURCE=${1:-}
+TARGET=${2:-${PROSPECT_API_DB:-${DATA_DIR:-/data}/prospects.db}}
+BACKUP_DIR=${BACKUP_DIR:-$(dirname -- "$SOURCE")}
+case "$SOURCE$TARGET" in *secret*|*credentials*|*.env*) echo 'refusing secret-like path' >&2; exit 2;; esac
+[ -f "$SOURCE" ] || { echo 'restore source does not exist' >&2; exit 2; }
+case "$TARGET" in /*) ;; *) echo 'restore target must be an absolute path' >&2; exit 2;; esac
+mkdir -p -- "$BACKUP_DIR"
+if [ -f "$TARGET" ]; then
+  pre_restore=$("$(dirname -- "$0")/backup_sqlite.sh" "$TARGET" "$BACKUP_DIR" 30)
+  pre_restore_db=$(printf '%s\n' "$pre_restore" | tail -n 1)
+  pre_restore_name="$BACKUP_DIR/pre-restore-$(date -u +%Y%m%dT%H%M%SZ)-$$.db"
+  mv -- "$pre_restore_db" "$pre_restore_name"
+  if [ -f "$pre_restore_db.sha256" ]; then mv -- "$pre_restore_db.sha256" "$pre_restore_name.sha256"; fi
+fi
+python3 - "$SOURCE" "$TARGET" <<'PY'
+import hashlib, os, sqlite3, sys, tempfile
+from pathlib import Path
+source, target = Path(sys.argv[1]).resolve(), Path(sys.argv[2]).resolve()
+if not source.is_file(): raise SystemExit('restore source must be a regular file')
+sidecar = Path(str(source)+'.sha256')
+if sidecar.exists():
+    expected = sidecar.read_text(encoding='ascii').split()[0]
+    actual = hashlib.sha256(source.read_bytes()).hexdigest()
+    if expected != actual: raise SystemExit('restore checksum mismatch')
+target.parent.mkdir(parents=True, exist_ok=True)
+fd, tmp = tempfile.mkstemp(prefix='.restore-', suffix='.tmp', dir=target.parent); os.close(fd)
+try:
+    with sqlite3.connect(f'file:{source}?mode=ro', uri=True) as src, sqlite3.connect(tmp) as dst:
+        src.backup(dst); result = dst.execute('PRAGMA integrity_check').fetchone()[0]
+        if result != 'ok': raise SystemExit('restored database integrity check failed')
+        dst.commit()
+    os.chmod(tmp, 0o600); os.replace(tmp, target)
+    print(target)
+finally:
+    Path(tmp).unlink(missing_ok=True)
+PY

scripts/rollback.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+set -eu
+cat >&2 <<'EOF'
+Rollback is a documented, non-destructive procedure. No rollback command is executed.
+
+1. Identify the last known-good immutable image digest/tag and configuration revision.
+2. Confirm the database backup is recent and run scripts/healthcheck.sh against the candidate.
+3. Review the exact rendered config: docker compose config.
+4. Change the deployment's image tag/digest in the deployment system (or pin IMAGE_TAG), then restart the service through the approved change process.
+5. Verify /api/v1/health/ready, representative authenticated reads, logs, and outreach_enabled=false.
+6. Record the rollback reason, old/new image digests, config revision, backup/checksum, and operator.
+
+This helper intentionally does not stop containers, delete images, restore databases, or alter production state.
+EOF