mDMS/infra/paperless/push_system_prompt.py

"""
Render SYSTEM_PROMPT.txt with the live correspondent list and push it to
the paperless-ai container's /app/data/.env on mDock.

The repo SYSTEM_PROMPT.txt is the template (with the placeholder
{{CORRESPONDENTS_LIST}}). This script:

  1. Reads the current correspondents from the Paperless API.
  2. Filters out names that must never appear as correspondent
     (recipients of m's mail — see RECIPIENT_EXCLUDE).
  3. Renders the prompt by substituting the placeholder.
  4. Reads the live /app/data/.env from the paperless-ai container.
  5. Replaces the SYSTEM_PROMPT=`…` block.
  6. Backs up the old .env (.bak.<ts>) and writes the new one.
  7. Restarts the paperless-ai container.

Dry-run is the default: prints the would-be rendered prompt without
writing.

Usage:
    python3 push_system_prompt.py             # dry run
    python3 push_system_prompt.py --apply     # write + restart

Migrated into m/mDMS from m/otto on 2026-05-16 (mDMS#3).
"""
import argparse
import datetime
import json
import os
import subprocess
import sys


PAPERLESS_HOST = "mdock"
PAPERLESS_AI_CONTAINER = "paperless-ai"
PAPERLESS_WEB_CONTAINER = "paperless-webserver-1"
ENV_PATH = "/app/data/.env"
HERE = os.path.dirname(os.path.abspath(__file__))
TEMPLATE_PATH = os.path.join(HERE, "SYSTEM_PROMPT.txt")
PLACEHOLDER = "{{CORRESPONDENTS_LIST}}"

# Names that are m or his household — recipients, never correspondents.
# Substring match, case-insensitive. Keep the actual correspondent records
# in Paperless (data integrity for historical doc assignments), but never
# show them to the LLM as candidate senders.
RECIPIENT_EXCLUDE = ("matthias siebels", "mathias siebels")


def get_token() -> str:
    out = subprocess.run(
        ["ssh", PAPERLESS_HOST,
         f"docker exec {PAPERLESS_AI_CONTAINER} sh -c "
         f"'grep ^PAPERLESS_API_TOKEN {ENV_PATH} | cut -d= -f2'"],
        capture_output=True, text=True, timeout=15,
    )
    return out.stdout.strip()


def fetch_correspondents(token: str) -> list[str]:
    cmd = (
        f"docker exec {PAPERLESS_WEB_CONTAINER} "
        f"curl -s -H 'Authorization: Token {token}' "
        f"'http://localhost:8000/api/correspondents/?page_size=500'"
    )
    out = subprocess.run(
        ["ssh", PAPERLESS_HOST, cmd],
        capture_output=True, text=True, timeout=30,
    )
    if out.returncode != 0:
        raise RuntimeError(f"fetch failed: {out.stderr}")
    data = json.loads(out.stdout)
    names = [c["name"] for c in data["results"]]
    filtered = [n for n in names
                if not any(x in n.lower() for x in RECIPIENT_EXCLUDE)]
    dropped = sorted(set(names) - set(filtered))
    if dropped:
        print(f"filtered out recipient-names: {dropped}")
    return sorted(filtered, key=lambda s: s.lower())


def render_prompt(template: str, names: list[str]) -> str:
    listing = "\n".join(f"- {n}" for n in names)
    return template.replace(PLACEHOLDER, listing)


def read_remote_env() -> str:
    out = subprocess.run(
        ["ssh", PAPERLESS_HOST,
         f"docker exec {PAPERLESS_AI_CONTAINER} cat {ENV_PATH}"],
        capture_output=True, text=True, timeout=15,
    )
    if out.returncode != 0:
        raise RuntimeError(f"cat failed: {out.stderr}")
    return out.stdout


def replace_system_prompt(env: str, new_prompt: str) -> str:
    """Replace the SYSTEM_PROMPT=`…` block with the new one.

    Paperless-AI's .env uses backtick-delimited values for multi-line
    settings (JS .env loader convention; bash would not accept this).
    """
    lines = env.splitlines(keepends=True)
    out = []
    inside = False
    replaced = False
    for line in lines:
        if not inside and line.startswith("SYSTEM_PROMPT="):
            out.append(f"SYSTEM_PROMPT=`{new_prompt.rstrip()}`\n")
            replaced = True
            stripped_value = line[len("SYSTEM_PROMPT="):].rstrip("\n")
            if stripped_value.startswith("`") and stripped_value.count("`") >= 2:
                continue
            inside = True
            continue
        if inside:
            if "`" in line:
                inside = False
            continue
        out.append(line)
    if not replaced:
        raise SystemExit("SYSTEM_PROMPT= line not found in .env")
    return "".join(out)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--apply", action="store_true",
                    help="Write new .env and restart paperless-ai")
    args = ap.parse_args()

    with open(TEMPLATE_PATH) as f:
        template = f.read()
    if PLACEHOLDER not in template:
        sys.exit(f"template missing placeholder {PLACEHOLDER}")

    token = get_token()
    names = fetch_correspondents(token)
    print(f"fetched {len(names)} live correspondents (after recipient filter)")
    rendered = render_prompt(template, names)
    print(f"rendered prompt: {len(rendered)} chars, {len(rendered.splitlines())} lines")

    env_before = read_remote_env()
    env_after = replace_system_prompt(env_before, rendered)
    if env_before == env_after:
        print("no change — live prompt already matches rendered template")
        return

    if not args.apply:
        print("--- new SYSTEM_PROMPT block ---")
        for line in env_after.splitlines():
            if line.startswith("SYSTEM_PROMPT="):
                print(line[:200] + ("…" if len(line) > 200 else ""))
        print()
        print("DRY RUN — re-run with --apply to write + restart paperless-ai")
        return

    ts = datetime.datetime.utcnow().strftime("%Y%m%dT%H%M%S")
    backup = f"{ENV_PATH}.bak.{ts}"
    subprocess.run(
        ["ssh", PAPERLESS_HOST,
         f"docker exec {PAPERLESS_AI_CONTAINER} cp {ENV_PATH} {backup}"],
        check=True, timeout=15,
    )
    print(f"backup: {backup}")

    write_cmd = (
        f"docker exec -i {PAPERLESS_AI_CONTAINER} "
        f"sh -c 'cat > {ENV_PATH}'"
    )
    proc = subprocess.run(
        ["ssh", PAPERLESS_HOST, write_cmd],
        input=env_after, capture_output=True, text=True, timeout=30,
    )
    if proc.returncode != 0:
        sys.exit(f"write failed: {proc.stderr}")
    print(f"wrote {len(env_after)} bytes to {ENV_PATH}")

    subprocess.run(
        ["ssh", PAPERLESS_HOST, f"docker restart {PAPERLESS_AI_CONTAINER}"],
        check=True, timeout=60,
    )
    print(f"restarted {PAPERLESS_AI_CONTAINER}")


if __name__ == "__main__":
    main()