Merge: t-paliad-311 — backup exporter drift-resistant + 4 broken ORDER BY cols fixed (m/paliad#140)
This commit is contained in:
99
internal/services/backup_service_live_test.go
Normal file
99
internal/services/backup_service_live_test.go
Normal file
@@ -0,0 +1,99 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jmoiron/sqlx"
|
||||
_ "github.com/lib/pq"
|
||||
)
|
||||
|
||||
// TestResolveOrgSheets_LiveSchemaSnapshot probes the live paliad schema
|
||||
// the way the backup runner does at the start of every run, then asserts
|
||||
// that every spec the registry declares either keeps all its ORDER BY
|
||||
// columns or — if any are missing — composes a fallback SELECT that the
|
||||
// DB can still execute. Catches the m/paliad#140 class of bug
|
||||
// (hardcoded ORDER BY against a renamed column) before deploy.
|
||||
//
|
||||
// Skipped when TEST_DATABASE_URL is unset. Read-only: opens a
|
||||
// REPEATABLE READ tx, never writes.
|
||||
func TestResolveOrgSheets_LiveSchemaSnapshot(t *testing.T) {
|
||||
url := os.Getenv("TEST_DATABASE_URL")
|
||||
if url == "" {
|
||||
t.Skip("TEST_DATABASE_URL not set — skipping live DB test")
|
||||
}
|
||||
pool, err := sqlx.Connect("postgres", url)
|
||||
if err != nil {
|
||||
t.Fatalf("connect: %v", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
specs := orgSheetSpecs()
|
||||
sheets, err := resolveOrgSheets(ctx, pool, specs)
|
||||
if err != nil {
|
||||
t.Fatalf("resolveOrgSheets: %v", err)
|
||||
}
|
||||
if len(sheets) != len(specs) {
|
||||
t.Fatalf("resolved %d sheets, want %d", len(sheets), len(specs))
|
||||
}
|
||||
|
||||
// Each resolved SELECT must run cleanly against the live schema.
|
||||
// We LIMIT 1 inside a sub-SELECT so we don't materialise the full
|
||||
// table (some are large) but still exercise the ORDER BY clause.
|
||||
for _, sq := range sheets {
|
||||
wrapped := `SELECT * FROM (` + sq.SQL + `) _wrap LIMIT 1`
|
||||
if _, err := pool.QueryxContext(ctx, wrapped, sq.Args...); err != nil {
|
||||
t.Errorf("sheet %q SQL failed: %v\nSQL: %s", sq.SheetName, err, sq.SQL)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestWriteOrg_LiveSmoke runs the full ExportService.WriteOrg pipeline
|
||||
// against a real DB: schema probe, REPEATABLE READ tx, every sheet
|
||||
// query, xlsx + json + per-sheet CSV assembly, outer zip framing.
|
||||
// Discards the bytes — this is a "does it crash" smoke, the bug class
|
||||
// it catches is exactly the one from m/paliad#140 (hardcoded ORDER BY
|
||||
// against a missing column).
|
||||
//
|
||||
// Skipped when TEST_DATABASE_URL is unset.
|
||||
func TestWriteOrg_LiveSmoke(t *testing.T) {
|
||||
url := os.Getenv("TEST_DATABASE_URL")
|
||||
if url == "" {
|
||||
t.Skip("TEST_DATABASE_URL not set — skipping live DB test")
|
||||
}
|
||||
pool, err := sqlx.Connect("postgres", url)
|
||||
if err != nil {
|
||||
t.Fatalf("connect: %v", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
svc := NewExportService(pool, "test-firm")
|
||||
var buf bytes.Buffer
|
||||
meta, err := svc.WriteOrg(context.Background(), &buf, ExportSpec{
|
||||
ActorID: uuid.New(),
|
||||
ActorEmail: "backup-smoke@test.local",
|
||||
ActorLabel: "Backup Smoke",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("WriteOrg: %v", err)
|
||||
}
|
||||
if buf.Len() == 0 {
|
||||
t.Fatalf("WriteOrg wrote no bytes")
|
||||
}
|
||||
// Spot-check meta fills.
|
||||
if meta.Scope != ExportScopeOrg {
|
||||
t.Errorf("meta.Scope = %q, want %q", meta.Scope, ExportScopeOrg)
|
||||
}
|
||||
if len(meta.RowCounts) != len(orgSheetSpecs()) {
|
||||
t.Errorf("meta.RowCounts has %d entries, want %d (one per sheet)", len(meta.RowCounts), len(orgSheetSpecs()))
|
||||
}
|
||||
// The bytes are a zip; the first 4 bytes are PK\x03\x04 for a non-empty zip.
|
||||
if buf.Len() >= 4 && !strings.HasPrefix(buf.String()[:4], "PK\x03\x04") {
|
||||
t.Errorf("bundle bytes don't look like a zip (first bytes: %x)", buf.Bytes()[:4])
|
||||
}
|
||||
}
|
||||
@@ -6,8 +6,10 @@ package services
|
||||
// it would live in backup_service_live_test.go under TEST_DATABASE_URL.
|
||||
// This file covers the bits that don't need a database:
|
||||
//
|
||||
// - orgSheetQueries registry shape: no duplicates, no excluded
|
||||
// - orgSheetSpecs registry shape: no duplicates, no excluded
|
||||
// paliadin sheets, predictable prefix split between entity and ref.
|
||||
// - composeOrgSheetSQL drift-resistance: missing ORDER BY cols drop,
|
||||
// SQL override path bypasses the builder, all-missing → no clause.
|
||||
// - LocalDiskStore Put / Get / Delete round-trip, key validation,
|
||||
// URI traversal rejection.
|
||||
|
||||
@@ -22,60 +24,216 @@ import (
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// orgSheetQueries registry
|
||||
// orgSheetSpecs registry
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestOrgSheetQueries_NoDuplicates(t *testing.T) {
|
||||
func TestOrgSheetSpecs_NoDuplicates(t *testing.T) {
|
||||
seen := map[string]bool{}
|
||||
for _, sq := range orgSheetQueries() {
|
||||
if seen[sq.SheetName] {
|
||||
t.Fatalf("duplicate sheet name in orgSheetQueries: %q", sq.SheetName)
|
||||
for _, sp := range orgSheetSpecs() {
|
||||
if seen[sp.SheetName] {
|
||||
t.Fatalf("duplicate sheet name in orgSheetSpecs: %q", sp.SheetName)
|
||||
}
|
||||
seen[sq.SheetName] = true
|
||||
seen[sp.SheetName] = true
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrgSheetQueries_ExcludesPaliadinTables(t *testing.T) {
|
||||
func TestOrgSheetSpecs_ExcludesPaliadinTables(t *testing.T) {
|
||||
// m's t-paliad-214 Q5 decision + this design's §11 Q3 default:
|
||||
// paliadin_turns and paliadin_aichat_conversation must be ABSENT
|
||||
// from the registry (structural exclusion, not just column-drop).
|
||||
for _, sq := range orgSheetQueries() {
|
||||
name := sq.SheetName
|
||||
for _, sp := range orgSheetSpecs() {
|
||||
name := sp.SheetName
|
||||
if strings.Contains(name, "paliadin") {
|
||||
t.Fatalf("orgSheetQueries leaked paliadin sheet: %q (m's Q3 mandates structural exclusion)", name)
|
||||
t.Fatalf("orgSheetSpecs leaked paliadin sheet: %q (m's Q3 mandates structural exclusion)", name)
|
||||
}
|
||||
// Belt-and-braces: SQL bodies should not reference the tables
|
||||
// either (no UNION joins, no subqueries pulling them in).
|
||||
if strings.Contains(sq.SQL, "paliadin_turns") || strings.Contains(sq.SQL, "paliadin_aichat_conversation") {
|
||||
t.Fatalf("orgSheetQueries[%q] SQL references a paliadin table: %s", name, sq.SQL)
|
||||
if strings.Contains(sp.Table, "paliadin") {
|
||||
t.Fatalf("orgSheetSpecs[%q].Table references a paliadin table: %s", name, sp.Table)
|
||||
}
|
||||
// Belt-and-braces: SQL override bodies (the few sheets that
|
||||
// bypass the Table+OrderBy builder) also can't pull paliadin
|
||||
// tables in through UNION/subquery.
|
||||
if strings.Contains(sp.SQL, "paliadin_turns") || strings.Contains(sp.SQL, "paliadin_aichat_conversation") {
|
||||
t.Fatalf("orgSheetSpecs[%q] SQL references a paliadin table: %s", name, sp.SQL)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrgSheetQueries_RefSheetsPrefixed(t *testing.T) {
|
||||
func TestOrgSheetSpecs_RefSheetsPrefixed(t *testing.T) {
|
||||
// Every sheet whose data is read-only reference material is
|
||||
// expected to use the `ref__` prefix. The writer's downstream
|
||||
// consumers rely on this convention to group reference data
|
||||
// visually in the workbook.
|
||||
for _, sq := range orgSheetQueries() {
|
||||
if !strings.HasPrefix(sq.SheetName, "ref__") {
|
||||
for _, sp := range orgSheetSpecs() {
|
||||
if !strings.HasPrefix(sp.SheetName, "ref__") {
|
||||
continue
|
||||
}
|
||||
// Reference sheets shouldn't carry per-row WHERE clauses (they
|
||||
// dump the whole reference table for portability).
|
||||
if strings.Contains(strings.ToUpper(sq.SQL), "WHERE") {
|
||||
t.Fatalf("orgSheetQueries[%q] is ref__ but has a WHERE clause; reference sheets dump the whole table", sq.SheetName)
|
||||
// dump the whole reference table for portability). Only
|
||||
// applies to the SQL-override path; the Table+OrderBy builder
|
||||
// never emits a WHERE.
|
||||
if sp.SQL != "" && strings.Contains(strings.ToUpper(sp.SQL), "WHERE") {
|
||||
t.Fatalf("orgSheetSpecs[%q] is ref__ but has a WHERE clause; reference sheets dump the whole table", sp.SheetName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrgSheetQueries_OrderByForDeterminism(t *testing.T) {
|
||||
// Every sheet must specify an ORDER BY so the byte-deterministic
|
||||
// contract from t-paliad-214 §3 holds across runs.
|
||||
for _, sq := range orgSheetQueries() {
|
||||
if !strings.Contains(strings.ToUpper(sq.SQL), "ORDER BY") {
|
||||
t.Fatalf("orgSheetQueries[%q] missing ORDER BY (determinism contract): %s", sq.SheetName, sq.SQL)
|
||||
func TestOrgSheetSpecs_OrderByForDeterminism(t *testing.T) {
|
||||
// Every sheet must declare a stable sort: either OrderBy on the
|
||||
// Table+OrderBy path, or ORDER BY in the SQL override. Keeps the
|
||||
// byte-deterministic contract from t-paliad-214 §3 across runs.
|
||||
//
|
||||
// (Drift removes ORDER BY columns at runtime, but only ones that
|
||||
// no longer exist in the schema — the spec-level declaration is
|
||||
// still required so we know what *should* be ordered.)
|
||||
for _, sp := range orgSheetSpecs() {
|
||||
if sp.SQL != "" {
|
||||
if !strings.Contains(strings.ToUpper(sp.SQL), "ORDER BY") {
|
||||
t.Fatalf("orgSheetSpecs[%q] SQL override missing ORDER BY (determinism contract): %s", sp.SheetName, sp.SQL)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if len(sp.OrderBy) == 0 {
|
||||
t.Fatalf("orgSheetSpecs[%q] has no OrderBy and no SQL override (determinism contract)", sp.SheetName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// composeOrgSheetSQL — drift-resistant SQL builder
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestComposeOrgSheetSQL_AllColumnsPresent(t *testing.T) {
|
||||
spec := orgSheetSpec{
|
||||
SheetName: "appointments",
|
||||
Table: "paliad.appointments",
|
||||
OrderBy: []string{"id"},
|
||||
}
|
||||
cols := map[string]map[string]struct{}{
|
||||
"appointments": {"id": {}, "project_id": {}},
|
||||
}
|
||||
got, dropped := composeOrgSheetSQL(spec, cols)
|
||||
want := "SELECT * FROM paliad.appointments ORDER BY id"
|
||||
if got != want {
|
||||
t.Fatalf("got SQL %q, want %q", got, want)
|
||||
}
|
||||
if len(dropped) != 0 {
|
||||
t.Fatalf("expected no dropped columns, got %v", dropped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComposeOrgSheetSQL_DropsMissingOrderByColumn(t *testing.T) {
|
||||
// The original bug from m/paliad#138 reproduced in unit form:
|
||||
// orderBy references a column the table doesn't have.
|
||||
spec := orgSheetSpec{
|
||||
SheetName: "appointment_caldav_targets",
|
||||
Table: "paliad.appointment_caldav_targets",
|
||||
OrderBy: []string{"appointment_id", "calendar_binding_id"}, // wrong: real col is binding_id
|
||||
}
|
||||
cols := map[string]map[string]struct{}{
|
||||
"appointment_caldav_targets": {
|
||||
"appointment_id": {},
|
||||
"binding_id": {},
|
||||
},
|
||||
}
|
||||
got, dropped := composeOrgSheetSQL(spec, cols)
|
||||
want := "SELECT * FROM paliad.appointment_caldav_targets ORDER BY appointment_id"
|
||||
if got != want {
|
||||
t.Fatalf("got SQL %q, want %q", got, want)
|
||||
}
|
||||
if len(dropped) != 1 || dropped[0] != "calendar_binding_id" {
|
||||
t.Fatalf("expected dropped=[calendar_binding_id], got %v", dropped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComposeOrgSheetSQL_AllOrderByMissing_NoClause(t *testing.T) {
|
||||
// If every declared ORDER BY column is gone, the builder still
|
||||
// produces a runnable SELECT — without ORDER BY. The export
|
||||
// succeeds; the order across runs is no longer deterministic for
|
||||
// this sheet until the spec is updated. WARN log alerts the
|
||||
// operator (verified in TestResolveOrgSheets_LogsWarnings).
|
||||
spec := orgSheetSpec{
|
||||
SheetName: "ghost",
|
||||
Table: "paliad.ghost",
|
||||
OrderBy: []string{"missing_a", "missing_b"},
|
||||
}
|
||||
cols := map[string]map[string]struct{}{
|
||||
"ghost": {"unrelated": {}},
|
||||
}
|
||||
got, dropped := composeOrgSheetSQL(spec, cols)
|
||||
want := "SELECT * FROM paliad.ghost"
|
||||
if got != want {
|
||||
t.Fatalf("got SQL %q, want %q", got, want)
|
||||
}
|
||||
if len(dropped) != 2 {
|
||||
t.Fatalf("expected 2 dropped columns, got %v", dropped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComposeOrgSheetSQL_SQLOverride_BypassesBuilder(t *testing.T) {
|
||||
// When a sheet declares SQL, the builder MUST NOT touch it — even
|
||||
// if the column knowledge would suggest a change. Custom
|
||||
// projections (documents drops ai_extracted) and special-case
|
||||
// joins both rely on this.
|
||||
spec := orgSheetSpec{
|
||||
SheetName: "documents",
|
||||
Table: "paliad.documents", // should be ignored
|
||||
OrderBy: []string{"id"}, // should be ignored
|
||||
SQL: "SELECT id, title FROM paliad.documents ORDER BY id",
|
||||
}
|
||||
cols := map[string]map[string]struct{}{
|
||||
"documents": {}, // empty → would drop everything if builder ran
|
||||
}
|
||||
got, dropped := composeOrgSheetSQL(spec, cols)
|
||||
if got != spec.SQL {
|
||||
t.Fatalf("SQL override mutated: got %q, want %q", got, spec.SQL)
|
||||
}
|
||||
if len(dropped) != 0 {
|
||||
t.Fatalf("override path should never report drops; got %v", dropped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComposeOrgSheetSQL_UnknownTable_DropsAllOrderBy(t *testing.T) {
|
||||
// A table missing entirely from the schema snapshot is treated as
|
||||
// "no columns known" — every ORDER BY column gets dropped, but
|
||||
// the SELECT still emits (so a stale registry doesn't crash the
|
||||
// backup; the operator gets WARNs to fix it).
|
||||
spec := orgSheetSpec{
|
||||
SheetName: "renamed_table",
|
||||
Table: "paliad.renamed_table",
|
||||
OrderBy: []string{"id"},
|
||||
}
|
||||
got, dropped := composeOrgSheetSQL(spec, map[string]map[string]struct{}{})
|
||||
want := "SELECT * FROM paliad.renamed_table"
|
||||
if got != want {
|
||||
t.Fatalf("got SQL %q, want %q", got, want)
|
||||
}
|
||||
if len(dropped) != 1 || dropped[0] != "id" {
|
||||
t.Fatalf("expected dropped=[id], got %v", dropped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComposeOrgSheetSQL_PreservesOrderByOrder(t *testing.T) {
|
||||
// Multi-column OrderBy must keep its declared order, with kept
|
||||
// columns concatenated in the same sequence. Determinism contract
|
||||
// from t-paliad-214 §3 depends on this.
|
||||
spec := orgSheetSpec{
|
||||
SheetName: "partner_unit_members",
|
||||
Table: "paliad.partner_unit_members",
|
||||
OrderBy: []string{"partner_unit_id", "missing_middle", "user_id"},
|
||||
}
|
||||
cols := map[string]map[string]struct{}{
|
||||
"partner_unit_members": {
|
||||
"partner_unit_id": {},
|
||||
"user_id": {},
|
||||
},
|
||||
}
|
||||
got, dropped := composeOrgSheetSQL(spec, cols)
|
||||
want := "SELECT * FROM paliad.partner_unit_members ORDER BY partner_unit_id, user_id"
|
||||
if got != want {
|
||||
t.Fatalf("got SQL %q, want %q", got, want)
|
||||
}
|
||||
if len(dropped) != 1 || dropped[0] != "missing_middle" {
|
||||
t.Fatalf("expected dropped=[missing_middle], got %v", dropped)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -46,6 +46,7 @@ import (
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -297,7 +298,10 @@ func (s *ExportService) WriteOrg(ctx context.Context, w io.Writer, spec ExportSp
|
||||
// is just bookkeeping that releases the snapshot.
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
sheets := orgSheetQueries()
|
||||
sheets, err := resolveOrgSheets(ctx, tx, orgSheetSpecs())
|
||||
if err != nil {
|
||||
return meta, err
|
||||
}
|
||||
if err := s.writeBundle(ctx, tx, w, sheets, &meta); err != nil {
|
||||
return meta, err
|
||||
}
|
||||
@@ -1560,73 +1564,249 @@ SELECT 'partner_unit_default'::text AS source,
|
||||
// secret|token|password|api_key|private_key on every sheet as a
|
||||
// belt-and-braces filter. user_caldav_config.password_encrypted is
|
||||
// explicitly named in DropColumns too.
|
||||
func orgSheetQueries() []sheetQuery {
|
||||
return []sheetQuery{
|
||||
//
|
||||
// Drift-resistance (m/paliad#140): each spec declares its desired
|
||||
// ORDER BY columns as a list. At backup time the exporter probes
|
||||
// information_schema.columns for the live schema; any ORDER BY column
|
||||
// that no longer exists is dropped (logged WARN). This way a column
|
||||
// rename or removal never breaks a backup — the worst case is a sheet
|
||||
// that loses sort stability until the spec is updated. A sheet whose
|
||||
// ORDER BY columns are all gone still exports, just in pg's natural
|
||||
// (unspecified) order.
|
||||
//
|
||||
// Custom column projections (e.g. documents drops ai_extracted) live
|
||||
// in the SQL override field; if set, it bypasses the Table+OrderBy
|
||||
// builder entirely. Use it sparingly — every override re-introduces
|
||||
// drift risk for that sheet.
|
||||
|
||||
// orgSheetSpec declares one org-scope sheet for the drift-resistant
|
||||
// builder. Either set SQL (free-form override) or set Table+OrderBy
|
||||
// (let the builder compose `SELECT * FROM <Table> ORDER BY <existing>`).
|
||||
type orgSheetSpec struct {
|
||||
// SheetName lands in the workbook sheet and the JSON top-level key.
|
||||
SheetName string
|
||||
// Table is schema-qualified (e.g. "paliad.appointments"). Used only
|
||||
// when SQL is empty. The schema/table form must be valid SQL
|
||||
// identifiers — the builder splits on the dot, no quoting.
|
||||
Table string
|
||||
// OrderBy is the *desired* sort columns. Missing columns are
|
||||
// dropped silently-with-a-WARN at build time; remaining columns
|
||||
// keep their declared order. Empty/all-missing → no ORDER BY (still
|
||||
// deterministic-within-a-snapshot under the REPEATABLE READ tx, but
|
||||
// the order across runs may differ).
|
||||
OrderBy []string
|
||||
// SQL is an explicit override; if non-empty, Table+OrderBy are
|
||||
// ignored entirely. Use only when the projection cannot be
|
||||
// expressed as SELECT * (e.g. documents drops the ai_extracted
|
||||
// jsonb column).
|
||||
SQL string
|
||||
// Args are positional arguments. Only meaningful with SQL override;
|
||||
// the Table+OrderBy path takes no args.
|
||||
Args []any
|
||||
// DropColumns is an explicit list of column names to drop from the
|
||||
// result regardless of the PII deny-regex.
|
||||
DropColumns []string
|
||||
}
|
||||
|
||||
func orgSheetSpecs() []orgSheetSpec {
|
||||
return []orgSheetSpec{
|
||||
// --- entity sheets (alphabetical) ---
|
||||
{SheetName: "appointment_caldav_targets", SQL: `SELECT * FROM paliad.appointment_caldav_targets ORDER BY appointment_id, binding_id`},
|
||||
{SheetName: "appointments", SQL: `SELECT * FROM paliad.appointments ORDER BY id`},
|
||||
{SheetName: "approval_policies", SQL: `SELECT * FROM paliad.approval_policies ORDER BY id`},
|
||||
{SheetName: "approval_requests", SQL: `SELECT * FROM paliad.approval_requests ORDER BY id`},
|
||||
{SheetName: "appointment_caldav_targets", Table: "paliad.appointment_caldav_targets", OrderBy: []string{"appointment_id", "binding_id"}},
|
||||
{SheetName: "appointments", Table: "paliad.appointments", OrderBy: []string{"id"}},
|
||||
{SheetName: "approval_policies", Table: "paliad.approval_policies", OrderBy: []string{"id"}},
|
||||
{SheetName: "approval_requests", Table: "paliad.approval_requests", OrderBy: []string{"id"}},
|
||||
// backups is self-reflexive — including it makes "what backups
|
||||
// have we taken" recoverable from any prior backup. Tiny table.
|
||||
{SheetName: "backups", SQL: `SELECT * FROM paliad.backups ORDER BY started_at, id`},
|
||||
{SheetName: "caldav_sync_log", SQL: `SELECT * FROM paliad.caldav_sync_log ORDER BY occurred_at, id`},
|
||||
{SheetName: "checklist_instances", SQL: `SELECT * FROM paliad.checklist_instances ORDER BY id`},
|
||||
{SheetName: "checklist_shares", SQL: `SELECT * FROM paliad.checklist_shares ORDER BY id`},
|
||||
{SheetName: "checklists", SQL: `SELECT * FROM paliad.checklists ORDER BY id`},
|
||||
{SheetName: "deadline_rule_audit", SQL: `SELECT * FROM paliad.deadline_rule_audit ORDER BY changed_at, id`},
|
||||
{SheetName: "deadlines", SQL: `SELECT * FROM paliad.deadlines ORDER BY id`},
|
||||
{SheetName: "backups", Table: "paliad.backups", OrderBy: []string{"started_at", "id"}},
|
||||
{SheetName: "caldav_sync_log", Table: "paliad.caldav_sync_log", OrderBy: []string{"occurred_at", "id"}},
|
||||
{SheetName: "checklist_instances", Table: "paliad.checklist_instances", OrderBy: []string{"id"}},
|
||||
{SheetName: "checklist_shares", Table: "paliad.checklist_shares", OrderBy: []string{"id"}},
|
||||
{SheetName: "checklists", Table: "paliad.checklists", OrderBy: []string{"id"}},
|
||||
{SheetName: "deadline_rule_audit", Table: "paliad.deadline_rule_audit", OrderBy: []string{"changed_at", "id"}},
|
||||
{SheetName: "deadlines", Table: "paliad.deadlines", OrderBy: []string{"id"}},
|
||||
// documents: ai_extracted jsonb dropped (verbose AI prompts;
|
||||
// matches the personal/project precedent). Binaries are not in
|
||||
// the export — only metadata.
|
||||
// the export — only metadata. Uses SQL override because the
|
||||
// projection isn't SELECT *.
|
||||
{
|
||||
SheetName: "documents",
|
||||
SQL: `SELECT id, project_id, title, doc_type, file_path, file_size, mime_type, uploaded_by, created_at, updated_at
|
||||
FROM paliad.documents
|
||||
ORDER BY id`,
|
||||
},
|
||||
{SheetName: "email_broadcasts", SQL: `SELECT * FROM paliad.email_broadcasts ORDER BY id`},
|
||||
{SheetName: "email_template_versions", SQL: `SELECT * FROM paliad.email_template_versions ORDER BY id`},
|
||||
{SheetName: "email_templates", SQL: `SELECT * FROM paliad.email_templates ORDER BY id`},
|
||||
{SheetName: "firm_dashboard_default", SQL: `SELECT * FROM paliad.firm_dashboard_default ORDER BY id`},
|
||||
{SheetName: "invitations", SQL: `SELECT * FROM paliad.invitations ORDER BY sent_at, id`},
|
||||
{SheetName: "notes", SQL: `SELECT * FROM paliad.notes ORDER BY id`},
|
||||
{SheetName: "parties", SQL: `SELECT * FROM paliad.parties ORDER BY id`},
|
||||
{SheetName: "partner_unit_events", SQL: `SELECT * FROM paliad.partner_unit_events ORDER BY id`},
|
||||
{SheetName: "partner_unit_members", SQL: `SELECT * FROM paliad.partner_unit_members ORDER BY partner_unit_id, user_id`},
|
||||
{SheetName: "partner_units", SQL: `SELECT * FROM paliad.partner_units ORDER BY id`},
|
||||
{SheetName: "policy_audit_log", SQL: `SELECT * FROM paliad.policy_audit_log ORDER BY changed_at, id`},
|
||||
{SheetName: "project_events", SQL: `SELECT * FROM paliad.project_events ORDER BY id`},
|
||||
{SheetName: "project_partner_units", SQL: `SELECT * FROM paliad.project_partner_units ORDER BY project_id, partner_unit_id`},
|
||||
{SheetName: "project_teams", SQL: `SELECT * FROM paliad.project_teams ORDER BY project_id, user_id`},
|
||||
{SheetName: "projects", SQL: `SELECT * FROM paliad.projects ORDER BY id`},
|
||||
{SheetName: "reminder_log", SQL: `SELECT * FROM paliad.reminder_log ORDER BY sent_at, id`},
|
||||
{SheetName: "submission_drafts", SQL: `SELECT * FROM paliad.submission_drafts ORDER BY id`},
|
||||
{SheetName: "system_audit_log", SQL: `SELECT * FROM paliad.system_audit_log ORDER BY created_at, id`},
|
||||
{SheetName: "email_broadcasts", Table: "paliad.email_broadcasts", OrderBy: []string{"id"}},
|
||||
{SheetName: "email_template_versions", Table: "paliad.email_template_versions", OrderBy: []string{"id"}},
|
||||
{SheetName: "email_templates", Table: "paliad.email_templates", OrderBy: []string{"key", "lang"}},
|
||||
{SheetName: "firm_dashboard_default", Table: "paliad.firm_dashboard_default", OrderBy: []string{"id"}},
|
||||
{SheetName: "invitations", Table: "paliad.invitations", OrderBy: []string{"sent_at", "id"}},
|
||||
{SheetName: "notes", Table: "paliad.notes", OrderBy: []string{"id"}},
|
||||
{SheetName: "parties", Table: "paliad.parties", OrderBy: []string{"id"}},
|
||||
{SheetName: "partner_unit_events", Table: "paliad.partner_unit_events", OrderBy: []string{"id"}},
|
||||
{SheetName: "partner_unit_members", Table: "paliad.partner_unit_members", OrderBy: []string{"partner_unit_id", "user_id"}},
|
||||
{SheetName: "partner_units", Table: "paliad.partner_units", OrderBy: []string{"id"}},
|
||||
{SheetName: "policy_audit_log", Table: "paliad.policy_audit_log", OrderBy: []string{"created_at", "id"}},
|
||||
{SheetName: "project_events", Table: "paliad.project_events", OrderBy: []string{"id"}},
|
||||
{SheetName: "project_partner_units", Table: "paliad.project_partner_units", OrderBy: []string{"project_id", "partner_unit_id"}},
|
||||
{SheetName: "project_teams", Table: "paliad.project_teams", OrderBy: []string{"project_id", "user_id"}},
|
||||
{SheetName: "projects", Table: "paliad.projects", OrderBy: []string{"id"}},
|
||||
{SheetName: "reminder_log", Table: "paliad.reminder_log", OrderBy: []string{"sent_at", "id"}},
|
||||
{SheetName: "submission_drafts", Table: "paliad.submission_drafts", OrderBy: []string{"id"}},
|
||||
{SheetName: "system_audit_log", Table: "paliad.system_audit_log", OrderBy: []string{"created_at", "id"}},
|
||||
{
|
||||
SheetName: "user_caldav_config",
|
||||
SQL: `SELECT * FROM paliad.user_caldav_config ORDER BY user_id`,
|
||||
Table: "paliad.user_caldav_config",
|
||||
OrderBy: []string{"user_id"},
|
||||
DropColumns: []string{"password_encrypted"}, // belt-and-braces; piiColumnDenyRegex also catches it
|
||||
},
|
||||
{SheetName: "user_calendar_bindings", SQL: `SELECT * FROM paliad.user_calendar_bindings ORDER BY user_id, calendar_path`},
|
||||
{SheetName: "user_card_layouts", SQL: `SELECT * FROM paliad.user_card_layouts ORDER BY id`},
|
||||
{SheetName: "user_dashboard_layouts", SQL: `SELECT * FROM paliad.user_dashboard_layouts ORDER BY user_id`},
|
||||
{SheetName: "user_pinned_projects", SQL: `SELECT * FROM paliad.user_pinned_projects ORDER BY user_id, project_id`},
|
||||
{SheetName: "user_views", SQL: `SELECT * FROM paliad.user_views ORDER BY id`},
|
||||
{SheetName: "users", SQL: `SELECT * FROM paliad.users ORDER BY id`},
|
||||
{SheetName: "user_calendar_bindings", Table: "paliad.user_calendar_bindings", OrderBy: []string{"user_id", "calendar_path"}},
|
||||
{SheetName: "user_card_layouts", Table: "paliad.user_card_layouts", OrderBy: []string{"id"}},
|
||||
{SheetName: "user_dashboard_layouts", Table: "paliad.user_dashboard_layouts", OrderBy: []string{"user_id"}},
|
||||
{SheetName: "user_pinned_projects", Table: "paliad.user_pinned_projects", OrderBy: []string{"user_id", "project_id"}},
|
||||
{SheetName: "user_views", Table: "paliad.user_views", OrderBy: []string{"id"}},
|
||||
{SheetName: "users", Table: "paliad.users", OrderBy: []string{"id"}},
|
||||
|
||||
// --- reference data (alphabetical, prefixed ref__) ---
|
||||
{SheetName: "ref__countries", SQL: `SELECT * FROM paliad.countries ORDER BY code`},
|
||||
{SheetName: "ref__courts", SQL: `SELECT * FROM paliad.courts ORDER BY id`},
|
||||
{SheetName: "ref__deadline_concept_event_types", SQL: `SELECT * FROM paliad.deadline_concept_event_types ORDER BY concept_id, event_type_id`},
|
||||
{SheetName: "ref__deadline_concepts", SQL: `SELECT * FROM paliad.deadline_concepts ORDER BY id`},
|
||||
{SheetName: "ref__deadline_event_types", SQL: `SELECT * FROM paliad.deadline_event_types ORDER BY rule_id, event_type_id`},
|
||||
{SheetName: "ref__deadline_rules", SQL: `SELECT * FROM paliad.deadline_rules_unified ORDER BY id`},
|
||||
{SheetName: "ref__event_categories", SQL: `SELECT * FROM paliad.event_categories ORDER BY id`},
|
||||
{SheetName: "ref__event_category_concepts", SQL: `SELECT * FROM paliad.event_category_concepts ORDER BY category_id, concept_id`},
|
||||
{SheetName: "ref__event_types", SQL: `SELECT * FROM paliad.event_types ORDER BY id`},
|
||||
{SheetName: "ref__holidays", SQL: `SELECT * FROM paliad.holidays ORDER BY date, country`},
|
||||
{SheetName: "ref__proceeding_types", SQL: `SELECT * FROM paliad.proceeding_types ORDER BY id`},
|
||||
{SheetName: "ref__trigger_events", SQL: `SELECT * FROM paliad.trigger_events ORDER BY id`},
|
||||
{SheetName: "ref__countries", Table: "paliad.countries", OrderBy: []string{"code"}},
|
||||
{SheetName: "ref__courts", Table: "paliad.courts", OrderBy: []string{"id"}},
|
||||
{SheetName: "ref__deadline_concept_event_types", Table: "paliad.deadline_concept_event_types", OrderBy: []string{"concept_id", "event_type_id"}},
|
||||
{SheetName: "ref__deadline_concepts", Table: "paliad.deadline_concepts", OrderBy: []string{"id"}},
|
||||
{SheetName: "ref__deadline_event_types", Table: "paliad.deadline_event_types", OrderBy: []string{"deadline_id", "event_type_id"}},
|
||||
{SheetName: "ref__deadline_rules", Table: "paliad.deadline_rules_unified", OrderBy: []string{"id"}},
|
||||
{SheetName: "ref__event_categories", Table: "paliad.event_categories", OrderBy: []string{"id"}},
|
||||
{SheetName: "ref__event_category_concepts", Table: "paliad.event_category_concepts", OrderBy: []string{"event_category_id", "concept_id"}},
|
||||
{SheetName: "ref__event_types", Table: "paliad.event_types", OrderBy: []string{"id"}},
|
||||
{SheetName: "ref__holidays", Table: "paliad.holidays", OrderBy: []string{"date", "country"}},
|
||||
{SheetName: "ref__proceeding_types", Table: "paliad.proceeding_types", OrderBy: []string{"id"}},
|
||||
{SheetName: "ref__trigger_events", Table: "paliad.trigger_events", OrderBy: []string{"id"}},
|
||||
}
|
||||
}
|
||||
|
||||
// composeOrgSheetSQL turns one orgSheetSpec into the final SQL string,
|
||||
// using a per-table column set (typically loaded once per backup run
|
||||
// from information_schema.columns). Returns the SQL and the list of
|
||||
// ORDER BY columns that were dropped because they don't exist in the
|
||||
// live schema.
|
||||
//
|
||||
// Pure function — no DB access — so the missing-column behaviour is
|
||||
// unit-testable without a fixture database.
|
||||
//
|
||||
// Rules:
|
||||
// - If spec.SQL is non-empty, return it unchanged (override path).
|
||||
// - Otherwise build `SELECT * FROM <Table> [ORDER BY <kept-cols>]`.
|
||||
// - Columns are kept in their declared order; missing ones recorded
|
||||
// in `dropped` and omitted from ORDER BY.
|
||||
// - If no ORDER BY columns survive, the ORDER BY clause is omitted.
|
||||
//
|
||||
// knownCols maps unqualified table names (e.g. "appointments") to the
|
||||
// set of columns they have. A table missing from knownCols is treated
|
||||
// as "no columns known" — every declared ORDER BY column gets dropped.
|
||||
func composeOrgSheetSQL(spec orgSheetSpec, knownCols map[string]map[string]struct{}) (sqlText string, dropped []string) {
|
||||
if spec.SQL != "" {
|
||||
return spec.SQL, nil
|
||||
}
|
||||
unqualified := spec.Table
|
||||
if i := strings.IndexByte(unqualified, '.'); i >= 0 {
|
||||
unqualified = unqualified[i+1:]
|
||||
}
|
||||
cols := knownCols[unqualified]
|
||||
kept := make([]string, 0, len(spec.OrderBy))
|
||||
for _, c := range spec.OrderBy {
|
||||
if _, ok := cols[c]; ok {
|
||||
kept = append(kept, c)
|
||||
} else {
|
||||
dropped = append(dropped, c)
|
||||
}
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString("SELECT * FROM ")
|
||||
b.WriteString(spec.Table)
|
||||
if len(kept) > 0 {
|
||||
b.WriteString(" ORDER BY ")
|
||||
b.WriteString(strings.Join(kept, ", "))
|
||||
}
|
||||
return b.String(), dropped
|
||||
}
|
||||
|
||||
// loadOrgSheetColumns probes information_schema.columns once for every
|
||||
// table referenced by Table+OrderBy specs. Returns a lookup
|
||||
// {table_name → {column_name → {}}} restricted to the paliad schema.
|
||||
//
|
||||
// The queryer is whatever runs the backup's read snapshot — typically
|
||||
// the REPEATABLE READ tx opened in WriteOrg, so the schema snapshot
|
||||
// matches the row snapshot.
|
||||
func loadOrgSheetColumns(ctx context.Context, queryer sqlx.QueryerContext, specs []orgSheetSpec) (map[string]map[string]struct{}, error) {
|
||||
tableSet := map[string]struct{}{}
|
||||
for _, sp := range specs {
|
||||
if sp.Table == "" {
|
||||
continue // SQL-override sheets carry their own column refs
|
||||
}
|
||||
t := sp.Table
|
||||
if i := strings.IndexByte(t, '.'); i >= 0 {
|
||||
t = t[i+1:]
|
||||
}
|
||||
tableSet[t] = struct{}{}
|
||||
}
|
||||
if len(tableSet) == 0 {
|
||||
return map[string]map[string]struct{}{}, nil
|
||||
}
|
||||
tables := make([]string, 0, len(tableSet))
|
||||
for t := range tableSet {
|
||||
tables = append(tables, t)
|
||||
}
|
||||
rows, err := queryer.QueryxContext(ctx, `
|
||||
SELECT table_name, column_name
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'paliad'
|
||||
AND table_name = ANY($1)
|
||||
`, tables)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("probe paliad columns: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make(map[string]map[string]struct{}, len(tableSet))
|
||||
for rows.Next() {
|
||||
var table, column string
|
||||
if err := rows.Scan(&table, &column); err != nil {
|
||||
return nil, fmt.Errorf("scan paliad columns: %w", err)
|
||||
}
|
||||
set, ok := out[table]
|
||||
if !ok {
|
||||
set = map[string]struct{}{}
|
||||
out[table] = set
|
||||
}
|
||||
set[column] = struct{}{}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate paliad columns: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// resolveOrgSheets materialises an org-scope spec list into the
|
||||
// concrete []sheetQuery that writeBundle expects. Composes each
|
||||
// spec's SQL via composeOrgSheetSQL using a schema snapshot loaded
|
||||
// from the same queryer. Logs WARN per dropped ORDER BY column.
|
||||
func resolveOrgSheets(ctx context.Context, queryer sqlx.QueryerContext, specs []orgSheetSpec) ([]sheetQuery, error) {
|
||||
knownCols, err := loadOrgSheetColumns(ctx, queryer, specs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]sheetQuery, 0, len(specs))
|
||||
for _, sp := range specs {
|
||||
sqlText, dropped := composeOrgSheetSQL(sp, knownCols)
|
||||
for _, c := range dropped {
|
||||
slog.Warn("backup: ORDER BY column dropped (not in schema)",
|
||||
"sheet", sp.SheetName,
|
||||
"table", sp.Table,
|
||||
"column", c,
|
||||
)
|
||||
}
|
||||
out = append(out, sheetQuery{
|
||||
SheetName: sp.SheetName,
|
||||
SQL: sqlText,
|
||||
Args: sp.Args,
|
||||
DropColumns: sp.DropColumns,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user