mAi: #8 - imagen.jobs queue + worker subcommand (flexsiebels write path)
Async write path for the flexsiebels owner-mode UI: flexsiebels INSERTs into imagen.jobs, the worker on mRiver claims pending rows via LISTEN/NOTIFY + 5s safety poll, runs the same generate pipeline imagen generate uses, and writes the result through internal/cloud into imagen.images. - Schema migration imagen_jobs_init: table + status CHECK + two indexes + owner-scoped RLS + grants + AFTER INSERT trigger publishing on the imagen_jobs channel via pg_notify. - internal/worker: DB-agnostic loop over a Queue interface. Drains the whole pending backlog on each wake. Job-scoped contexts are derived from Background so SIGTERM lets the in-flight generation finish (no half-state). ResetStaleRunning at startup unsticks rows left over from a previous crash. Eight unit tests cover the done / failed / missing-id / drain / NOTIFY-wake / shutdown / transient-error paths against a fake queue (no real Postgres in CI). - cmd/imagen/worker.go: pgx-backed Queue (one dedicated conn for LISTEN + UPDATE), plus the workerPipeline that reuses buildBackend + attachUsageSink + prompt.Apply + buildWriter + maybeCloudSync. The per-job owner_user_id overrides the env-level fallback so each row in imagen.images is attributed correctly. - maybeCloudSync now returns (*cloud.SyncResult, error) so the worker can link imagen.jobs.image_id to the inserted imagen.images row. The CLI generate path keeps printing its stderr summary unchanged. - scripts/imagen-worker.service + .env.example for the systemd --user unit on mRiver. EnvironmentFile lives in ~/.dotfiles and is never committed. - docs/setup-worker-mriver.md walks through installation + the spec's SQL-INSERT smoke; docs/architecture.md grows an "async write path" section. - worker_integration_test.go (env-guarded by IMAGEN_WORKER_INTEGRATION=1) drives one real job through the full pipeline against msupabase using the mock backend, then verifies imagen.images + Storage object landed and the row flipped to done with image_id linked. Verified end-to-end: pickup latency ~7ms, total 74ms, failure path captures error text.
This commit is contained in:
@@ -132,9 +132,11 @@ func runGenerate(ctx context.Context, args []string) error {
|
||||
fmt.Fprintln(os.Stderr, "sidecar:", paths.SidecarPath)
|
||||
}
|
||||
|
||||
if err := maybeCloudSync(ctx, cfg, noCloud, paths, in, res, w, h); err != nil {
|
||||
if result, err := maybeCloudSync(ctx, cfg, noCloud, "", paths, in, res, w, h); err != nil {
|
||||
// cloud-sync failures are warnings — the image already wrote.
|
||||
fmt.Fprintln(os.Stderr, "imagen: cloud sync:", err)
|
||||
} else if result != nil && result.ImageID != "" {
|
||||
fmt.Fprintf(os.Stderr, "cloud: imagen.images.id=%s storage_path=%s\n", result.ImageID, result.StoragePath)
|
||||
}
|
||||
|
||||
if err := maybePreview(cfg, previewOn, previewOff, paths.ImagePath, rawPrompt); err != nil {
|
||||
@@ -167,39 +169,45 @@ func resolveCloudSyncMode(cfg *config.Config, noCloudFlag bool, env string) (str
|
||||
}
|
||||
|
||||
// maybeCloudSync resolves the effective mode and, if it says yes, uploads
|
||||
// the PNG and inserts the row. Always non-fatal — the image already wrote.
|
||||
func maybeCloudSync(ctx context.Context, cfg *config.Config, noCloud bool, paths *output.Outputs, in output.Inputs, res *backend.Result, width, height int) error {
|
||||
// the PNG and inserts the row. Returns the SyncResult on success so callers
|
||||
// that need the imagen.images.id (e.g. the worker linking a job row) can pick
|
||||
// it up. ownerOverride, when non-empty, wins over config + env — the worker
|
||||
// passes the job row's owner_user_id so each job is attributed correctly.
|
||||
func maybeCloudSync(ctx context.Context, cfg *config.Config, noCloud bool, ownerOverride string, paths *output.Outputs, in output.Inputs, res *backend.Result, width, height int) (*cloud.SyncResult, error) {
|
||||
mode, err := resolveCloudSyncMode(cfg, noCloud, os.Getenv("IMAGEN_CLOUD_SYNC"))
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
if mode == "off" {
|
||||
return nil
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
sink, ok := cloud.NewFromEnv()
|
||||
if !ok {
|
||||
if mode == "on" {
|
||||
return fmt.Errorf("cloud_sync=on but SUPABASE_URL / SUPABASE_SERVICE_KEY not set in env")
|
||||
return nil, fmt.Errorf("cloud_sync=on but SUPABASE_URL / SUPABASE_SERVICE_KEY not set in env")
|
||||
}
|
||||
// auto + missing env = silent skip.
|
||||
return nil
|
||||
return nil, nil
|
||||
}
|
||||
// Config-supplied owner_user_id takes precedence over $IMAGEN_OWNER_USER_ID.
|
||||
if cfg != nil && cfg.OwnerUserID != "" {
|
||||
switch {
|
||||
case ownerOverride != "":
|
||||
sink.OwnerUserID = ownerOverride
|
||||
case cfg != nil && cfg.OwnerUserID != "":
|
||||
// Config-supplied owner_user_id takes precedence over $IMAGEN_OWNER_USER_ID.
|
||||
sink.OwnerUserID = cfg.OwnerUserID
|
||||
}
|
||||
if sink.OwnerUserID == "" {
|
||||
if mode == "on" {
|
||||
return fmt.Errorf("cloud_sync=on but owner_user_id not set in config and $IMAGEN_OWNER_USER_ID is empty")
|
||||
return nil, fmt.Errorf("cloud_sync=on but owner_user_id not set in config and $IMAGEN_OWNER_USER_ID is empty")
|
||||
}
|
||||
// auto + missing UUID = silent skip.
|
||||
return nil
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
pngBytes, readErr := os.ReadFile(paths.ImagePath)
|
||||
if readErr != nil {
|
||||
return fmt.Errorf("read local image: %w", readErr)
|
||||
return nil, fmt.Errorf("read local image: %w", readErr)
|
||||
}
|
||||
|
||||
// Reuse the writer's date/slug/seed so storage_path mirrors the local
|
||||
@@ -257,14 +265,7 @@ func maybeCloudSync(ctx context.Context, cfg *config.Config, noCloud bool, paths
|
||||
}
|
||||
syncCtx, cancel := context.WithTimeout(ctx, 60*time.Second)
|
||||
defer cancel()
|
||||
result, err := sink.Sync(syncCtx, syncReq)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if result != nil && result.ImageID != "" {
|
||||
fmt.Fprintf(os.Stderr, "cloud: imagen.images.id=%s storage_path=%s\n", result.ImageID, result.StoragePath)
|
||||
}
|
||||
return nil
|
||||
return sink.Sync(syncCtx, syncReq)
|
||||
}
|
||||
|
||||
func metaString(m map[string]any, key string) string {
|
||||
|
||||
Reference in New Issue
Block a user