package host import ( "bytes" "database/sql" "encoding/json" "errors" "net/http" "net/http/httptest" "path/filepath" "sync/atomic" "testing" "github.com/carlosframework/platform/internal/hostbox" "github.com/carlosframework/platform/internal/hostguard" _ "modernc.org/sqlite" ) // A receipt lookup that fails says nothing about the request. A replay of the // retained request after a lost reply must never become a definite refusal. func TestPlan9JournalReplayLookupFailureIsUncertain(t *testing.T) { path := filepath.Join(t.TempDir(), "root.db") j, err := hostbox.OpenLaunchJournal(path) if err != nil { t.Fatal(err) } defer j.Close() s, err := hostbox.NewJournalService(j) if err != nil { t.Fatal(err) } in := hostbox.LaunchInventory{Target: hostguard.FenceTarget{Host: "app.example", Incarnation: "inc", Stamp: 1, Session: 1, Box: "box", Enrollment: "enroll", BootID: "boot", ControllerID: "controller"}, ControllerPID: 17, ProcessStart: "23", NoState: true, Groups: []hostbox.LaunchGroup{{RelativePath: "session", Device: 1, Inode: 2}}} q := plan9CanonicalJournalRequest(t, hostbox.JournalRequest{V: 1, Op: "create", Launch: in}) if _, err = s.Execute(t.Context(), "box/enroll", q); err != nil { t.Fatal(err) } root := ClaimRootHTTP{Journals: s} root.Reconcile.Box = "box" root.Reconcile.Enrollment = "enroll" peer := StopPeer{UID: 1001, PID: 17, ProcessStart: "23", BootID: "boot", ControllerID: "controller"} // The client resends the byte-identical request and ID it retained. replay := func() *httptest.ResponseRecorder { body, _ := json.Marshal(q) response := httptest.NewRecorder() root.serveJournal(response, httptest.NewRequest(http.MethodPost, "/claim/journal", bytes.NewReader(body)), peer) return response } // Break only the receipt lookup, from a second connection. other, err := sql.Open("sqlite", "file:"+path+"?_pragma=busy_timeout(5000)") if err != nil { t.Fatal(err) } defer other.Close() if _, err = other.Exec("ALTER TABLE journal_requests RENAME TO journal_requests_hidden"); err != nil { t.Fatal(err) } // 400/403/409 are definite refusals to the client; only 5xx stays uncertain. if response := replay(); response.Code < 500 { t.Fatalf("failed receipt lookup reported as definite result %d: %s", response.Code, response.Body.String()) } if _, err = other.Exec("ALTER TABLE journal_requests_hidden RENAME TO journal_requests"); err != nil { t.Fatal(err) } var out hostbox.JournalResponse response := replay() if err = json.Unmarshal(response.Body.Bytes(), &out); response.Code != http.StatusOK || err != nil || out.Error != "" || out.ID != q.ID { t.Fatalf("replay did not recover once lookup worked: %d %+v %v", response.Code, out, err) } var receipts int if err = other.QueryRow("SELECT COUNT(*) FROM journal_requests").Scan(&receipts); err != nil || receipts != 1 { t.Fatalf("duplicate effect or receipt: %d %v", receipts, err) } } // A byte-different request under a stored ID is a proven conflict. func TestPlan9JournalReplayMismatchIsConflict(t *testing.T) { j, err := hostbox.OpenLaunchJournal(filepath.Join(t.TempDir(), "root.db")) if err != nil { t.Fatal(err) } defer j.Close() s, err := hostbox.NewJournalService(j) if err != nil { t.Fatal(err) } in := hostbox.LaunchInventory{Target: hostguard.FenceTarget{Host: "app.example", Incarnation: "inc", Stamp: 1, Session: 1, Box: "box", Enrollment: "enroll", BootID: "boot", ControllerID: "controller"}, ControllerPID: 17, ProcessStart: "23", NoState: true, Groups: []hostbox.LaunchGroup{{RelativePath: "session", Device: 1, Inode: 2}}} q := plan9CanonicalJournalRequest(t, hostbox.JournalRequest{V: 1, Op: "create", Launch: in}) if _, err = s.Execute(t.Context(), "box/enroll", q); err != nil { t.Fatal(err) } if _, _, err = s.Replay(t.Context(), "other/peer", q); !errors.Is(err, hostbox.ErrChanged) { t.Fatalf("different owner not a proven mismatch: %v", err) } } // A request committed and its reply is lost. A new rootJournalClient replays // the retained request. While the receipt lookup fails it must report // JournalUncertainError, never a definite refusal; once the lookup works the // same request replays without a second effect. func TestPlan9JournalRestartedClientLostReplyLookupFailure(t *testing.T) { path := filepath.Join(t.TempDir(), "root.db") j, err := hostbox.OpenLaunchJournal(path) if err != nil { t.Fatal(err) } defer j.Close() s, err := hostbox.NewJournalService(j) if err != nil { t.Fatal(err) } root := ClaimRootHTTP{Journals: s} root.Reconcile.Box = "box" root.Reconcile.Enrollment = "enroll" peer := StopPeer{UID: 1001, PID: 17, ProcessStart: "23", BootID: "boot", ControllerID: "controller"} in := hostbox.LaunchInventory{Target: hostguard.FenceTarget{Host: "app.example", Incarnation: "inc", Stamp: 1, Session: 1, Box: "box", Enrollment: "enroll", BootID: "boot", ControllerID: "controller"}, ControllerPID: 17, ProcessStart: "23", NoState: true, Groups: []hostbox.LaunchGroup{{RelativePath: "session", Device: 1, Inode: 2}}} if err = j.Create(t.Context(), in); err != nil { t.Fatal(err) } if err = j.RecordProducer(t.Context(), in.Target, "issued"); err != nil { t.Fatal(err) } var loseReply atomic.Bool var requests atomic.Int32 handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1) if !loseReply.Load() { root.serveJournal(w, r, peer) return } // Commit, then drop the connection before any reply is written. root.serveJournal(httptest.NewRecorder(), r, peer) if conn, _, err := w.(http.Hijacker).Hijack(); err == nil { conn.Close() } }) countReceipts := func(db *sql.DB) int { var n int if err := db.QueryRow("SELECT COUNT(*) FROM journal_requests").Scan(&n); err != nil { t.Fatal(err) } return n } other, err := sql.Open("sqlite", "file:"+path+"?_pragma=busy_timeout(5000)") if err != nil { t.Fatal(err) } defer other.Close() // First client: the request commits and every reply is lost. loseReply.Store(true) first, _ := hostbox.RemoteJournals(plan9JournalSocket(t, handler)) var uncertain *hostbox.JournalUncertainError if _, err = first.CompleteProducer(t.Context(), in.Target, "issued", hostbox.ProducerCompletion{}); !errors.As(err, &uncertain) { t.Fatalf("lost reply was not uncertain: %v", err) } if n := countReceipts(other); n != 1 { t.Fatalf("request did not commit exactly once: %d", n) } // Restarted client: receipt lookup fails on the server. loseReply.Store(false) if _, err = other.Exec("ALTER TABLE journal_requests RENAME TO journal_requests_hidden"); err != nil { t.Fatal(err) } before := requests.Load() restarted, _ := hostbox.RemoteJournals(plan9JournalSocket(t, handler)) retained := uncertain _, err = restarted.ReplayUncertain(t.Context(), retained) if !errors.As(err, &uncertain) { t.Fatalf("failed lookup was not JournalUncertainError (definite refusal?): %v", err) } if requests.Load()-before != 2 { t.Fatalf("restarted client attempts = %d, want 2", requests.Load()-before) } // Lookup restored: another fresh client replays the stored result. if _, err = other.Exec("ALTER TABLE journal_requests_hidden RENAME TO journal_requests"); err != nil { t.Fatal(err) } again, _ := hostbox.RemoteJournals(plan9JournalSocket(t, handler)) if _, err = again.ReplayUncertain(t.Context(), retained); err != nil { t.Fatalf("replay after restore failed: %v", err) } if n := countReceipts(other); n != 1 { t.Fatalf("duplicate effect or receipt: %d", n) } } // HTTP classification of Replay errors: a proven mismatch is 409, an invalid // request is 400, and only ErrChanged maps to 409. func TestPlan9JournalReplayErrorStatuses(t *testing.T) { j, err := hostbox.OpenLaunchJournal(filepath.Join(t.TempDir(), "root.db")) if err != nil { t.Fatal(err) } defer j.Close() s, err := hostbox.NewJournalService(j) if err != nil { t.Fatal(err) } in := hostbox.LaunchInventory{Target: hostguard.FenceTarget{Host: "app.example", Incarnation: "inc", Stamp: 1, Session: 1, Box: "box", Enrollment: "enroll", BootID: "boot", ControllerID: "controller"}, ControllerPID: 17, ProcessStart: "23", NoState: true, Groups: []hostbox.LaunchGroup{{RelativePath: "session", Device: 1, Inode: 2}}} q := plan9CanonicalJournalRequest(t, hostbox.JournalRequest{V: 1, Op: "create", Launch: in}) if _, err = s.Execute(t.Context(), "box/enroll", q); err != nil { t.Fatal(err) } peer := StopPeer{UID: 1001, PID: 17, ProcessStart: "23", BootID: "boot", ControllerID: "controller"} serve := func(box string, q hostbox.JournalRequest) int { root := ClaimRootHTTP{Journals: s} root.Reconcile.Box = box root.Reconcile.Enrollment = "enroll" body, _ := json.Marshal(q) response := httptest.NewRecorder() root.serveJournal(response, httptest.NewRequest(http.MethodPost, "/claim/journal", bytes.NewReader(body)), peer) return response.Code } if got := serve("other", q); got != http.StatusConflict { t.Fatalf("other owner under stored ID = %d, want 409", got) } tampered := q tampered.Launch.ControllerPID++ if got := serve("box", tampered); got != http.StatusConflict { t.Fatalf("changed bytes under stored ID = %d, want 409", got) } invalid := q invalid.V = 0 if got := serve("box", invalid); got != http.StatusBadRequest { t.Fatalf("invalid request = %d, want 400", got) } }