package hostbox import ( "context" "errors" "reflect" "strings" "testing" "time" ) func parkRequest(t *testing.T) JournalRequest { t.Helper() p := ServingParkRequest{Successor: ServingArtifactIdentity{Host: "app.example", Box: "box", Region: "ie", Incarnation: "inc", Location: "instances/app.example", Generation: "G2", Stamp: 2, Session: 2}} p.Predecessor = p.Successor p.Predecessor.Stamp, p.Predecessor.Session, p.Predecessor.Generation = 1, 1, "G" return plan9Mutation(t, JournalRequest{Op: "serving-park", ServingPark: &p}) } func TestServingParkJournalPendingBeforeEffectAndReplay(t *testing.T) { s, path, _ := journalFixture(t) q := parkRequest(t) result := ServingParkResult{ID: strings.Repeat("a", 64), Successor: q.ServingPark.Successor} calls := 0 run := func(ctx context.Context) (ServingParkResult, error) { calls++ // This read would block if the callback held the write transaction. out, found, err := s.Replay(ctx, "peer", q) if err != nil || !found || out.Error != "uncertain" { t.Fatalf("effect without durable pending receipt: %+v %t %v", out, found, err) } if calls == 1 { return result, errors.New("lost retirement save reply") } return result, nil } if _, err := s.ExecuteServingPark(t.Context(), "peer", q, run); !errors.Is(err, ErrJournalUncertain) { t.Fatal(err) } if err := s.Launches.Close(); err != nil { t.Fatal(err) } j, err := OpenLaunchJournal(path) if err != nil { t.Fatal(err) } t.Cleanup(func() { j.Close() }) s, err = NewJournalService(j) if err != nil { t.Fatal(err) } out, err := s.ExecuteServingPark(t.Context(), "peer", q, run) if err != nil || out.ServingPark == nil || *out.ServingPark != result || calls != 2 { t.Fatalf("pending replay: %+v %v calls=%d", out, err, calls) } s.Now = func() time.Time { return time.Now().Add(48 * time.Hour) } replayed, err := s.ExecuteServingPark(t.Context(), "peer", q, run) if err != nil || !reflect.DeepEqual(replayed, out) || calls != 2 { t.Fatalf("completed replay executed again: %+v %v calls=%d", replayed, err, calls) } if _, err = s.ExecuteServingPark(t.Context(), "other-enrollment", q, run); !errors.Is(err, ErrChanged) || calls != 2 { t.Fatalf("foreign owner replay: %v", err) } } func TestServingParkJournalAgeAndPrunedReplay(t *testing.T) { for _, pending := range []bool{false, true} { t.Run(map[bool]string{false: "new", true: "pending"}[pending], func(t *testing.T) { s, _, _ := journalFixture(t) q := parkRequest(t) calls := 0 run := func(context.Context) (ServingParkResult, error) { calls++ return ServingParkResult{}, errors.New("partial rename") } if pending { _, _ = s.ExecuteServingPark(t.Context(), "peer", q, run) } want := calls for _, shift := range []time.Duration{-16 * time.Minute, 16 * time.Minute} { s.Now = func() time.Time { return time.Unix(0, q.IssuedAt).Add(shift) } if _, err := s.ExecuteServingPark(t.Context(), "peer", q, run); !errors.Is(err, ErrJournalUncertain) || calls != want { t.Fatalf("expired request ran: %v calls=%d", err, calls) } } }) } s, _, _ := journalFixture(t) q := parkRequest(t) result := ServingParkResult{ID: strings.Repeat("a", 64), Successor: q.ServingPark.Successor} run := func(context.Context) (ServingParkResult, error) { return result, nil } if _, err := s.ExecuteServingPark(t.Context(), "peer", q, run); err != nil { t.Fatal(err) } // A new request prunes the old receipt. Roll the clock back afterwards: // the persisted rejection boundary still prevents executing the old ID. now := time.Unix(0, q.IssuedAt).Add(48 * time.Hour) s.Now = func() time.Time { return now } newQ, err := journalPrepare(JournalRequest{Op: q.Op, ServingPark: q.ServingPark}, now) if err != nil { t.Fatal(err) } if _, err = s.ExecuteServingPark(t.Context(), "peer", newQ, run); err != nil { t.Fatal(err) } s.Now = func() time.Time { return time.Unix(0, q.IssuedAt) } if _, err = s.ExecuteServingPark(t.Context(), "peer", q, func(context.Context) (ServingParkResult, error) { t.Fatal("pruned request reran"); return result, nil }); !errors.Is(err, ErrJournalUncertain) { t.Fatal(err) } } func TestServingParkJournalRejectsMixedAndChangedSelectors(t *testing.T) { s, _, _ := journalFixture(t) for name, change := range map[string]func(*JournalRequest){ "version": func(q *JournalRequest) { q.V = 2 }, "unknown": func(q *JournalRequest) { q.Op = "serving-park-delete" }, "path-field": func(q *JournalRequest) { q.RecordKey = "/victim" }, "launch": func(q *JournalRequest) { q.Launch.NoState = true }, "target": func(q *JournalRequest) { q.Target.Host = "victim" }, "successor": func(q *JournalRequest) { q.ServingPark.Successor.Generation = "forged" }, "predecessor": func(q *JournalRequest) { q.ServingPark.Predecessor.Generation = "forged" }, "legacy": func(q *JournalRequest) { q.ServingPark.Legacy.Empty = true }, "retirement": func(q *JournalRequest) { q.ServingPark.Retirement = strings.Repeat("f", 64) }, "pressure": func(q *JournalRequest) { q.Op = "serving-park-pressure" }, } { t.Run(name, func(t *testing.T) { q := parkRequest(t) change(&q) if _, err := s.ExecuteServingPark(t.Context(), "peer", q, func(context.Context) (ServingParkResult, error) { t.Fatal("invalid request reached effects") return ServingParkResult{}, nil }); err == nil { t.Fatal("accepted invalid request") } }) } } func TestServingParkClientRetainsExactUncertainRequest(t *testing.T) { var sent JournalRequest client := ServingParkClient{Call: func(_ context.Context, q JournalRequest) (JournalResponse, error) { sent = q return JournalResponse{}, errors.New("lost reply") }} p := parkRequest(t).ServingPark _, err := client.Park(t.Context(), p.Successor, p.Predecessor) var uncertain *JournalUncertainError if !errors.As(err, &uncertain) || !reflect.DeepEqual(uncertain.Request, sent) || sent.V != 4 { t.Fatalf("lost request: %v", err) } client.Call = func(_ context.Context, q JournalRequest) (JournalResponse, error) { if !reflect.DeepEqual(q, sent) { t.Fatal("request changed on replay") } return JournalResponse{V: 1, ID: q.ID, Error: "refused"}, nil } if _, err = client.ReplayUncertain(t.Context(), uncertain); !errors.As(err, &uncertain) { t.Fatalf("late refusal became definite: %v", err) } client.Call = func(_ context.Context, q JournalRequest) (JournalResponse, error) { return JournalResponse{V: 1, ID: q.ID}, nil } if _, err = client.Park(t.Context(), p.Successor, p.Predecessor); !errors.As(err, &uncertain) { t.Fatalf("missing result lost request: %v", err) } } func TestServingParkJournalReceiptFailureKeepsUncertainty(t *testing.T) { for _, point := range []string{"before-effect", "after-effect"} { t.Run(point, func(t *testing.T) { s, _, _ := journalFixture(t) q := parkRequest(t) verb := "INSERT" if point == "after-effect" { verb = "UPDATE" } if _, err := s.Launches.db.Exec("CREATE TRIGGER fail_park_receipt BEFORE " + verb + " ON journal_requests BEGIN SELECT RAISE(ABORT, 'receipt unavailable'); END"); err != nil { t.Fatal(err) } calls := 0 run := func(context.Context) (ServingParkResult, error) { calls++ return ServingParkResult{ID: strings.Repeat("a", 64), Successor: q.ServingPark.Successor}, nil } _, err := s.ExecuteServingPark(t.Context(), "peer", q, run) if err == nil { t.Fatal("receipt failure succeeded") } if point == "before-effect" && calls != 0 { t.Fatal("effect without pending receipt") } if point == "after-effect" && (calls != 1 || !errors.Is(err, ErrJournalUncertain)) { t.Fatalf("lost outcome: %v calls=%d", err, calls) } if _, err = s.Launches.db.Exec("DROP TRIGGER fail_park_receipt"); err != nil { t.Fatal(err) } if _, err = s.ExecuteServingPark(t.Context(), "peer", q, run); err != nil { t.Fatal(err) } }) } }