//go:build unix package hostbox import ( "context" "crypto/ed25519" "encoding/json" "errors" "github.com/carlosframework/platform/internal/genselect" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/release" "testing" "time" ) type plan12LossOwner struct{ calls int } func (p *plan12LossOwner) ValidateLossRecovery(c hostclaim.Claim, l hostclaim.LossRecord, a hostclaim.Admission) error { p.calls++ if c.Op.Kind != hostclaim.KindRestoring || l.OpID != c.Op.ID || a.Box != "box" { return hostclaim.ErrLossRecord } return nil } func TestPlan12LossAdmission(t *testing.T) { plan12LossAdmissionCases(t, []string{"ok", "source-cut", "covered-source", "no-source", "clean-return", "sealed", "missing-owner", "missing-source", "wrong-source", "wrong-op", "wrong-signature", "stale", "restore-mismatch", "bypass"}) } func TestLossBeforeTargetAdmissionRecovers(t *testing.T) { plan12LossAdmissionCases(t, []string{"source-cut", "covered-source", "no-source"}) } func plan12LossAdmissionCases(t *testing.T, modes []string) { for _, mode := range modes { t.Run(mode, func(t *testing.T) { _, store, o := workFixture(t) var err error o, err = hostclaim.Update(t.Context(), store, o, func(c *hostclaim.Claim) error { c.Op = hostclaim.Op{Kind: hostclaim.KindRestoring, ID: "op", Phase: hostclaim.PhaseProbing, Exec: "box", Participants: []string{"box"}, Intent: map[string]string{"target": "target"}, Journal: map[string]string{"loss.loss_1": `{"kind":"move-target-lost","target":"target"}`}} c.Holder.Session, c.Holder.Generation, c.Holder.WriterBox = 2, "target-gen", "target" if mode == "source-cut" || mode == "covered-source" || mode == "no-source" { c.Holder.WriterBox = "box" } if mode == "no-source" { c.Holder.Session, c.Holder.Generation, c.Holder.WriterBox = 0, "", "" } c.LossRefs = []string{"loss_1"} return nil }) if err != nil { t.Fatal(err) } pub, priv, err := ed25519.GenerateKey(nil) if err != nil { t.Fatal(err) } now := time.Now().UTC() r := hostclaim.LossRecord{V: 1, ID: "loss_1", State: "prepared", Actor: "operator", AcceptedAt: now, Host: o.Host, HolderIncarnation: o.Claim.Holder.Incarnation, HolderStamp: o.Claim.Holder.Stamp, HolderSession: 2, ExpectedClaimGen: o.Claim.Gen, ExpectedClaimETag: o.ETag, OpID: "op", OpKind: hostclaim.KindLanding, OpPhase: "probing", Box: "target", Enrollment: "enroll", QuarantineRevision: 1, CloudObservation: hostclaim.LossCloudObservation{ObservedAt: now, Account: "123", Region: "eu-west-1", InstanceID: "i-1", State: "stopped", RequestID: "request"}, OperatorAttestation: hostclaim.LossOperatorAttestation{MaintainComputeFence: true, ColdQuarantineStartOnly: true}, FencedAt: now, ConfirmedConsequences: hostclaim.LossConsequences{RevokeExecution: true, RemoveParticipation: true, RecoveryIntent: json.RawMessage(`{"kind":"move-target-lost"}`)}, Timers: hostclaim.TimerLoss{Knowledge: hostclaim.TimersUnknown}, WriteSources: []hostclaim.WriteSource{{Reason: "target", Box: "target", Session: 2, Generation: "target-gen", Region: "uk", Baseline: "target-snapshot"}}} if mode == "source-cut" { r.WriteSources[0].Reason, r.WriteSources[0].Box = "source-cut", "box" } if mode == "no-source" || mode == "covered-source" { r.WriteSources = nil r.HolderSession = 0 } if mode == "wrong-op" { r.OpID = "different" } created, err := hostclaim.CreateLossRecord(t.Context(), store, release.RootSigner(priv), r) if err != nil { t.Fatal(err) } r.State, r.ResultClaimGen = "committed", o.Claim.Gen if _, err = hostclaim.ReplaceLossRecord(t.Context(), store, release.RootSigner(priv), created, r); err != nil { t.Fatal(err) } owner := &plan12LossOwner{} w := Writer{Claims: store, Box: "box", Region: "ie", AdmissionEvidence: func(_ context.Context, obs hostclaim.Observed, admission hostclaim.Admission, baseline *genselect.Plan) error { if baseline == nil || baseline.MaxTXID != 3 { t.Fatal("restore baseline changed", baseline) } want := uint64(7) if mode == "no-source" || mode == "covered-source" || mode == "wrong-source" || hostclaim.OperationLossConsumed(obs.Claim, "loss_1") { want = 3 } if admission.RemoteMaxTXID != want { t.Fatalf("consumer maximum=%d want=%d", admission.RemoteMaxTXID, want) } return nil }, Loss: &LossAdmission{PublicKey: pub, Owner: owner}} w.Loss.Sources = func(_ context.Context, _ hostclaim.Observed, r hostclaim.LossRecord, _ hostclaim.Admission, _ *genselect.Plan) ([]hostclaim.ValidatedWriteSource, error) { if mode == "no-source" || mode == "covered-source" { return nil, nil } f := r.WriteSources[0] v := hostclaim.ValidatedWriteSource{Reason: f.Reason, Box: f.Box, Session: f.Session, Generation: f.Generation, Region: f.Region, Baseline: f.Baseline, RemoteMaxTXID: 7} if mode == "wrong-source" { v.Session++ } return []hostclaim.ValidatedWriteSource{v}, nil } if mode == "missing-owner" { w.Loss.Owner = nil } if mode == "missing-source" { w.Loss.Sources = nil } if mode == "wrong-signature" { w.Loss.PublicKey, _, _ = ed25519.GenerateKey(nil) } in := hostclaim.Admission{ObservedOpID: "op", ObservedPhase: hostclaim.PhaseProbing, ObservedSession: 2, ObservedGeneration: "target-gen", Box: "box", Mode: "cold", NewGeneration: "returned", Region: "ie", PredecessorRegion: "uk", RemoteMaxTXID: 3} if mode == "no-source" { in.ObservedSession, in.ObservedGeneration = 0, "" } if mode == "clean-return" || mode == "sealed" { in.Coverage = mode } if mode == "stale" { _, err = hostclaim.Update(t.Context(), store, o, func(c *hostclaim.Claim) error { return nil }) if err != nil { t.Fatal(err) } } if mode == "restore-mismatch" { in.RemoteMaxTXID = 4 } var admitted hostclaim.Observed if mode == "bypass" { admitted, err = hostclaim.Update(t.Context(), store, o, hostclaim.AdmitWriter(in)) } else { admitted, err = w.Admit(t.Context(), o, in, restorePlan(o.Claim.Holder.Location, "", 3)) } if mode != "ok" && mode != "source-cut" && mode != "covered-source" && mode != "no-source" { if err == nil { t.Fatal("accepted", mode) } if mode == "stale" && !errors.Is(err, hostclaim.ErrConflict) { t.Fatal(err) } return } if err != nil { t.Fatal(err) } a := admitted.Claim.Holder.Admissions[0] if a.LossID != "loss_1" || a.Coverage != "" || (mode != "no-source" && mode != "covered-source" && (a.WritesLost == nil || a.WritesLost.Session != 2 || a.WritesLost.RemoteMaxTXID != 7)) || owner.calls != 1 { t.Fatalf("%+v owner=%d", a, owner.calls) } if mode == "covered-source" && (a.WritesLost == nil || a.WritesLost.RemoteMaxTXID != 3) { t.Fatalf("restore maximum lost: %+v", a) } if mode == "no-source" && a.WritesLost != nil { t.Fatalf("invented predecessor: %+v", a) } // Restart loses the writer object, but not the claim's consumption marker. restarted := Writer{Claims: store, Box: "box", Region: "ie", AdmissionEvidence: w.AdmissionEvidence} fresh, err := hostclaim.Read(t.Context(), store, o.Host) if err != nil { t.Fatal(err) } in.ObservedSession, in.ObservedGeneration, in.NewGeneration = fresh.Claim.Holder.Session, fresh.Claim.Holder.Generation, "second-cold" in.PredecessorRegion = "ie" in.LossID = "loss_1" again, err := restarted.Admit(t.Context(), fresh, in, &genselect.Plan{MaxTXID: 3}) if err != nil { t.Fatal("second cold admission after restart", err) } if again.Claim.Holder.Admissions[0].LossID != "loss_1" || again.Claim.Holder.Admissions[len(again.Claim.Holder.Admissions)-1].LossID != "" { t.Fatal("loss consumed again") } }) } }