package hostclaim import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "reflect" "slices" "strconv" "strings" "testing" "time" "github.com/carlosframework/platform/internal/boxid" "github.com/carlosframework/platform/internal/release" "github.com/carlosframework/platform/internal/termfence" ) const ( relSource = "adi-knitting" relTarget = "bri-knitting" relSpare = "cai-knitting" relSpare2 = "dov-knitting" ) var relObserved = time.Date(2026, 10, 7, 9, 0, 0, 0, time.UTC) func relEnrollment(box string) string { return strings.Repeat(string(box[0]), 32) } // relAuthority accepts any fence record for a box in this region. type relAuthority struct{} func (relAuthority) ValidateFence(_ context.Context, f termfence.Record) (FenceBinding, error) { if f.Enrollment != relEnrollment(f.Box) { return FenceBinding{}, ErrFence } return FenceBinding{Region: "ie", RosterRevision: 9, RosterETag: "roster"}, nil } type relReleases struct{ err error } func (r relReleases) VerifyRelease(context.Context, Claim, Release) error { return r.err } func putFence(t *testing.T, s release.Store, box string) FenceRef { t.Helper() f := termfence.Record{V: 1, Box: box, Enrollment: relEnrollment(box), State: "terminated", Cloud: boxid.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-" + box}, ObservedAt: relObserved.Format(time.RFC3339Nano), Via: "describe", RequestID: "req", WrittenBy: termfence.WrittenBy{Box: "worker", Enrollment: strings.Repeat("e", 32)}} raw, err := json.Marshal(f) if err != nil { t.Fatal(err) } sum := sha256.Sum256(raw) ref := FenceRef{Enrollment: f.Enrollment, Key: "control/box-fences/" + f.Enrollment + ".json", SHA256: hex.EncodeToString(sum[:])} if _, err := s.Put(context.Background(), ref.Key, raw, ""); err != nil && !errors.Is(err, release.ErrCASConflict) { t.Fatal(err) } return ref } // relServing is a serving claim on relSource whose source has admitted one // cold writer session (generation g1). func relServing(t *testing.T) (release.Store, Observed) { t.Helper() ctx := context.Background() s := release.NewDirStore(t.TempDir()) o := servingClaim(t, ctx, s) o, err := Update(ctx, s, o, AdmitWriter(Admission{Box: relSource, Mode: "cold", NewGeneration: "g1", Region: "ie", KeepStamp: true})) if err != nil { t.Fatal(err) } return s, o } func relStart(reason string, fence *FenceRef) RelocationStart { return RelocationStart{OpID: "rel-1", By: "worker", Now: relObserved, Region: "ie", Reason: reason, SourceBox: relSource, SourceEnrollment: relEnrollment(relSource), TargetBox: relTarget, TargetEnrollment: relEnrollment(relTarget), DirectoryRevision: 9, AdmissionID: "adm-1", Fence: fence} } func step(t *testing.T, s release.Store, o Observed, tr func(*Claim) error) Observed { t.Helper() next, err := Update(context.Background(), s, o, tr) if err != nil { t.Fatal(err) } return next } func ack(box string) func(*Claim) error { return AcknowledgeExecution(box, ExecutionAcknowledgement{Op: "rel-1", Enrollment: relEnrollment(box), BootID: "boot", ControllerID: "ctl", LeaseETag: "lease", Expires: relObserved.Add(time.Hour)}) } // relRelease is the app release the test pins name (R9). const relRelease = "v1.4.2" // relPin is a pin selected against o: its key names o's claim generation. func relPin(o Observed, maxTXID uint64, seq int64, at time.Time) RecoveryPin { return RecoveryPin{Key: RecoveryPinKey(o.Claim.Host, "rel-1", o.Claim.Gen), SHA256: strings.Repeat("a", 64), Generation: "g1", MaxTXID: maxTXID, FilesManifest: "instances/x/files/g1/manifest.json", FilesSeq: seq, PinnedAt: at, Empty: maxTXID == 0, Release: relRelease} } func relPrepared(gen string, maxTXID uint64, filesSeq int64) PreparedAuthority { a := PreparedAuthority{Generation: gen, SnapshotTXID: maxTXID, FilesSeq: 0, SourceFilesSeq: filesSeq, FilesManifestSHA256: strings.Repeat("f", 64)} if maxTXID > 0 { a.InstallManifest, a.SHA256 = "m", strings.Repeat("c", 64) } return a } // journalN is journalUint for a journal the test's own transitions wrote. func journalN(op Op, key string) uint64 { n, err := journalUint(op, key) if err != nil { panic(err) } return n } func localInstall(c *Claim) LocalInstall { return LocalInstall{Box: c.Op.Exec, Enrollment: relEnrollment(c.Op.Exec), InstallStamp: journalN(c.Op, "install_stamp"), FilesManifestVersion: 3, JournalSHA256: strings.Repeat("b", 64)} } // toProbing drives an operation at pinning (with its pin) through // installing, installed and the local install receipt into probing. func toProbing(t *testing.T, s release.Store, o Observed, prepared string, maxTXID uint64) Observed { t.Helper() target := Target{Box: o.Claim.Op.Intent["target_box"], Enrollment: o.Claim.Op.Intent["target_enrollment"]} o = step(t, s, o, ToInstalling("rel-1", target, prepared)) var pin RecoveryPin readJournal(o.Claim.Op, "pin", &pin) o = step(t, s, o, ToInstalled("rel-1", relPrepared(prepared, maxTXID, pin.FilesSeq))) o = step(t, s, o, RecordLocalInstall("rel-1", localInstall(o.Claim))) return step(t, s, o, ToProbing("rel-1", relObserved)) } func TestRelocationDrainEndToEnd(t *testing.T) { ctx := context.Background() s, o := relServing(t) o, err := StartRelocation(ctx, s, o, relStart(ReasonDrain, nil), relAuthority{}) if err != nil { t.Fatal(err) } if c := o.Claim; c.Op.Exec != relSource || c.Op.Phase != PhaseStarting || !slices.Equal(c.Op.Participants, []string{relSource, relTarget}) || c.Op.Intent["stamp_from"] == "" { t.Fatalf("start: %+v", c.Op) } if _, err := Update(ctx, s, o, ToReleasing("rel-1", relObserved)); !errors.Is(err, ErrRelocation) { t.Fatalf("barrier without acknowledgements: %v", err) } o = step(t, s, o, ack(relSource)) o = step(t, s, o, ack(relTarget)) o = step(t, s, o, ToReleasing("rel-1", relObserved)) if o.Claim.Op.Journal["releasing_since"] == "" { t.Fatal("releasing did not start its deadline") } h := o.Claim.Holder rel := Release{ReleasedBy: relSource, Enrollment: relEnrollment(relSource), Generation: h.Generation, Session: h.Session, FinalTXID: 40, ManifestSeq: 7, Receipt: "control/stop/r.json", ReceiptSHA256: strings.Repeat("d", 64)} o = step(t, s, o, JournalRelease("rel-1", rel)) if _, err := PinAfterRelease(ctx, s, o, "rel-1", relReleases{err: errors.New("bad receipt")}, nil); err == nil { t.Fatal("pinned on an unverified receipt") } if o, err = PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, nil); err != nil { t.Fatal(err) } stamp := o.Claim.Holder.Stamp if o.Claim.Op.Exec != "none" || o.Claim.Op.Phase != PhasePinning { t.Fatalf("pinning: %+v", o.Claim.Op) } if _, err := Update(ctx, s, o, JournalPin("rel-1", relPin(o, 39, 7, relObserved))); !errors.Is(err, ErrRelocation) { t.Fatalf("pin short of the release: %v", err) } stale := relPin(o, 40, 7, relObserved) stale.Key = RecoveryPinKey(o.Claim.Host, "rel-1", o.Claim.Gen-1) if _, err := Update(ctx, s, o, JournalPin("rel-1", stale)); !errors.Is(err, ErrRelocation) { t.Fatalf("a pin selected against an earlier observation was journaled: %v", err) } o = step(t, s, o, JournalPin("rel-1", relPin(o, 40, 7, relObserved))) if o.Claim.Holder.Stamp != stamp { t.Fatal("pinning changed the stamp") } o = step(t, s, o, ToInstalling("rel-1", Target{relTarget, relEnrollment(relTarget)}, "P1")) for name, a := range map[string]PreparedAuthority{ "no files manifest": {Generation: "P1", InstallManifest: "m", SHA256: strings.Repeat("c", 64), SnapshotTXID: 40}, "another files seq": {Generation: "P1", InstallManifest: "m", SHA256: strings.Repeat("c", 64), SnapshotTXID: 40, FilesSeq: 6, FilesManifestSHA256: strings.Repeat("f", 64)}, "no install manifest": {Generation: "P1", SnapshotTXID: 40, FilesSeq: 7, FilesManifestSHA256: strings.Repeat("f", 64)}, "another snapshot txid": {Generation: "P1", InstallManifest: "m", SHA256: strings.Repeat("c", 64), SnapshotTXID: 41, FilesSeq: 7, FilesManifestSHA256: strings.Repeat("f", 64)}, } { if _, err := Update(ctx, s, o, ToInstalled("rel-1", a)); !errors.Is(err, ErrRelocation) { t.Fatalf("%s: installed on an incomplete build: %v", name, err) } } o = step(t, s, o, ToInstalled("rel-1", relPrepared("P1", 40, 7))) o = step(t, s, o, RecordLocalInstall("rel-1", localInstall(o.Claim))) o = step(t, s, o, ToProbing("rel-1", relObserved)) install := journalN(o.Claim.Op, "install_stamp") if o.Claim.Op.Exec != relTarget || o.Claim.Holder.Stamp != install || !slices.Contains(o.Claim.Touched.Boxes, relTarget) { t.Fatalf("installation: exec %s stamp %d install %d", o.Claim.Op.Exec, o.Claim.Holder.Stamp, install) } o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseRelocatingProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: relTarget, Mode: "cold", NewGeneration: "P1", KeepStamp: true, PredecessorCertified: true, Region: "ie", PredecessorRegion: "ie"})) last := o.Claim.Holder.Admissions[len(o.Claim.Holder.Admissions)-1] if last.Coverage != "" || last.WritesLost != nil || o.Claim.Holder.Stamp != install { t.Fatalf("certified admission: %+v stamp %d", last, o.Claim.Holder.Stamp) } o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relTarget, Session: o.Claim.Holder.Session})) o = step(t, s, o, RecordProbeStop("rel-1", ProbeStop{Box: relTarget, Session: o.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)})) o = step(t, s, o, CompleteRelocation("rel-1", counter())) c := o.Claim if c.Op.Kind != KindServing || c.Op.ID != "" || c.Op.Exec != relTarget || !slices.Equal(c.Op.Participants, []string{relTarget}) { t.Fatalf("serving: %+v", c.Op) } if len(c.Obligations) != 1 || c.Obligations[0].Kind != "park-local" || c.Obligations[0].Box != relSource || c.Obligations[0].Op != "rel-1" { t.Fatalf("the source's park-local: %+v", c.Obligations) } } func TestRelocationFailoverCitesTheFenceAndDeclaresTheLoss(t *testing.T) { ctx := context.Background() for _, qualified := range []bool{false, true} { s, o := relServing(t) o = step(t, s, o, func(c *Claim) error { // a chore the dead source already owed c.Obligations = append(c.Obligations, Obligation{ID: "old", Op: "old-op", Kind: "park-local", Box: relSource, Retirement: "r", Owner: "box:" + relSource}) return nil }) ref := putFence(t, s, relSource) sourceSession := o.Claim.Holder.Session in := relStart(ReasonFailover, &ref) in.Discharge = &LocalDischarge{Qualified: qualified, Records: []string{"control/box-volumes/x/1.json"}} o, err := StartRelocation(ctx, s, o, in, relAuthority{}) if err != nil { t.Fatal(err) } c := o.Claim if c.Op.Exec != "none" || !slices.Equal(c.Op.Participants, []string{relTarget}) || !slices.Contains(c.Lost, relSource) || c.Op.Journal["fence."+relSource] == "" { t.Fatalf("failover start: %+v lost %v", c.Op, c.Lost) } if owes := slices.ContainsFunc(c.Obligations, func(o Obligation) bool { return o.Box == relSource }); owes == qualified { t.Fatalf("discharge qualified=%v left chores %+v", qualified, c.Obligations) } if _, err := Update(ctx, s, o, ToReleasing("rel-1", relObserved)); err == nil { t.Fatal("a failover entered releasing") } o = step(t, s, o, ack(relTarget)) o = step(t, s, o, PinFailover("rel-1")) if _, err := Update(ctx, s, o, JournalPin("rel-1", relPin(o, 40, 7, relObserved.Add(29*time.Second)))); !errors.Is(err, ErrRelocation) { t.Fatalf("pinned inside the 30-second window: %v", err) } o = step(t, s, o, JournalPin("rel-1", relPin(o, 40, 7, relObserved.Add(FailoverPinDelay)))) o = toProbing(t, s, o, "P1", 40) probe := Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseRelocatingProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: relTarget, Mode: "cold", NewGeneration: "P1", KeepStamp: true, PredecessorCertified: true, Region: "ie", PredecessorRegion: "ie"} if _, err := Update(ctx, s, o, AdmitWriter(probe)); !errors.Is(err, ErrWrongState) { t.Fatalf("a failover admission certified its predecessor without a release: %v", err) } probe.PredecessorCertified = false o = step(t, s, o, AdmitWriter(probe)) last := o.Claim.Holder.Admissions[len(o.Claim.Holder.Admissions)-1] if last.WritesLost == nil || last.WritesLost.Box != relSource || last.WritesLost.Generation != "g1" { t.Fatalf("failover must declare the source's session: %+v", last) } o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relTarget, Session: o.Claim.Holder.Session})) o = step(t, s, o, RecordProbeStop("rel-1", ProbeStop{Box: relTarget, Session: o.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)})) o = step(t, s, o, CompleteRelocation("rel-1", counter())) owes := slices.ContainsFunc(o.Claim.Obligations, func(ob Obligation) bool { return ob.Kind == "park-local" && ob.Box == relSource }) if owes == qualified { t.Fatalf("qualified=%v: source chores at serving %+v", qualified, o.Claim.Obligations) } // D4: the loss outlives the operation, with the console still to // materialize it (nothing here stands in for the console). c = o.Claim if len(c.EvidenceRefs) != 1 || len(c.LossRefs) != 1 || c.EvidenceRefs[0].ID != c.LossRefs[0] { t.Fatalf("retained losses after serving: %+v refs %v", c.EvidenceRefs, c.LossRefs) } l := c.EvidenceRefs[0] if l.Box != relSource || l.Fence != ref || l.ObservedAt != relObserved.Format(time.RFC3339Nano) || l.Op != "rel-1" || l.Role != "source" || l.Session != sourceSession || l.Generation != "g1" || l.Discharge.Qualified != qualified || l.Timers != TimersUnknown || l.Reported { t.Fatalf("retained loss: %+v", l) } done, ok, _ := ReadCompletedRelocation(c) if len(c.Op.Journal) != 1 || !ok || done.Op != "rel-1" || done.Stop.Session != c.Holder.Session || done.Phase != PhaseRelocatingProbing || !slices.Contains(c.Lost, relSource) { t.Fatalf("serving kept the journal, or lost the box: %+v %v", c.Op, c.Lost) } } } func TestRelocationSourceLossBecomesFailover(t *testing.T) { ctx := context.Background() for _, tc := range []struct { name string prepare func(*testing.T, release.Store, Observed) Observed wantPhase string wantExec string wantParts []string }{ {"starting before acknowledging", func(t *testing.T, s release.Store, o Observed) Observed { return o }, PhaseStarting, "none", []string{relTarget}}, {"starting after acknowledging", func(t *testing.T, s release.Store, o Observed) Observed { return step(t, s, o, ack(relSource)) }, PhaseStarting, "none", []string{relTarget}}, {"releasing before its receipt", func(t *testing.T, s release.Store, o Observed) Observed { o = step(t, s, step(t, s, o, ack(relSource)), ack(relTarget)) return step(t, s, o, ToReleasing("rel-1", relObserved)) }, PhasePinning, "none", []string{relTarget}}, {"releasing after its receipt", func(t *testing.T, s release.Store, o Observed) Observed { o = step(t, s, step(t, s, o, ack(relSource)), ack(relTarget)) o = step(t, s, o, ToReleasing("rel-1", relObserved)) h := o.Claim.Holder return step(t, s, o, JournalRelease("rel-1", Release{ReleasedBy: relSource, Enrollment: relEnrollment(relSource), Generation: h.Generation, Session: h.Session, FinalTXID: 1, Receipt: "r", ReceiptSHA256: strings.Repeat("d", 64)})) }, PhaseReleasing, relSource, []string{relSource, relTarget}}, } { t.Run(tc.name, func(t *testing.T) { s, o := relServing(t) o, err := StartRelocation(ctx, s, o, relStart(ReasonDrain, nil), relAuthority{}) if err != nil { t.Fatal(err) } o = tc.prepare(t, s, o) stamp := o.Claim.Holder.Stamp o, err = FenceSource(ctx, s, o, "rel-1", putFence(t, s, relSource), nil, relAuthority{}) if err != nil { t.Fatal(err) } c := o.Claim if c.Op.Phase != tc.wantPhase || c.Op.Exec != tc.wantExec || !slices.Equal(c.Op.Participants, tc.wantParts) || c.Holder.Stamp != stamp || !slices.Contains(c.Lost, relSource) { t.Fatalf("phase %s exec %s parts %v stamp %d lost %v", c.Op.Phase, c.Op.Exec, c.Op.Participants, c.Holder.Stamp, c.Lost) } if c.Op.Intent["reason"] != ReasonDrain { t.Fatal("the intent was rewritten") } }) } } // relAtPhase starts a drain and drives it to phase. "probing+P" is probing // after the first admission opened P. func relAtPhase(t *testing.T, phase string) (release.Store, Observed) { t.Helper() ctx := context.Background() s, o := relServing(t) o, err := StartRelocation(ctx, s, o, relStart(ReasonDrain, nil), relAuthority{}) if err != nil { t.Fatal(err) } o = step(t, s, step(t, s, o, ack(relSource)), ack(relTarget)) if phase == PhaseStarting { return s, o } o = step(t, s, o, ToReleasing("rel-1", relObserved)) if phase == PhaseReleasing { return s, o } h := o.Claim.Holder o = step(t, s, o, JournalRelease("rel-1", Release{ReleasedBy: relSource, Enrollment: relEnrollment(relSource), Generation: h.Generation, Session: h.Session, FinalTXID: 40, ManifestSeq: 7, Receipt: "r", ReceiptSHA256: strings.Repeat("d", 64)})) if o, err = PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, nil); err != nil { t.Fatal(err) } o = step(t, s, o, JournalPin("rel-1", relPin(o, 40, 7, relObserved))) if phase == PhasePinning { return s, o } o = step(t, s, o, ToInstalling("rel-1", Target{relTarget, relEnrollment(relTarget)}, "P1")) if phase == PhaseInstalling { return s, o } o = step(t, s, o, ToInstalled("rel-1", relPrepared("P1", 40, 7))) if phase == PhaseInstalled { return s, o } o = step(t, s, o, RecordLocalInstall("rel-1", localInstall(o.Claim))) o = step(t, s, o, ToProbing("rel-1", relObserved)) if phase == PhaseRelocatingProbing { return s, o } o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseRelocatingProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: relTarget, Mode: "cold", NewGeneration: "P1", KeepStamp: true, PredecessorCertified: true, Region: "ie", PredecessorRegion: "ie"})) return s, o } func TestRelocationReplacementTable(t *testing.T) { ctx := context.Background() for _, tc := range []struct { at, wantPhase, wantExec string newStamp, keepP bool }{ {PhaseStarting, PhaseStarting, relSource, false, false}, {PhaseReleasing, PhaseReleasing, relSource, false, false}, {PhasePinning, PhasePinning, "none", false, false}, {PhaseInstalling, PhaseInstalling, relSpare, true, false}, {PhaseInstalled, PhaseInstalled, relSpare, true, true}, {PhaseRelocatingProbing, PhaseInstalled, relSpare, true, true}, {"probing+P", PhasePinning, "none", false, false}, } { t.Run(tc.at, func(t *testing.T) { s, o := relAtPhase(t, tc.at) before := o.Claim.Holder.Stamp o, err := ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, relTarget), nil, Target{relSpare, relEnrollment(relSpare)}, "P2", relAuthority{}) if err != nil { t.Fatal(err) } c := o.Claim if c.Op.Phase != tc.wantPhase || c.Op.Exec != tc.wantExec || c.Op.Intent["target_box"] != relSpare || !slices.Contains(c.Lost, relTarget) || slices.Contains(c.Op.Participants, relTarget) { t.Fatalf("phase %s exec %s intent %v lost %v parts %v", c.Op.Phase, c.Op.Exec, c.Op.Intent, c.Lost, c.Op.Participants) } if (c.Holder.Stamp != before) != tc.newStamp { t.Fatalf("stamp %d -> %d", before, c.Holder.Stamp) } if tc.keepP && c.Op.Journal["prepared"] != "P1" { t.Fatalf("the unopened P must be reused: %q", c.Op.Journal["prepared"]) } if c.Op.Journal["local_install"] != "" { t.Fatal("a replacement inherited its predecessor's receipt") } if tc.wantPhase == PhaseInstalled { if _, err := Update(ctx, s, o, ToProbing("rel-1", relObserved)); err == nil { t.Fatal("probing without the replacement's own local install") } } }) } } func TestRelocationTwoReplacementsThenStuck(t *testing.T) { ctx := context.Background() s, o := relAtPhase(t, PhasePinning) var err error for _, next := range []string{relSpare, relSpare2} { old := o.Claim.Op.Intent["target_box"] if o, err = ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, old), nil, Target{next, relEnrollment(next)}, "", relAuthority{}); err != nil { t.Fatal(err) } } if _, err = ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, relSpare2), nil, Target{"eve-knitting", relEnrollment("eve-knitting")}, "", relAuthority{}); !errors.Is(err, ErrReplacementsExceeded) { t.Fatalf("third replacement: %v", err) } o = step(t, s, o, MarkStuck("rel-1", "two target replacements used")) if journalN(o.Claim.Op, "replacements") != 2 { t.Fatal("stuck reset the count") } if _, err := Update(ctx, s, o, ToInstalling("rel-1", Target{relSpare2, relEnrollment(relSpare2)}, "P9")); !errors.Is(err, ErrRelocation) { t.Fatalf("a stuck operation moved: %v", err) } } // Final review: a malformed replacements count is an error, never 0 // (absence or malformed data is never evidence). Absent is 0. func TestRelocationMalformedReplacementsRefuses(t *testing.T) { ctx := context.Background() s, o := relAtPhase(t, PhasePinning) if n, err := Replacements(o.Claim); err != nil || n != 0 { t.Fatalf("absent: %d %v", n, err) } o = step(t, s, o, func(c *Claim) error { c.Op.Journal["replacements"] = "two"; return nil }) if _, err := Replacements(o.Claim); !errors.Is(err, ErrRelocation) { t.Fatalf("malformed count: %v", err) } if _, err := ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, relTarget), nil, Target{relSpare, relEnrollment(relSpare)}, "", relAuthority{}); !errors.Is(err, ErrRelocation) || errors.Is(err, ErrReplacementsExceeded) { t.Fatalf("a replacement over a malformed count: %v", err) } after, err := Read(ctx, s, o.Host) if err != nil { t.Fatal(err) } if after.ETag != o.ETag || after.Claim.Op.Intent["target_box"] != relTarget { t.Fatalf("the claim moved: %+v", after.Claim.Op) } } func TestRelocationReplacementInReleasingAcknowledgesBeforePinning(t *testing.T) { ctx := context.Background() s, o := relAtPhase(t, PhaseReleasing) h := o.Claim.Holder o = step(t, s, o, JournalRelease("rel-1", Release{ReleasedBy: relSource, Enrollment: relEnrollment(relSource), Generation: h.Generation, Session: h.Session, FinalTXID: 40, ManifestSeq: 7, Receipt: "r", ReceiptSHA256: strings.Repeat("d", 64)})) o, err := ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, relTarget), nil, Target{relSpare, relEnrollment(relSpare)}, "", relAuthority{}) if err != nil || o.Claim.Op.Phase != PhaseReleasing { t.Fatalf("replacement in releasing: %v %+v", err, o.Claim.Op) } if _, err := PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, nil); !errors.Is(err, ErrRelocation) { t.Fatalf("left releasing before the replacement acknowledged: %v", err) } o = step(t, s, o, ack(relSpare)) if o, err = PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, nil); err != nil || o.Claim.Op.Phase != PhasePinning { t.Fatalf("after the acknowledgement: %v %+v", err, o.Claim.Op) } } func TestRelocationExhaustedTargetIsFencedIntoAStuckPinning(t *testing.T) { ctx := context.Background() exhaust := func(t *testing.T, s release.Store, o Observed) Observed { var err error for _, next := range []string{relSpare, relSpare2} { old := o.Claim.Op.Intent["target_box"] if o, err = ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, old), nil, Target{next, relEnrollment(next)}, "", relAuthority{}); err != nil { t.Fatal(err) } } return o } t.Run("before exhaustion it is a replacement", func(t *testing.T) { s, o := relAtPhase(t, PhasePinning) if _, err := FenceExhaustedTarget(ctx, s, o, "rel-1", putFence(t, s, relTarget), nil, relAuthority{}); !errors.Is(err, ErrRelocation) { t.Fatalf("fenced as exhausted with replacements left: %v", err) } }) // Round 8 finding 1: while the live source is still op.exec, the // target's fence consumes the target and records stuck; the source's // execution identity, phase and release evidence stay. for _, phase := range []string{PhaseStarting, PhaseReleasing} { t.Run("dead while the source executes in "+phase, func(t *testing.T) { s, o := relAtPhase(t, phase) if phase == PhaseReleasing { h := o.Claim.Holder o = step(t, s, o, JournalRelease("rel-1", Release{ReleasedBy: relSource, Enrollment: relEnrollment(relSource), Generation: h.Generation, Session: h.Session, FinalTXID: 40, ManifestSeq: 7, Receipt: "r", ReceiptSHA256: strings.Repeat("d", 64)})) } o = exhaust(t, s, o) before := o.Claim ref := putFence(t, s, relSpare2) o, err := FenceExhaustedTarget(ctx, s, o, "rel-1", ref, nil, relAuthority{}) if err != nil { t.Fatal(err) } c := o.Claim if c.Op.Phase != phase || c.Op.Exec != relSource || c.Holder.Stamp != before.Holder.Stamp || c.Op.Journal["stuck"] == "" || !slices.Contains(c.Op.Participants, relSource) || slices.Contains(c.Op.Participants, relSpare2) || !slices.Contains(c.Lost, relSpare2) { t.Fatalf("the live source was revoked: %+v lost %v", c.Op, c.Lost) } for _, k := range []string{"release", "release.phase", "releasing_since", "exec_ack." + relSource} { if c.Op.Journal[k] != before.Op.Journal[k] { t.Fatalf("%s changed: %q -> %q", k, before.Op.Journal[k], c.Op.Journal[k]) } } if phase == PhaseReleasing && c.Op.Journal["release"] == "" { t.Fatal("the source's release evidence was dropped") } if l := c.EvidenceRefs[len(c.EvidenceRefs)-1]; l.Box != relSpare2 || l.Fence != ref { t.Fatalf("the dead target's loss: %+v", c.EvidenceRefs) } if err := c.Validate(); err != nil { t.Fatal(err) } }) } t.Run("dead after its local install, P unopened", func(t *testing.T) { s, o := relAtPhase(t, PhasePinning) o = exhaust(t, s, o) o = step(t, s, o, ack(relSpare2)) o = step(t, s, o, ToInstalling("rel-1", Target{relSpare2, relEnrollment(relSpare2)}, "P1")) o = step(t, s, o, ToInstalled("rel-1", relPrepared("P1", 40, 7))) o = step(t, s, o, RecordLocalInstall("rel-1", localInstall(o.Claim))) if _, err := Update(ctx, s, o, MarkStuck("rel-1", "x")); !errors.Is(err, ErrRelocation) { t.Fatalf("MarkStuck between installed and a probe: %v", err) } ref := putFence(t, s, relSpare2) o, err := FenceExhaustedTarget(ctx, s, o, "rel-1", ref, nil, relAuthority{}) if err != nil { t.Fatal(err) } c := o.Claim if c.Op.Phase != PhasePinning || c.Op.Exec != "none" || c.Op.Journal["stuck"] == "" || slices.Contains(c.Op.Participants, relSpare2) || !slices.Contains(c.Lost, relSpare2) || c.Op.Journal["local_install"] != "" || c.Op.Journal["pin"] == "" || c.Op.Journal["prepared"] != "P1" || c.Op.Journal["prepared_authority"] == "" { t.Fatalf("exhausted and dead: %+v lost %v", c.Op, c.Lost) } if len(c.EvidenceRefs) != 3 || c.EvidenceRefs[2].Box != relSpare2 || c.EvidenceRefs[2].Fence != ref { t.Fatalf("the dead target's loss: %+v", c.EvidenceRefs) } if _, err := Update(ctx, s, o, ToInstalling("rel-1", Target{relSpare2, relEnrollment(relSpare2)}, "P2")); !errors.Is(err, ErrRelocation) { t.Fatalf("a stuck operation moved: %v", err) } if _, err := FenceExhaustedTarget(ctx, s, o, "rel-1", ref, nil, relAuthority{}); err == nil { t.Fatal("the fence was consumed twice") } }) t.Run("dead after P was opened", func(t *testing.T) { s, o := relAtPhase(t, "probing+P") o = exhaust(t, s, o) // after P's admission each replacement is a nested pinning o = step(t, s, o, ack(relSpare2)) o = step(t, s, o, JournalPin("rel-1", relPin(o, 60, 9, relObserved.Add(FailoverPinDelay)))) o = step(t, s, o, ToInstalling("rel-1", Target{relSpare2, relEnrollment(relSpare2)}, "P3")) o = step(t, s, o, ToInstalled("rel-1", relPrepared("P3", 60, 9))) o = step(t, s, o, RecordLocalInstall("rel-1", localInstall(o.Claim))) o = step(t, s, o, ToProbing("rel-1", relObserved)) o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseRelocatingProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: relSpare2, Mode: "cold", NewGeneration: "P3", KeepStamp: true, Region: "ie", PredecessorRegion: "ie"})) o, err := FenceExhaustedTarget(ctx, s, o, "rel-1", putFence(t, s, relSpare2), nil, relAuthority{}) if err != nil { t.Fatal(err) } c := o.Claim if c.Op.Phase != PhasePinning || c.Op.Exec != "none" || c.Op.Journal["stuck"] == "" || c.Op.Journal["pin"] != "" || c.Op.Journal["prepared"] != "" { t.Fatalf("nested stuck pinning: %+v", c.Op) } }) } func TestRelocationProbeFailureIsANestedRelocation(t *testing.T) { ctx := context.Background() s, o := relAtPhase(t, "probing+P") o = step(t, s, o, ToProbeFailed("rel-1", relObserved)) if o.Claim.Op.Journal["probe_failed_since"] == "" { t.Fatal("probe-failed did not start its release deadline") } h := o.Claim.Holder rel := Release{ReleasedBy: relTarget, Enrollment: relEnrollment(relTarget), Generation: h.Generation, Session: h.Session, FinalTXID: 55, ManifestSeq: 9, Receipt: "r2", ReceiptSHA256: strings.Repeat("d", 64)} o = step(t, s, o, JournalRelease("rel-1", rel)) if _, err := PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, &Target{relTarget, relEnrollment(relTarget)}); err == nil { t.Fatal("the failed target was named again") } o, err := PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, &Target{relSpare, relEnrollment(relSpare)}) if err != nil { t.Fatal(err) } if o.Claim.Op.Phase != PhasePinning || o.Claim.Op.Exec != "none" { t.Fatalf("nested relocation: %+v", o.Claim.Op) } if _, err := Update(ctx, s, o, JournalPin("rel-1", relPin(o, 54, 9, relObserved))); !errors.Is(err, ErrRelocation) { t.Fatalf("pin lost the probe's writes: %v", err) } o = step(t, s, o, JournalPin("rel-1", relPin(o, 55, 9, relObserved))) o = step(t, s, o, ack(relSpare)) o = toProbing(t, s, o, "P2", 55) // released coverage on P2's opening: the predecessor is the failed // target's probe session, whose release this operation journaled. o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseRelocatingProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: relSpare, Mode: "cold", NewGeneration: "P2", KeepStamp: true, PredecessorCertified: true, Region: "ie", PredecessorRegion: "ie"})) o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relSpare, Session: o.Claim.Holder.Session})) o = step(t, s, o, RecordProbeStop("rel-1", ProbeStop{Box: relSpare, Session: o.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)})) o = step(t, s, o, CompleteRelocation("rel-1", counter())) var owed []string for _, ob := range o.Claim.Obligations { owed = append(owed, ob.Box) } slices.Sort(owed) if !slices.Equal(owed, []string{relSource, relTarget}) { t.Fatalf("park-local owed by the source and the abandoned target: %v", owed) } } func TestRelocationAdmissionRules(t *testing.T) { ctx := context.Background() probe := func(o Observed, gen string, certified bool) Admission { return Admission{ObservedOpID: "rel-1", ObservedPhase: o.Claim.Op.Phase, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: gen, PredecessorCertified: certified, Region: "ie", PredecessorRegion: "ie"} } for _, tc := range []struct { name, at string admit func(Observed) Admission }{ {"the target admits in installing", PhaseInstalling, func(o Observed) Admission { return probe(o, "P1", false) }}, {"the target admits in installed", PhaseInstalled, func(o Observed) Admission { return probe(o, "P1", false) }}, {"the first probe skips P", PhaseRelocatingProbing, func(o Observed) Admission { return probe(o, "other", false) }}, {"P is opened twice", "probing+P", func(o Observed) Admission { return probe(o, "P1", false) }}, {"certified after P was opened", "probing+P", func(o Observed) Admission { return probe(o, "P3", true) }}, {"released coverage (claim Plan 11 grants none)", PhaseRelocatingProbing, func(o Observed) Admission { a := probe(o, "P1", false) a.Coverage = "released" return a }}, {"warm probe", PhaseRelocatingProbing, func(o Observed) Admission { a := probe(o, "", false) a.Mode, a.PredecessorCertified = "warm", true return a }}, {"the source certifies in starting", PhaseStarting, func(o Observed) Admission { return probe(o, "g2", true) }}, // Claim Plan 11 R1 as accepted: the source's ordinary admission in // starting stays refused until its admission-fence authority exists. {"the source wakes in starting", PhaseStarting, func(o Observed) Admission { return probe(o, "g2", false) }}, } { t.Run(tc.name, func(t *testing.T) { s, o := relAtPhase(t, tc.at) if _, err := Update(ctx, s, o, AdmitWriter(tc.admit(o))); !errors.Is(err, ErrWrongState) { t.Fatalf("admitted: %v", err) } }) } t.Run("certified names another predecessor", func(t *testing.T) { s, o := relAtPhase(t, PhaseRelocatingProbing) o.Claim.Holder.Admissions[len(o.Claim.Holder.Admissions)-1].Session++ if _, err := Update(ctx, s, o, AdmitWriter(probe(o, "P1", true))); !errors.Is(err, ErrWrongState) { t.Fatalf("admitted: %v", err) } }) t.Run("certified names another predecessor box", func(t *testing.T) { s, o := relAtPhase(t, PhaseRelocatingProbing) o.Claim.Holder.Admissions[len(o.Claim.Holder.Admissions)-1].Box = relSpare if _, err := Update(ctx, s, o, AdmitWriter(probe(o, "P1", true))); !errors.Is(err, ErrWrongState) { t.Fatalf("admitted: %v", err) } }) t.Run("a third attempt reopens P", func(t *testing.T) { s, o := relAtPhase(t, "probing+P") o = step(t, s, o, AdmitWriter(probe(o, "P3", false))) // attempt two: a fresh generation if _, err := Update(ctx, s, o, AdmitWriter(probe(o, "P1", false))); !errors.Is(err, ErrWrongState) { t.Fatalf("P was opened again: %v", err) } }) t.Run("released outside relocating", func(t *testing.T) { s, o := relServing(t) if _, err := Update(ctx, s, o, AdmitWriter(Admission{ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: relSource, Mode: "cold", NewGeneration: "g2", Coverage: "released", Region: "ie"})); !errors.Is(err, ErrWrongState) { t.Fatalf("admitted: %v", err) } }) t.Run("a stuck executor does not probe", func(t *testing.T) { // The same probe opens P on an operation that is not stuck. s, o := relAtPhase(t, PhaseRelocatingProbing) if _, err := Update(ctx, s, o, AdmitWriter(probe(o, "P1", false))); err != nil { t.Fatalf("the probe on a live operation: %v", err) } s, o = relAtPhase(t, PhaseRelocatingProbing) o = step(t, s, o, MarkStuck("rel-1", "operator")) if _, err := Update(ctx, s, o, AdmitWriter(probe(o, "P1", false))); !errors.Is(err, ErrWrongState) { t.Fatalf("admitted: %v", err) } }) t.Run("with no P journaled a probe certifies nothing", func(t *testing.T) { s, o := relAtPhase(t, PhaseRelocatingProbing) o = step(t, s, o, func(c *Claim) error { delete(c.Op.Journal, "prepared"); return nil }) if _, err := Update(ctx, s, o, AdmitWriter(probe(o, "P9", true))); !errors.Is(err, ErrWrongState) { t.Fatalf("a certified probe with no P was admitted: %v", err) } if _, err := Update(ctx, s, o, AdmitWriter(probe(o, "P9", false))); err != nil { t.Fatalf("main's uncertified cold probe: %v", err) } }) } func TestRelocationEmptyPinNeedsItsEvidence(t *testing.T) { ctx := context.Background() t.Run("a release at txid 0 vouches", func(t *testing.T) { s, o := relAtPhase(t, PhaseReleasing) h := o.Claim.Holder o = step(t, s, o, JournalRelease("rel-1", Release{ReleasedBy: relSource, Enrollment: relEnrollment(relSource), Generation: h.Generation, Session: h.Session, FinalTXID: 0, ManifestSeq: 0, Receipt: "r", ReceiptSHA256: strings.Repeat("d", 64)})) o, err := PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, nil) if err != nil { t.Fatal(err) } o = step(t, s, o, JournalPin("rel-1", relPin(o, 0, 0, relObserved))) o = step(t, s, o, ToInstalling("rel-1", Target{relTarget, relEnrollment(relTarget)}, "P1")) if _, err := Update(ctx, s, o, ToInstalled("rel-1", relPrepared("P1", 40, 0))); !errors.Is(err, ErrRelocation) { t.Fatalf("an empty pin accepted a database build: %v", err) } if _, err := Update(ctx, s, o, ToInstalled("rel-1", PreparedAuthority{Generation: "P1", FilesSeq: 0})); !errors.Is(err, ErrRelocation) { t.Fatalf("an empty P without its files manifest: %v", err) } step(t, s, o, ToInstalled("rel-1", relPrepared("P1", 0, 0))) }) t.Run("a release at txid 40 does not", func(t *testing.T) { s, o := relAtPhase(t, PhaseReleasing) h := o.Claim.Holder o = step(t, s, o, JournalRelease("rel-1", Release{ReleasedBy: relSource, Enrollment: relEnrollment(relSource), Generation: h.Generation, Session: h.Session, FinalTXID: 40, Receipt: "r", ReceiptSHA256: strings.Repeat("d", 64)})) o, err := PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, nil) if err != nil { t.Fatal(err) } if _, err := Update(ctx, s, o, JournalPin("rel-1", relPin(o, 0, 0, relObserved))); !errors.Is(err, ErrRelocation) { t.Fatalf("an empty pin against a release at 40: %v", err) } }) t.Run("a failover of a host that had a writer does not", func(t *testing.T) { s, o := relServing(t) ref := putFence(t, s, relSource) o, err := StartRelocation(ctx, s, o, relStart(ReasonFailover, &ref), relAuthority{}) if err != nil { t.Fatal(err) } o = step(t, s, step(t, s, o, ack(relTarget)), PinFailover("rel-1")) if _, err := Update(ctx, s, o, JournalPin("rel-1", relPin(o, 0, 0, relObserved.Add(FailoverPinDelay)))); !errors.Is(err, ErrRelocation) { t.Fatalf("a listing that found nothing was taken as empty: %v", err) } }) t.Run("a failover of a never-woken host does", func(t *testing.T) { s := release.NewDirStore(t.TempDir()) o := servingClaim(t, ctx, s) // session 0: no writer was ever admitted ref := putFence(t, s, relSource) o, err := StartRelocation(ctx, s, o, relStart(ReasonFailover, &ref), relAuthority{}) if err != nil { t.Fatal(err) } o = step(t, s, step(t, s, o, ack(relTarget)), PinFailover("rel-1")) step(t, s, o, JournalPin("rel-1", relPin(o, 0, 0, relObserved.Add(FailoverPinDelay)))) }) } func TestRelocationProbeAttemptsAreBounded(t *testing.T) { ctx := context.Background() s, o := relAtPhase(t, PhaseRelocatingProbing) for i := 0; i < MaxProbeAttempts; i++ { o = step(t, s, o, BeginProbeAttempt("rel-1", relTarget)) } if journalN(o.Claim.Op, "probe_attempts."+relTarget) != MaxProbeAttempts { t.Fatalf("count: %v", o.Claim.Op.Journal) } if _, err := Update(ctx, s, o, BeginProbeAttempt("rel-1", relTarget)); !errors.Is(err, ErrProbeAttemptsExhausted) { t.Fatalf("a fourth attempt: %v", err) } if _, err := Update(ctx, s, o, BeginProbeAttempt("rel-1", relSpare)); !errors.Is(err, ErrRelocation) { t.Fatalf("a non-executor counted an attempt: %v", err) } // A replacement target has its own count. o, err := ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, relTarget), nil, Target{relSpare, relEnrollment(relSpare)}, "", relAuthority{}) if err != nil { t.Fatal(err) } if journalN(o.Claim.Op, "probe_attempts."+relSpare) != 0 { t.Fatal("the replacement inherited attempts") } } func TestRelocationDischargeCompletesDiscoveryLikeARetirement(t *testing.T) { ctx := context.Background() s, o := relServing(t) o = step(t, s, o, func(c *Claim) error { c.Touched.DiscoveryOp = "disc" c.Obligations = append(c.Obligations, Obligation{ID: "d1", Op: "disc", Kind: "park-local", Box: relSource, Retirement: "r", Owner: "box:" + relSource}) return nil }) ref := putFence(t, s, relSource) in := relStart(ReasonFailover, &ref) in.Discharge = &LocalDischarge{Qualified: true, Records: []string{"control/box-volumes/x/1.json"}} o, err := StartRelocation(ctx, s, o, in, relAuthority{}) if err != nil { t.Fatal(err) } if len(o.Claim.Obligations) != 0 || !o.Claim.Touched.Complete || o.Claim.Touched.DiscoveryOp != "" { t.Fatalf("discharging the final discovery chore: %+v touched %+v", o.Claim.Obligations, o.Claim.Touched) } } func TestRelocationLossRetainsAFormerWritersSession(t *testing.T) { // The source is fenced after the target's probe admission made the // target the writer: the loss still names the source's last session. ctx := context.Background() s, o := relAtPhase(t, "probing+P") before := o.Claim.Holder var sourceAdmission AdmissionRecord for _, a := range before.Admissions { if a.Box == relSource { sourceAdmission = a } } o, err := FenceSource(ctx, s, o, "rel-1", putFence(t, s, relSource), nil, relAuthority{}) if err != nil { t.Fatal(err) } l := o.Claim.EvidenceRefs[0] if l.Box != relSource || l.Session != sourceAdmission.Session || l.Generation != sourceAdmission.Generation || l.LastAdmission == nil || !reflect.DeepEqual(*l.LastAdmission, sourceAdmission) || l.Incarnation != before.Incarnation || l.Stamp != before.Stamp { t.Fatalf("retained: %+v, source admission %+v", l, sourceAdmission) } if err := o.Claim.Validate(); err != nil { t.Fatal(err) } } func TestRelocationRetainsEveryFencedBoxAcrossCompletion(t *testing.T) { // A drain whose target is terminated in installing: the replacement // completes, and the dead target's loss is still on the claim. ctx := context.Background() s, o := relAtPhase(t, PhaseInstalling) ref := putFence(t, s, relTarget) o, err := ReplaceTarget(ctx, s, o, "rel-1", ref, nil, Target{relSpare, relEnrollment(relSpare)}, "P2", relAuthority{}) if err != nil { t.Fatal(err) } o = step(t, s, o, ack(relSpare)) o = step(t, s, o, ToInstalled("rel-1", relPrepared("P2", 40, 7))) o = step(t, s, o, RecordLocalInstall("rel-1", localInstall(o.Claim))) o = step(t, s, o, ToProbing("rel-1", relObserved)) o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseRelocatingProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: relSpare, Mode: "cold", NewGeneration: "P2", KeepStamp: true, PredecessorCertified: true, Region: "ie", PredecessorRegion: "ie"})) o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relSpare, Session: o.Claim.Holder.Session})) o = step(t, s, o, RecordProbeStop("rel-1", ProbeStop{Box: relSpare, Session: o.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)})) o = step(t, s, o, CompleteRelocation("rel-1", counter())) c := o.Claim if len(c.EvidenceRefs) != 1 || c.EvidenceRefs[0].Box != relTarget || c.EvidenceRefs[0].Role != "target" || c.EvidenceRefs[0].Session != 0 || !slices.Contains(c.LossRefs, c.EvidenceRefs[0].ID) { t.Fatalf("the abandoned target's loss: %+v refs %v", c.EvidenceRefs, c.LossRefs) } if err := c.Validate(); err != nil { t.Fatal(err) } } func TestRelocationStartGuard(t *testing.T) { serving := func(obs ...Obligation) *Claim { return &Claim{Holder: &Holder{Incarnation: "i"}, Op: Op{Kind: KindServing, Exec: relSource}, Obligations: obs} } for _, tc := range []struct { name string ob Obligation fenced bool want error }{ {"park-generation in the region (L11)", Obligation{Kind: "park-generation", Region: "ie"}, false, nil}, {"park-replica in the region", Obligation{Kind: "park-replica", Region: "ie"}, false, ErrPendingChore}, {"park-replica elsewhere", Obligation{Kind: "park-replica", Region: "uk"}, false, nil}, {"park-local on the target", Obligation{Kind: "park-local", Box: relTarget}, true, ErrPendingChore}, {"park-local on a live source", Obligation{Kind: "park-local", Box: relSource}, false, ErrPendingChore}, {"park-local on a fenced source", Obligation{Kind: "park-local", Box: relSource}, true, nil}, } { t.Run(tc.name, func(t *testing.T) { if err := RelocationStartGuard(serving(tc.ob), "", "ie", relSource, relTarget, tc.fenced); !errors.Is(err, tc.want) { t.Fatalf("got %v, want %v", err, tc.want) } }) } } func TestRelocationLocalInstallReceiptIsBound(t *testing.T) { ctx := context.Background() for _, tc := range []struct { name string mutate func(*LocalInstall) }{ {"another box", func(l *LocalInstall) { l.Box = relSpare }}, {"another enrollment", func(l *LocalInstall) { l.Enrollment = relEnrollment(relSpare) }}, {"a later writer stamp", func(l *LocalInstall) { l.InstallStamp++ }}, } { t.Run(tc.name, func(t *testing.T) { s, o := relAtPhase(t, PhaseInstalled) l := localInstall(o.Claim) tc.mutate(&l) if _, err := Update(ctx, s, o, RecordLocalInstall("rel-1", l)); !errors.Is(err, ErrRelocation) { t.Fatalf("recorded: %v", err) } }) } s, o := relAtPhase(t, PhaseInstalled) l := localInstall(o.Claim) o = step(t, s, o, RecordLocalInstall("rel-1", l)) if _, err := Update(ctx, s, o, RecordLocalInstall("rel-1", l)); !errors.Is(err, ErrAlreadyRecorded) { t.Fatalf("replay: %v", err) } if _, err := Update(ctx, s, o, MarkStuck("rel-1", "operator")); !errors.Is(err, ErrRelocation) { t.Fatalf("stuck between installed and a probe: %v", err) } // B1-2: probing is refused on another box's receipt, even one in the journal. foreign := l foreign.Box = relSpare raw, _ := json.Marshal(foreign) o.Claim.Op.Journal["local_install"] = string(raw) if _, err := Update(ctx, s, o, ToProbing("rel-1", relObserved)); !errors.Is(err, ErrRelocation) { t.Fatalf("probing on another box's receipt: %v", err) } } // Round 5 finding 1: a spare that still owes a local retirement for the // host is never named a replacement target, whatever phase the // replacement resumes at (installing and installed hand it op.exec in the // same CAS); a clean spare is. func TestRelocationReplacementRefusesABoxOwingALocalRetirement(t *testing.T) { ctx := context.Background() for _, phase := range []string{PhaseStarting, PhaseInstalling, PhaseInstalled} { t.Run(phase, func(t *testing.T) { s, o := relAtPhase(t, phase) o = step(t, s, o, func(c *Claim) error { c.Obligations = append(c.Obligations, Obligation{ID: "old", Op: "old-op", Kind: "park-local", Box: relSpare, Retirement: "r", Owner: "box:" + relSpare}) return nil }) ref := putFence(t, s, relTarget) if _, err := ReplaceTarget(ctx, s, o, "rel-1", ref, nil, Target{relSpare, relEnrollment(relSpare)}, "P2", relAuthority{}); !errors.Is(err, ErrPendingChore) { t.Fatalf("a box owing park-local was named: %v", err) } if _, err := Update(ctx, s, o, NameTarget("rel-1", Target{relSpare, relEnrollment(relSpare)})); err == nil { t.Fatal("NameTarget named a box owing park-local") } o, err := ReplaceTarget(ctx, s, o, "rel-1", ref, nil, Target{relSpare2, relEnrollment(relSpare2)}, "P2", relAuthority{}) if err != nil { t.Fatal(err) } c := o.Claim if c.Op.Intent["target_box"] != relSpare2 || slices.Contains(c.Op.Participants, relSpare) || slices.Contains(c.Touched.Boxes, relSpare) { t.Fatalf("the clean spare was not chosen: %+v touched %v", c.Op, c.Touched.Boxes) } if phase != PhaseStarting && c.Op.Exec != relSpare2 { t.Fatalf("exec %s", c.Op.Exec) } }) } } // Round 5 finding 2: the automatic fence passes the dead box's regional // work (park-generation, park-replica owned by box:) to its region in // the same CAS, as DeclareLost does for a declared loss, keeping every // other field; another box's work is untouched; local discharge, qualified // or not, changes nothing about it. func TestRelocationFenceReassignsTheDeadBoxsRegionalWork(t *testing.T) { ctx := context.Background() gen := Obligation{ID: "pg", Op: "old-op", Kind: "park-generation", Region: "ie", Generation: "g0", Retirement: "rg", Owner: "box:" + relSource} rep := Obligation{ID: "pr", Op: "old-op", Kind: "park-replica", Region: "uk", Retirement: "rr", Owner: "box:" + relSource} other := Obligation{ID: "po", Op: "old-op", Kind: "park-generation", Region: "ie", Generation: "g0b", Retirement: "ro", Owner: "box:" + relSpare} local := Obligation{ID: "pl", Op: "old-op", Kind: "park-local", Box: relSource, Retirement: "rl", Owner: "box:" + relSource} for _, qualified := range []bool{false, true} { s, o := relServing(t) o = step(t, s, o, func(c *Claim) error { c.Obligations = append(c.Obligations, gen, rep, other, local) return nil }) ref := putFence(t, s, relSource) in := relStart(ReasonFailover, &ref) in.Discharge = &LocalDischarge{Qualified: qualified, Records: []string{"control/box-volumes/x/1.json"}} o, err := StartRelocation(ctx, s, o, in, relAuthority{}) if err != nil { t.Fatal(err) } byID := map[string]Obligation{} for _, ob := range o.Claim.Obligations { byID[ob.ID] = ob } wantGen, wantRep := gen, rep wantGen.Owner, wantRep.Owner = "region:ie", "region:uk" if !reflect.DeepEqual(byID["pg"], wantGen) || !reflect.DeepEqual(byID["pr"], wantRep) || !reflect.DeepEqual(byID["po"], other) { t.Fatalf("qualified=%v: regional work %+v", qualified, o.Claim.Obligations) } if _, owes := byID["pl"]; owes == qualified { t.Fatalf("qualified=%v: the local chore %+v", qualified, o.Claim.Obligations) } want := []ObligationRef{{ID: "pg", Op: "old-op", Retirement: "rg"}, {ID: "pr", Op: "old-op", Retirement: "rr"}} if len(o.Claim.EvidenceRefs) != 1 || !reflect.DeepEqual(o.Claim.EvidenceRefs[0].Reassigned, want) { t.Fatalf("qualified=%v: the retained loss does not name the reassignment: %+v", qualified, o.Claim.EvidenceRefs) } if err := o.Claim.Validate(); err != nil { t.Fatal(err) } } } // Round 11's sweep: P's opening is journaled with the admission that made // it, so it survives AdmitWriter's trim of holder.admissions to eight. The // removal below stands in for that trim (generation.go drops a covered, // loss-archived head once more than eight are kept); with P's admission // gone from the list, P is still opened: never reopened, and the local // install is never remade. func TestRelocationPOpenedOutlivesTheAdmissionTrim(t *testing.T) { ctx := context.Background() s, o := relAtPhase(t, "probing+P") if o.Claim.Op.Journal["prepared_opened"] != "P1" { t.Fatalf("the opening was not journaled: %+v", o.Claim.Op.Journal) } o = step(t, s, o, func(c *Claim) error { c.Holder.Admissions = slices.DeleteFunc(c.Holder.Admissions, func(a AdmissionRecord) bool { return a.Generation == "P1" }) return nil }) if !PreparedOpened(o.Claim) { t.Fatal("P forgotten with its admission") } again := Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseRelocatingProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: relTarget, Mode: "cold", NewGeneration: "P1", KeepStamp: true, Region: "ie", PredecessorRegion: "ie"} if _, err := Update(ctx, s, o, AdmitWriter(again)); !errors.Is(err, ErrWrongState) { t.Fatalf("P reopened after the trim: %v", err) } if _, err := Update(ctx, s, o, RecordLocalInstall("rel-1", localInstall(o.Claim))); err == nil { t.Fatal("the local install was remade after P was opened") } } // Round 14 finding 1: P's prepared manifest is at sequence 0 whatever the // source's; the pin's sequence (7 here) is its provenance. ToInstalled // refuses a prepared manifest claimed at the source's sequence, and one // built from another source revision, and binds the one it accepts to // the installation stamp. func TestRelocationPreparedManifestIsSequenceZero(t *testing.T) { ctx := context.Background() s, o := relAtPhase(t, PhaseInstalling) var pin RecoveryPin readJournal(o.Claim.Op, "pin", &pin) if pin.FilesSeq != 7 { t.Fatalf("pin at %d", pin.FilesSeq) } atSource := relPrepared("P1", 40, 7) atSource.FilesSeq = 7 if _, err := Update(ctx, s, o, ToInstalled("rel-1", atSource)); !errors.Is(err, ErrRelocation) { t.Fatalf("a prepared manifest at the source's sequence: %v", err) } if _, err := Update(ctx, s, o, ToInstalled("rel-1", relPrepared("P1", 40, 6))); !errors.Is(err, ErrRelocation) { t.Fatalf("a prepared manifest from another source revision: %v", err) } o = step(t, s, o, ToInstalled("rel-1", relPrepared("P1", 40, 7))) var a PreparedAuthority readJournal(o.Claim.Op, "prepared_authority", &a) if a.FilesSeq != 0 || a.SourceFilesSeq != 7 || a.InstallStamp == 0 || a.InstallStamp != journalN(o.Claim.Op, "install_stamp") { t.Fatalf("journaled authority: %+v", a) } } // Two-box model finding F3: a target replaced in installing leaves its P // unopened; the replacement builds P'. The serving CAS queues a // park-generation for P in the region, and none for the opened P'. func TestRelocationParksEveryAbandonedPreparedGeneration(t *testing.T) { ctx := context.Background() s, o := relAtPhase(t, PhaseInstalling) o, err := ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, relTarget), nil, Target{relSpare, relEnrollment(relSpare)}, "P2", relAuthority{}) if err != nil { t.Fatal(err) } var abandoned []string if !readJournal(o.Claim.Op, "abandoned_prepared", &abandoned) || !slices.Equal(abandoned, []string{"P1"}) { t.Fatalf("abandoned: %v", abandoned) } o = step(t, s, o, ack(relSpare)) o = step(t, s, o, ToInstalled("rel-1", relPrepared("P2", 40, 7))) o = step(t, s, o, RecordLocalInstall("rel-1", localInstall(o.Claim))) o = step(t, s, o, ToProbing("rel-1", relObserved)) o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseRelocatingProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: relSpare, Mode: "cold", NewGeneration: "P2", KeepStamp: true, PredecessorCertified: true, Region: "ie", PredecessorRegion: "ie"})) o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relSpare, Session: o.Claim.Holder.Session})) o = step(t, s, o, RecordProbeStop("rel-1", ProbeStop{Box: relSpare, Session: o.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)})) o = step(t, s, o, CompleteRelocation("rel-1", counter())) var parks []string for _, ob := range o.Claim.Obligations { if ob.Kind == "park-generation" && ob.Region == "ie" && ob.Owner == "region:ie" { parks = append(parks, ob.Generation) } } if !slices.Contains(parks, "P1") || slices.Contains(parks, "P2") { t.Fatalf("park-generation chores: %v (%+v)", parks, o.Claim.Obligations) } if err := o.Claim.Validate(); err != nil { t.Fatal(err) } } // Round 17 finding 3: the target's probe deadline passes before any // admission (P1 unopened), it releases in probe-failed, then it is // terminated; the replacement's default branch drops P1. It is recorded // first, so the completion of the nested relocation parks it. func TestRelocationParksAPDroppedFromProbeFailed(t *testing.T) { ctx := context.Background() s, o := relAtPhase(t, PhaseRelocatingProbing) if PreparedOpened(o.Claim) { t.Fatal("P1 was opened") } o = step(t, s, o, ToProbeFailed("rel-1", relObserved)) o, err := ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, relTarget), nil, Target{relSpare, relEnrollment(relSpare)}, "", relAuthority{}) if err != nil { t.Fatal(err) } if o.Claim.Op.Phase != PhasePinning || o.Claim.Op.Journal["prepared"] != "" { t.Fatalf("nested: %+v", o.Claim.Op) } var abandoned []string if !readJournal(o.Claim.Op, "abandoned_prepared", &abandoned) || !slices.Equal(abandoned, []string{"P1"}) { t.Fatalf("abandoned: %v", abandoned) } o = step(t, s, o, ack(relSpare)) o = step(t, s, o, JournalPin("rel-1", relPin(o, 40, 7, relObserved.Add(FailoverPinDelay)))) o = toProbing(t, s, o, "P2", 40) o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseRelocatingProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: relSpare, Mode: "cold", NewGeneration: "P2", KeepStamp: true, Region: "ie", PredecessorRegion: "ie"})) o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relSpare, Session: o.Claim.Holder.Session})) o = step(t, s, o, RecordProbeStop("rel-1", ProbeStop{Box: relSpare, Session: o.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)})) o = step(t, s, o, CompleteRelocation("rel-1", counter())) parked := false for _, ob := range o.Claim.Obligations { if ob.Kind == "park-generation" && ob.Generation == "P1" && ob.Region == "ie" { parked = true } } if !parked { t.Fatalf("P1 was not parked: %+v", o.Claim.Obligations) } } // Fix round 1: a move converted to relocating journals the source's fence // without its observation time. The claim refuses a pin until FenceSource // has consumed that fence, and then holds it FailoverPinDelay past the // fence's observed_at (H3), as it does for a failover. func TestRelocationConvertedMovePinsOnlyAfterFenceSourceAndTheHold(t *testing.T) { ctx := context.Background() s, o, ref, a := c7Fixture(t) o, err := ConvertMovingToRelocating(ctx, s, o, "move-1", ref, a, &c7Guard{}) if err != nil { t.Fatal(err) } observed, err := time.Parse(time.RFC3339Nano, a.want.ObservedAt) if err != nil { t.Fatal(err) } pin := func(o Observed, at time.Time) RecoveryPin { return RecoveryPin{Key: RecoveryPinKey(o.Claim.Host, "move-1", o.Claim.Gen), SHA256: strings.Repeat("a", 64), Generation: "gen-current", MaxTXID: 40, FilesManifest: "instances/x/files/gen-current/manifest.json", FilesSeq: 7, PinnedAt: at, Release: relRelease} } for _, late := range []time.Duration{time.Second, time.Hour} { if _, err := Update(ctx, s, o, JournalPin("move-1", pin(o, observed.Add(late)))); !errors.Is(err, ErrRelocation) { t.Fatalf("pinned %s after a fence FenceSource never consumed: %v", late, err) } } if o, err = FenceSource(ctx, s, o, "move-1", ref, nil, a); err != nil { t.Fatal(err) } if o.Claim.Op.Journal["fence.adi-knitting.observed_at"] != a.want.ObservedAt || !slices.Contains(o.Claim.Lost, "adi-knitting") { t.Fatalf("FenceSource on the converted move: %+v lost %v", o.Claim.Op.Journal, o.Claim.Lost) } if _, err := Update(ctx, s, o, JournalPin("move-1", pin(o, observed.Add(FailoverPinDelay-time.Second)))); !errors.Is(err, ErrRelocation) { t.Fatalf("pinned inside the 30-second window: %v", err) } o = step(t, s, o, JournalPin("move-1", pin(o, observed.Add(FailoverPinDelay)))) if !readJournal(o.Claim.Op, "pin", new(RecoveryPin)) { t.Fatalf("pin not journaled: %+v", o.Claim.Op.Journal) } } // Task 5 fix round 1: a converted move's own target (in the region the move // was going to) leaves the barrier when NameTarget names the same-region // target, in the same CAS. Otherwise a move target that never acknowledged, // or died, holds the relocation in pinning for ever: nothing in this // region can consume its fence (moves §7.10, §5.8's failover participants). func TestRelocationNameTargetDropsTheConvertedMovesTarget(t *testing.T) { ctx := context.Background() s, o, ref, a := c7Fixture(t) o, err := ConvertMovingToRelocating(ctx, s, o, "move-1", ref, a, &c7Guard{}) if err != nil { t.Fatal(err) } if !slices.Equal(o.Claim.Op.Participants, []string{"bri-knitting"}) || o.Claim.Op.Intent["move_target_box"] != "bri-knitting" { t.Fatalf("not the converted shape this test is about: %+v", o.Claim.Op) } gen := o.Claim.Gen o = step(t, s, o, NameTarget("move-1", Target{Box: "cai-knitting", Enrollment: strings.Repeat("e", 32)})) if !slices.Equal(o.Claim.Op.Participants, []string{"cai-knitting"}) || o.Claim.Op.Intent["target_box"] != "cai-knitting" || o.Claim.Gen != gen+1 { t.Fatalf("the move's target is still on the barrier: %+v", o.Claim.Op) } if o.Claim.Op.Intent["move_target_box"] != "bri-knitting" || o.Claim.Op.Journal["replacements"] != "" { t.Fatalf("the move's history or the replacement count changed: %+v", o.Claim.Op) } } // R9 (amadan #236): install_release is the journaled pin's release, written // in the installing CAS beside install_stamp, copied again by a re-targeted // installation, and never present without install_stamp. func TestRelocationInstallReleaseFollowsThePin(t *testing.T) { ctx := context.Background() pair := func(t *testing.T, c *Claim) { t.Helper() _, stamped := c.Op.Journal["install_stamp"] _, released := c.Op.Journal["install_release"] if stamped != released { t.Fatalf("install_stamp present %v, install_release present %v", stamped, released) } } t.Run("the pin needs a release", func(t *testing.T) { s, o := relAtPhase(t, PhaseReleasing) h := o.Claim.Holder o = step(t, s, o, JournalRelease("rel-1", Release{ReleasedBy: relSource, Enrollment: relEnrollment(relSource), Generation: h.Generation, Session: h.Session, FinalTXID: 40, ManifestSeq: 7, Receipt: "r", ReceiptSHA256: strings.Repeat("d", 64)})) o, err := PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, nil) if err != nil { t.Fatal(err) } bare := relPin(o, 40, 7, relObserved) bare.Release = "" if _, err := Update(ctx, s, o, JournalPin("rel-1", bare)); !errors.Is(err, ErrRelocation) { t.Fatalf("a pin without a release was journaled: %v", err) } o = step(t, s, o, JournalPin("rel-1", relPin(o, 40, 7, relObserved))) pair(t, o.Claim) // A journaled pin that lost its release (malformed) installs nothing. broken := o broken.Claim = cloneForTest(t, o.Claim) broken.Claim.Op.Journal["pin"] = strings.Replace(broken.Claim.Op.Journal["pin"], relRelease, "", 1) if _, err := Update(ctx, s, broken, ToInstalling("rel-1", Target{relTarget, relEnrollment(relTarget)}, "P1")); !errors.Is(err, ErrRelocation) { t.Fatalf("installed without the pin's release: %v", err) } }) t.Run("installing copies it", func(t *testing.T) { _, o := relAtPhase(t, PhaseInstalling) c := o.Claim if c.Op.Journal["install_release"] != relRelease || c.Op.Journal["install_stamp"] != strconv.FormatUint(c.Gen, 10) { t.Fatalf("install_release %q install_stamp %q gen %d", c.Op.Journal["install_release"], c.Op.Journal["install_stamp"], c.Gen) } }) for _, at := range []string{PhaseInstalling, PhaseInstalled, PhaseRelocatingProbing} { t.Run("a re-targeted installation from "+at+" keeps it", func(t *testing.T) { s, o := relAtPhase(t, at) stamp := o.Claim.Op.Journal["install_stamp"] o, err := ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, relTarget), nil, Target{relSpare, relEnrollment(relSpare)}, "P2", relAuthority{}) if err != nil { t.Fatal(err) } c := o.Claim if c.Op.Exec != relSpare || c.Op.Journal["install_stamp"] == stamp || c.Op.Journal["install_release"] != relRelease { t.Fatalf("exec %s install_stamp %q (was %q) install_release %q", c.Op.Exec, c.Op.Journal["install_stamp"], stamp, c.Op.Journal["install_release"]) } pair(t, c) }) } t.Run("a nested pin's release replaces both together", func(t *testing.T) { s, o := relAtPhase(t, "probing+P") o, err := ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, relTarget), nil, Target{relSpare, relEnrollment(relSpare)}, "", relAuthority{}) if err != nil { t.Fatal(err) } // Nested pinning: the pin is gone, the last installation's pair is // kept together (op.exec is none, so nothing can stage under it). if o.Claim.Op.Phase != PhasePinning || o.Claim.Op.Exec != "none" || o.Claim.Op.Journal["pin"] != "" { t.Fatalf("not a nested pinning: %+v", o.Claim.Op) } pair(t, o.Claim) next := relPin(o, 40, 7, relObserved.Add(FailoverPinDelay)) next.Release = "v1.5.0" o = step(t, s, o, JournalPin("rel-1", next)) o = step(t, s, o, ack(relSpare)) o = toProbing(t, s, o, "P3", 40) if o.Claim.Op.Journal["install_release"] != "v1.5.0" || o.Claim.Op.Journal["install_stamp"] != strconv.FormatUint(o.Claim.Holder.Stamp, 10) { t.Fatalf("the new installation's pair: %q %q", o.Claim.Op.Journal["install_release"], o.Claim.Op.Journal["install_stamp"]) } pair(t, o.Claim) }) t.Run("the serving CAS drops both", func(t *testing.T) { s, o := relAtPhase(t, "probing+P") o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relTarget, Session: o.Claim.Holder.Session})) o = step(t, s, o, RecordProbeStop("rel-1", ProbeStop{Box: relTarget, Session: o.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)})) o = step(t, s, o, CompleteRelocation("rel-1", counter())) pair(t, o.Claim) if o.Claim.Op.Journal["install_stamp"] != "" { t.Fatalf("serving kept the installation: %+v", o.Claim.Op) } }) } func cloneForTest(t *testing.T, c *Claim) *Claim { t.Helper() out, err := cloneClaim(c) if err != nil { t.Fatal(err) } return out } // NameTarget in starting is a converted committed landing's alone: a // failover with no executor. A drain in starting never names one. func TestRelocationNameTargetInStartingIsAFailoversAlone(t *testing.T) { _, o := relAtPhase(t, PhaseStarting) c := cloneForTest(t, o.Claim) delete(c.Op.Intent, "target_box") if err := NameTarget("rel-1", Target{relSpare, relEnrollment(relSpare)})(c); !errors.Is(err, ErrRelocation) { t.Fatalf("a drain named a target in starting: %v", err) } } // Stage B final review A3: a repair of the local install replaces the // journaled receipt with the repaired one, in installed and in probing // before P is admitted, only over exactly the journaled receipt and only // for the same box, enrollment and install stamp; never after P opens. func TestReplaceLocalInstallOnlyOverTheJournaledReceiptBeforePOpens(t *testing.T) { ctx := context.Background() for _, phase := range []string{PhaseInstalled, PhaseRelocatingProbing} { s, o := relAtPhase(t, phase) old := localInstall(o.Claim) if phase == PhaseInstalled { o = step(t, s, o, RecordLocalInstall("rel-1", old)) } repaired := old repaired.JournalSHA256 = strings.Repeat("c", 64) other := repaired other.JournalSHA256 = strings.Repeat("d", 64) if _, err := Update(ctx, s, o, ReplaceLocalInstall("rel-1", other, repaired)); !errors.Is(err, ErrRelocation) { t.Fatalf("%s: replaced over a receipt that is not journaled: %v", phase, err) } for name, bad := range map[string]func(*LocalInstall){ "another box": func(l *LocalInstall) { l.Box = relSource }, "another enrollment": func(l *LocalInstall) { l.Enrollment = relEnrollment(relSource) }, "another stamp": func(l *LocalInstall) { l.InstallStamp++ }, "a malformed digest": func(l *LocalInstall) { l.JournalSHA256 = "x" }, } { l := repaired bad(&l) if _, err := Update(ctx, s, o, ReplaceLocalInstall("rel-1", old, l)); !errors.Is(err, ErrRelocation) { t.Fatalf("%s: %s: %v", phase, name, err) } } if _, err := Update(ctx, s, o, ReplaceLocalInstall("rel-1", old, old)); !errors.Is(err, ErrAlreadyRecorded) { t.Fatalf("%s: the same receipt: %v", phase, err) } o = step(t, s, o, ReplaceLocalInstall("rel-1", old, repaired)) var got LocalInstall if !readJournal(o.Claim.Op, "local_install", &got) || got != repaired || o.Claim.Op.Phase != phase { t.Fatalf("%s: journaled %+v in %s", phase, got, o.Claim.Op.Phase) } if _, err := Update(ctx, s, o, ReplaceLocalInstall("rel-1", old, other)); !errors.Is(err, ErrRelocation) { t.Fatalf("%s: replaced over the superseded receipt: %v", phase, err) } } s, o := relAtPhase(t, "probing+P") old := localInstall(o.Claim) repaired := old repaired.JournalSHA256 = strings.Repeat("c", 64) if _, err := Update(ctx, s, o, ReplaceLocalInstall("rel-1", old, repaired)); !errors.Is(err, ErrRelocation) { t.Fatalf("replaced after P opened: %v", err) } }