package hostops import ( "context" "errors" "fmt" "github.com/carlosframework/platform/internal/alerting" "os" "strings" "time" "github.com/carlosframework/platform/internal/hostclaim" "github.com/carlosframework/platform/internal/release" ) // ListClaims is discovery only. Every action subsequently reads its own key. func ListClaims(ctx context.Context, s release.Store) ([]string, error) { keys, err := s.List(ctx, "control/hosts/") if err != nil { return nil, err } var hosts []string for _, key := range keys { tail, ok := strings.CutPrefix(key, "control/hosts/") if !ok { return nil, ErrIdentity } if strings.Contains(tail, "/") || !strings.HasSuffix(tail, ".json") { continue } host := strings.TrimSuffix(tail, ".json") if _, err := hostclaim.Key(host); err != nil { return nil, err } hosts = append(hosts, host) } return hosts, nil } type Recovery struct { // ScheduleLossPages triggers a separate worker after materialization. ScheduleLossPages func(context.Context) ReportAutomaticLosses func(context.Context, string) error MoveFenceAuthority hostclaim.FenceAuthority Moves func(context.Context, *MoveRecoveryGrant) error // MoveLogs, when set, runs once a pass, after every claim: the console's // delivery of the move log entries its move pass queued, through the // pass's MoveLogWriter (move_log_writer.go). MoveLogs func(context.Context, *MoveLogWriter) error SweepAdmissions func(context.Context) error Controller Controller // These are required integrations, not no-op defaults. Purge func(context.Context, hostclaim.Observed) error Placement func(context.Context, hostclaim.Observed) error SweepRecords func(context.Context) error SweepPurges func(context.Context) error } func (r Recovery) Pass(ctx context.Context) error { c := r.Controller if err := c.ready(); err != nil { return err } if r.Purge == nil || r.Placement == nil || r.SweepRecords == nil || r.SweepPurges == nil || r.SweepAdmissions == nil { return ErrContract } hosts, err := ListClaims(ctx, c.Store) if err != nil { return err } var all []error for _, host := range hosts { if err := ctx.Err(); err != nil { return errors.Join(append(all, err)...) } obs, err := hostclaim.Read(ctx, c.Store, host) if err != nil { all = append(all, err) continue } if obs.Absent { continue } if r.ReportAutomaticLosses != nil && len(obs.Claim.EvidenceRefs) > 0 { if err := r.ReportAutomaticLosses(ctx, host); err != nil { all = append(all, err) if pageErr := alerting.Page(ctx, alerting.LevelWarning, hostclaim.ErrLossRecord.Error(), fmt.Sprintf("%s: %v", host, err)); pageErr != nil { all = append(all, pageErr) } } obs, err = hostclaim.Read(ctx, c.Store, host) if err != nil { all = append(all, err) continue } } if obs.Absent || obs.Claim == nil { continue } op := obs.Claim.Op // Zero time is unknown age, never old enough. Freeing empty parking is safe // independently of age; this includes Plan 4's holderless sweep retries. if moveRecoveryKind(op.Kind) { if r.Moves == nil { continue } err = runMoveRecovery(ctx, obs, r.Moves, r.MoveFenceAuthority) } else if op.Kind == hostclaim.KindParking && len(obs.Claim.Obligations) == 0 { err = c.Free(ctx, host, op.ID) } else if op.Kind != "" && op.Kind != hostclaim.KindServing && !op.StartedAt.IsZero() && c.Now().After(op.StartedAt.Add(5*time.Minute)) { switch op.Kind { case hostclaim.KindCreating: err = c.RecoverCreate(ctx, host) case hostclaim.KindDeleting: err = c.Retire(ctx, host, op.ID) case hostclaim.KindPurging: err = r.Purge(ctx, obs) case hostclaim.KindPlacing: err = r.Placement(ctx, obs) } } if err != nil && !errors.Is(err, ErrPending) { all = append(all, err) } } if r.ScheduleLossPages != nil { r.ScheduleLossPages(ctx) } if r.MoveLogs != nil { if err := runMoveLogDrain(ctx, r.MoveLogs); err != nil && !errors.Is(err, ErrPending) { all = append(all, err) } } // Records have their own uncapped scan: late orphans outlive the one-hour // cancellation hint; records without claims need the shared adoption path. if err := r.SweepRecords(ctx); err != nil { all = append(all, err) } // Reservations can exist without any claim (crash just after reserve). if err := r.SweepPurges(ctx); err != nil { all = append(all, err) } if err := r.SweepAdmissions(ctx); err != nil { all = append(all, err) } return errors.Join(all...) } type App struct{ Account, Name string } func Apps(ctx context.Context, s release.Store) ([]App, error) { accounts, err := s.ListDirs(ctx, "apps/") if err != nil { return nil, err } var out []App for _, prefix := range accounts { account := strings.TrimSuffix(strings.TrimPrefix(prefix, "apps/"), "/") if prefix != "apps/"+account+"/" { return nil, ErrIdentity } // The retained legacy layout is apps//..., so its first segment is // an app name, not necessarily an account. Skip names that are valid apps // but invalid accounts before treating this as a qualified account root. // A letters-only legacy name can pass both validators, so preserve its // children as candidates and let the exact record and purge parsers // validate them. The path alone cannot tell that case from a real account // containing apps with those legal names; never infer deletion authority // over retained legacy content from this prefix shape. if !release.ValidAccount(account) { if release.ValidApp(account) && prefix == release.LegacyAppPrefix(account) { continue } return nil, ErrIdentity } apps, err := s.ListDirs(ctx, prefix) if err != nil { return nil, err } for _, ap := range apps { app := strings.TrimSuffix(strings.TrimPrefix(ap, prefix), "/") if _, err := PurgeKey(account, app); err != nil || ap != prefix+app+"/" { return nil, ErrIdentity } out = append(out, App{Account: account, Name: app}) } } return out, nil } func (c Controller) SweepAllRecords(ctx context.Context) error { if err := c.ready(); err != nil { return err } apps, err := Apps(ctx, c.Store) if err != nil { return err } var all []error for _, app := range apps { records, err := Records(ctx, c.Store, app.Account, app.Name) if err != nil { all = append(all, err) continue } for _, listed := range records { record, err := ReadRecord(ctx, c.Store, listed.Holder.Key) if errors.Is(err, os.ErrNotExist) { continue } if err != nil { all = append(all, err) continue } obs, err := hostclaim.Read(ctx, c.Store, record.Holder.Host) if err != nil { all = append(all, err) continue } if obs.Absent { err = ErrPending } else { _, err = RemoveOrphan(ctx, c.Store, record) } if err != nil { all = append(all, err) } } } return errors.Join(all...) } func SweepAllPurges(ctx context.Context, s release.Store, replay func(context.Context, App, Purge) error) error { if replay == nil { return ErrContract } apps, err := Apps(ctx, s) if err != nil { return err } var all []error for _, app := range apps { p, _, err := ReadPurge(ctx, s, app.Account, app.Name) if errors.Is(err, os.ErrNotExist) { continue } if err != nil { all = append(all, err) continue } if err := replay(ctx, app, p); err != nil { all = append(all, err) } } return errors.Join(all...) }