package host import ( "context" "errors" "fmt" "log" "sort" "strings" "time" "github.com/carlosframework/platform/internal/hostbox" ) const servingParkRecoveryLimit = 128 var errServingParkRetagPending = errors.New("the earlier park is incomplete; retain its evidence and inspect the root service journal before retrying") type servingParkRecoveryResult struct { Host, ID string Err error } type servingParkRecoveryPage struct { Results []servingParkRecoveryResult Next string More bool } // The cursor bounds record processing and lets a later pass reach records // beyond a blocked host. Store.Names still enumerates the private directory. func (l *servingParkLedger) recoveryIDs(ctx context.Context, after string, limit int) ([]string, bool, error) { if ctx == nil || l == nil || l.state == nil || limit < 1 || limit > servingParkRecoveryLimit { return nil, false, hostbox.ErrUnsupported } if err := ctx.Err(); err != nil { return nil, false, err } names, err := l.state.Names() if err != nil { return nil, false, err } var ids []string for _, name := range names { if strings.HasPrefix(name, "retirement-") && strings.HasSuffix(name, ".json") { id := strings.TrimSuffix(strings.TrimPrefix(name, "retirement-"), ".json") if id > after { ids = append(ids, id) } } } sort.Strings(ids) more := len(ids) > limit if more { ids = ids[:limit] } return ids, more, ctx.Err() } // Completed deletion needs no live claim or agent. Its original durable cause // and recorded identities remain the authority after a host is removed. func (l *servingParkLedger) recoverCompleted(ctx context.Context, id string) error { return l.withNamespace(ctx, func() error { r, err := l.read(id) if err != nil { return err } unlock, err := l.state.Lock(ctx, "slot-"+servingParkKey(r.Authorization.Successor.Box, r.Authorization.Successor.Host)+".lock") if err != nil { return err } defer unlock() unlockRecord, err := l.state.Lock(ctx, "retirement-"+id+".lock") if err != nil { return err } defer unlockRecord() if !r.Complete { return hostbox.ErrChanged } if r.Deleting { if !r.Closed { err = l.deleteLocked(ctx, &r, r.Cause) } else { // Older records can be closed with their empty event retained. err = l.fs.Delete(&r, func() error { if err := ctx.Err(); err != nil { return err } return l.save(r) }) } if err != nil { return err } } // Completion may be durable while clearing Pending lost its reply. // Last and ID are replay references and survive this cleanup. slot, err := l.slot(r.Authorization) if err != nil { return err } if slot.Pending == id { if err = ctx.Err(); err != nil { return err } slot.Pending = "" if err = l.write("slot-"+servingParkKey(slot.Box, slot.Host)+".json", slot); err != nil { return err } } return l.collectClosedLocked(ctx, &r, slot) }) } func (p *servingParkRuntime) recoverPending(ctx context.Context) error { if p == nil { return nil } call, done, err := p.begin(ctx) if err != nil { return err } defer done() recovery, ok := p.authority.(interface { recoverServingParks(context.Context, string, int) (servingParkRecoveryPage, error) }) if !ok { return hostbox.ErrUnsupported } p.recoveryMu.Lock() defer p.recoveryMu.Unlock() page, err := recovery.recoverServingParks(call, p.recoveryAfter, servingParkRecoveryLimit) if err == nil { p.recoveryAfter = "" if page.More { p.recoveryAfter = page.Next } } var failures []error if err != nil { failures = append(failures, hostbox.ErrChanged) } for _, result := range page.Results { if result.Err != nil { // Draft retirement.reap_failed and reason.park_pending from the index. reason := "a local park operation failed; inspect the root service journal" if errors.Is(result.Err, errServingParkRetagPending) { reason = errServingParkRetagPending.Error() } else if errors.Is(result.Err, hostbox.ErrChanged) || errors.Is(result.Err, ErrClaimStopReceipt) { reason = "the parked artifact or its parent no longer matches the root-recorded identity" } if result.Host == "unknown" { reason = "the private retirement record could not be validated; retain the data for investigation" } failures = append(failures, fmt.Errorf("Host %q: serving database retirement %q retained: %s.", result.Host, result.ID, reason)) } } if len(failures) > 0 && time.Since(p.lastRecoveryLog) >= time.Hour { log.Print(errors.Join(failures...)) p.lastRecoveryLog = time.Now() } return errors.Join(err, call.Err()) }