// The scan sidecar (spec §5): apps///scan.json, the coverage // rollup, and the object an audit answer is quoted from. // // It is a SEPARATE OBJECT from store.json on purpose. The sweep writes once // per page over minutes; the create, grant and rotate doors write store.json // too. Sharing one object would put a page-write in a compare-and-swap race // with every repair walk, and losing that race is the normal case rather than // the exception. With its own object and a single writer, the whole CAS story // shrinks to "another sweep is ahead of me, stand down". package objstore import ( "context" "encoding/json" "errors" "fmt" "os" "time" "github.com/carlosframework/platform/internal/release" ) const SidecarSchema = 1 // ErrSidecarMismatch is a sidecar whose body disagrees with the path it was // read from. var ErrSidecarMismatch = errors.New("objstore: scan sidecar names another app") func ScanKey(acct, app string) string { return release.AppPrefix(acct, app) + "scan.json" } // Sweep is one reconciliation pass over one bucket. // // Scanned and Unscanned are NOT complements. Skipped, Denied and Failed are // recorded verdicts that are not coverage — an object too large to stream was // never judged, and calling it clean would be a lie an auditor is entitled to // catch. That is why the audit sentence names two numbers. type Sweep struct { StartedAt time.Time `json:"started_at"` At time.Time `json:"at"` Engine string `json:"engine"` // Signature is the engine's signature-database version, verbatim. A clean // verdict from three-month-old signatures should not be indistinguishable // from a fresh one. Signature string `json:"signature,omitempty"` Objects int64 `json:"objects"` Scanned int64 `json:"scanned"` Infected int64 `json:"infected"` Skipped int64 `json:"skipped"` Denied int64 `json:"denied"` Failed int64 `json:"failed"` Unscanned int64 `json:"unscanned"` // Fresh and Reused split Scanned by WHEN the verdict was reached. A clean // tag is reused for as long as the object's ETag matches, so a completed // sweep can be built almost entirely from judgements made months ago, // against a signature database that is not the one named above. Without // this split, Signature reads as a claim about every object in the rollup, // and it is not one. Fresh int64 `json:"fresh"` Reused int64 `json:"reused"` Bytes int64 `json:"bytes"` // After is the last key of the last written page — the resume point. // A key, not a ListObjectsV2 continuation token: a token's lifetime across // processes is unspecified and, on Ceph, unmeasured, while a key is // portable to any S3-compatible store. After string `json:"after,omitempty"` } // ScanSidecar holds the in-flight sweep and the last completed one. // // TWO SWEEPS, NOT ONE. A single field would have to mean both "in progress" // and "the answer", and the tick after a sweep completed would overwrite the // completed counts with page-one partials — destroying the compliance artifact // on every cycle. Current accumulates; on its last page it moves to // LastComplete, and LastComplete is the only thing an audit answer quotes. type ScanSidecar struct { Schema int `json:"schema"` Account string `json:"account"` App string `json:"app"` // NotApplicable is a store whose member attested client-side encryption. // A scanner cannot read ciphertext, so no sweep runs and no coverage is // claimed — reporting "all clean" for such a store would be worse than // reporting nothing. NotApplicable bool `json:"not_applicable,omitempty"` Current *Sweep `json:"current,omitempty"` LastComplete *Sweep `json:"last_complete,omitempty"` } // GetSidecar reads the sidecar. Absent is (nil, "", nil) — no sidecar is a // state, not an error, the same idiom Get uses for an absent record. func GetSidecar(ctx context.Context, s release.Store, acct, app string) (*ScanSidecar, string, error) { if !release.ValidAccount(acct) { return nil, "", fmt.Errorf("objstore: invalid account %q", acct) } if !release.ValidApp(app) { return nil, "", fmt.Errorf("objstore: invalid app %q", app) } raw, etag, err := s.Get(ctx, ScanKey(acct, app)) if err != nil { if errors.Is(err, os.ErrNotExist) { return nil, "", nil } return nil, "", err } var sc ScanSidecar if err := json.Unmarshal(raw, &sc); err != nil { return nil, "", fmt.Errorf("objstore: %s: %w", ScanKey(acct, app), err) } // An unknown schema is refused rather than guessed at — the rule // release.ReadAppTombstone already applies to its own shape. if sc.Schema != SidecarSchema { return nil, "", fmt.Errorf("objstore: %s: schema %d is not %d", ScanKey(acct, app), sc.Schema, SidecarSchema) } // `accounts migrate` copies every key under an app's prefix, so a sidecar // can arrive under an account it was never written for, carrying rollups // for a bucket named after the old one. Refusing beats reporting another // account's coverage as this one's. if sc.Account != acct || sc.App != app { return nil, "", fmt.Errorf("%w: body says %s/%s, path says %s/%s", ErrSidecarMismatch, sc.Account, sc.App, acct, app) } return &sc, etag, nil } // PutSidecar writes with the caller's CAS position; etag "" is create-only. // It stamps Schema, Account and App the way Put stamps Schema and App, so a // caller cannot write a body that disagrees with its own path. func claimRawPutSidecar(ctx context.Context, s release.Store, acct, app string, sc *ScanSidecar, etag string) (string, error) { sc.Schema = SidecarSchema sc.Account = acct sc.App = app body, err := json.MarshalIndent(sc, "", " ") if err != nil { return "", err } return s.Put(ctx, ScanKey(acct, app), append(body, '\n'), etag) } func PutSidecar(ctx context.Context, s release.Store, acct, app string, sc *ScanSidecar, etag string) (claimResult0 string, claimResult1 error) { claimResult1 = release.WithAppMutation(ctx, s, acct, app, "objstore.PutSidecar", func(claimContext context.Context) error { claimResult0, claimResult1 = claimRawPutSidecar(claimContext, s, acct, app, sc, etag) return claimResult1 }) return }