package runs import ( "bytes" "context" "crypto/rand" "errors" "regexp" "strings" "testing" "testing/iotest" "time" "github.com/carlosframework/platform/internal/release" ) func TestIDsAndKeys(t *testing.T) { id, err := NewID(rand.Reader) if err != nil || !regexp.MustCompile(`^run_[a-z2-7]{26}$`).MatchString(id) || !ValidID(id) { t.Fatalf("id %q, %v", id, err) } for _, bad := range []string{"", "run_", "run_ABCDEFGHIJKLMNOPQRSTUVWXYZ", "run_abcdefghijklmnopqrstuvwxy/", "mv_abcdefghijklmnopqrstuvwxyz"} { if ValidID(bad) { t.Errorf("%q accepted", bad) } } if got := Prefix("bab", "shop", id); got != "apps/bab/shop/runs/"+id+"/" { t.Errorf("Prefix = %q", got) } k := ChunkKey("bab", "shop", id, 17, StreamErr) if k != OutPrefix("bab", "shop", id)+"00000017-e" { t.Errorf("ChunkKey = %q", k) } seq, stream, ok := ParseChunkKey(k, OutPrefix("bab", "shop", id)) if !ok || seq != 17 || stream != StreamErr { t.Errorf("ParseChunkKey = %d %q %v", seq, stream, ok) } for _, bad := range []string{OutPrefix("bab", "shop", id) + "00000017-x", OutPrefix("bab", "shop", id) + "17-o", "elsewhere"} { if _, _, ok := ParseChunkKey(bad, OutPrefix("bab", "shop", id)); ok { t.Errorf("ParseChunkKey(%q) accepted", bad) } } } func TestCheckArgvAndTimeout(t *testing.T) { cases := []struct { argv []string want error }{ {nil, ErrNoArgs}, {make([]string, 257), ErrTooManyArgs}, {[]string{strings.Repeat("a", 64<<10+1)}, ErrArgvTooLarge}, {[]string{strings.Repeat("a", 64<<10-1), "b"}, nil}, // "at most 64 KB in total": the limit itself passes {[]string{strings.Repeat("a", 32<<10), strings.Repeat("b", 32<<10+1)}, ErrArgvTooLarge}, {[]string{"a\x00b"}, ErrArgNUL}, {[]string{"import", "--dry-run"}, nil}, } for i := range cases[1].argv { cases[1].argv[i] = "x" } for _, c := range cases { if err := CheckArgv(c.argv); !errors.Is(err, c.want) && !(c.want == nil && err == nil) { t.Errorf("CheckArgv(%d args) = %v, want %v", len(c.argv), err, c.want) } } for d, ok := range map[time.Duration]bool{0: false, 999 * time.Millisecond: false, time.Second: true, 6 * time.Hour: true, 6*time.Hour + time.Second: false} { if (CheckTimeout(d) == nil) != ok { t.Errorf("CheckTimeout(%s) = %v", d, CheckTimeout(d)) } } } func TestCheckFiles(t *testing.T) { if err := CheckFiles([]MetaFile{{Name: "a.csv", Size: 1}, {Name: "b.json", Size: 2}}); err != nil { t.Fatal(err) } if err := CheckFiles([]MetaFile{{Name: "a", Size: MaxFileBytes}, {Name: "b", Size: 1}}); !errors.Is(err, ErrFilesTooLarge) { t.Errorf("over the cap: %v", err) } for _, name := range []string{"", ".", "..", "a/b", "../x", "a\x00b", "a\nb", strings.Repeat("n", 256)} { var fe *FileNameError if err := CheckFiles([]MetaFile{{Name: name, Size: 1}}); !errors.Is(err, ErrFileName) || !errors.As(err, &fe) || fe.Name != name { t.Errorf("name %q: %v", name, err) } } if err := CheckFiles([]MetaFile{{Name: "a", Size: 1}, {Name: "a", Size: 1}}); !errors.Is(err, ErrFileTwice) { t.Errorf("twice: %v", err) } if err := CheckFiles([]MetaFile{{Name: "a", Size: -1}}); !errors.Is(err, ErrFilesTooLarge) { t.Errorf("a negative size must be refused: %v", err) } } func TestStoreHelpers(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) id := "run_abcdefghijklmnopqrstuvwxyz" if r, err := ReadResult(ctx, s, "bab", "shop", id); r != nil || err != nil { t.Fatalf("absent result = %v, %v", r, err) } for seq := int64(1); seq <= 5; seq++ { if _, err := s.Put(ctx, ChunkKey("bab", "shop", id, seq, StreamOut), []byte("x"), ""); err != nil { t.Fatal(err) } } keys, err := ListChunks(ctx, s, "bab", "shop", id, 2, 2) if err != nil || len(keys) != 2 || !strings.HasSuffix(keys[0], "00000003-o") { t.Fatalf("ListChunks = %q, %v", keys, err) } if last, err := LastChunkSeq(ctx, s, "bab", "shop", id); err != nil || last != 5 { t.Fatalf("LastChunkSeq = %d, %v", last, err) } first := Result{Exit: 3, LastSeq: 5} if err := WriteResult(ctx, s, "bab", "shop", id, first); err != nil { t.Fatal(err) } if err := WriteResult(ctx, s, "bab", "shop", id, Result{Interrupted: InterruptedLost}); err != nil { t.Fatalf("a second result is not an error: %v", err) } if r, err := ReadResult(ctx, s, "bab", "shop", id); err != nil || r.Exit != 3 || r.Interrupted != "" { t.Fatalf("the first result must win: %+v, %v", r, err) } if err := DeleteRun(ctx, s, "bab", "shop", id); err != nil { t.Fatal(err) } if left, _ := s.List(ctx, Prefix("bab", "shop", id)); len(left) != 0 { t.Fatalf("DeleteRun left %q", left) } } // A run id becomes one path element of an object key and of a directory on // the box, so ValidID must refuse anything that could be more or less. func TestValidIDIsOneCleanPathElement(t *testing.T) { good := "run_abcdefghijklmnopqrstuvwxyz" if !ValidID(good) { t.Fatalf("%q refused", good) } for _, bad := range []string{ "", ".", "..", "run_", "/", "run_/", "../" + good, good + "/", good + "/..", good + "\n", "\n" + good, good + "\x00", good + " ", "run_abcdefghijklmnopqrstuvwxy", good + "a", "run_abcdefghijklmnopqrstuvwx\\z", "run_abcdefghijklmnopqrstuvwx.z", "run_abcdefghijklmnopqrstuvwxé", "RUN_abcdefghijklmnopqrstuvwxyz", "run_abcdefghijklmnopqrstuvwx01", } { if ValidID(bad) { t.Errorf("ValidID(%q) accepted", bad) } } } func TestNewIDDrawsFromTheReader(t *testing.T) { // 16 bytes in, 128 bits of the id out: a different draw is a different id. a, err := NewID(bytes.NewReader(bytes.Repeat([]byte{0x00}, 16))) if err != nil || a != "run_"+strings.Repeat("a", 26) { t.Fatalf("zero draw = %q, %v", a, err) } b, err := NewID(bytes.NewReader(append(bytes.Repeat([]byte{0x00}, 15), 0x01))) if err != nil || b == a || !ValidID(b) { t.Fatalf("last bit draw = %q, %v", b, err) } if _, err := NewID(bytes.NewReader(make([]byte, 15))); err == nil { t.Error("a short read minted an id") } if _, err := NewID(iotest.ErrReader(errors.New("no entropy"))); err == nil { t.Error("a failing reader minted an id") } } func TestMintIDIsRandomAndValid(t *testing.T) { seen := map[string]bool{} for i := 0; i < 2000; i++ { id, err := MintID() if err != nil || !ValidID(id) { t.Fatalf("MintID = %q, %v", id, err) } if seen[id] { t.Fatalf("MintID repeated %q", id) } seen[id] = true } } func TestParseChunkKeyIsCanonicalOnly(t *testing.T) { out := OutPrefix("bab", "shop", "run_abcdefghijklmnopqrstuvwxyz") for _, name := range []string{"+0000017-o", "-0000017-o", "0000000-o", "00000000-o", " 0000017-o", "0x000017-o", "0000017 -o"} { if _, _, ok := ParseChunkKey(out+name, out); ok { t.Errorf("ParseChunkKey(%q) accepted", name) } } } func TestStoreHelpersRefuseAnIdThatIsNotOne(t *testing.T) { ctx := context.Background() s := release.NewDirStore(t.TempDir()) if _, err := s.Put(ctx, "apps/bab/shop/runs/keep", []byte("x"), ""); err != nil { t.Fatal(err) } for _, id := range []string{"", "..", "../x", "run_x/../..", "run_abcdefghijklmnopqrstuvwxyz/"} { if _, err := ReadRequest(ctx, s, "bab", "shop", id); !errors.Is(err, ErrBadID) { t.Errorf("ReadRequest(%q) = %v", id, err) } if _, err := ReadResult(ctx, s, "bab", "shop", id); !errors.Is(err, ErrBadID) { t.Errorf("ReadResult(%q) = %v", id, err) } if err := WriteResult(ctx, s, "bab", "shop", id, Result{}); !errors.Is(err, ErrBadID) { t.Errorf("WriteResult(%q) = %v", id, err) } if _, err := ListChunks(ctx, s, "bab", "shop", id, 0, 0); !errors.Is(err, ErrBadID) { t.Errorf("ListChunks(%q) = %v", id, err) } if _, err := LastChunkSeq(ctx, s, "bab", "shop", id); !errors.Is(err, ErrBadID) { t.Errorf("LastChunkSeq(%q) = %v", id, err) } if err := DeleteRun(ctx, s, "bab", "shop", id); !errors.Is(err, ErrBadID) { t.Errorf("DeleteRun(%q) = %v", id, err) } } if left, _ := s.List(ctx, "apps/bab/shop/runs/"); len(left) != 1 { t.Fatalf("a refused delete touched the store: %q", left) } } // acct and app are concatenated into every key, so a slash or ".." in either // would leave the app's subtree. Nothing may be read or written for them. func TestStoreHelpersRefuseAnAccountOrAppThatIsNotOne(t *testing.T) { ctx := context.Background() dir := t.TempDir() s := release.NewDirStore(dir) id := "run_abcdefghijklmnopqrstuvwxyz" victim := RequestKey("bab", "x", id) if _, err := s.Put(ctx, victim, []byte(`{}`), ""); err != nil { t.Fatal(err) } before, _ := s.List(ctx, "") for _, c := range [][2]string{{"bab/shop", "x"}, {"bab", "../x"}, {"bab", "../../y"}, {"", "shop"}, {"bab", ""}, {"bab", "a/b"}, {"..", "x"}, {"BAB", "x"}, {"bab", "x\n"}} { acct, app := c[0], c[1] if _, err := ReadRequest(ctx, s, acct, app, id); !errors.Is(err, ErrBadApp) { t.Errorf("ReadRequest(%q, %q) = %v", acct, app, err) } if _, err := ReadResult(ctx, s, acct, app, id); !errors.Is(err, ErrBadApp) { t.Errorf("ReadResult(%q, %q) = %v", acct, app, err) } if err := WriteResult(ctx, s, acct, app, id, Result{}); !errors.Is(err, ErrBadApp) { t.Errorf("WriteResult(%q, %q) = %v", acct, app, err) } if _, err := ListChunks(ctx, s, acct, app, id, 0, 0); !errors.Is(err, ErrBadApp) { t.Errorf("ListChunks(%q, %q) = %v", acct, app, err) } if _, err := LastChunkSeq(ctx, s, acct, app, id); !errors.Is(err, ErrBadApp) { t.Errorf("LastChunkSeq(%q, %q) = %v", acct, app, err) } if err := DeleteRun(ctx, s, acct, app, id); !errors.Is(err, ErrBadApp) { t.Errorf("DeleteRun(%q, %q) = %v", acct, app, err) } } if after, _ := s.List(ctx, ""); strings.Join(after, "\n") != strings.Join(before, "\n") { t.Fatalf("a refused call changed the store: %q -> %q", before, after) } } // DeleteRun removes the marker last and request.json just before it, so a // run with any object left still has the marker that keeps its row single. func TestDeleteRunKeepsTheMarkerAndRequestUntilTheRestIsGone(t *testing.T) { ctx := context.Background() inner := release.NewDirStore(t.TempDir()) id, _ := MintID() for _, k := range []string{AuditedKey("bab", "shop", id), RequestKey("bab", "shop", id), ResultKey("bab", "shop", id), ChunkKey("bab", "shop", id, 1, StreamOut)} { if _, err := inner.Put(ctx, k, []byte("x"), ""); err != nil { t.Fatal(err) } } s := &failDelete{Store: inner, fail: ResultKey("bab", "shop", id)} if err := DeleteRun(ctx, s, "bab", "shop", id); err == nil { t.Fatal("a failed delete must be returned") } left, _ := inner.List(ctx, Prefix("bab", "shop", id)) if len(left) != 3 { // result, request, audited t.Fatalf("left %q, want result.json, request.json and audited", left) } s.fail = "" if err := DeleteRun(ctx, s, "bab", "shop", id); err != nil { t.Fatal(err) } if left, _ := inner.List(ctx, Prefix("bab", "shop", id)); len(left) != 0 { t.Fatalf("left %q", left) } } // A chunk that will not delete stops DeleteRun before result.json, so a // follower of a half-deleted run still reads how it ended. Then the result // goes before request.json. func TestDeleteRunKeepsTheResultUntilTheOutputIsGone(t *testing.T) { ctx := context.Background() inner := release.NewDirStore(t.TempDir()) id, _ := MintID() chunk := ChunkKey("bab", "shop", id, 1, StreamOut) for _, k := range []string{AuditedKey("bab", "shop", id), RequestKey("bab", "shop", id), ResultKey("bab", "shop", id), chunk, InputKey("bab", "shop", id, "data.csv")} { if _, err := inner.Put(ctx, k, []byte("{}"), ""); err != nil { t.Fatal(err) } } s := &failDelete{Store: inner, fail: chunk} if err := DeleteRun(ctx, s, "bab", "shop", id); err == nil { t.Fatal("a failed delete must be returned") } if r, err := ReadResult(ctx, inner, "bab", "shop", id); r == nil || err != nil { t.Fatalf("result.json went before the output: %v, %v", r, err) } s.fail = RequestKey("bab", "shop", id) if err := DeleteRun(ctx, s, "bab", "shop", id); err == nil { t.Fatal("a failed delete must be returned") } if r, _ := ReadResult(ctx, inner, "bab", "shop", id); r != nil { t.Fatal("result.json must go before request.json") } } type failDelete struct { release.Store fail string } func (f *failDelete) Delete(ctx context.Context, key string) error { if key == f.fail { return errors.New("delete refused") } return f.Store.Delete(ctx, key) } func TestReleaseTimeoutMaxIsTheRunMax(t *testing.T) { if got := time.Duration(release.MaxReleaseTimeoutSec) * time.Second; got != MaxTimeout { t.Fatalf("release.MaxReleaseTimeoutSec = %v, runs.MaxTimeout = %v: spec §5.2.1 caps release at the carlos run maximum", got, MaxTimeout) } }