//go:build browser package gallery import ( "context" "fmt" "net/http" "sort" "strings" "testing" "time" "github.com/chromedp/chromedp" "amadan.net/rastrillo/fiesta" "amadan.net/rastrillo/fiesta/internal/axe" "amadan.net/rastrillo/fiesta/internal/harness" ) // The gallery's own pages under axe. a11y_browser_test.go scans every // state's frame document, which is where the examples' markup lives; // this file scans the pages around them, which carry the design // system's explanations: the families' headings and sentences, the // rules at the head of the primitives, and the notes under the states. // a11yPages is what the gallery scan reads in every theme, scheme and // locale it covers: one page per page kind, read off pageKinds() so a // kind added later is scanned the day it lands (the families' pages, // form.html among them, included), then the pages no kind covers. // button carries notes on its states and the widget, dropdown the // idiom's note on one state, signin the summary, a note per screen and // the warning callout, format-money a format's blurb, layout-sidebar a // shell framed at the page width with its open button, modal the // primitive with its live link; then two documents a reader opens from // the Demos, the demo application's index and the modal route. func a11yPages(s *site) []string { var out []string for _, pk := range s.pageKinds() { out = append(out, pk.File) } return append(out, "button.html", "dropdown.html", "signin.html", "format-money.html", "layout-sidebar.html", "modal.html", "demo.html", "demo-modal.html") } // TestA11yScansTheGalleryDay, Plain and Signal are the gate over the gallery's own pages, // which carry the design system's explanations around the samples: // a11yPages in day, plain and signal, with data-theme light and dark, // in English and in Arabic, at 1280 (where the rail, the bar and the // Overview's content are all on the page), with the WCAG 2.2 AA tag // set. Expected: no violations. The rules axe could not decide are // logged per leg, never silently dropped. It extends Fiesta's // TestTheGalleryPagesPassAxe, which scanned seven pages in English and // the Arabic index, to every page kind and the Demos. The frames' // documents are TestEveryExamplePassesAxe's. // // Preservation of the pages scan, already passing when written. The // named mutation is .ds-lead's colour set to --rst-line in // gallery.css, which axe reports as color-contrast on every page with a // lead. The rail's accessible name removed is not one: two unnamed // navigation landmarks are axe's landmark-unique, a best-practice rule // outside the WCAG 2.2 AA tag set this gate runs. // // It is one test per theme, so the suite's slices can run them on // shards of their own. Each checks that the themes are exactly the // three the tests name, so a theme added later cannot go unscanned, // and counts its own scans, as the one test counted all of them. func TestA11yScansTheGalleryDay(t *testing.T) { a11yScansTheGallery(t, "day") } func TestA11yScansTheGalleryPlain(t *testing.T) { a11yScansTheGallery(t, "plain") } func TestA11yScansTheGallerySignal(t *testing.T) { a11yScansTheGallery(t, "signal") } // a11yScanThemes are the themes the three tests above scan, one each. var a11yScanThemes = []string{"day", "plain", "signal"} // a11yScansTheGallery is the gallery's pages scanned in one theme. func a11yScansTheGallery(t *testing.T, only string) { if got := fiesta.ThemeNames(); strings.Join(got, " ") != strings.Join(a11yScanThemes, " ") { t.Fatalf("the themes are %v and the gallery's pages are scanned in %v, one test each; a theme needs a test of its own", got, a11yScanThemes) } rig := harness.New(t, func(string) http.Handler { return treeHandler(t) }) ctx, cancel := context.WithTimeout(rig.Context(), 18*time.Minute) defer cancel() pages := a11yPages(newTestSite(t, fiesta.FS(), mountPath)) scans, failed := 0, 0 for _, theme := range []string{only} { for _, locale := range []string{"en", "ar"} { name := theme + " " + locale incomplete := map[string]int{} start := time.Now() for _, file := range pages { dir := theme + "/" + locale + "/" where := name + " " + file mustRun(t, ctx, chromedp.EmulateViewport(1280, 900), chromedp.Navigate(rig.Origin+href(dir+file)), chromedp.WaitReady("body"), ) // The body's ground is read back after each flip, and // the two must differ: a dark scan of a page still // painted light would be the light scan twice. ground := map[string]string{} for _, scheme := range []string{"light", "dark"} { var bg string mustRun(t, ctx, chromedp.Evaluate(fmt.Sprintf(`document.documentElement.setAttribute("data-theme", %q); getComputedStyle(document.body).backgroundColor`, scheme), &bg)) ground[scheme] = bg res, err := axe.Scan(ctx, "", axe.DefaultTags()) if err != nil { t.Fatalf("%s %s: %v", where, scheme, err) } scans++ for _, id := range res.Incomplete { incomplete[id]++ } failed += report(t, where+" "+scheme, res.Violations) } if ground["light"] == ground["dark"] { t.Fatalf("%s: the body paints %s under both schemes, so the flip to dark did not take", where, ground["light"]) } } var ids []string for id, n := range incomplete { ids = append(ids, fmt.Sprintf("%s x%d", id, n)) } sort.Strings(ids) t.Logf("%s: %d pages scanned in %s; needs review, axe could not decide: %v", name, len(pages), time.Since(start).Round(time.Second), ids) } } if want := 2 * len(pages) * 2; scans != want { t.Fatalf("%d scans, want %d", scans, want) } t.Logf("%d scans, %d violations", scans, failed) rig.Screen("main", "the last page") } // shellDemo is one layout example's document in the tree, in theme and // English, at the state the manifest opens it on: a shell demo. func shellDemo(t *testing.T, theme, name string) string { t.Helper() for _, e := range manifest(t).Examples { if e.Name == name { if !e.Document { t.Fatalf("%s is not a layout document; it has no shell demo", name) } return frameFile(theme, "en", name, e.Default) } } t.Fatalf("the manifest has no %s", name) return "" } // shownJS reports whether the first element matching a selector has a // box on screen. The patency check and its control both read it, so // the two are the same instrument pointed at two moments. It asks // checkVisibility as well as for a box: the content of a closed // details still lays out when asked (the engine hides it with // content-visibility), so its box alone reads as shown. const shownJS = `((sel) => { const el = document.querySelector(sel); if (!el) return false; const r = el.getBoundingClientRect(); return el.checkVisibility() && r.width > 0 && r.height > 0; })(%q)` // settleShellJS waits out the document's finite animations and any // view transition. The cards fade in and a narrow navigation between // two shell documents slides; axe reading either mid-flight measures // text at partial opacity, blended colours no reader sees once it has // landed. Finite animations only: a spinner's never finishes. const settleShellJS = `(async () => { if (document.activeViewTransition) await document.activeViewTransition.finished.catch(() => null); await Promise.all(document.getAnimations() .filter(a => a.effect && a.effect.getComputedTiming().endTime !== Infinity) .map(a => a.finished.catch(() => null))); return true; })()` // targetSizeJS sets whether the engine on the page runs axe's // target-size rule. The engine's own run is kept aside the first time, // so a scan with the rule is always the engine as shipped. const targetSizeJS = `((on) => { if (!window.__axeRun) window.__axeRun = window.axe.run.bind(window.axe); window.axe.run = on ? window.__axeRun : (target, opts) => window.__axeRun(target, Object.assign({}, opts, {rules: Object.assign({}, opts && opts.rules, {"target-size": {enabled: false}})})); return true; })(%t)` // TestA11yScansTheShellsCollapsed is the scan the others do not cover: // every shell demo below its 800px breakpoint. // // The other scans run the shell documents at 1280, where the topbar's // collapse control does not exist at all and the sidebar and console // show their rails beside the page. So the views a reader on a phone // meets first are views axe would never see. The topbar is scanned // with its card open. The sidebar and console are two views of every // URL below 800px, so both of their documents are scanned, in every // theme and scheme: the index (the rail is the page), with the // console's card and the sidebar's profile card open as well, and the // content page (the back control and the page, no rail). Then the // sidebar's profile card at the rail's foot on a desktop, open with // the language menu inside it, in every theme and scheme. // // It also measures what axe will not: target size. The topbar's // summary against WCAG 2.2's 24x24 floor, and the back control against // the 44px the touch block promises (390px is inside its width half). // // Ported from Rastrillo onto Fiesta's layout documents: layout-topbar, // layout-sidebar and layout-console are the index documents and the // two -page examples the content pages. A scheme is painted as // Fiesta's other scans paint it, by data-theme on the root, and each // paint is checked to have changed the ground. Every disclosure is // checked shut before the click and open after it, so a summary that // stopped opening fails rather than being scanned shut and reported // clean. // // A preservation port, already passing when written: Fiesta's shells // were clean collapsed. Its two mutations, each one rule appended to // frameStyle and reverted: [rst-shell-back] a { color: #bbb } is // color-contrast on every content page in light, which no other scan // sees, the back control being hidden above 800px; and the back // control held to 30px tall fails the 44px check on every content // page. func TestA11yScansTheShellsCollapsed(t *testing.T) { rig := harness.New(t, func(string) http.Handler { return treeHandler(t) }) ctx, cancel := context.WithTimeout(rig.Context(), 15*time.Minute) defer cancel() total, scans := 0, 0 incomplete := map[string]int{} shown := func(where, sel string) bool { t.Helper() var ok bool if err := chromedp.Run(ctx, chromedp.Evaluate(fmt.Sprintf(shownJS, sel), &ok)); err != nil { t.Fatalf("%s: reading whether %q is on screen: %v", where, sel, err) } return ok } load := func(where, file string, w, h int64, ready string) { t.Helper() if err := chromedp.Run(ctx, chromedp.EmulateViewport(w, h), chromedp.Navigate(rig.Origin+href(file)), chromedp.WaitVisible(ready, chromedp.ByQuery), ); err != nil { t.Fatalf("%s: loading: %v", where, err) } } // paint puts the document in scheme, and proves the flip took: the // body's ground under the other scheme differs from its ground // under this one, or a dark scan would be the light scan twice. paint := func(where, scheme string) { t.Helper() other := map[string]string{"light": "dark", "dark": "light"}[scheme] var away, here string const flip = `document.documentElement.setAttribute("data-theme", %q); getComputedStyle(document.body).backgroundColor` mustRun(t, ctx, chromedp.Evaluate(fmt.Sprintf(flip, other), &away), chromedp.Evaluate(fmt.Sprintf(flip, scheme), &here)) if away == here { t.Fatalf("%s: the body paints %s under both schemes, so the flip to %s did not take", where, here, scheme) } } // scan runs axe over the whole document in scheme, with or without // the target-size rule. scan := func(where, scheme string, targetSize bool) { t.Helper() var loaded bool mustRun(t, ctx, chromedp.Evaluate(settleShellJS, nil, awaitPromise), chromedp.Evaluate(`typeof window.axe !== "undefined"`, &loaded)) if !loaded { mustRun(t, ctx, chromedp.Evaluate(axe.Script(), nil)) } mustRun(t, ctx, chromedp.Evaluate(fmt.Sprintf(targetSizeJS, targetSize), nil)) paint(where, scheme) res, err := axe.Scan(ctx, "", axe.DefaultTags()) if err != nil { t.Fatalf("%s: %v", where, err) } scans++ for _, id := range res.Incomplete { incomplete[id]++ } total += report(t, where, res.Violations) } // box is a control's size and visible label. box := func(sel string) (w, h int, text string) { t.Helper() var got struct { W, H int Text string } at(t, ctx, fmt.Sprintf(`((sel) => { const s = document.querySelector(sel); const r = s.getBoundingClientRect(); return JSON.stringify({W: Math.round(r.width), H: Math.round(r.height), Text: s.innerText.trim()}); })(%q)`, sel), &got) return got.W, got.H, got.Text } // open clicks a disclosure and asserts what it reveals, with the // control first: the same reading one click earlier, where the // answer is known. A patency check hard-wired true passes after the // click and fails here. open := func(where, summary, revealed string) { t.Helper() if shown(where, revealed) { t.Errorf("%s: %q is on screen with the disclosure still shut, so it is not behind the disclosure and its being shown proves nothing about the click", where, revealed) } if err := chromedp.Run(ctx, chromedp.Click(summary, chromedp.ByQuery)); err != nil { t.Fatalf("%s: opening: %v", where, err) } if !shown(where, revealed) { t.Fatalf("%s: clicking the disclosure did not bring %q on screen; axe would scan the collapsed document as the disclosed one", where, revealed) } } schemes := []string{"light", "dark"} themes := fiesta.ThemeNames() // The topbar: its card, open. for _, scheme := range schemes { where := themes[0] + "/en topbar shell at 390px, card open (" + scheme + ")" load(where, shellDemo(t, themes[0], "layout-topbar"), 390, 780, "[rst-shell-menu] > summary") open(where, "[rst-shell-menu] > summary", "[rst-shell-tail] [rst-shell-nav] a") scan(where, scheme, true) w, h, text := box("[rst-shell-menu] > summary") if w < 24 || h < 24 { t.Errorf("%s: the disclosure's target is %dx%dpx, under WCAG 2.2 SC 2.5.8's 24x24", where, w, h) } if text == "" { t.Errorf("%s: the disclosure has no visible label; the icon beside it is aria-hidden, so the control would have no accessible name at all", where) } } // The sidebar and console: both documents, every theme and scheme. for _, shell := range []string{"sidebar", "console"} { for _, theme := range themes { for _, scheme := range schemes { where := theme + "/en " + shell + " index at 390px (" + scheme + ")" load(where, shellDemo(t, theme, "layout-"+shell), 390, 780, "[rst-shell-rail]") // Patency: the document is the index before axe reads it. if rail, main := shown(where, "[rst-shell-rail] [rst-shell-nav] a"), shown(where, "[rst-shell-main]"); !rail || main { t.Fatalf("%s: not the index (rail shown %v, main shown %v); axe would scan the wrong view", where, rail, main) } scan(where, scheme, true) // The whole document with the card open, every rule but // target-size. With the card open everything outside it // lies under the summary's light-dismiss layer, which axe // cannot see (it works out what covers a target from // element boxes): a rail row half under the card reads as // a target cut short when no tap can reach it at all. The // rows' size is measured with the card shut, just above. if shell == "console" { where := theme + "/en console index at 390px, card open (" + scheme + ")" // On the index the account's links are laid out flat // in the card, so a link is what the click reveals. open(where, "[rst-shell-menu] > summary", "[rst-shell-tail] [rst-shell-account] [rst-dropdown-menu] a") scan(where, scheme, false) } else { where := theme + "/en sidebar index at 390px, profile card open (" + scheme + ")" open(where, "[rst-shell-profile] > summary", "[rst-shell-profile] [rst-shell-who] [rst-person-name]") open(where+", language open", "[rst-shell-profile] [rst-locale] > summary", "[rst-shell-profile] [rst-locale] [rst-dropdown-menu] a") scan(where, scheme, false) } where = theme + "/en " + shell + " content page at 390px (" + scheme + ")" load(where, shellDemo(t, theme, "layout-"+shell+"-page"), 390, 780, "[rst-shell-back] a") if shown(where, "[rst-shell-rail] [rst-shell-nav] a") { t.Fatalf("%s: the rail's nav is on screen on a content page; axe would scan the index", where) } scan(where, scheme, true) w, h, text := box("[rst-shell-back] a") if w < 44 || h < 44 { t.Errorf("%s: the back control is %dx%dpx; the touch block promises 44x44", where, w, h) } if text == "" { t.Errorf("%s: the back control has no visible label", where) } } } } // The sidebar's profile menu on a desktop, the drop-up at the rail's // foot, open with the language inside it, in every theme and scheme. // target-size is off for the same layer as above: the page under // the card is behind it. for _, theme := range themes { for _, scheme := range schemes { where := theme + "/en sidebar content page at 1280px, profile card open (" + scheme + ")" load(where, shellDemo(t, theme, "layout-sidebar-page"), 1280, 720, "[rst-shell-profile] > summary") open(where, "[rst-shell-profile] > summary", "[rst-shell-profile] [rst-shell-who] [rst-person-name]") open(where+", language open", "[rst-shell-profile] [rst-locale] > summary", "[rst-shell-profile] [rst-locale] [rst-dropdown-menu] a") scan(where, scheme, false) } } if want := 2 + 2*len(themes)*2*3 + len(themes)*2; scans != want { t.Errorf("%d scans; want %d", scans, want) } var ids []string for id, n := range incomplete { ids = append(ids, fmt.Sprintf("%s x%d", id, n)) } sort.Strings(ids) t.Logf("%d scans at 390px and the desktop profile card, %d violations; needs review, axe could not decide: %v", scans, total, ids) }