diff --git a/.gitea/workflows/sonarqube.yml b/.gitea/workflows/sonarqube.yml index cc750a8..01143d4 100755 --- a/.gitea/workflows/sonarqube.yml +++ b/.gitea/workflows/sonarqube.yml @@ -178,6 +178,14 @@ jobs: # summaries first. Without this step continue-on-error above would turn a # failing suite into a passing job, which is the one way this change could # do real damage. + # + # always() is load-bearing. A step whose `if:` omits it still implicitly + # requires every previous step to have succeeded, so this was skipped in + # exactly the case it exists for: the summarise step above used to exit 1 + # on a failing suite, which failed the job first and left this gate as dead + # code. The job then reported its failure under a name that describes + # summarising rather than testing. The summarisers exit 0 now; this is what + # fails the run. See #142. - name: Fail if either suite failed - if: steps.unit.outcome == 'failure' || steps.e2e.outcome == 'failure' + if: always() && (steps.unit.outcome == 'failure' || steps.e2e.outcome == 'failure') run: exit 1 diff --git a/scripts/summarize-jest.js b/scripts/summarize-jest.js index 2797862..d705ebd 100644 --- a/scripts/summarize-jest.js +++ b/scripts/summarize-jest.js @@ -8,6 +8,7 @@ const [, , resultsPath, label] = process.argv; // fails on the real step, and a stack trace here would only bury it. if (!fs.existsSync(resultsPath)) { report([`## ${label || 'Test'} Results`, '', `No results file at \`${resultsPath}\` — the test step failed before writing one.`, '']); + console.log(`${label || 'Test'}: no results file at ${resultsPath} — the test step failed before writing one`); process.exit(0); } @@ -44,7 +45,20 @@ if (data.numFailedTests > 0) { report(lines); -process.exit(data.numFailedTests > 0 ? 1 : 0); +headline(); + +// Exits 0 whatever it found, for the same reason as summarize-playwright.js: +// summarising and gating are two jobs, and the workflow's `Fail if either suite +// failed` step is the one that should fail a run. +process.exit(0); + +/** One line of counts on stdout, so the log says what happened, not only the Summary tab. */ +function headline() { + const counts = + `${label || 'Test'}: ${data.numPassedTests} passed, ${data.numFailedTests} failed, ` + + `${data.numPendingTests} skipped`; + console.log(data.numFailedTests > 0 ? `${counts} — see the job summary for which` : counts); +} function report(out) { const summaryPath = process.env.GITEA_STEP_SUMMARY || process.env.GITHUB_STEP_SUMMARY; diff --git a/scripts/summarize-playwright.js b/scripts/summarize-playwright.js index c9f0bd5..82fe5bd 100644 --- a/scripts/summarize-playwright.js +++ b/scripts/summarize-playwright.js @@ -8,6 +8,7 @@ const [, , resultsPath] = process.argv; // fails on the real step, and a stack trace here would only bury it. if (!fs.existsSync(resultsPath)) { report([`## Playwright E2E Results`, '', `No results file at \`${resultsPath}\` — the test step failed before writing one.`, '']); + console.log(`Playwright: no results file at ${resultsPath} — the test step failed before writing one`); process.exit(0); } @@ -55,7 +56,32 @@ if (failures.length) { report(lines); -process.exit((stats.unexpected || 0) > 0 ? 1 : 0); +headline(); + +// Exits 0 whatever it found. Summarising and gating are two jobs, and this +// script doing both meant the workflow failed at a step named "Summarize +// end-to-end tests", which reads as a broken summary script rather than as +// failing tests. The workflow's own `Fail if either suite failed` step fails the +// job now. It was written for exactly that and was being skipped, because a step +// whose `if:` omits always() still requires its predecessors to have succeeded, +// and this script had already failed the job before it could run. +process.exit(0); + +/** + * One line of counts on stdout, alongside the markdown. + * + * `report` writes to GITEA_STEP_SUMMARY when it is set, which under Actions is + * always — so the table and the failure list landed in the Summary tab while the + * log showed a bare exit with no output at all. This puts the numbers in the log + * too, at the point the run stops. + */ +function headline() { + const failed = stats.unexpected || 0; + const counts = + `Playwright: ${stats.expected || 0} passed, ${failed} failed, ` + + `${stats.flaky || 0} flaky, ${stats.skipped || 0} skipped`; + console.log(failed > 0 ? `${counts} — see the job summary for which` : counts); +} function report(out) { const summaryPath = process.env.GITEA_STEP_SUMMARY || process.env.GITHUB_STEP_SUMMARY;