mirrors/etcd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

testgrid: print out all failed tests for visibility.

Browse Source 
Signed-off-by: Siyuan Zhang <sizhang@google.com>
This commit is contained in:
Siyuan Zhang 2024-04-12 17:52:36 +00:00
parent 18fd7d67a3
commit b03368485c
3 changed files with 23 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
scripts/measure-testgrid-flakiness.sh
View File

@ -10,6 +10,12 @@ then
fi
pushd ./tools/testgrid-analysis
go run main.go flaky --create-issue --dashboard=sig-etcd-periodics --tab=ci-etcd-e2e-amd64
go run main.go flaky --create-issue --dashboard=sig-etcd-periodics --tab=ci-etcd-unit-test-amd64
# ci-etcd-e2e-amd64 and ci-etcd-unit-test-amd64 runs 6 times a day. Keeping a rolling window of 14 days.
go run main.go flaky --create-issue --dashboard=sig-etcd-periodics --tab=ci-etcd-e2e-amd64 --max-days=14
go run main.go flaky --create-issue --dashboard=sig-etcd-periodics --tab=ci-etcd-unit-test-amd64 --max-days=14
# do not create issues for presubmit tests
go run main.go flaky --dashboard=sig-etcd-presubmits --tab=pull-etcd-e2e-amd64
go run main.go flaky --dashboard=sig-etcd-presubmits --tab=pull-etcd-unit-test
popd

11
tools/testgrid-analysis/cmd/data.go
View File

@ -20,6 +20,7 @@ import (
"net/http"
"os"
"strings"
"time"
apipb "github.com/GoogleCloudPlatform/testgrid/pb/api/v1"
statuspb "github.com/GoogleCloudPlatform/testgrid/pb/test_status"
@ -77,6 +78,7 @@ func processRow(dashboard, tab string, row *apipb.ListRowsResponse_Row, allTests
if !strings.HasPrefix(row.Name, "go.etcd.io") {
return &t
}
earliestTimeToConsider := time.Now().AddDate(0, 0, -1*maxDays)
total := 0
failed := 0
logs := []string{}
@ -89,13 +91,19 @@ func processRow(dashboard, tab string, row *apipb.ListRowsResponse_Row, allTests
}
continue
}
header := headers[i]
if maxDays > 0 && header.Started.AsTime().Before(earliestTimeToConsider) {
continue
}
total += 1
if _, ok := failureTestStatusesInt[cell.Result]; ok {
failed += 1
header := headers[i]
// markdown table format of | commit | log |
logs = append(logs, fmt.Sprintf("| %s | %s | https://prow.k8s.io/view/gs/kubernetes-jenkins/logs/%s/%s |", strings.Join(header.Extra, ","), header.Started.AsTime().String(), tab, header.Build))
}
if maxRuns > 0 && total >= maxRuns {
break
}
}
t.FailedRuns = failed
t.TotalRuns = total
@ -106,6 +114,7 @@ func processRow(dashboard, tab string, row *apipb.ListRowsResponse_Row, allTests
t.IssueBody = fmt.Sprintf("## %s Test: %s \nTest failed %.1f%% (%d/%d) of the time\n\nfailure logs are:\n| commit | started | log |\n| --- | --- | --- |\n%s\n",
dashboardUrl, t.FullName, t.FailureRate*100, t.FailedRuns, t.TotalRuns, strings.Join(t.FailureLogs, "\n"))
t.IssueBody += "\nPlease follow the [instructions in the contributing guide](https://github.com/etcd-io/etcd/blob/main/CONTRIBUTING.md#check-for-flaky-tests) to reproduce the issue.\n"
fmt.Printf("%s failed %.1f%% (%d/%d) of the time\n", t.FullName, t.FailureRate*100, t.FailedRuns, t.TotalRuns)
}
return &t
}

6
tools/testgrid-analysis/cmd/flaky.go
View File

@ -31,6 +31,8 @@ var flakyCmd = &cobra.Command{
var (
flakyThreshold float32
minRuns int
maxRuns int
maxDays int
createGithubIssue bool
githubOwner string
githubRepo string
@ -44,6 +46,8 @@ func init() {
flakyCmd.Flags().BoolVar(&createGithubIssue, "create-issue", false, "create Github issue for each flaky test")
flakyCmd.Flags().Float32Var(&flakyThreshold, "flaky-threshold", 0.1, "fraction threshold of test failures for a test to be considered flaky")
flakyCmd.Flags().IntVar(&minRuns, "min-runs", 20, "minimum test runs for a test to be included in flaky analysis")
flakyCmd.Flags().IntVar(&maxRuns, "max-runs", 0, "maximum test runs for a test to be included in flaky analysis, 0 to include all")
flakyCmd.Flags().IntVar(&maxDays, "max-days", 0, "maximum days of results before today to be included in flaky analysis, 0 to include all")
flakyCmd.Flags().StringVar(&githubOwner, "github-owner", "etcd-io", "the github organization to create the issue for")
flakyCmd.Flags().StringVar(&githubRepo, "github-repo", "etcd", "the github repo to create the issue for")
}
@ -59,7 +63,7 @@ func flakyFunc(cmd *cobra.Command, args []string) {
}
}
fmt.Println(lineSep)
fmt.Printf("Detected total %d flaky tests for %s#%s\n", len(flakyTests), dashboard, tab)
fmt.Printf("Detected total %d flaky tests above the %.0f%% threshold for %s#%s\n", len(flakyTests), flakyThreshold*100, dashboard, tab)
fmt.Println(lineSep)
if len(flakyTests) == 0 {
return