mirrors/etcd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

functional-tester: plog for milli-second timestamp

Browse Source 
Standard log package by default only prints out the second-scale
so the 3rd party log feeder mixes the order of the events, which makes
the debugging hard. This replaces it with capnslog and make them consistent
with all other etcd log formats.
This commit is contained in:
Gyu-Ho Lee
2016-02-17 18:39:05 -08:00
parent 40d3e0daff
commit 7aa62ec595
6 changed files with 59 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
tools/functional-tester/etcd-tester/tester.go
View File

@ -15,7 +15,6 @@
package main
import (
"log"
"sync"
"time"
)
@ -46,49 +45,49 @@ func (tt *tester) runLoop() {
tt.status.setCase(j)
if err := tt.cluster.WaitHealth(); err != nil {
log.Printf("etcd-tester: [round#%d case#%d] wait full health error: %v", i, j, err)
plog.Printf("etcd-tester: [round#%d case#%d] wait full health error: %v", i, j, err)
if err := tt.cleanup(i, j); err != nil {
log.Printf("etcd-tester: [round#%d case#%d] cleanup error: %v", i, j, err)
plog.Printf("etcd-tester: [round#%d case#%d] cleanup error: %v", i, j, err)
return
}
continue
}
log.Printf("etcd-tester: [round#%d case#%d] start failure %s", i, j, f.Desc())
plog.Printf("etcd-tester: [round#%d case#%d] start failure %s", i, j, f.Desc())
log.Printf("etcd-tester: [round#%d case#%d] start injecting failure...", i, j)
plog.Printf("etcd-tester: [round#%d case#%d] start injecting failure...", i, j)
if err := f.Inject(tt.cluster, i); err != nil {
log.Printf("etcd-tester: [round#%d case#%d] injection error: %v", i, j, err)
plog.Printf("etcd-tester: [round#%d case#%d] injection error: %v", i, j, err)
if err := tt.cleanup(i, j); err != nil {
log.Printf("etcd-tester: [round#%d case#%d] cleanup error: %v", i, j, err)
plog.Printf("etcd-tester: [round#%d case#%d] cleanup error: %v", i, j, err)
return
}
continue
}
log.Printf("etcd-tester: [round#%d case#%d] injected failure", i, j)
plog.Printf("etcd-tester: [round#%d case#%d] injected failure", i, j)
log.Printf("etcd-tester: [round#%d case#%d] start recovering failure...", i, j)
plog.Printf("etcd-tester: [round#%d case#%d] start recovering failure...", i, j)
if err := f.Recover(tt.cluster, i); err != nil {
log.Printf("etcd-tester: [round#%d case#%d] recovery error: %v", i, j, err)
plog.Printf("etcd-tester: [round#%d case#%d] recovery error: %v", i, j, err)
if err := tt.cleanup(i, j); err != nil {
log.Printf("etcd-tester: [round#%d case#%d] cleanup error: %v", i, j, err)
plog.Printf("etcd-tester: [round#%d case#%d] cleanup error: %v", i, j, err)
return
}
continue
}
log.Printf("etcd-tester: [round#%d case#%d] recovered failure", i, j)
plog.Printf("etcd-tester: [round#%d case#%d] recovered failure", i, j)
if tt.cluster.v2Only {
log.Printf("etcd-tester: [round#%d case#%d] succeed!", i, j)
plog.Printf("etcd-tester: [round#%d case#%d] succeed!", i, j)
continue
}
log.Printf("etcd-tester: [round#%d case#%d] canceling the stressers...", i, j)
plog.Printf("etcd-tester: [round#%d case#%d] canceling the stressers...", i, j)
for _, s := range tt.cluster.Stressers {
s.Cancel()
}
log.Printf("etcd-tester: [round#%d case#%d] canceled stressers", i, j)
plog.Printf("etcd-tester: [round#%d case#%d] canceled stressers", i, j)
log.Printf("etcd-tester: [round#%d case#%d] checking current revisions...", i, j)
plog.Printf("etcd-tester: [round#%d case#%d] checking current revisions...", i, j)
var (
revs map[string]int64
hashes map[string]int64
@ -100,55 +99,55 @@ func (tt *tester) runLoop() {
revs, hashes, rerr = tt.cluster.getRevisionHash()
if rerr != nil {
log.Printf("etcd-tester: [round#%d case#%d.%d] failed to get current revisions (%v)", i, j, k, rerr)
plog.Printf("etcd-tester: [round#%d case#%d.%d] failed to get current revisions (%v)", i, j, k, rerr)
continue
}
if currentRevision, ok = getSameValue(revs); ok {
break
}
log.Printf("etcd-tester: [round#%d case#%d.%d] inconsistent current revisions %+v", i, j, k, revs)
plog.Printf("etcd-tester: [round#%d case#%d.%d] inconsistent current revisions %+v", i, j, k, revs)
}
if !ok || rerr != nil {
log.Printf("etcd-tester: [round#%d case#%d] checking current revisions failed (%v)", i, j, revs)
plog.Printf("etcd-tester: [round#%d case#%d] checking current revisions failed (%v)", i, j, revs)
if err := tt.cleanup(i, j); err != nil {
log.Printf("etcd-tester: [round#%d case#%d] cleanup error: %v", i, j, err)
plog.Printf("etcd-tester: [round#%d case#%d] cleanup error: %v", i, j, err)
return
}
continue
}
log.Printf("etcd-tester: [round#%d case#%d] all members are consistent with current revisions", i, j)
plog.Printf("etcd-tester: [round#%d case#%d] all members are consistent with current revisions", i, j)
log.Printf("etcd-tester: [round#%d case#%d] checking current storage hashes...", i, j)
plog.Printf("etcd-tester: [round#%d case#%d] checking current storage hashes...", i, j)
if _, ok = getSameValue(hashes); !ok {
log.Printf("etcd-tester: [round#%d case#%d] checking current storage hashes failed (%v)", i, j, hashes)
plog.Printf("etcd-tester: [round#%d case#%d] checking current storage hashes failed (%v)", i, j, hashes)
if err := tt.cleanup(i, j); err != nil {
log.Printf("etcd-tester: [round#%d case#%d] cleanup error: %v", i, j, err)
plog.Printf("etcd-tester: [round#%d case#%d] cleanup error: %v", i, j, err)
return
}
continue
}
log.Printf("etcd-tester: [round#%d case#%d] all members are consistent with storage hashes", i, j)
plog.Printf("etcd-tester: [round#%d case#%d] all members are consistent with storage hashes", i, j)
log.Printf("etcd-tester: [round#%d case#%d] restarting the stressers...", i, j)
plog.Printf("etcd-tester: [round#%d case#%d] restarting the stressers...", i, j)
for _, s := range tt.cluster.Stressers {
go s.Stress()
}
log.Printf("etcd-tester: [round#%d case#%d] succeed!", i, j)
plog.Printf("etcd-tester: [round#%d case#%d] succeed!", i, j)
}
revToCompact := max(0, currentRevision-10000)
log.Printf("etcd-tester: [round#%d] compacting storage at %d (current revision %d)", i, revToCompact, currentRevision)
plog.Printf("etcd-tester: [round#%d] compacting storage at %d (current revision %d)", i, revToCompact, currentRevision)
if err := tt.cluster.compactKV(revToCompact); err != nil {
log.Printf("etcd-tester: [round#%d] compactKV error (%v)", i, err)
plog.Printf("etcd-tester: [round#%d] compactKV error (%v)", i, err)
if err := tt.cleanup(i, 0); err != nil {
log.Printf("etcd-tester: [round#%d] cleanup error: %v", i, err)
plog.Printf("etcd-tester: [round#%d] cleanup error: %v", i, err)
return
}
continue
}
log.Printf("etcd-tester: [round#%d] compacted storage", i)
plog.Printf("etcd-tester: [round#%d] compacted storage", i)
// TODO: make sure compaction is finished
time.Sleep(30 * time.Second)
@ -159,7 +158,7 @@ func (tt *tester) cleanup(i, j int) error {
roundFailedTotalCounter.Inc()
caseFailedTotalCounter.WithLabelValues(tt.failures[j].Desc()).Inc()
log.Printf("etcd-tester: [round#%d case#%d] cleaning up...", i, j)
plog.Printf("etcd-tester: [round#%d case#%d] cleaning up...", i, j)
if err := tt.cluster.Cleanup(); err != nil {
return err
}