diff --git a/server/config/config.go b/server/config/config.go index 7874d8ee0..021b42f35 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -192,6 +192,9 @@ type ServerConfig struct { // ExperimentalBootstrapDefragThresholdMegabytes is the minimum number of megabytes needed to be freed for etcd server to // consider running defrag during bootstrap. Needs to be set to non-zero value to take effect. ExperimentalBootstrapDefragThresholdMegabytes uint `json:"experimental-bootstrap-defrag-threshold-megabytes"` + + // V2Deprecation defines a phase of v2store deprecation process. + V2Deprecation V2DeprecationEnum `json:"v2-deprecation"` } // VerifyBootstrap sanity-checks the initial config for bootstrap case diff --git a/server/embed/v2_deprecation.go b/server/config/v2_deprecation.go similarity index 50% rename from server/embed/v2_deprecation.go rename to server/config/v2_deprecation.go index 2fb9ba374..828bd9a8f 100644 --- a/server/embed/v2_deprecation.go +++ b/server/config/v2_deprecation.go @@ -12,35 +12,39 @@ // See the License for the specific language governing permissions and // limitations under the License. -package embed +package config type V2DeprecationEnum string const ( -// Default in v3.5. Issues a warning if v2store have meaningful content. -V2_DEPR_0_NOT_YET = V2DeprecationEnum("not-yet") -// Default in v3.6. Meaningful v2 state is not allowed. -// The V2 files are maintained for v3.5 rollback. -V2_DEPR_1_WRITE_ONLY = V2DeprecationEnum("write-only") -// V2store is WIPED if found !!! -V2_DEPR_1_WRITE_ONLY_DROP = V2DeprecationEnum("write-only-drop-data") -// V2store is neither written nor read. Usage of this configuration is blocking -// ability to rollback to etcd v3.5. -V2_DEPR_2_GONE = V2DeprecationEnum("gone") + // Default in v3.5. Issues a warning if v2store have meaningful content. + V2_DEPR_0_NOT_YET = V2DeprecationEnum("not-yet") + // Default in v3.6. Meaningful v2 state is not allowed. + // The V2 files are maintained for v3.5 rollback. + V2_DEPR_1_WRITE_ONLY = V2DeprecationEnum("write-only") + // V2store is WIPED if found !!! + V2_DEPR_1_WRITE_ONLY_DROP = V2DeprecationEnum("write-only-drop-data") + // V2store is neither written nor read. Usage of this configuration is blocking + // ability to rollback to etcd v3.5. + V2_DEPR_2_GONE = V2DeprecationEnum("gone") -V2_DEPR_DEFAULT = V2_DEPR_0_NOT_YET + V2_DEPR_DEFAULT = V2_DEPR_0_NOT_YET ) func (e V2DeprecationEnum) IsAtLeast(v2d V2DeprecationEnum) bool { - return e.level() >= v2d.level() + return e.level() >= v2d.level() } func (e V2DeprecationEnum) level() int { switch e { - case V2_DEPR_0_NOT_YET: return 0 - case V2_DEPR_1_WRITE_ONLY: return 1 - case V2_DEPR_1_WRITE_ONLY_DROP: return 2 - case V2_DEPR_2_GONE: return 3 + case V2_DEPR_0_NOT_YET: + return 0 + case V2_DEPR_1_WRITE_ONLY: + return 1 + case V2_DEPR_1_WRITE_ONLY_DROP: + return 2 + case V2_DEPR_2_GONE: + return 3 } panic("Unknown V2DeprecationEnum: " + e) -} \ No newline at end of file +} diff --git a/server/embed/v2_deprecation_test.go b/server/config/v2_deprecation_test.go similarity index 94% rename from server/embed/v2_deprecation_test.go rename to server/config/v2_deprecation_test.go index f9533bd5a..c8d911d60 100644 --- a/server/embed/v2_deprecation_test.go +++ b/server/config/v2_deprecation_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package embed +package config import "testing" @@ -32,7 +32,7 @@ func TestV2DeprecationEnum_IsAtLeast(t *testing.T) { {V2_DEPR_1_WRITE_ONLY_DROP, V2_DEPR_1_WRITE_ONLY, true}, } for _, tt := range tests { - t.Run(string(tt.e) + " >= " + string(tt.v2d), func(t *testing.T) { + t.Run(string(tt.e)+" >= "+string(tt.v2d), func(t *testing.T) { if got := tt.e.IsAtLeast(tt.v2d); got != tt.want { t.Errorf("IsAtLeast() = %v, want %v", got, tt.want) } diff --git a/server/embed/config.go b/server/embed/config.go index bf24b0483..9e8583a49 100644 --- a/server/embed/config.go +++ b/server/embed/config.go @@ -33,6 +33,7 @@ import ( "go.etcd.io/etcd/client/pkg/v3/types" "go.etcd.io/etcd/pkg/v3/flags" "go.etcd.io/etcd/pkg/v3/netutil" + "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/server/v3/etcdserver/api/v3compactor" @@ -402,7 +403,7 @@ type Config struct { ExperimentalTxnModeWriteWithSharedBuffer bool `json:"experimental-txn-mode-write-with-shared-buffer"` // V2Deprecation describes phase of API & Storage V2 support - V2Deprecation V2DeprecationEnum `json:"v2-deprecation"` + V2Deprecation config.V2DeprecationEnum `json:"v2-deprecation"` } // configYAML holds the config suitable for yaml parsing @@ -498,7 +499,7 @@ func NewConfig() *Config { ExperimentalMemoryMlock: false, ExperimentalTxnModeWriteWithSharedBuffer: true, - V2Deprecation: V2_DEPR_DEFAULT, + V2Deprecation: config.V2_DEPR_DEFAULT, } cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name) return cfg @@ -800,12 +801,11 @@ func (cfg Config) InitialClusterFromName(name string) (ret string) { func (cfg Config) IsNewCluster() bool { return cfg.ClusterState == ClusterStateFlagNew } func (cfg Config) ElectionTicks() int { return int(cfg.ElectionMs / cfg.TickMs) } -func (cfg Config) V2DeprecationEffective() V2DeprecationEnum { +func (cfg Config) V2DeprecationEffective() config.V2DeprecationEnum { if cfg.V2Deprecation == "" { - return V2_DEPR_DEFAULT - } else { - return cfg.V2Deprecation + return config.V2_DEPR_DEFAULT } + return cfg.V2Deprecation } func (cfg Config) defaultPeerHost() bool { diff --git a/server/embed/etcd.go b/server/embed/etcd.go index d38091573..79b473c6f 100644 --- a/server/embed/etcd.go +++ b/server/embed/etcd.go @@ -226,6 +226,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { ExperimentalMemoryMlock: cfg.ExperimentalMemoryMlock, ExperimentalTxnModeWriteWithSharedBuffer: cfg.ExperimentalTxnModeWriteWithSharedBuffer, ExperimentalBootstrapDefragThresholdMegabytes: cfg.ExperimentalBootstrapDefragThresholdMegabytes, + V2Deprecation: cfg.V2DeprecationEffective(), } if srvcfg.ExperimentalEnableDistributedTracing { @@ -696,6 +697,9 @@ func (e *Etcd) serveClients() (err error) { // Start a client server goroutine for each listen address var h http.Handler if e.Config().EnableV2 { + if e.Config().V2DeprecationEffective().IsAtLeast(config.V2_DEPR_1_WRITE_ONLY) { + return fmt.Errorf("--enable-v2 and --v2-deprecation=%s are mutually exclusive", e.Config().V2DeprecationEffective()) + } e.cfg.logger.Warn("Flag `enable-v2` is deprecated and will get removed in etcd 3.6.") if len(e.Config().ExperimentalEnableV2V3) > 0 { e.cfg.logger.Warn("Flag `experimental-enable-v2v3` is deprecated and will get removed in etcd 3.6.") diff --git a/server/etcdmain/config.go b/server/etcdmain/config.go index 0072109d6..f9c91d9f9 100644 --- a/server/etcdmain/config.go +++ b/server/etcdmain/config.go @@ -27,6 +27,7 @@ import ( "go.etcd.io/etcd/api/v3/version" "go.etcd.io/etcd/client/pkg/v3/logutil" "go.etcd.io/etcd/pkg/v3/flags" + cconfig "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/embed" "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp" @@ -121,10 +122,10 @@ func newConfig() *config { proxyFlagOn, ), v2deprecation: flags.NewSelectiveStringsValue( - string(embed.V2_DEPR_0_NOT_YET), - string(embed.V2_DEPR_1_WRITE_ONLY), - string(embed.V2_DEPR_1_WRITE_ONLY_DROP), - string(embed.V2_DEPR_2_GONE)), + string(cconfig.V2_DEPR_0_NOT_YET), + string(cconfig.V2_DEPR_1_WRITE_ONLY), + string(cconfig.V2_DEPR_1_WRITE_ONLY_DROP), + string(cconfig.V2_DEPR_2_GONE)), } fs := cfg.cf.flagSet @@ -343,7 +344,7 @@ func (cfg *config) parse(arguments []string) error { } if cfg.ec.V2Deprecation == "" { - cfg.ec.V2Deprecation = embed.V2_DEPR_DEFAULT + cfg.ec.V2Deprecation = cconfig.V2_DEPR_DEFAULT } // now logger is set up @@ -400,7 +401,7 @@ func (cfg *config) configFromCmdLine() error { cfg.cp.Fallback = cfg.cf.fallback.String() cfg.cp.Proxy = cfg.cf.proxy.String() - cfg.ec.V2Deprecation = embed.V2DeprecationEnum(cfg.cf.v2deprecation.String()) + cfg.ec.V2Deprecation = cconfig.V2DeprecationEnum(cfg.cf.v2deprecation.String()) // disable default advertise-client-urls if lcurls is set missingAC := flags.IsSet(cfg.cf.flagSet, "listen-client-urls") && !flags.IsSet(cfg.cf.flagSet, "advertise-client-urls") diff --git a/server/etcdmain/help.go b/server/etcdmain/help.go index ca8ac039d..dc5b55fae 100644 --- a/server/etcdmain/help.go +++ b/server/etcdmain/help.go @@ -18,6 +18,7 @@ import ( "fmt" "strconv" + cconfig "go.etcd.io/etcd/server/v3/config" "go.etcd.io/etcd/server/v3/embed" "golang.org/x/crypto/bcrypt" ) @@ -124,8 +125,8 @@ Clustering: Interpret 'auto-compaction-retention' one of: periodic|revision. 'periodic' for duration based retention, defaulting to hours if no time unit is provided (e.g. '5m'). 'revision' for revision number based retention. --enable-v2 '` + strconv.FormatBool(embed.DefaultEnableV2) + `' Accept etcd V2 client requests. Deprecated and to be decommissioned in v3.6. - --v2-deprecation '` + string(embed.V2_DEPR_DEFAULT) + `' - Phase of v2store deprecation. Allows to optin for higher compatibility mode. + --v2-deprecation '` + string(cconfig.V2_DEPR_DEFAULT) + `' + Phase of v2store deprecation. Allows to opt-in for higher compatibility mode. Supported values: 'not-yet' // Issues a warning if v2store have meaningful content (default in v3.5) 'write-only' // Custom v2 state is not allowed (planned default in v3.6) diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go index ab62888c5..791740df9 100644 --- a/server/etcdserver/server.go +++ b/server/etcdserver/server.go @@ -479,6 +479,11 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) { cfg.Logger.Panic("failed to recover from snapshot", zap.Error(err)) } + if err = assertNoV2StoreContent(cfg.Logger, st, cfg.V2Deprecation); err != nil { + cfg.Logger.Error("illegal v2store content", zap.Error(err)) + return nil, err + } + cfg.Logger.Info( "recovered v2 store from snapshot", zap.Uint64("snapshot-index", snapshot.Metadata.Index), @@ -496,6 +501,8 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) { zap.Int64("backend-size-in-use-bytes", s2), zap.String("backend-size-in-use", humanize.Bytes(uint64(s2))), ) + } else { + cfg.Logger.Info("No snapshot found. Recovering WAL from scratch!") } if !cfg.ForceNewCluster { @@ -662,6 +669,23 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) { return srv, nil } +// assertNoV2StoreContent -> depending on the deprecation stage, warns or report an error +// if the v2store contains custom content. +func assertNoV2StoreContent(lg *zap.Logger, st v2store.Store, deprecationStage config.V2DeprecationEnum) error { + metaOnly, err := membership.IsMetaStoreOnly(st) + if err != nil { + return err + } + if metaOnly { + return nil + } + if deprecationStage.IsAtLeast(config.V2_DEPR_1_WRITE_ONLY) { + return fmt.Errorf("detected disallowed custom content in v2store for stage --v2-deprecation=%s", deprecationStage) + } + lg.Warn("detected custom v2store content. Etcd v3.5 is the last version allowing to access it using API v2. Please remove the content.") + return nil +} + func (s *EtcdServer) Logger() *zap.Logger { s.lgMu.RLock() l := s.lg @@ -1252,6 +1276,10 @@ func (s *EtcdServer) applySnapshot(ep *etcdProgress, apply *apply) { lg.Panic("failed to restore v2 store", zap.Error(err)) } + if err := assertNoV2StoreContent(lg, s.v2store, s.Cfg.V2Deprecation); err != nil { + lg.Panic("illegal v2store content", zap.Error(err)) + } + lg.Info("restored v2 store") s.cluster.SetBackend(newbe) diff --git a/tests/integration/cluster.go b/tests/integration/cluster.go index 755d8520a..c5302e429 100644 --- a/tests/integration/cluster.go +++ b/tests/integration/cluster.go @@ -706,6 +706,8 @@ func mustNewMember(t testutil.TB, mcfg memberConfig) *member { m.InitialCorruptCheck = true m.WarningApplyDuration = embed.DefaultWarningApplyDuration + m.V2Deprecation = config.V2_DEPR_DEFAULT + m.Logger = memberLogger(t, mcfg.name) t.Cleanup(func() { // if we didn't cleanup the logger, the consecutive test