smallzstd: new package that constructs zstd small encoders/decoders.

It's just a config wrapper that passes "use less memory at the
expense of compression" parameters by default, so that we don't
accidentally construct resource-hungry (de)compressors.

Also includes a benchmark that measures the memory cost of the
small variants vs. the stock variants. The savings are significant
on both compressors (~8x less memory) and decompressors (~1.4x less,
not including the savings from the significantly smaller
window on the compression side - with those savings included it's
more like ~140x smaller).

BenchmarkSmallEncoder-8            	   56174	     19354 ns/op	      31 B/op	       0 allocs/op
BenchmarkSmallEncoderWithBuild-8   	    2900	    382940 ns/op	 1746547 B/op	      36 allocs/op
BenchmarkStockEncoder-8            	   48921	     25761 ns/op	     286 B/op	       0 allocs/op
BenchmarkStockEncoderWithBuild-8   	     426	   2630241 ns/op	13843842 B/op	     124 allocs/op
BenchmarkSmallDecoder-8            	  123814	      9344 ns/op	       0 B/op	       0 allocs/op
BenchmarkSmallDecoderWithBuild-8   	   41547	     27455 ns/op	   27694 B/op	      31 allocs/op
BenchmarkStockDecoder-8            	  129832	      9417 ns/op	       1 B/op	       0 allocs/op
BenchmarkStockDecoderWithBuild-8   	   25561	     51751 ns/op	   39607 B/op	      92 allocs/op

Signed-off-by: David Anderson <danderson@tailscale.com>

smallzstd/zstd_test.go

@@ -0,0 +1,131 @@
+// Copyright (c) 2020 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package smallzstd
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/klauspost/compress/zstd"
+)
+
+func BenchmarkSmallEncoder(b *testing.B) {
+	benchEncoder(b, func() (*zstd.Encoder, error) { return NewEncoder(nil) })
+}
+
+func BenchmarkSmallEncoderWithBuild(b *testing.B) {
+	benchEncoderWithConstruction(b, func() (*zstd.Encoder, error) { return NewEncoder(nil) })
+}
+
+func BenchmarkStockEncoder(b *testing.B) {
+	benchEncoder(b, func() (*zstd.Encoder, error) { return zstd.NewWriter(nil) })
+}
+
+func BenchmarkStockEncoderWithBuild(b *testing.B) {
+	benchEncoderWithConstruction(b, func() (*zstd.Encoder, error) { return zstd.NewWriter(nil) })
+}
+
+func BenchmarkSmallDecoder(b *testing.B) {
+	benchDecoder(b, func() (*zstd.Decoder, error) { return NewDecoder(nil) })
+}
+
+func BenchmarkSmallDecoderWithBuild(b *testing.B) {
+	benchDecoderWithConstruction(b, func() (*zstd.Decoder, error) { return NewDecoder(nil) })
+}
+
+func BenchmarkStockDecoder(b *testing.B) {
+	benchDecoder(b, func() (*zstd.Decoder, error) { return zstd.NewReader(nil) })
+}
+
+func BenchmarkStockDecoderWithBuild(b *testing.B) {
+	benchDecoderWithConstruction(b, func() (*zstd.Decoder, error) { return zstd.NewReader(nil) })
+}
+
+func benchEncoder(b *testing.B, mk func() (*zstd.Encoder, error)) {
+	b.ReportAllocs()
+
+	in := testdata(b)
+	out := make([]byte, 0, 10<<10) // 10kiB
+
+	e, err := mk()
+	if err != nil {
+		b.Fatalf("making encoder: %v", err)
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		e.EncodeAll(in, out)
+	}
+}
+
+func benchEncoderWithConstruction(b *testing.B, mk func() (*zstd.Encoder, error)) {
+	b.ReportAllocs()
+
+	in := testdata(b)
+	out := make([]byte, 0, 10<<10) // 10kiB
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		e, err := mk()
+		if err != nil {
+			b.Fatalf("making encoder: %v", err)
+		}
+
+		e.EncodeAll(in, out)
+	}
+}
+
+func benchDecoder(b *testing.B, mk func() (*zstd.Decoder, error)) {
+	b.ReportAllocs()
+
+	in := compressedTestdata(b)
+	out := make([]byte, 0, 10<<10)
+
+	d, err := mk()
+	if err != nil {
+		b.Fatalf("creating decoder: %v", err)
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		d.DecodeAll(in, out)
+	}
+}
+
+func benchDecoderWithConstruction(b *testing.B, mk func() (*zstd.Decoder, error)) {
+	b.ReportAllocs()
+
+	in := compressedTestdata(b)
+	out := make([]byte, 0, 10<<10)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		d, err := mk()
+		if err != nil {
+			b.Fatalf("creating decoder: %v", err)
+		}
+
+		d.DecodeAll(in, out)
+	}
+}
+
+func testdata(b *testing.B) []byte {
+	b.Helper()
+	in, err := ioutil.ReadFile("testdata")
+	if err != nil {
+		b.Fatalf("reading testdata: %v", err)
+	}
+	return in
+}
+
+func compressedTestdata(b *testing.B) []byte {
+	b.Helper()
+	uncomp := testdata(b)
+	e, err := NewEncoder(nil)
+	if err != nil {
+		b.Fatalf("creating encoder: %v", err)
+	}
+	return e.EncodeAll(uncomp, nil)
+}