mirrors/tailscale
1
0
Fork 0
Code Issues Pull Requests Projects Wiki Activity

tstest/integration/vms: build and run NixOS (#2190)

Browse Source 
Okay, so, at a high level testing NixOS is a lot different than
other distros due to NixOS' determinism. Normally NixOS wants packages to
be defined in either an overlay, a custom packageOverrides or even
yolo-inline as a part of the system configuration. This is going to have
us take a different approach compared to other distributions. The overall
plan here is as following:

1. make the binaries as normal
2. template in their paths as raw strings to the nixos system module
3. run `nixos-generators -f qcow -o $CACHE_DIR/tailscale/nixos/version -c generated-config.nix`
4. pass that to the steps that make the virtual machine

It doesn't really make sense for us to use a premade virtual machine image
for this as that will make it harder to deterministically create the image.

Nix commands generate a lot of output, so their output is hidden behind the
`-verbose-nix-output` flag.

This unfortunately makes this test suite have a hard dependency on
Nix/NixOS, however the test suite has only ever been run on NixOS (and I
am not sure if it runs on other distros at all), so this probably isn't too
big of an issue.

Signed-off-by: Christine Dodrill <xe@tailscale.com>
This commit is contained in:
Christine Dodrill
2021-06-28 06:45:45 -07:00
committed by GitHub
parent 72a0b5f042
commit b131a74f99
4 changed files with 250 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
tstest/integration/vms/vms_test.go
View File

@ -100,6 +100,8 @@ func TestDownloadImages(t *testing.T) {
t.Skip("not running integration tests (need --run-vm-tests)")
}
bins := integration.BuildTestBinaries(t)
for _, d := range distros {
distro := d
t.Run(distro.name, func(t *testing.T) {
@ -107,9 +109,13 @@ func TestDownloadImages(t *testing.T) {
t.Skipf("distro name %q doesn't match regex: %s", distro.name, distroRex)
}
if strings.HasPrefix(distro.name, "nixos") {
t.Skip("NixOS is built on the fly, no need to download it")
}
t.Parallel()
fetchDistro(t, distro)
fetchDistro(t, distro, bins)
})
}
}
@ -158,6 +164,14 @@ var distros = []Distro{
{"ubuntu-20-04", "https://cloud-images.ubuntu.com/focal/20210603/focal-server-cloudimg-amd64.img", "1c0969323b058ba8b91fec245527069c2f0502fc119b9138b213b6bfebd965cb", 512, "apt", "systemd"},
{"ubuntu-20-10", "https://cloud-images.ubuntu.com/groovy/20210604/groovy-server-cloudimg-amd64.img", "2196df5f153faf96443e5502bfdbcaa0baaefbaec614348fec344a241855b0ef", 512, "apt", "systemd"},
{"ubuntu-21-04", "https://cloud-images.ubuntu.com/hirsute/20210603/hirsute-server-cloudimg-amd64.img", "bf07f36fc99ff521d3426e7d257e28f0c81feebc9780b0c4f4e25ae594ff4d3b", 512, "apt", "systemd"},
// NOTE(Xe): We build fresh NixOS images for every test run, so the URL being
// used here is actually the URL of the NixOS channel being built from and the
// shasum is meaningless. This `channel:name` syntax is documented at [1].
//
// [1]: https://nixos.org/manual/nix/unstable/command-ref/env-common.html
{"nixos-unstable", "channel:nixos-unstable", "lolfakesha", 512, "nix", "systemd"},
{"nixos-21-05", "channel:nixos-21.05", "lolfakesha", 512, "nix", "systemd"},
}
// fetchFromS3 fetches a distribution image from Amazon S3 or reports whether
@ -212,7 +226,7 @@ func fetchFromS3(t *testing.T, fout *os.File, d Distro) bool {
// fetchDistro fetches a distribution from the internet if it doesn't already exist locally. It
// also validates the sha256 sum from a known good hash.
func fetchDistro(t *testing.T, resultDistro Distro) {
func fetchDistro(t *testing.T, resultDistro Distro, bins *integration.Binaries) string {
t.Helper()
cdir, err := os.UserCacheDir()
@ -221,6 +235,10 @@ func fetchDistro(t *testing.T, resultDistro Distro) {
}
cdir = filepath.Join(cdir, "tailscale", "vm-test")
if strings.HasPrefix(resultDistro.name, "nixos") {
return makeNixOSImage(t, resultDistro, cdir, bins)
}
qcowPath := filepath.Join(cdir, "qcow2", resultDistro.sha256sum)
_, err = os.Stat(qcowPath)
@ -267,6 +285,8 @@ func fetchDistro(t *testing.T, resultDistro Distro) {
}
}
}
return qcowPath
}
func checkCachedImageHash(t *testing.T, d Distro, cacheDir string) (gotHash string) {
@ -311,18 +331,12 @@ func run(t *testing.T, dir, prog string, args ...string) {
// mkLayeredQcow makes a layered qcow image that allows us to keep the upstream
// VM images pristine and only do our changes on an overlay.
func mkLayeredQcow(t *testing.T, tdir string, d Distro) {
func mkLayeredQcow(t *testing.T, tdir string, d Distro, qcowBase string) {
t.Helper()
cdir, err := os.UserCacheDir()
if err != nil {
t.Fatalf("can't find cache dir: %v", err)
}
cdir = filepath.Join(cdir, "tailscale", "vm-test")
run(t, tdir, "qemu-img", "create",
"-f", "qcow2",
"-o", "backing_file="+filepath.Join(cdir, "qcow2", d.sha256sum),
"-o", "backing_file="+qcowBase,
filepath.Join(tdir, d.name+".qcow2"),
)
}
@ -413,7 +427,7 @@ func mkSeed(t *testing.T, d Distro, sshKey, hostURL, tdir string, port int) {
// mkVM makes a KVM-accelerated virtual machine and prepares it for introduction
// to the testcontrol server. The function it returns is for killing the virtual
// machine when it is time for it to die.
func mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir string) {
func mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir string, bins *integration.Binaries) {
t.Helper()
cdir, err := os.UserCacheDir()
@ -428,8 +442,7 @@ func mkVM(t *testing.T, n int, d Distro, sshKey, hostURL, tdir string) {
t.Fatal(err)
}
fetchDistro(t, d)
mkLayeredQcow(t, tdir, d)
mkLayeredQcow(t, tdir, d, fetchDistro(t, d, bins))
mkSeed(t, d, sshKey, hostURL, tdir, port)
driveArg := fmt.Sprintf("file=%s,if=virtio", filepath.Join(tdir, d.name+".qcow2"))
@ -634,7 +647,7 @@ func TestVMIntegrationEndToEnd(t *testing.T) {
}
defer ramsem.Release(int64(distro.mem))
mkVM(t, n, distro, string(pubkey), loginServer, dir)
mkVM(t, n, distro, string(pubkey), loginServer, dir, bins)
var ipm ipMapping
t.Run("wait-for-start", func(t *testing.T) {
@ -747,6 +760,10 @@ func testDistro(t *testing.T, loginServer string, d Distro, signer ssh.Signer, i
}
func copyBinaries(t *testing.T, d Distro, conn *ssh.Client, bins *integration.Binaries) {
if strings.HasPrefix(d.name, "nixos") {
return
}
cli, err := sftp.NewClient(conn)
if err != nil {
t.Fatalf("can't connect over sftp to copy binaries: %v", err)