From def265083b56a19b7f813a054d217af09fc0a7ba Mon Sep 17 00:00:00 2001 From: yaruk-byte Date: Fri, 17 Jul 2026 12:29:12 -0700 Subject: [PATCH] tstest/integration: run a smoke test against a Windows tailscaled service (#20382) * tstest/integration: run a test against a real Windows tailscaled service Updates #20381 Signed-off-by: Yaruk Asghar * tstest/integration: serialize Windows service tests and clean up state Updates #20381 Signed-off-by: Yaruk Asghar * tstest/integration: address review feedback on Windows service tests Updates #20381 Signed-off-by: Yaruk Asghar * tstest/integration: use background context for service teardown Updates #20381 Signed-off-by: Yaruk Asghar * tstest/integration: run Windows service test via the normal windows CI run Updates #20381 Signed-off-by: Yaruk Asghar * tstest/integration: address review feedback on Windows service tests Updates #20381 Signed-off-by: Yaruk Asghar --------- Signed-off-by: Yaruk Asghar --- tsconst/wintun/wintun.go | 24 ++ tstest/integration/integration.go | 126 ++++++++--- tstest/integration/integration_test.go | 8 +- tstest/integration/service_notwindows.go | 18 ++ tstest/integration/service_windows.go | 270 +++++++++++++++++++++++ 5 files changed, 414 insertions(+), 32 deletions(-) create mode 100644 tsconst/wintun/wintun.go create mode 100644 tstest/integration/service_notwindows.go create mode 100644 tstest/integration/service_windows.go diff --git a/tsconst/wintun/wintun.go b/tsconst/wintun/wintun.go new file mode 100644 index 000000000..a2676b96e --- /dev/null +++ b/tsconst/wintun/wintun.go @@ -0,0 +1,24 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +// Package wintun is the single source of truth for the pinned wintun.dll release +// the Tailscale Windows client is built and tested against. +package wintun + +const ( + // Version is the pinned wintun release. + Version = "0.14.1" + // URL is the wintun.net download for Version. + URL = "https://www.wintun.net/builds/wintun-" + Version + ".zip" + // SHA256 is the hex-encoded sha256 of the zip at URL. + SHA256 = "07c256185d6ee3652e09fa55c0b673e2624b565e02c4b9091c79ca7d2f24ef51" +) + +// DLLZipPath returns the path of the wintun.dll for goArch within the release zip. +func DLLZipPath(goArch string) string { + arch := goArch + if goArch == "386" { + arch = "x86" + } + return "wintun/bin/" + arch + "/wintun.dll" +} diff --git a/tstest/integration/integration.go b/tstest/integration/integration.go index 6b6df74c1..fe62bc5a2 100644 --- a/tstest/integration/integration.go +++ b/tstest/integration/integration.go @@ -40,6 +40,7 @@ "tailscale.com/ipn/ipnstate" "tailscale.com/ipn/store" "tailscale.com/net/stun/stuntest" + "tailscale.com/paths" "tailscale.com/safesocket" "tailscale.com/syncs" "tailscale.com/tailcfg" @@ -49,6 +50,7 @@ "tailscale.com/types/logger" "tailscale.com/types/logid" "tailscale.com/types/nettype" + "tailscale.com/util/cibuild" "tailscale.com/util/rands" "tailscale.com/util/zstdframe" "tailscale.com/version" @@ -57,6 +59,10 @@ var ( verboseTailscaled = flag.Bool("verbose-tailscaled", false, "verbose tailscaled logging") verboseTailscale = flag.Bool("verbose-tailscale", false, "verbose tailscale CLI logging") + + // runWindowsServiceTests enables the Windows service-mode integration tests. + // On by default in CI; tests opt in via NewTestEnv(t, canRunAsServiceOnWindows()). + runWindowsServiceTests = flag.Bool("run-windows-service-tests", cibuild.On(), "run Windows service-mode integration tests") ) // MainError is an error that's set if an error conditions happens outside of a @@ -98,7 +104,7 @@ type BinaryInfo struct { // was cleaned up. func (b BinaryInfo) CopyTo(dir string) (BinaryInfo, error) { ret := b - ret.Path = filepath.Join(dir, path.Base(b.Path)) + ret.Path = filepath.Join(dir, filepath.Base(b.Path)) switch runtime.GOOS { case "linux": @@ -496,6 +502,7 @@ type Entry struct { type TestEnv struct { t testing.TB tunMode bool + windowsService bool // run tailscaled as a Windows service cli string daemon string loopbackPort *int @@ -534,11 +541,40 @@ func (f ConfigureControl) ModifyTestEnv(te *TestEnv) { f(te.Control) } +// canRunAsServiceOnWindowsOpt is the TestEnvOpt returned by canRunAsServiceOnWindows. +type canRunAsServiceOnWindowsOpt struct{} + +func (canRunAsServiceOnWindowsOpt) ModifyTestEnv(te *TestEnv) { + // Only run as a service on Windows; on other platforms the test runs + // the normal userspace daemon with a faked Windows GOOS, as it always has. + if runtime.GOOS == "windows" { + te.windowsService = true + } +} + +// canRunAsServiceOnWindows enables the test to run on Windows. +// TODO(#20464): remove this and explicitly skip tests that need more work +// before they can run on Windows, instead of requiring tests to opt in with this option. +func canRunAsServiceOnWindows() TestEnvOpt { return canRunAsServiceOnWindowsOpt{} } + // NewTestEnv starts a bunch of services and returns a new test environment. // NewTestEnv arranges for the environment's resources to be cleaned up on exit. func NewTestEnv(t testing.TB, opts ...TestEnvOpt) *TestEnv { + // Integration tests skip on Windows unless a test opts in via canRunAsServiceOnWindows. + // Pre-scan the opts before starting any servers so a skip leaks nothing. + canRunAsService := false + for _, o := range opts { + if _, ok := o.(canRunAsServiceOnWindowsOpt); ok { + canRunAsService = true + } + } if runtime.GOOS == "windows" { - t.Skip("not tested/working on Windows yet") + if !canRunAsService { + t.Skip("integration tests skip on Windows unless the test calls canRunAsServiceOnWindows") + } + if !*runWindowsServiceTests { + t.Skip("Windows service tests disabled (--run-windows-service-tests=false)") + } } derpMap := RunDERPAndSTUN(t, logger.Discard, "127.0.0.1") logc := new(LogCatcher) @@ -608,11 +644,18 @@ func NewTestNode(t *testing.T, env *TestEnv) *TestNode { sockFile = filepath.Join(os.TempDir(), rands.HexString(8)+".sock") t.Cleanup(func() { os.Remove(sockFile) }) } + stateFile := filepath.Join(dir, "tailscaled.state") // matches what cmd/tailscaled uses + if env.windowsService { + // A LocalSystem service ignores --socket/--statedir and uses the + // default pipe and state path; point the harness at those. + sockFile = paths.DefaultTailscaledSocket() + stateFile = paths.DefaultTailscaledStateFile() + } n := &TestNode{ env: env, dir: dir, sockFile: sockFile, - stateFile: filepath.Join(dir, "tailscaled.state"), // matches what cmd/tailscaled uses + stateFile: stateFile, } // Look for a data race or panic. @@ -775,9 +818,17 @@ func (op *nodeOutputParser) parseLinesLocked() { type Daemon struct { Process *os.Process + + // svc is set when the daemon is a Windows service (no owned Process); + // MustCleanShutdown then stops it via the SCM. + svc *TestNode } func (d *Daemon) MustCleanShutdown(t testing.TB) { + if d.svc != nil { + d.svc.stopService() + return + } d.Process.Signal(os.Interrupt) ps, err := d.Process.Wait() if err != nil { @@ -809,6 +860,40 @@ func (n *TestNode) awaitTailscaledRunnable() error { return nil } +// daemonEnv returns the extra environment variables to use when starting tailscaled. +// The ipnGOOS argument overrides [envknob.GOOS]. +func (n *TestNode) daemonEnv(ipnGOOS string) []string { + env := []string{ + "TS_DEBUG_PERMIT_HTTP_C2N=1", + "TS_LOG_TARGET=" + n.env.LogCatcherServer.URL, + "HTTP_PROXY=" + n.env.TrafficTrapServer.URL, + "HTTPS_PROXY=" + n.env.TrafficTrapServer.URL, + "TS_DEBUG_FAKE_GOOS=" + ipnGOOS, + "TS_LOGS_DIR=" + n.dir, + "TS_NETCHECK_GENERATE_204_URL=" + n.env.ControlServer.URL + "/generate_204", + "TS_ASSUME_NETWORK_UP_FOR_TEST=1", // don't pause control client in airplane mode (no wifi, etc) + "TS_PANIC_IF_HIT_MAIN_CONTROL=1", + "TS_DISABLE_PORTMAPPER=1", // shouldn't be needed; test is all localhost + "TS_DEBUG_LOG_RATE=all", + } + if n.allowUpdates { + env = append(env, "TS_TEST_ALLOW_AUTO_UPDATE=1") + } + if n.env.loopbackPort != nil { + env = append(env, "TS_DEBUG_NETSTACK_LOOPBACK_PORT="+strconv.Itoa(*n.env.loopbackPort)) + } + if n.env.neverDirectUDP { + env = append(env, "TS_DEBUG_NEVER_DIRECT_UDP=1") + } + if n.env.relayServerUseLoopback { + env = append(env, "TS_DEBUG_RELAY_SERVER_ADDRS=::1,127.0.0.1") + } + if version.IsRace() { + env = append(env, "GORACE=halt_on_error=1") + } + return env +} + // StartDaemon starts the node's tailscaled, failing if it fails to start. // StartDaemon ensures that the process will exit when the test completes. func (n *TestNode) StartDaemon() *Daemon { @@ -822,6 +907,12 @@ func (n *TestNode) StartDaemonAsIPNGOOS(ipnGOOS string) *Daemon { t.Fatalf("awaitTailscaledRunnable: %v", err) } + if n.env.windowsService { + // TODO(#20443): plumb service logs here so races/panics/DEBUG-ADDR are seen in service mode. + n.tailscaledParser = &nodeOutputParser{n: n} + return n.startWindowsServiceDaemon() + } + cmd := exec.Command(n.env.daemon) cmd.Args = append(cmd.Args, "--statedir="+n.dir, @@ -843,34 +934,7 @@ func (n *TestNode) StartDaemonAsIPNGOOS(ipnGOOS string) *Daemon { if n.encryptState { cmd.Args = append(cmd.Args, "--encrypt-state") } - cmd.Env = append(os.Environ(), - "TS_DEBUG_PERMIT_HTTP_C2N=1", - "TS_LOG_TARGET="+n.env.LogCatcherServer.URL, - "HTTP_PROXY="+n.env.TrafficTrapServer.URL, - "HTTPS_PROXY="+n.env.TrafficTrapServer.URL, - "TS_DEBUG_FAKE_GOOS="+ipnGOOS, - "TS_LOGS_DIR="+t.TempDir(), - "TS_NETCHECK_GENERATE_204_URL="+n.env.ControlServer.URL+"/generate_204", - "TS_ASSUME_NETWORK_UP_FOR_TEST=1", // don't pause control client in airplane mode (no wifi, etc) - "TS_PANIC_IF_HIT_MAIN_CONTROL=1", - "TS_DISABLE_PORTMAPPER=1", // shouldn't be needed; test is all localhost - "TS_DEBUG_LOG_RATE=all", - ) - if n.allowUpdates { - cmd.Env = append(cmd.Env, "TS_TEST_ALLOW_AUTO_UPDATE=1") - } - if n.env.loopbackPort != nil { - cmd.Env = append(cmd.Env, "TS_DEBUG_NETSTACK_LOOPBACK_PORT="+strconv.Itoa(*n.env.loopbackPort)) - } - if n.env.neverDirectUDP { - cmd.Env = append(cmd.Env, "TS_DEBUG_NEVER_DIRECT_UDP=1") - } - if n.env.relayServerUseLoopback { - cmd.Env = append(cmd.Env, "TS_DEBUG_RELAY_SERVER_ADDRS=::1,127.0.0.1") - } - if version.IsRace() { - cmd.Env = append(cmd.Env, "GORACE=halt_on_error=1") - } + cmd.Env = append(os.Environ(), n.daemonEnv(ipnGOOS)...) n.tailscaledParser = &nodeOutputParser{n: n} cmd.Stderr = n.tailscaledParser if *verboseTailscaled { diff --git a/tstest/integration/integration_test.go b/tstest/integration/integration_test.go index 192ecdd12..8a894c7cf 100644 --- a/tstest/integration/integration_test.go +++ b/tstest/integration/integration_test.go @@ -35,6 +35,7 @@ "go4.org/mem" "tailscale.com/client/local" "tailscale.com/cmd/testwrapper/flakytest" + "tailscale.com/envknob" "tailscale.com/feature" _ "tailscale.com/feature/clientupdate" "tailscale.com/health" @@ -57,6 +58,11 @@ func TestMain(m *testing.M) { // Have to disable UPnP which hits the network, otherwise it fails due to HTTP proxy. os.Setenv("TS_DISABLE_UPNP", "true") flag.Parse() + if *runWindowsServiceTests && runtime.GOOS == "windows" { + // On Windows the service is a singleton, so its tests run serially. + // envknob.Setenv refreshes the TS_SERIAL_TESTS that tstest.Parallel reads. + envknob.Setenv("TS_SERIAL_TESTS", "true") + } v := m.Run() if v != 0 { os.Exit(v) @@ -1310,7 +1316,7 @@ func TestNoControlConnWhenDown(t *testing.T) { // without the GUI to kick off a Start. func TestOneNodeUpWindowsStyle(t *testing.T) { tstest.Parallel(t) - env := NewTestEnv(t) + env := NewTestEnv(t, canRunAsServiceOnWindows()) n1 := NewTestNode(t, env) n1.upFlagGOOS = "windows" diff --git a/tstest/integration/service_notwindows.go b/tstest/integration/service_notwindows.go new file mode 100644 index 000000000..729977232 --- /dev/null +++ b/tstest/integration/service_notwindows.go @@ -0,0 +1,18 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !windows + +package integration + +// Non-Windows stubs for the Windows service backend; never called, since +// windowsService is only set on Windows. + +func (n *TestNode) startWindowsServiceDaemon() *Daemon { + n.env.t.Fatal("Windows service daemon is only supported on Windows") + return nil +} + +func (n *TestNode) stopService() { + n.env.t.Fatal("Windows service daemon is only supported on Windows") +} diff --git a/tstest/integration/service_windows.go b/tstest/integration/service_windows.go new file mode 100644 index 000000000..6e24d5ea9 --- /dev/null +++ b/tstest/integration/service_windows.go @@ -0,0 +1,270 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build windows + +package integration + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "golang.org/x/sys/windows/svc" + "golang.org/x/sys/windows/svc/mgr" + "tailscale.com/tsconst/wintun" +) + +// serviceName is the Windows service tailscaled installs itself as. +const serviceName = "Tailscale" + +// startWindowsServiceDaemon installs and starts tailscaled as a Windows service. +func (n *TestNode) startWindowsServiceDaemon() *Daemon { + t := n.env.t + t.Helper() + + // A pre-existing Tailscale service means this isn't a disposable/CI machine; + // fail rather than uninstall it. Stale state is wiped below, not fatal. + if serviceExists(t) { + t.Fatal("existing Tailscale service found; run only on a disposable/CI machine") + } + + n.cleanupServiceState() + stageWintun(t, filepath.Dir(n.env.daemon)) + n.writeServiceEnvFile() + + if out, err := exec.CommandContext(t.Context(), n.env.daemon, "install-system-daemon").CombinedOutput(); err != nil { + t.Fatalf("install-system-daemon: %v\n%s", err, out) + } + // Teardown (LIFO): stop, uninstall, then wipe state for the next test. + t.Cleanup(func() { + n.stopService() + n.uninstallService() + n.cleanupServiceState() + }) + + n.startService() + n.waitServiceReady(90 * time.Second) + return &Daemon{svc: n} +} + +// startService starts the service and waits for the SCM to report it Running. +func (n *TestNode) startService() { + t := n.env.t + t.Helper() + m := connectSCM(t) + defer m.Disconnect() + s, err := m.OpenService(serviceName) + if err != nil { + t.Fatalf("open service %q: %v", serviceName, err) + } + defer s.Close() + if err := s.Start(); err != nil { + t.Fatalf("start service %q: %v", serviceName, err) + } + n.waitServiceState(s, svc.Running, 60*time.Second) +} + +// stopService requests a stop and waits until the service is Stopped, failing +// the test if it doesn't stop in time. +func (n *TestNode) stopService() { + t := n.env.t + t.Helper() + m := connectSCM(t) + defer m.Disconnect() + s, err := m.OpenService(serviceName) + if err != nil { + return // not installed; nothing to stop + } + defer s.Close() + st, err := s.Query() + if err != nil { + t.Fatalf("query service %q: %v", serviceName, err) + } + if st.State == svc.Stopped { + return + } + if _, err := s.Control(svc.Stop); err != nil { + t.Fatalf("stop service %q: %v", serviceName, err) + } + n.waitServiceState(s, svc.Stopped, 60*time.Second) +} + +// uninstallService removes the service via tailscaled's uninstall-system-daemon +// and waits until it's gone; a missing service is fine. +func (n *TestNode) uninstallService() { + t := n.env.t + t.Helper() + if !serviceExists(t) { + return + } + // Not t.Context(): this also runs from t.Cleanup, where it's canceled. + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if out, err := exec.CommandContext(ctx, n.env.daemon, "uninstall-system-daemon").CombinedOutput(); err != nil { + t.Fatalf("uninstall-system-daemon: %v\n%s", err, out) + } + n.waitServiceGone(30 * time.Second) +} + +// writeServiceEnvFile writes the harness env to the file tailscaled reads at +// startup, since a service doesn't inherit the test process's environment. +func (n *TestNode) writeServiceEnvFile() { + t := n.env.t + t.Helper() + dir := serviceStateDir() + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("creating %s: %v", dir, err) + } + dst := filepath.Join(dir, "tailscaled-env.txt") + body := strings.Join(n.daemonEnv("windows"), "\n") + "\n" + if err := os.WriteFile(dst, []byte(body), 0o644); err != nil { + t.Fatalf("writing %s: %v", dst, err) + } +} + +// cleanupServiceState removes the service's global state dir so the next test +// doesn't inherit a prior node's identity; call only after the service stops. +func (n *TestNode) cleanupServiceState() { + t := n.env.t + t.Helper() + dir := serviceStateDir() + if err := os.RemoveAll(dir); err != nil { + t.Logf("removing %s: %v", dir, err) + } +} + +// serviceStateDir is the global state dir a Tailscale service uses. +func serviceStateDir() string { + return filepath.Join(os.Getenv("ProgramData"), "Tailscale") +} + +// waitServiceReady polls the LocalAPI until BackendState leaves NoState, +// guarding the post-start race (tailscale/tailscale#8695). +func (n *TestNode) waitServiceReady(timeout time.Duration) { + t := n.env.t + t.Helper() + deadline := time.Now().Add(timeout) + var last error + for time.Now().Before(deadline) { + st, err := n.Status() + if err == nil && st.BackendState != "" && st.BackendState != "NoState" { + return + } + last = err + time.Sleep(2 * time.Second) + } + t.Fatalf("service LocalAPI not ready within %v (last err: %v)", timeout, last) +} + +// waitServiceState polls s until it reaches want, failing the test on timeout. +func (n *TestNode) waitServiceState(s *mgr.Service, want svc.State, timeout time.Duration) { + t := n.env.t + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + st, err := s.Query() + if err != nil { + t.Fatalf("query service %q: %v", serviceName, err) + } + if st.State == want { + return + } + time.Sleep(time.Second) + } + t.Fatalf("service %q did not reach state %d within %v", serviceName, want, timeout) +} + +// waitServiceGone polls until the service no longer exists. +func (n *TestNode) waitServiceGone(timeout time.Duration) { + t := n.env.t + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if !serviceExists(t) { + return + } + time.Sleep(time.Second) + } + t.Fatalf("service %q still present after %v", serviceName, timeout) +} + +// serviceExists reports whether the Tailscale service is currently installed. +func serviceExists(t testing.TB) bool { + t.Helper() + m := connectSCM(t) + defer m.Disconnect() + s, err := m.OpenService(serviceName) + if err != nil { + return false + } + s.Close() + return true +} + +// connectSCM connects to the Windows service manager, failing the test on error. +func connectSCM(t testing.TB) *mgr.Mgr { + t.Helper() + m, err := mgr.Connect() + if err != nil { + t.Fatalf("connect to service manager: %v", err) + } + return m +} + +// stageWintun downloads and verifies wintun.dll into dir; tailscaled loads it +// from its executable's dir (cmd/tailscaled.fullyQualifiedWintunPath). +func stageWintun(t testing.TB, dir string) { + t.Helper() + req, err := http.NewRequestWithContext(t.Context(), "GET", wintun.URL, nil) + if err != nil { + t.Fatalf("wintun request: %v", err) + } + res, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("downloading wintun: %v", err) + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + t.Fatalf("downloading %s: HTTP %s", wintun.URL, res.Status) + } + zipBytes, err := io.ReadAll(res.Body) + if err != nil { + t.Fatalf("reading wintun zip: %v", err) + } + if sum := sha256.Sum256(zipBytes); hex.EncodeToString(sum[:]) != wintun.SHA256 { + t.Fatalf("wintun zip sha256 = %s, want %s", hex.EncodeToString(sum[:]), wintun.SHA256) + } + zr, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes))) + if err != nil { + t.Fatalf("opening wintun zip: %v", err) + } + member := wintun.DLLZipPath(runtime.GOARCH) + f, err := zr.Open(member) + if err != nil { + t.Fatalf("wintun zip missing %q: %v", member, err) + } + defer f.Close() + out, err := os.Create(filepath.Join(dir, "wintun.dll")) + if err != nil { + t.Fatalf("creating wintun.dll: %v", err) + } + if _, err := io.Copy(out, f); err != nil { + out.Close() + t.Fatalf("extracting wintun.dll: %v", err) + } + if err := out.Close(); err != nil { + t.Fatalf("closing wintun.dll: %v", err) + } +}