cmd/k8s-operator: workload identity support for multi-tailnet (#20016)

This commit modifies the reconciler for the `Tailnet` custom resource
to allow referenced secrets to specify an `audience` field. If a
referenced secret contains both an `audience` and `client_id` we assume
the user's intention is to use workload identity.

In that case, we configure the tailscale API client to authenticate
using the Kubernetes token request API against the operator's service
account. This requires the operator to be aware of its own service
account name.

A small change has also been made to the messages added to the `Tailnet`
CRD's status field in the even that it is missing scopes to make it
clearer that certain scopes may not be applied.

Closes: #19090
Updates: #19471

Signed-off-by: David Bond <davidsbond93@gmail.com>

cmd/k8s-operator/deploy/chart/templates/deployment.yaml

@@ -81,6 +81,10 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+            - name: OPERATOR_SERVICE_ACCOUNT_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.serviceAccountName
             - name: OPERATOR_LOGIN_SERVER
               value: {{ .Values.loginServer }}
             - name: OPERATOR_INGRESS_CLASS_NAME

cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml

@@ -76,6 +76,10 @@ rules:
 - apiGroups: [""]
   resources: ["secrets", "serviceaccounts", "configmaps"]
   verbs: ["create","delete","deletecollection","get","list","patch","update","watch"]
+- apiGroups: [""]
+  resources: ["serviceaccounts/token"]
+  resourceNames: ["operator"]
+  verbs: ["create"]
 - apiGroups: [""]
   resources: ["pods"]
   verbs: ["get","list","watch", "update"]

cmd/k8s-operator/deploy/crds/tailscale.com_tailnets.yaml

@@ -58,15 +58,18 @@ spec:
                 - credentials
               properties:
                 credentials:
-                  description: Denotes the location of the OAuth credentials to use for authenticating with this Tailnet.
+                  description: Denotes the location of the credentials to use for authenticating with this Tailnet.
                   type: object
                   required:
                     - secretName
                   properties:
                     secretName:
                       description: |-
-                        The name of the secret containing the OAuth credentials. This secret must contain two fields "client_id" and
-                        "client_secret".
+                        The name of the secret containing the credentials used to authenticate with this Tailnet. The secret must always
+                        contain a "client_id" field. To authenticate with a static OAuth client, also set "client_secret". To authenticate
+                        via workload identity federation, set "audience" to the audience value expected by the Tailscale OAuth
+                        client; the operator will mint a ServiceAccount token for itself with that audience and exchange it for an API
+                        token. "client_secret" and "audience" are mutually exclusive.
                       type: string
                 loginUrl:
                   description: URL of the control plane to be used by all resources managed by the operator using this Tailnet.

cmd/k8s-operator/deploy/manifests/operator.yaml

@@ -6151,12 +6151,15 @@ spec:
                             https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
                         properties:
                             credentials:
-                                description: Denotes the location of the OAuth credentials to use for authenticating with this Tailnet.
+                                description: Denotes the location of the credentials to use for authenticating with this Tailnet.
                                 properties:
                                     secretName:
                                         description: |-
-                                            The name of the secret containing the OAuth credentials. This secret must contain two fields "client_id" and
-                                            "client_secret".
+                                            The name of the secret containing the credentials used to authenticate with this Tailnet. The secret must always
+                                            contain a "client_id" field. To authenticate with a static OAuth client, also set "client_secret". To authenticate
+                                            via workload identity federation, set "audience" to the audience value expected by the Tailscale OAuth
+                                            client; the operator will mint a ServiceAccount token for itself with that audience and exchange it for an API
+                                            token. "client_secret" and "audience" are mutually exclusive.
                                         type: string
                                 required:
                                     - secretName
@@ -6409,6 +6412,14 @@ rules:
         - patch
         - update
         - watch
+    - apiGroups:
+        - ""
+      resourceNames:
+        - operator
+      resources:
+        - serviceaccounts/token
+      verbs:
+        - create
     - apiGroups:
         - ""
       resources:
@@ -6560,6 +6571,10 @@ spec:
                       valueFrom:
                         fieldRef:
                             fieldPath: metadata.namespace
+                    - name: OPERATOR_SERVICE_ACCOUNT_NAME
+                      valueFrom:
+                        fieldRef:
+                            fieldPath: spec.serviceAccountName
                     - name: OPERATOR_LOGIN_SERVER
                       value: null
                     - name: OPERATOR_INGRESS_CLASS_NAME

cmd/k8s-operator/operator.go

@@ -97,6 +97,7 @@ func main() {
 		isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false)
 		loginServer           = strings.TrimSuffix(defaultEnv("OPERATOR_LOGIN_SERVER", ""), "/")
 		ingressClassName      = defaultEnv("OPERATOR_INGRESS_CLASS_NAME", "tailscale")
+		operatorSAName        = defaultEnv("OPERATOR_SERVICE_ACCOUNT_NAME", "operator")
 	)
 
 	var opts []kzap.Opts
@@ -157,6 +158,7 @@ func main() {
 		tsServer:                      s,
 		tsClient:                      tsc,
 		tailscaleNamespace:            tsNamespace,
+		operatorSAName:                operatorSAName,
 		restConfig:                    restConfig,
 		proxyImage:                    image,
 		k8sProxyImage:                 k8sProxyImage,
@@ -349,6 +351,7 @@ func runReconcilers(opts reconcilerOpts) {
 	tailnetOptions := tailnet.ReconcilerOptions{
 		Client:             mgr.GetClient(),
 		TailscaleNamespace: opts.tailscaleNamespace,
+		OperatorSAName:     opts.operatorSAName,
 		Clock:              tstime.DefaultClock{},
 		Logger:             opts.log,
 		Registry:           clients,
@@ -818,6 +821,10 @@ type reconcilerOpts struct {
 	// ingressClassName is the name of the ingress class used by reconcilers of Ingress resources. This defaults
 	// to "tailscale" but can be customised.
 	ingressClassName string
+	// operatorSAName is the name of the ServiceAccount that the operator pod runs as. It is used as the target
+	// ServiceAccount when minting tokens via the Kubernetes TokenRequest API for Tailnets that authenticate using
+	// workload identity federation.
+	operatorSAName string
 }
 
 // enqueueAllIngressEgressProxySvcsinNS returns a reconcile request for each

k8s-operator/api.md

@@ -1273,7 +1273,7 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
-| `secretName` _string_ | The name of the secret containing the OAuth credentials. This secret must contain two fields "client_id" and<br />"client_secret". |  |  |
+| `secretName` _string_ | The name of the secret containing the credentials used to authenticate with this Tailnet. The secret must always<br />contain a "client_id" field. To authenticate with a static OAuth client, also set "client_secret". To authenticate<br />via workload identity federation, set "audience" to the audience value expected by the Tailscale OAuth<br />client; the operator will mint a ServiceAccount token for itself with that audience and exchange it for an API<br />token. "client_secret" and "audience" are mutually exclusive. |  |  |
 
 
 #### TailnetDevice
@@ -1328,7 +1328,7 @@ _Appears in:_
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
 | `loginUrl` _string_ | URL of the control plane to be used by all resources managed by the operator using this Tailnet. |  |  |
-| `credentials` _[TailnetCredentials](#tailnetcredentials)_ | Denotes the location of the OAuth credentials to use for authenticating with this Tailnet. |  |  |
+| `credentials` _[TailnetCredentials](#tailnetcredentials)_ | Denotes the location of the credentials to use for authenticating with this Tailnet. |  |  |
 
 
 #### TailnetStatus

k8s-operator/apis/v1alpha1/types_tailnet.go

@@ -48,13 +48,16 @@ type TailnetSpec struct {
 	// URL of the control plane to be used by all resources managed by the operator using this Tailnet.
 	// +optional
 	LoginURL string `json:"loginUrl,omitempty"`
-	// Denotes the location of the OAuth credentials to use for authenticating with this Tailnet.
+	// Denotes the location of the credentials to use for authenticating with this Tailnet.
 	Credentials TailnetCredentials `json:"credentials"`
 }
 
 type TailnetCredentials struct {
-	// The name of the secret containing the OAuth credentials. This secret must contain two fields "client_id" and
-	// "client_secret".
+	// The name of the secret containing the credentials used to authenticate with this Tailnet. The secret must always
+	// contain a "client_id" field. To authenticate with a static OAuth client, also set "client_secret". To authenticate
+	// via workload identity federation, set "audience" to the audience value expected by the Tailscale OAuth
+	// client; the operator will mint a ServiceAccount token for itself with that audience and exchange it for an API
+	// token. "client_secret" and "audience" are mutually exclusive.
 	SecretName string `json:"secretName"`
 }
 

k8s-operator/reconciler/tailnet/tailnet.go

@@ -4,8 +4,8 @@
 //go:build !plan9
 
 // Package tailnet provides reconciliation logic for the Tailnet custom resource definition. It is responsible for
-// ensuring the referenced OAuth credentials are valid and have the required scopes to be able to generate authentication
-// keys, manage devices & manage VIP services.
+// ensuring the referenced credentials (either an OAuth client or a workload identity federation configuration) are
+// valid and have the required scopes to be able to generate authentication keys, manage devices & manage VIP services.
 package tailnet
 
 import (
@@ -17,6 +17,7 @@
 	"time"
 
 	"go.uber.org/zap"
+	authnv1 "k8s.io/api/authentication/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -45,6 +46,7 @@
 		client.Client
 
 		tailscaleNamespace string
+		operatorSAName     string
 		clock              tstime.Clock
 		logger             *zap.SugaredLogger
 		clientFunc         func(*tsapi.Tailnet, *corev1.Secret) tsclient.Client
@@ -59,9 +61,13 @@
 	ReconcilerOptions struct {
 		// The client for interacting with the Kubernetes API.
 		Client client.Client
-		// The namespace the operator is installed in. This reconciler expects Tailnet OAuth credentials to be stored
+		// The namespace the operator is installed in. This reconciler expects Tailnet credentials to be stored
 		// in Secret resources within this namespace.
 		TailscaleNamespace string
+		// The name of the ServiceAccount the operator runs as, in TailscaleNamespace. This is used as the target
+		// ServiceAccount when minting tokens via the Kubernetes TokenRequest API for Tailnets that authenticate
+		// using workload identity federation.
+		OperatorSAName string
 		// Controls which clock to use for performing time-based functions. This is typically modified for use
 		// in tests.
 		Clock tstime.Clock
@@ -92,6 +98,7 @@ func NewReconciler(options ReconcilerOptions) *Reconciler {
 	return &Reconciler{
 		Client:             options.Client,
 		tailscaleNamespace: options.TailscaleNamespace,
+		operatorSAName:     options.OperatorSAName,
 		clock:              options.Clock,
 		logger:             options.Logger.Named(reconcilerName),
 		clientFunc:         options.ClientFunc,
@@ -243,10 +250,11 @@ func (r *Reconciler) createOrUpdate(ctx context.Context, tailnet *tsapi.Tailnet)
 	return reconcile.Result{}, nil
 }
 
-// Constants for OAuth credential fields within the Secret referenced by the Tailnet.
+// Constants for credential fields within the Secret referenced by the Tailnet.
 const (
 	clientIDKey     = "client_id"
 	clientSecretKey = "client_secret"
+	audienceKey     = "audience"
 )
 
 func (r *Reconciler) createClient(tailnet *tsapi.Tailnet, secret *corev1.Secret) (tsclient.Client, error) {
@@ -264,16 +272,61 @@ func (r *Reconciler) createClient(tailnet *tsapi.Tailnet, secret *corev1.Secret)
 		return nil, fmt.Errorf("failed to parse base URL %q: %w", baseURL, err)
 	}
 
+	var auth tailscale.Auth
+
+	clientID := string(secret.Data[clientIDKey])
+	audience := string(secret.Data[audienceKey])
+	clientSecret := string(secret.Data[clientSecretKey])
+
+	switch {
+	case audience != "":
+		// If the audience field is present, we assume workload identity as the authentication method.
+		auth = &tailscale.IdentityFederation{
+			ClientID:    clientID,
+			IDTokenFunc: r.createToken(audience),
+		}
+	case clientSecret != "":
+		// For a client secret, we assume oauth.
+		auth = &tailscale.OAuth{
+			ClientID:     clientID,
+			ClientSecret: clientSecret,
+		}
+	default:
+		// We shouldn't land here as previous functions will have ensured the validity of the secret, but we
+		// error here anyway as we won't know what to do:
+		return nil, errors.New("unable to determine authentication method")
+	}
+
 	return tsclient.Wrap(&tailscale.Client{
 		BaseURL:   base,
 		UserAgent: "tailscale-k8s-operator",
-		Auth: &tailscale.OAuth{
-			ClientID:     string(secret.Data[clientIDKey]),
-			ClientSecret: string(secret.Data[clientSecretKey]),
-		},
+		Auth:      auth,
 	}), nil
 }
 
+func (r *Reconciler) createToken(audience string) func() (string, error) {
+	return func() (string, error) {
+		serviceAccount := &corev1.ServiceAccount{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      r.operatorSAName,
+				Namespace: r.tailscaleNamespace,
+			},
+		}
+
+		tokenRequest := &authnv1.TokenRequest{
+			Spec: authnv1.TokenRequestSpec{
+				Audiences: []string{audience},
+			},
+		}
+
+		if err := r.SubResource("token").Create(context.Background(), serviceAccount, tokenRequest); err != nil {
+			return "", fmt.Errorf("failed to mint service account token for %q in namespace %q: %w", r.operatorSAName, r.tailscaleNamespace, err)
+		}
+
+		return tokenRequest.Status.Token, nil
+	}
+}
+
 func (r *Reconciler) ensurePermissions(ctx context.Context, tsClient tsclient.Client, tailnet *tsapi.Tailnet) bool {
 	// Perform basic list requests here to confirm that the OAuth credentials referenced on the Tailnet resource
 	// can perform the basic operations required for the operator to function. This has a caveat of only performing
@@ -281,15 +334,15 @@ func (r *Reconciler) ensurePermissions(ctx context.Context, tsClient tsclient.Cl
 	// has completely forgotten an entire scope that's required.
 	var errs error
 	if _, err := tsClient.Devices().List(ctx); err != nil {
-		errs = errors.Join(errs, fmt.Errorf("failed to list devices: %w", err))
+		errs = errors.Join(errs, fmt.Errorf("failed to list devices: %w (client may be missing the devices scope)", err))
 	}
 
 	if _, err := tsClient.Keys().List(ctx, false); err != nil {
-		errs = errors.Join(errs, fmt.Errorf("failed to list auth keys: %w", err))
+		errs = errors.Join(errs, fmt.Errorf("failed to list auth keys: %w (client may be missing the keys scope)", err))
 	}
 
 	if _, err := tsClient.VIPServices().List(ctx); err != nil {
-		errs = errors.Join(errs, fmt.Errorf("failed to list tailscale services: %w", err))
+		errs = errors.Join(errs, fmt.Errorf("failed to list tailscale services: %w (client may be missing the services scope)", err))
 	}
 
 	if errs != nil {
@@ -317,8 +370,8 @@ func (r *Reconciler) ensureSecret(tailnet *tsapi.Tailnet, secret *corev1.Secret)
 		message = fmt.Sprintf("Secret %q is empty", secret.Name)
 	case len(secret.Data[clientIDKey]) == 0:
 		message = fmt.Sprintf("Secret %q is missing the client_id field", secret.Name)
-	case len(secret.Data[clientSecretKey]) == 0:
-		message = fmt.Sprintf("Secret %q is missing the client_secret field", secret.Name)
+	case len(secret.Data[clientSecretKey]) == 0 && len(secret.Data[audienceKey]) == 0:
+		message = fmt.Sprintf("Secret %q must contain either a client_secret or an audience field", secret.Name)
 	}
 
 	if message == "" {

k8s-operator/reconciler/tailnet/tailnet_test.go

@@ -141,7 +141,7 @@ func TestReconciler_Reconcile(t *testing.T) {
 			},
 		},
 		{
-			Name: "invalid-status-missing-client-secret",
+			Name: "invalid-status-missing-client-secret-and-audience",
 			Request: reconcile.Request{
 				NamespacedName: types.NamespacedName{
 					Name: "test",
@@ -171,7 +171,7 @@ func TestReconciler_Reconcile(t *testing.T) {
 					Type:    string(tsapi.TailnetReady),
 					Status:  metav1.ConditionFalse,
 					Reason:  tailnet.ReasonInvalidSecret,
-					Message: `Secret "test" is missing the client_secret field`,
+					Message: `Secret "test" must contain either a client_secret or an audience field`,
 				},
 			},
 		},
@@ -210,7 +210,7 @@ func TestReconciler_Reconcile(t *testing.T) {
 					Type:    string(tsapi.TailnetReady),
 					Status:  metav1.ConditionFalse,
 					Reason:  tailnet.ReasonInvalidOAuth,
-					Message: `failed to list devices: EOF`,
+					Message: `failed to list devices: EOF (client may be missing the devices scope)`,
 				},
 			},
 		},
@@ -249,7 +249,7 @@ func TestReconciler_Reconcile(t *testing.T) {
 					Type:    string(tsapi.TailnetReady),
 					Status:  metav1.ConditionFalse,
 					Reason:  tailnet.ReasonInvalidOAuth,
-					Message: `failed to list tailscale services: EOF`,
+					Message: `failed to list tailscale services: EOF (client may be missing the services scope)`,
 				},
 			},
 		},
@@ -288,7 +288,7 @@ func TestReconciler_Reconcile(t *testing.T) {
 					Type:    string(tsapi.TailnetReady),
 					Status:  metav1.ConditionFalse,
 					Reason:  tailnet.ReasonInvalidOAuth,
-					Message: `failed to list auth keys: EOF`,
+					Message: `failed to list auth keys: EOF (client may be missing the keys scope)`,
 				},
 			},
 		},
@@ -331,6 +331,45 @@ func TestReconciler_Reconcile(t *testing.T) {
 				},
 			},
 		},
+		{
+			Name: "ready-workload-identity",
+			Request: reconcile.Request{
+				NamespacedName: types.NamespacedName{
+					Name: "default",
+				},
+			},
+			Tailnet: &tsapi.Tailnet{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "default",
+				},
+				Spec: tsapi.TailnetSpec{
+					Credentials: tsapi.TailnetCredentials{
+						SecretName: "test",
+					},
+				},
+			},
+			Secret: &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test",
+					Namespace: "tailscale",
+				},
+				Data: map[string][]byte{
+					"client_id": []byte("test"),
+					"audience":  []byte("https://tailscale.com"),
+				},
+			},
+			ClientFunc: func(_ *tsapi.Tailnet, _ *corev1.Secret) tsclient.Client {
+				return &MockTailnetClient{}
+			},
+			ExpectedConditions: []metav1.Condition{
+				{
+					Type:    string(tsapi.TailnetReady),
+					Status:  metav1.ConditionTrue,
+					Reason:  tailnet.ReasonValid,
+					Message: tailnet.ReasonValid,
+				},
+			},
+		},
 	}
 
 	for _, tc := range tt {