kandra: Use tbot application proxy + teleport-sd for service discovery.

This makes the Prometheus configuration no longer have to know
anything about which hosts run which exporters; instead, hosts
register the exporter in Teleport, and Prometheus asks Teleport which
instances of a given exporter it knows about.

puppet/kandra/files/tbot.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=tbot - Teleport Machine & Workload Identity Service
+After=network.target
+
+[Service]
+Type=simple
+User=teleport
+Group=teleport
+Restart=always
+RestartSec=5
+Environment="TELEPORT_ANONYMOUS_TELEMETRY=0"
+ExecStart=/usr/local/bin/tbot start -c /etc/tbot.yaml --pid-file=/var/lib/teleport/bot/pid
+ExecReload=/bin/kill -HUP $MAINPID
+PIDFile=/var/lib/teleport/bot/pid
+LimitNOFILE=524288
+
+[Install]
+WantedBy=multi-user.target

puppet/kandra/files/tbot.yaml

@@ -0,0 +1,24 @@
+version: v2
+proxy_server: teleport.zulipchat.net:443
+onboarding:
+  join_method: iam
+  # This "token" is defined in Teleport, and links to the AWS role
+  # https://teleport.zulipchat.net/web/tokens
+  # https://goteleport.com/docs/reference/deployment/join-methods/#aws-iam-role-iam
+  # https://goteleport.com/docs/installation/agents/aws-iam/
+  token: prometheus-scrape-bot
+  ca_pins:
+    - "sha256:062db37249ea74c450579da8f02043b317cb8a174d653bb5090a89752d68efe7"
+storage:
+  type: directory
+  path: /var/lib/teleport/bot
+services:
+  # Prometheus proxies all exporter traffic through this proxy
+  - type: application-proxy
+    listen: tcp://127.0.0.1:8080
+outputs:
+  # This is consumed by teleport-sd to list exporters
+  - type: identity
+    destination:
+      type: directory
+      path: /var/lib/teleport/bot-identity

puppet/kandra/manifests/profile/prometheus_server.pp

@@ -6,6 +6,10 @@ class kandra::profile::prometheus_server inherits kandra::profile::base {
 
   include kandra::prometheus::base
 
+  # Service discovery via Teleport
+  include kandra::teleport::tbot
+  include kandra::teleport::sd
+
   # This blackbox monitoring of the backup system runs locally
   include kandra::prometheus::wal_g
 
@@ -72,6 +76,7 @@ class kandra::profile::prometheus_server inherits kandra::profile::base {
       File[$bin],
       File[$data_dir],
       File['/etc/prometheus/prometheus.yaml'],
+      Service['tbot'],
     ],
     owner   => 'root',
     group   => 'root',

puppet/kandra/manifests/teleport/sd.pp

@@ -0,0 +1,31 @@
+# @summary Returns Prometheus scrape targets based on Teleport apps
+#
+# Only one instance is necessary.
+#
+class kandra::teleport::sd {
+  include zulip::supervisor
+  include kandra::teleport::tbot
+
+  $version = $zulip::common::versions['teleport-sd']['version']
+  $bin = "/srv/zulip-teleport-sd-${version}"
+
+  zulip::external_dep { 'teleport-sd':
+    version       => $version,
+    url           => "https://github.com/alexmv/teleport-sd/releases/download/v${version}/teleport-sd-linux-${zulip::common::goarch}",
+    cleanup_after => [Service[supervisor]],
+  }
+
+  file { "${zulip::common::supervisor_conf_dir}/teleport-sd.conf":
+    ensure  => file,
+    require => [
+      Package[teleport],
+      Package[supervisor],
+      File[$bin],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    content => template('kandra/supervisor/conf.d/teleport-sd.conf.erb'),
+    notify  => Service[supervisor],
+  }
+}

puppet/kandra/manifests/teleport/tbot.pp

@@ -0,0 +1,40 @@
+# @summary Provide a local application proxy
+# https://goteleport.com/docs/reference/machine-workload-identity/configuration/#application-proxy
+class kandra::teleport::tbot {
+  include kandra::teleport::base
+
+  file { '/etc/tbot.yaml':
+    ensure => file,
+    owner  => 'root',
+    group  => 'root',
+    mode   => '0644',
+    source => 'puppet:///modules/kandra/tbot.yaml',
+    notify => Service['tbot'],
+  }
+
+  file { '/etc/systemd/system/tbot.service':
+    require => [
+      Package[teleport],
+    ],
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0644',
+    source  => 'puppet:///modules/kandra/tbot.service',
+    notify  => [Exec['reload systemd'], Service['tbot']],
+  }
+
+  file { ['/var/lib/teleport/bot', '/var/lib/teleport/bot-identity']:
+    ensure  => directory,
+    owner   => 'teleport',
+    group   => 'teleport',
+    mode    => '0644',
+    require => Service['teleport'],
+    before  => Service['tbot'],
+  }
+
+  service {'tbot':
+    ensure  => running,
+    enable  => true,
+    require => [Service['teleport'], Exec['reload systemd']],
+  }
+}

puppet/kandra/templates/supervisor/conf.d/teleport-sd.conf.erb

@@ -0,0 +1,6 @@
+[program:teleport-sd]
+command=<%= @bin %> --listen 127.0.0.1:9092 --proxy teleport.zulipchat.net:443 --identity /var/lib/teleport/bot-identity/identity
+priority=15
+autostart=true
+autorestart=true
+user=teleport

puppet/zulip/manifests/common.pp

@@ -194,6 +194,15 @@ class zulip::common {
       },
     },
 
+    # https://github.com/alexmv/teleport-sd/releases
+    'teleport-sd' => {
+      'version' => '0.1.0',
+      'sha256' => {
+        'amd64'   => '2128d78dd602c6a24c13fbd759bf19d0c3cebaa732ad0824019ced53163cf455',
+        'aarch64' => 'df93ba9510cb8898ba0f15b293d7c6877173a8d06b1f284dcc42ffc7ff32978e',
+      },
+    },
+
     # https://github.com/timonwong/uwsgi_exporter/releases
     'uwsgi_exporter' => {
       'version' => '1.3.0',