Connect Prometheus to Thanos

Connect Prometheus to Thanos

Prerequisites

  • Deploy your SKS (Exoscale Kubernetes Managed Cluster)
  • Install kubectl and jq
  • Check you can access your cluster with kubectl (kubectl get pods -A)

Deploying Prometheus on your SKS Cluster

To deploy Prometheus, two components are needed: Prometheus Operator and Prometheus Agent.

1/ Deploy Prometheus Operator and Agent

Copy below script into a file deploy-quick.sh

NOTE
After script copy, please change the values in Create Thanos secret to set your Thanos “Receiver remote-write URI” and user password to access the receiver. The URL you will use should look like https://yyyyyy-exoscale-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.i.aivencloud.com:21706 . The namespace used to deploy Prometheus is “monitoring”, you can change in the script. This example is a configuration proposal; you can adjust it as needed.

#!/bin/bash

NAMESPACE=monitoring
kubectl create namespace $NAMESPACE || true
TMPDIR=$(mktemp -d)
LATEST=$(curl -s https://api.github.com/repos/prometheus-operator/prometheus-operator/releases/latest | jq -cr .tag_name)
curl -s "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/refs/tags/$LATEST/bundle.yaml" > "$TMPDIR/bundle.yaml"
# macOS: sed -i '' / Linux: sed -i
sed -i'' "s/namespace: default/namespace: $NAMESPACE/g" "$TMPDIR/bundle.yaml"
kubectl apply --server-side -f "$TMPDIR/bundle.yaml"

kubectl apply -f - <<'CRDEOF'
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
  name: prometheusagents.monitoring.coreos.com
spec:
  group: monitoring.coreos.com
  scope: Namespaced
  names:
    kind: PrometheusAgent
    plural: prometheusagents
  versions:
  - name: v1alpha1
    served: true
    storage: true
    schema:
      openAPIV3Schema:
        type: object
        properties:
          apiVersion:
            type: string
          kind:
            type: string
          metadata:
            type: object
          spec:
            type: object
            x-kubernetes-preserve-unknown-fields: true
          status:
            type: object
            x-kubernetes-preserve-unknown-fields: true
CRDEOF

kubectl apply -f - <<'RBACEOF'
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: prometheus-operator-extra-perms
rules:
- apiGroups: ["monitoring.coreos.com"]
  resources: ["prometheusagents", "prometheusagents/status", "prometheuses", "prometheuses/status"]
  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
- apiGroups: [""]
  resources: ["namespaces", "nodes", "nodes/metrics", "configmaps", "secrets", "pods", "services", "endpoints"]
  verbs: ["get", "list", "watch"]
- apiGroups: ["monitoring.coreos.com"]
  resources: ["servicemonitors", "podmonitors", "probes", "prometheusrules"]
  verbs: ["get", "list", "watch"]
- apiGroups: ["apps"]
  resources: ["statefulsets", "deployments"]
  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
- apiGroups: [""]
  resources: ["events"]
  verbs: ["get", "list", "watch", "create", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: prometheus-operator-extra-perms
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: prometheus-operator-extra-perms
subjects:
- kind: ServiceAccount
  name: prometheus-operator
  namespace: monitoring
RBACEOF

kubectl rollout restart deployment prometheus-operator -n monitoring 2>/dev/null || true

# Create Thanos secret
kubectl create secret generic thanos-credentials \
  --from-literal=url="https://yyyyyy-exoscale-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.i.aivencloud.com:21706" \
  --from-literal=username="avnadmin" \
  --from-literal=password="MyThanosPassword" \
  -n monitoring \
  --dry-run=client -o yaml | kubectl apply -f -

sleep 20

kubectl apply -f - <<'AGENTEOF'
apiVersion: monitoring.coreos.com/v1alpha1
kind: PrometheusAgent
metadata:
  name: prometheus-agent
  namespace: monitoring
spec:
  replicas: 1
  serviceAccountName: prometheus-operator
  scrapeInterval: 30s
  podMonitorSelector:
    matchLabels: {}
  probeSelector:
    matchLabels: {}
  serviceMonitorSelector:
    matchLabels: {}
  prometheusRulesSelector:
    matchLabels: {}
  resources:
    requests:
      cpu: 50m
      memory: 64Mi
    limits:
      cpu: 200m
      memory: 256Mi
  storage:
    emptyDir: {}
  retention: 1h
  walCompression: true
  remoteWrite:
  - url: https://yyyyyy-exoscale-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.i.aivencloud.com:21706/api/v1/receive
    basicAuth:
      username:
        name: thanos-credentials
        key: username
      password:
        name: thanos-credentials
        key: password
    writeRelabelConfigs:
    - sourceLabels: [__name__]
      regex: up|process_.*|go_.*
      action: keep
AGENTEOF

# Create ServiceMonitor to scrape prometheus-operator
kubectl apply -f - <<'MONITORF'
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: prometheus-operator
  namespace: monitoring
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: prometheus-operator
  endpoints:
  - port: http
    interval: 30s
MONITORF

Then launch the script

chmod 755 deploy-quick.sh
export KUBECONFIG=my_path/kubeconfig
./deploy-quick.sh

2/ Check Prometheus services and pods

%  kubectl get svc -n monitoring
NAME                        TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)    AGE
prometheus-agent-operated   ClusterIP   None         <none>        9090/TCP   3h9m
prometheus-operator         ClusterIP   None         <none>        8080/TCP   3h9m
% kubectl get pods -n monitoring        
NAME                                   READY   STATUS    RESTARTS   AGE
prom-agent-prometheus-agent-0          2/2     Running   0          3h15m
prometheus-operator-5dfd576fc9-jdxhb   1/1     Running   0          3h3m
% kubectl get prometheusagent -n monitoring
NAME               AGE
prometheus-agent   3h17m

NOTE
Now Prometheus is working and pushing the metrics to your Thanos DBaaS. You can now integrate it with Grafana to see your metrics: Integrate with Grafana

Last updated on