Connect Prometheus to Thanos
Prerequisites
- Deploy your SKS (Exoscale Kubernetes Managed Cluster)
- Install kubectl and jq
- Check you can access your cluster with kubectl (
kubectl get pods -A)
Deploying Prometheus on your SKS Cluster
To deploy Prometheus, two components are needed: Prometheus Operator and Prometheus Agent.
1/ Deploy Prometheus Operator and Agent
Copy below script into a file deploy-quick.sh
NOTE
After script copy, please change the values in Create Thanos secret to set your Thanos “Receiver remote-write URI” and user password to access the receiver. The URL you will use should look like https://yyyyyy-exoscale-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.i.aivencloud.com:21706 . The namespace used to deploy Prometheus is “monitoring”, you can change in the script. This example is a configuration proposal; you can adjust it as needed.
#!/bin/bash
NAMESPACE=monitoring
kubectl create namespace $NAMESPACE || true
TMPDIR=$(mktemp -d)
LATEST=$(curl -s https://api.github.com/repos/prometheus-operator/prometheus-operator/releases/latest | jq -cr .tag_name)
curl -s "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/refs/tags/$LATEST/bundle.yaml" > "$TMPDIR/bundle.yaml"
# macOS: sed -i '' / Linux: sed -i
sed -i'' "s/namespace: default/namespace: $NAMESPACE/g" "$TMPDIR/bundle.yaml"
kubectl apply --server-side -f "$TMPDIR/bundle.yaml"
kubectl apply -f - <<'CRDEOF'
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: prometheusagents.monitoring.coreos.com
spec:
group: monitoring.coreos.com
scope: Namespaced
names:
kind: PrometheusAgent
plural: prometheusagents
versions:
- name: v1alpha1
served: true
storage: true
schema:
openAPIV3Schema:
type: object
properties:
apiVersion:
type: string
kind:
type: string
metadata:
type: object
spec:
type: object
x-kubernetes-preserve-unknown-fields: true
status:
type: object
x-kubernetes-preserve-unknown-fields: true
CRDEOF
kubectl apply -f - <<'RBACEOF'
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: prometheus-operator-extra-perms
rules:
- apiGroups: ["monitoring.coreos.com"]
resources: ["prometheusagents", "prometheusagents/status", "prometheuses", "prometheuses/status"]
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
- apiGroups: [""]
resources: ["namespaces", "nodes", "nodes/metrics", "configmaps", "secrets", "pods", "services", "endpoints"]
verbs: ["get", "list", "watch"]
- apiGroups: ["monitoring.coreos.com"]
resources: ["servicemonitors", "podmonitors", "probes", "prometheusrules"]
verbs: ["get", "list", "watch"]
- apiGroups: ["apps"]
resources: ["statefulsets", "deployments"]
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
- apiGroups: [""]
resources: ["events"]
verbs: ["get", "list", "watch", "create", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: prometheus-operator-extra-perms
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: prometheus-operator-extra-perms
subjects:
- kind: ServiceAccount
name: prometheus-operator
namespace: monitoring
RBACEOF
kubectl rollout restart deployment prometheus-operator -n monitoring 2>/dev/null || true
# Create Thanos secret
kubectl create secret generic thanos-credentials \
--from-literal=url="https://yyyyyy-exoscale-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.i.aivencloud.com:21706" \
--from-literal=username="avnadmin" \
--from-literal=password="MyThanosPassword" \
-n monitoring \
--dry-run=client -o yaml | kubectl apply -f -
sleep 20
kubectl apply -f - <<'AGENTEOF'
apiVersion: monitoring.coreos.com/v1alpha1
kind: PrometheusAgent
metadata:
name: prometheus-agent
namespace: monitoring
spec:
replicas: 1
serviceAccountName: prometheus-operator
scrapeInterval: 30s
podMonitorSelector:
matchLabels: {}
probeSelector:
matchLabels: {}
serviceMonitorSelector:
matchLabels: {}
prometheusRulesSelector:
matchLabels: {}
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 200m
memory: 256Mi
storage:
emptyDir: {}
retention: 1h
walCompression: true
remoteWrite:
- url: https://yyyyyy-exoscale-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.i.aivencloud.com:21706/api/v1/receive
basicAuth:
username:
name: thanos-credentials
key: username
password:
name: thanos-credentials
key: password
writeRelabelConfigs:
- sourceLabels: [__name__]
regex: up|process_.*|go_.*
action: keep
AGENTEOF
# Create ServiceMonitor to scrape prometheus-operator
kubectl apply -f - <<'MONITORF'
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: prometheus-operator
namespace: monitoring
spec:
selector:
matchLabels:
app.kubernetes.io/name: prometheus-operator
endpoints:
- port: http
interval: 30s
MONITORFThen launch the script
chmod 755 deploy-quick.sh
export KUBECONFIG=my_path/kubeconfig
./deploy-quick.sh2/ Check Prometheus services and pods
% kubectl get svc -n monitoring
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
prometheus-agent-operated ClusterIP None <none> 9090/TCP 3h9m
prometheus-operator ClusterIP None <none> 8080/TCP 3h9m% kubectl get pods -n monitoring
NAME READY STATUS RESTARTS AGE
prom-agent-prometheus-agent-0 2/2 Running 0 3h15m
prometheus-operator-5dfd576fc9-jdxhb 1/1 Running 0 3h3m% kubectl get prometheusagent -n monitoring
NAME AGE
prometheus-agent 3h17mNOTE
Now Prometheus is working and pushing the metrics to your Thanos DBaaS. You can now integrate it with Grafana to see your metrics: Integrate with Grafana