Skip to content

Commit e716dbb

Browse files
ramius345etsauer
ramius345
authored and
etsauer
committed
federated prometheus instance (#19)
* federated prometheus instance * fix container image that doesnt work on a 3.11 cluster * fix cluster path in example * Update extra_prometheus_hosts to be secret based instead of from seed_hosts.yml * make secret documentation simpler * missed check * fix requirements yml * remove uneeded secrets file * fix wording in README.md * update requirements to reference redhat-cop
1 parent 61cd1ef commit e716dbb

18 files changed

Lines changed: 566 additions & 1 deletion

‎.applier/group_vars/seed-hosts.yml‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,60 @@
11
dashboard_namespace: custom-dashboards
2+
23
openshift_cluster_content:
4+
- galaxy_requirements:
5+
- "{{ inventory_dir }}/../mdt-secret-discovery-requirements.yml"
6+
- object: MDT Prometheus
7+
pre_steps:
8+
- role: mdt-quickstart/secret-discovery
9+
content:
10+
- name: MDT Prometheus htpasswd secret
11+
file: "{{ inventory_dir }}/../federated-prometheus/prometheus-htpasswd-secret.yml.j2"
12+
namespace: "{{ dashboard_namespace }}"
13+
tags:
14+
- dashboard-sdm
15+
- name: MDT Prometheus operator
16+
file: "{{ inventory_dir }}/../federated-prometheus/prometheus-operator-config.yml.j2"
17+
namespace: "{{ dashboard_namespace }}"
18+
tags:
19+
- dashboard-sdm
20+
- name: MDT Scrape config secret
21+
file: "{{ inventory_dir }}/../federated-prometheus/prometheus-scrape-config-secret.yml.j2"
22+
namespace: "{{ dashboard_namespace }}"
23+
tags:
24+
- dashboard-sdm
25+
- name: MDT Prometheus CR
26+
file: "{{ inventory_dir }}/../federated-prometheus/prometheus-cr.yml"
27+
namespace: "{{ dashboard_namespace }}"
28+
tags:
29+
- dashboard-sdm
30+
- name: MDT Prometheus Service
31+
file: "{{ inventory_dir }}/../federated-prometheus/prometheus-svc.yml"
32+
namespace: "{{ dashboard_namespace }}"
33+
tags:
34+
- dashboard-sdm
35+
- name: MDT Prometheus Route
36+
file: "{{ inventory_dir }}/../federated-prometheus/prometheus-route.yml"
37+
namespace: "{{ dashboard_namespace }}"
38+
tags:
39+
- dashboard-sdm
40+
- name: MDT Prometheus Oauth Cluster Role
41+
file: "{{ inventory_dir}}/../federated-prometheus/prometheus-oauth-cluster-role.yml"
42+
tags:
43+
- dashboard-sdm
44+
- name: MDT Prometheus Oauth Service Account
45+
file: "{{ inventory_dir }}/../federated-prometheus/prometheus-sa.yml"
46+
namespace: "{{ dashboard_namespace }}"
47+
tags:
48+
- dashboard sdm
49+
- name: MDT Prometheus Oauth ClusterRoleBinding
50+
file: "{{ inventory_dir }}/../federated-prometheus/prometheus-oauth-cluster-role-binding.yml.j2"
51+
tags:
52+
- dashboard-sdm
53+
- name: MDT Repoint Grafana Datasource
54+
file: "{{ inventory_dir }}/../federated-prometheus/grafana-mdt-datasources.yml.j2"
55+
namespace: "{{ dashboard_namespace }}"
56+
tags:
57+
- dashboard-sdm
358
- object: Dashboards - DevOps
459
content:
560
- name: Software Delivery Metrics Dashboard

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
dependencies
2-
temp
2+
temp
3+
galaxy

‎README.md‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,30 @@ ansible-playbook -i galaxy/openshift-toolkit/custom-dashboards/.applier galaxy/o
4242
ansible-playbook -i .applier/ galaxy/openshift-applier/playbooks/openshift-cluster-seed.yml
4343
```
4444

45+
### Adding extra prometheus instances
46+
47+
Edit the extra_prometheus_hosts.yml file. It is a yaml file with an array of entries with the following parameters:
48+
49+
* id - a description of the prometheus host (this will be used as a label to select metrics in the federated instance).
50+
* hostname - the fully qualified domain name or ip address of the host with the extra prometheus instance
51+
* password - the password used for the 'internal' basic auth account (this is provided by the k8s metrics prometheus instances in a secret).
52+
53+
For example:
54+
55+
```
56+
extra_prometheus_hosts:
57+
- id: "ci-1"
58+
hostname: "prometheus-k8s-openshift-monitoring.apps.example.com"
59+
password: "<redacted>"
60+
```
61+
Once you are finished adding your extra hosts, apply the file as the secret 'extra-prometheus-secrets'.
62+
63+
```
64+
oc create secret generic extra-prometheus-secrets --from-file extra_prometheus_hosts.yml
65+
```
66+
67+
```
68+
4569
### Cleaning Up
4670
4771
If you would like to undo the changes above:

‎extra_prometheus_hosts.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
extra_prometheus_hosts:
2+
- id: "ci-1"
3+
hostname: "prometheus-k8s-openshift-monitoring.apps.example.com"
4+
password: "password here"

‎federated-prometheus/grafana-mdt-datasources.yml.j2‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
apiVersion: v1
2+
data:
3+
prometheus.yaml: {{ new_grafana_datasources_mdt_data }}
4+
kind: Secret
5+
metadata:
6+
name: grafana-datasources
7+
type: Opaque

‎federated-prometheus/prometheus-cr.yml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
apiVersion: monitoring.coreos.com/v1
2+
kind: Prometheus
3+
metadata:
4+
name: prometheus-mdt
5+
labels:
6+
prometheus: prometheus-mdt
7+
spec:
8+
replicas: 2
9+
serviceAccountName: prometheus-mdt
10+
serviceMonitorSelector:
11+
matchLabels:
12+
junk: junk
13+
additionalScrapeConfigs:
14+
name: additional-scrape-configs
15+
key: prometheus-additional.yml
16+
secrets:
17+
- prometheus-mdt-tls
18+
- prometheus-mdt-htpasswd
19+
containers:
20+
- args:
21+
- -provider=openshift
22+
- -https-address=:9091
23+
- -http-address=
24+
- -email-domain=*
25+
- -upstream=http://localhost:9090
26+
- -htpasswd-file=/etc/proxy/htpasswd/auth
27+
- -openshift-service-account=prometheus-mdt
28+
- '-openshift-sar={"resource": "namespaces", "verb": "get"}'
29+
- '-openshift-delegate-urls={"/": {"resource": "namespaces", "verb": "get"}}'
30+
- -tls-cert=/etc/tls/private/tls.crt
31+
- -tls-key=/etc/tls/private/tls.key
32+
- -client-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token
33+
- -cookie-secret=bacon
34+
- -openshift-ca=/etc/pki/tls/cert.pem
35+
- -openshift-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
36+
- -skip-auth-regex=^/metrics
37+
image: registry.redhat.io/openshift3/oauth-proxy:v3.11
38+
name: prometheus-proxy
39+
ports:
40+
- containerPort: 9091
41+
name: web-tls
42+
resources: {}
43+
volumeMounts:
44+
- mountPath: /etc/tls/private
45+
name: secret-prometheus-mdt-tls
46+
- mountPath: /etc/proxy/htpasswd
47+
name: secret-prometheus-mdt-htpasswd
48+
49+

‎federated-prometheus/prometheus-htpasswd-secret.yml.j2‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
apiVersion: v1
2+
data:
3+
auth: "{{ mdt_prometheus_htpasswd_auth }}"
4+
kind: Secret
5+
metadata:
6+
labels:
7+
k8s-app: prometheus-mdt
8+
name: prometheus-mdt-htpasswd
9+
type: Opaque

‎federated-prometheus/prometheus-oauth-cluster-role-binding.yml.j2‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: rbac.authorization.k8s.io/v1
2+
kind: ClusterRoleBinding
3+
metadata:
4+
name: prometheus-mdt-oauth-crb
5+
roleRef:
6+
apiGroup: rbac.authorization.k8s.io
7+
kind: ClusterRole
8+
name: prometheus-mdt-oauth-clusterrole
9+
subjects:
10+
- kind: ServiceAccount
11+
name: prometheus-mdt
12+
namespace: {{ dashboard_namespace }}

‎federated-prometheus/prometheus-oauth-cluster-role.yml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
apiVersion: rbac.authorization.k8s.io/v1
2+
kind: ClusterRole
3+
metadata:
4+
name: prometheus-mdt-oauth-clusterrole
5+
rules:
6+
- apiGroups:
7+
- authentication.k8s.io
8+
resources:
9+
- tokenreviews
10+
verbs:
11+
- 'create'
12+
- apiGroups:
13+
- authorization.k8s.io
14+
resources:
15+
- subjectaccessreviews
16+
verbs:
17+
- 'create'

‎federated-prometheus/prometheus-operator-config.yml.j2‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
apiVersion: rbac.authorization.k8s.io/v1
2+
kind: ClusterRoleBinding
3+
metadata:
4+
labels:
5+
app.kubernetes.io/component: controller
6+
app.kubernetes.io/name: prometheus-operator-mdt
7+
app.kubernetes.io/version: v0.33.0
8+
name: prometheus-operator-mdt
9+
roleRef:
10+
apiGroup: rbac.authorization.k8s.io
11+
kind: ClusterRole
12+
name: prometheus-operator-mdt
13+
subjects:
14+
- kind: ServiceAccount
15+
name: prometheus-operator-mdt
16+
namespace: {{ dashboard_namespace }}
17+
---
18+
apiVersion: rbac.authorization.k8s.io/v1
19+
kind: ClusterRole
20+
metadata:
21+
labels:
22+
app.kubernetes.io/component: controller
23+
app.kubernetes.io/name: prometheus-operator-mdt
24+
app.kubernetes.io/version: v0.33.0
25+
name: prometheus-operator-mdt
26+
rules:
27+
- apiGroups:
28+
- apiextensions.k8s.io
29+
resources:
30+
- customresourcedefinitions
31+
verbs:
32+
- '*'
33+
- apiGroups:
34+
- monitoring.coreos.com
35+
resources:
36+
- alertmanagers
37+
- prometheuses
38+
- prometheuses/finalizers
39+
- alertmanagers/finalizers
40+
- servicemonitors
41+
- podmonitors
42+
- prometheusrules
43+
verbs:
44+
- '*'
45+
- apiGroups:
46+
- apps
47+
resources:
48+
- statefulsets
49+
verbs:
50+
- '*'
51+
- apiGroups:
52+
- ""
53+
resources:
54+
- configmaps
55+
- secrets
56+
verbs:
57+
- '*'
58+
- apiGroups:
59+
- ""
60+
resources:
61+
- pods
62+
verbs:
63+
- list
64+
- delete
65+
- apiGroups:
66+
- ""
67+
resources:
68+
- services
69+
- services/finalizers
70+
- endpoints
71+
verbs:
72+
- get
73+
- create
74+
- update
75+
- delete
76+
- apiGroups:
77+
- ""
78+
resources:
79+
- nodes
80+
verbs:
81+
- list
82+
- watch
83+
- apiGroups:
84+
- ""
85+
resources:
86+
- namespaces
87+
verbs:
88+
- get
89+
- list
90+
- watch
91+
---
92+
apiVersion: apps/v1
93+
kind: Deployment
94+
metadata:
95+
labels:
96+
app.kubernetes.io/component: controller
97+
app.kubernetes.io/name: prometheus-operator-mdt
98+
app.kubernetes.io/version: v0.33.0
99+
name: prometheus-operator-mdt
100+
namespace: {{ dashboard_namespace }}
101+
spec:
102+
replicas: 1
103+
selector:
104+
matchLabels:
105+
app.kubernetes.io/component: controller
106+
app.kubernetes.io/name: prometheus-operator-mdt
107+
template:
108+
metadata:
109+
labels:
110+
app.kubernetes.io/component: controller
111+
app.kubernetes.io/name: prometheus-operator-mdt
112+
app.kubernetes.io/version: v0.33.0
113+
spec:
114+
containers:
115+
- args:
116+
- --kubelet-service=kube-system/kubelet
117+
- --logtostderr=true
118+
- --config-reloader-image=quay.io/coreos/configmap-reload:v0.0.1
119+
- --prometheus-config-reloader=quay.io/coreos/prometheus-config-reloader:v0.33.0
120+
image: quay.io/coreos/prometheus-operator:v0.33.0
121+
name: prometheus-operator-mdt
122+
ports:
123+
- containerPort: 8080
124+
name: http
125+
resources:
126+
limits:
127+
cpu: 200m
128+
memory: 200Mi
129+
requests:
130+
cpu: 100m
131+
memory: 100Mi
132+
securityContext:
133+
allowPrivilegeEscalation: false
134+
# securityContext:
135+
# runAsNonRoot: true
136+
# runAsUser: 1000730001
137+
# runAsUser: 1000360001
138+
serviceAccountName: prometheus-operator-mdt
139+
---
140+
apiVersion: v1
141+
kind: ServiceAccount
142+
metadata:
143+
labels:
144+
app.kubernetes.io/component: controller
145+
app.kubernetes.io/name: prometheus-operator-mdt
146+
app.kubernetes.io/version: v0.33.0
147+
name: prometheus-operator-mdt
148+
namespace: {{ dashboard_namespace }}
149+
---
150+
apiVersion: v1
151+
kind: Service
152+
metadata:
153+
labels:
154+
app.kubernetes.io/component: controller
155+
app.kubernetes.io/name: prometheus-operator-mdt
156+
app.kubernetes.io/version: v0.33.0
157+
name: prometheus-operator-mdt
158+
namespace: {{ dashboard_namespace }}
159+
spec:
160+
clusterIP: None
161+
ports:
162+
- name: http
163+
port: 8080
164+
targetPort: http
165+
selector:
166+
app.kubernetes.io/component: controller
167+
app.kubernetes.io/name: prometheus-operator-mdt

0 commit comments

Comments
 (0)