From 7a0a530adda44cc90ec7e09782d95ed75bfbf67e Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@hackerspace-zone.labs.lowerlayer.nl>
Date: Fri, 11 Nov 2022 12:03:04 +0000
Subject: [PATCH 01/13] collapse everything into one docker-compose environment
 and enable prometheus logging

---
 .gitignore                                    |   3 +
 Makefile                                      |  87 ++++++++
 README.md                                     |   1 +
 env.production                                |  34 ++-
 grafana.yaml                                  |  41 ++++
 grafana/docker-compose.yaml                   |  26 ---
 grafana/env.production                        |   0
 grafana/keycloak.sh                           |   4 +
 .../nginx.conf                                |   6 +-
 grafana/setup                                 |  50 -----
 hedgedoc.yaml                                 |  58 +++++
 hedgedoc/docker-compose.yaml                  |  34 ---
 hedgedoc/env.production                       |   2 -
 hedgedoc/keycloak.sh                          |  50 +++++
 .../docs.conf.template => hedgedoc/nginx.conf |   8 +-
 hedgedoc/setup                                |  69 ------
 keycloak.yaml                                 |  63 ++++++
 keycloak/README.md                            |   5 +-
 keycloak/client-create                        |  44 ++--
 keycloak/client-delete                        |  40 ----
 keycloak/docker-compose.yaml                  |  43 ----
 keycloak/env.production                       |   0
 .../nginx.conf                                |   4 +-
 keycloak/setup                                | 119 -----------
 mastodon.yaml                                 | 198 ++++++++++++++++++
 mastodon/README.md                            |   3 +-
 mastodon/docker-compose.yaml                  | 131 ------------
 mastodon/entrypoint.sh                        |  38 ++++
 mastodon/env.production                       |   9 +-
 mastodon/keycloak.sh                          |   3 +
 mastodon/nginx.conf                           | 154 ++++++++++++++
 mastodon/setup                                |  78 -------
 nginx.yaml                                    |  30 +++
 nginx/Dockerfile                              |  30 +++
 nginx/certbot-renew                           |  34 ---
 ...000-default.conf.template => default.conf} |  41 +++-
 nginx/docker-compose.yaml                     |  28 ---
 nginx/docker-entrypoint.d/01-collectd.sh      |  12 ++
 nginx/docker-entrypoint.d/10-createkey.sh     |  31 +++
 .../20-envsubst-on-templates.sh               |  39 ++++
 nginx/docker-entrypoint.sh                    |  47 +++++
 nginx/env.production                          |   0
 nginx/{nginx => etc}/includes/challenge.conf  |   0
 .../includes/options-ssl-nginx.conf           |   0
 .../{nginx => etc}/includes/ssl-dhparams.pem  |   0
 nginx/{nginx => etc}/nginx.conf               |  25 +--
 nginx/nginx/templates/pixelfed.conf.template  |  30 ---
 nginx/nginx/templates/social.conf.template    |  41 ----
 nginx/setup                                   |  39 ----
 prometheus.yaml                               |  18 ++
 prometheus/prometheus.yaml                    |  21 ++
 start-all                                     |  39 ----
 stop-all                                      |   7 -
 53 files changed, 1038 insertions(+), 879 deletions(-)
 create mode 100644 Makefile
 create mode 100644 grafana.yaml
 delete mode 100644 grafana/docker-compose.yaml
 delete mode 100644 grafana/env.production
 create mode 100755 grafana/keycloak.sh
 rename nginx/nginx/templates/dashboard.conf.template => grafana/nginx.conf (90%)
 delete mode 100755 grafana/setup
 create mode 100644 hedgedoc.yaml
 delete mode 100644 hedgedoc/docker-compose.yaml
 delete mode 100644 hedgedoc/env.production
 create mode 100755 hedgedoc/keycloak.sh
 rename nginx/nginx/templates/docs.conf.template => hedgedoc/nginx.conf (90%)
 delete mode 100755 hedgedoc/setup
 create mode 100644 keycloak.yaml
 delete mode 100755 keycloak/client-delete
 delete mode 100644 keycloak/docker-compose.yaml
 delete mode 100644 keycloak/env.production
 rename nginx/nginx/templates/login.conf.template => keycloak/nginx.conf (83%)
 delete mode 100755 keycloak/setup
 create mode 100644 mastodon.yaml
 delete mode 100644 mastodon/docker-compose.yaml
 create mode 100755 mastodon/entrypoint.sh
 create mode 100755 mastodon/keycloak.sh
 create mode 100644 mastodon/nginx.conf
 delete mode 100755 mastodon/setup
 create mode 100644 nginx.yaml
 create mode 100644 nginx/Dockerfile
 delete mode 100755 nginx/certbot-renew
 rename nginx/{nginx/templates/000-default.conf.template => default.conf} (70%)
 delete mode 100644 nginx/docker-compose.yaml
 create mode 100755 nginx/docker-entrypoint.d/01-collectd.sh
 create mode 100755 nginx/docker-entrypoint.d/10-createkey.sh
 create mode 100755 nginx/docker-entrypoint.d/20-envsubst-on-templates.sh
 create mode 100755 nginx/docker-entrypoint.sh
 delete mode 100644 nginx/env.production
 rename nginx/{nginx => etc}/includes/challenge.conf (100%)
 rename nginx/{nginx => etc}/includes/options-ssl-nginx.conf (100%)
 rename nginx/{nginx => etc}/includes/ssl-dhparams.pem (100%)
 rename nginx/{nginx => etc}/nginx.conf (75%)
 delete mode 100644 nginx/nginx/templates/pixelfed.conf.template
 delete mode 100644 nginx/nginx/templates/social.conf.template
 delete mode 100755 nginx/setup
 create mode 100644 prometheus.yaml
 create mode 100644 prometheus/prometheus.yaml
 delete mode 100755 start-all
 delete mode 100755 stop-all

diff --git a/.gitignore b/.gitignore
index d9390d9..303e9a1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,6 @@
 data
 *.secrets
 env.smtp
+*.old
+*.log
+test
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..85a4244
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,87 @@
+MODULES += nginx
+MODULES += keycloak
+MODULES += hedgedoc
+MODULES += grafana
+MODULES += prometheus
+MODULES += mastodon
+#MODULES += pixelfed
+
+include env.production
+domain_name := $(DOMAIN_NAME)
+
+
+help:
+	@echo "usage: make run"
+UC = $(shell echo '$1' | tr '[:lower:]' '[:upper:]')
+
+DOCKER = \
+	$(foreach m,$(MODULES),. data/$m/secrets && ) \
+	docker-compose \
+		--env-file env.production \
+		$(foreach m,$(MODULES),--file ./$m.yaml) \
+
+run:
+	$(DOCKER) up
+down:
+	$(DOCKER) down
+nginx-shell:
+	$(DOCKER) exec nginx sh
+grafana-shell:
+	$(DOCKER) exec grafana bash
+hedgedoc-shell:
+	$(DOCKER) exec hedgedoc sh
+keycloak-shell:
+	$(DOCKER) exec keycloak sh
+mastodon-shell:
+	$(DOCKER) exec mastodon bash
+mastodon-streaming-shell:
+	$(DOCKER) exec mastodon-streaming bash
+nginx-build: data/nginx/secrets
+	$(DOCKER) build nginx
+
+certdir		= ./data/certbot/conf/live/${DOMAIN_NAME}
+
+run: secrets-setup
+
+secrets-setup: $(foreach m,$(MODULES),data/$m/secrets)
+
+# Create the per-subdomain secrets if they don't exist
+# not every service requires all of these features, but create them anyway
+GET_MODULE = $(call UC,$(word 2,$(subst /, ,$@)))
+RAND = $$(openssl rand -hex $1)
+
+data/%/secrets:
+	mkdir -p $(dir $@)
+	echo >$@ "# DO NOT CHECK IN"
+	echo >>$@ "export $(GET_MODULE)_ADMIN_PASSWORD=$(call RAND,8)"
+	echo >>$@ "export $(GET_MODULE)_CLIENT_SECRET=$(call RAND,20)"
+	echo >>$@ "export $(GET_MODULE)_SESSION_SECRET=$(call RAND,20)"
+
+keycloak-setup: secrets-setup
+	$(DOCKER) run keycloak-setup
+
+certbot:
+	$(DOCKER) \
+		run --entrypoint '/bin/sh -c "\
+		rm -rf /etc/letsencrypt ; \
+		certbot certonly \
+			--webroot \
+			--webroot-path /var/www/certbot \
+			--email "admin@$(DOMAIN_NAME)" \
+			--rsa-key-size "2048" \
+			--agree-tos \
+			--no-eff-email \
+			--force-renewal \
+			-d $(DOMAIN_NAME) \
+			$(foreach m,$(MODULES),\
+				-d $($(call UC,$m)_HOSTNAME).$(DOMAIN_NAME)) \
+		"' certbot
+
+nginx-reload:
+	$(DOCKER) restart nginx
+
+
+config:
+	$(DOCKER) config
+
+FORCE:
diff --git a/README.md b/README.md
index 10df55f..7243285 100644
--- a/README.md
+++ b/README.md
@@ -8,6 +8,7 @@ Infrastructure for the self-hosted, single-sign-on, community-run services.
 
 ```
 apt install jq docker-compose
+apt install prometheus
 ```
 
 * Setup each of the services. `keycloak` and `nginx` are required to start the others:
diff --git a/env.production b/env.production
index 0f083c1..1d807cb 100644
--- a/env.production
+++ b/env.production
@@ -1,12 +1,26 @@
-DOMAIN_NAME=hackerspace.zone
+# Fill in with your top-level domain name and desired OAUTH realm name
+DOMAIN_NAME=dev.v.st
 REALM=hackerspace
 
-KEYCLOAK_HOSTNAME=login.hackerspace.zone
-HEDGEDOC_HOSTNAME=docs.hackerspace.zone
-MASTODON_HOSTNAME=social.hackerspace.zone
-NEXTCLOUD_HOSTNAME=cloud.hackerspace.zone
-GRAFANA_HOSTNAME=dashboard.hackerspace.zone
-GITEA_HOSTNAME=git.hackerspace.zone
-MATRIX_HOSTNAME=matrix.hackerspace.zone
-MOBILIZON_HOSTNAME=events.hackerspace.zone
-PIXELFED_HOSTNAME=pixelfed.hackerspace.zone
+# Fill in with your SMTP server, if you have one
+SMTP_SERVER=
+SMTP_USER=
+SMTP_PASSWORD=
+SMTP_PORT=
+
+# You can leave these as is or change them if you like
+NGINX_HOSTNAME=www
+KEYCLOAK_HOSTNAME=login
+HEDGEDOC_HOSTNAME=docs
+MASTODON_HOSTNAME=social
+NEXTCLOUD_HOSTNAME=cloud
+GRAFANA_HOSTNAME=dashboard
+GITEA_HOSTNAME=git
+MATRIX_HOSTNAME=matrix
+MOBILIZON_HOSTNAME=events
+PIXELFED_HOSTNAME=pixelfed
+PROMETHEUS_HOSTNAME=metrics
+
+AUTH_URL=https://${KEYCLOAK_HOSTNAME}.${DOMAIN_NAME}/realms/${REALM}/protocol/openid-connect/auth
+TOKEN_URL=https://${KEYCLOAK_HOSTNAME}.${DOMAIN_NAME}/realms/${REALM}/protocol/openid-connect/token
+USERINFO_URL=https://${KEYCLOAK_HOSTNAME}.${DOMAIN_NAME}/realms/${REALM}/protocol/openid-connect/userinfo
diff --git a/grafana.yaml b/grafana.yaml
new file mode 100644
index 0000000..24cea0c
--- /dev/null
+++ b/grafana.yaml
@@ -0,0 +1,41 @@
+version: "3"
+services:
+  grafana:
+    image: grafana/grafana-oss:8.5.1
+    container_name: grafana
+    user: "0:0"
+    environment:
+      GF_AUTH_GENERIC_OAUTH_ENABLED: 'True'
+      GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP: 'True' # otherwise no login is possible
+      #GF_AUTH_GENERIC_OAUTH_TEAM_IDS: ''
+      #GF_AUTH_GENERIC_OAUTH_ALLOWED_ORGANIZATIONS: ''
+      #GF_AUTH_GENERIC_OAUTH_ALLOWED_DOMAINS: '<domains>'
+      #GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD} # ignored?
+      GF_AUTH_GENERIC_OAUTH_NAME: Keycloak
+      GF_AUTH_GENERIC_OAUTH_CLIENT_ID: grafana
+      GF_AUTH_GENERIC_OAUTH_SCOPES: openid profile email
+      GF_SERVER_ROOT_URL: https://${GRAFANA_HOSTNAME}.${DOMAIN_NAME}/
+      GF_SERVER_DOMAIN: ${GRAFANA_HOSTNAME}.${DOMAIN_NAME}
+      GF_AUTH_GENERIC_OAUTH_AUTH_URL: ${AUTH_URL}
+      GF_AUTH_GENERIC_OAUTH_TOKEN_URL: ${TOKEN_URL}
+      GF_AUTH_GENERIC_OAUTH_API_URL: ${USERINFO_URL}
+      GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ${GRAFANA_CLIENT_SECRET}
+    # reset the admin password on every run, since otherwise it defaults to admin/admin
+    entrypoint: ["sh", "-c", "grafana-cli admin reset-admin-password ${GRAFANA_ADMIN_PASSWORD} && /run.sh"]
+    volumes:
+      - ./data/grafana:/var/lib/grafana
+    restart: always
+    #    ports:
+    #  - 3000:3000
+
+  # add the grafana nginx configuration into the nginx volume
+  nginx:
+    volumes:
+      - ./grafana/nginx.conf:/etc/nginx/templates/grafana.conf.template:ro
+
+  # add the grafana client secrets to the keycloak-setup volume
+  keycloak-setup:
+    env_file:
+      - data/grafana/secrets
+    volumes:
+      - ./grafana/keycloak.sh:/keycloak-setup/grafana.sh:ro
diff --git a/grafana/docker-compose.yaml b/grafana/docker-compose.yaml
deleted file mode 100644
index e1e2b6a..0000000
--- a/grafana/docker-compose.yaml
+++ /dev/null
@@ -1,26 +0,0 @@
-version: "3"
-
-services:
-  grafana:
-    image: grafana/grafana-oss:8.5.1
-    user: "0:0"
-    environment:
-      GF_AUTH_GENERIC_OAUTH_ENABLED: 'True'
-      GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP: 'True' # otherwise no login is possible
-      #GF_AUTH_GENERIC_OAUTH_TEAM_IDS: ''
-      #GF_AUTH_GENERIC_OAUTH_ALLOWED_ORGANIZATIONS: ''
-      #GF_AUTH_GENERIC_OAUTH_ALLOWED_DOMAINS: '<domains>'
-      GF_AUTH_GENERIC_OAUTH_NAME: Keycloak
-      GF_AUTH_GENERIC_OAUTH_CLIENT_ID: grafana
-      GF_AUTH_GENERIC_OAUTH_SCOPES: openid profile email
-      # GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET is in env.secrets
-      # auth URLs are in the env.secrets since they have hostname expansion
-    volumes:
-      - ../data/grafana:/var/lib/grafana
-    restart: always
-    ports:
-      - 8000:3000
-    env_file:
-      - ../env.production
-      - env.production
-      - ../data/grafana/env.secrets
diff --git a/grafana/env.production b/grafana/env.production
deleted file mode 100644
index e69de29..0000000
diff --git a/grafana/keycloak.sh b/grafana/keycloak.sh
new file mode 100755
index 0000000..377850d
--- /dev/null
+++ b/grafana/keycloak.sh
@@ -0,0 +1,4 @@
+#!/bin/bash -x
+# Setup the grafana client connection
+
+client-create grafana "$GRAFANA_HOSTNAME.$DOMAIN_NAME" "$GRAFANA_CLIENT_SECRET" </dev/null
diff --git a/nginx/nginx/templates/dashboard.conf.template b/grafana/nginx.conf
similarity index 90%
rename from nginx/nginx/templates/dashboard.conf.template
rename to grafana/nginx.conf
index c4327d2..5fd36b5 100644
--- a/nginx/nginx/templates/dashboard.conf.template
+++ b/grafana/nginx.conf
@@ -4,7 +4,7 @@ map $http_upgrade $connection_upgrade {
 }
 
 server {
-	server_name ${GRAFANA_HOSTNAME};
+	server_name ${GRAFANA_HOSTNAME} ${GRAFANA_HOSTNAME}.${DOMAIN_NAME};
 	client_max_body_size 128m;
 
 	sendfile on;
@@ -24,7 +24,7 @@ server {
 	chunked_transfer_encoding on;
 
 	location / {
-		proxy_pass http://host.docker.internal:8000;
+		proxy_pass http://grafana:3000;
 		proxy_set_header Host $host;
 		proxy_set_header X-Real-IP $remote_addr;
 		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -32,7 +32,7 @@ server {
 	}
 
         location /socket.io/ {
-                proxy_pass http://host.docker.internal:8000;
+                proxy_pass http://grafana:3000;
                 proxy_set_header Host $host; 
                 proxy_set_header X-Real-IP $remote_addr; 
                 proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 
diff --git a/grafana/setup b/grafana/setup
deleted file mode 100755
index b806956..0000000
--- a/grafana/setup
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/bin/bash
-die() { echo >&2 "$@" ; exit 1 ; }
-
-DIRNAME="$(dirname $0)"
-cd "$DIRNAME"
-source ../env.production || die "no top level env?"
-source env.production || die "no local env?"
-
-BASE="https://$KEYCLOAK_HOSTNAME/realms/$REALM/protocol/openid-connect"
-SECRETS="../data/grafana/env.secrets"
-
-if [ -r "$SECRETS" ]; then
-	docker-compose up -d || die "grafana: unable to start container"
-	exit 0
-fi
-
-docker-compose down 2>/dev/null
-
-GRAFANA_CLIENT_SECRET="$(openssl rand -hex 32)"
-GRAFANA_ADMIN_PASSWORD="$(openssl rand -hex 4)"
-
-echo "Generating secrets: admin password $GRAFANA_ADMIN_PASSWORD"
-mkdir -p "$(dirname "$SECRETS")"
-cat <<EOF > "$SECRETS"
-# Do not check in!
-GF_SECURITY_ADMIN_PASSWORD=$GRAFANA_ADMIN_PASSWORD
-GF_SERVER_ROOT_URL=https://$GRAFANA_HOSTNAME/
-GF_SERVER_DOMAIN=$GRAFANA_HOSTNAME
-GF_AUTH_GENERIC_OAUTH_AUTH_URL=$BASE/auth
-GF_AUTH_GENERIC_OAUTH_TOKEN_URL=$BASE/token
-GF_AUTH_GENERIC_OAUTH_API_URL=$BASE/userinfo
-GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=$GRAFANA_CLIENT_SECRET
-EOF
-
-
-../keycloak/client-delete 'grafana' 2>/dev/null
-
-../keycloak/client-create << EOF || die "unable to create client id"
-{
-	"clientId": "grafana",
-	"rootUrl": "https://$GRAFANA_HOSTNAME/",
-	"adminUrl": "https://$GRAFANA_HOSTNAME/",
-	"redirectUris": [ "https://$GRAFANA_HOSTNAME/*" ],
-	"webOrigins": [ "https://$GRAFANA_HOSTNAME" ],
-	"clientAuthenticatorType": "client-secret",
-	"secret": "$GRAFANA_CLIENT_SECRET"
-}
-EOF
-
-docker-compose up -d || die "grafana: unable to bring up container"
diff --git a/hedgedoc.yaml b/hedgedoc.yaml
new file mode 100644
index 0000000..6828dd4
--- /dev/null
+++ b/hedgedoc.yaml
@@ -0,0 +1,58 @@
+version: '3.9'
+services:
+  hedgedoc-db:
+    image: postgres:13.4-alpine
+    container_name: hedgedoc-db
+    environment:
+      - POSTGRES_USER=hedgedoc
+      - POSTGRES_PASSWORD=password
+      - POSTGRES_DB=hedgedoc
+    volumes:
+      - ./data/hedgedoc/database:/var/lib/postgresql/data
+    restart: always
+
+  hedgedoc:
+    # Make sure to use the latest release from https://hedgedoc.org/latest-release
+    image: quay.io/hedgedoc/hedgedoc:1.9.4
+    container_name: hedgedoc
+    environment:
+      #- CMD_CSP_ENABLE=false
+      - CMD_DB_URL=postgres://hedgedoc:password@hedgedoc-db:5432/hedgedoc
+      - CMD_PROTOCOL_USESSL=true
+      - CMD_ALLOW_ANONYMOUS=false  # anonymous user's can't create notes
+      - CMD_ALLOW_ANONYMOUS_EDITS=true # but they can be invited to edit notes
+      - CMD_ALLOW_FREEURL=true # users can create arbitrary names
+      - CMD_EMAIL=false # only oauth logins
+      - CMD_DOMAIN=${HEDGEDOC_HOSTNAME}.${DOMAIN_NAME}
+      - CMD_OAUTH2_AUTHORIZATION_URL=${AUTH_URL}
+      - CMD_OAUTH2_TOKEN_URL=${TOKEN_URL}
+      - CMD_OAUTH2_USER_PROFILE_URL=${USERINFO_URL}
+      - CMD_OAUTH2_USER_PROFILE_USERNAME_ATTR=preferred_username
+      - CMD_OAUTH2_USER_PROFILE_DISPLAY_NAME_ATTR=name
+      - CMD_OAUTH2_USER_PROFILE_EMAIL_ATTR=email
+      - CMD_OAUTH2_CLIENT_ID=hedgedoc
+      - CMD_OAUTH2_PROVIDERNAME=Keycloak
+      - CMD_OAUTH2_CLIENT_SECRET=${HEDGEDOC_CLIENT_SECRET}
+      - CMD_SESSION_SECRET=${HEDGEDOC_SESSION_SECRET}
+    env_file:
+      - env.production
+    volumes:
+      - ./data/hedgedoc/uploads:/hedgedoc/public/uploads
+        #    ports:
+        #- "3000:3000"
+    restart: always
+    depends_on:
+      - hedgedoc-db
+      - keycloak
+
+  # add the hedgedoc nginx configuration into the nginx volume
+  nginx:
+    volumes:
+      - ./hedgedoc/nginx.conf:/etc/nginx/templates/hedgedoc.conf.template:ro
+
+  # add the hedgedoc client secrets to the keycloak-setup volume
+  keycloak-setup:
+    env_file:
+      - data/hedgedoc/secrets
+    volumes:
+      - ./hedgedoc/keycloak.sh:/keycloak-setup/hedgedoc.sh:ro
diff --git a/hedgedoc/docker-compose.yaml b/hedgedoc/docker-compose.yaml
deleted file mode 100644
index daf7342..0000000
--- a/hedgedoc/docker-compose.yaml
+++ /dev/null
@@ -1,34 +0,0 @@
-version: '3'
-services:
-  database:
-    image: postgres:13.4-alpine
-    environment:
-      - POSTGRES_USER=hedgedoc
-      - POSTGRES_PASSWORD=password
-      - POSTGRES_DB=hedgedoc
-    volumes:
-      - ../data/hedgedoc/database:/var/lib/postgresql/data
-    restart: always
-  hedgedoc:
-    # Make sure to use the latest release from https://hedgedoc.org/latest-release
-    image: quay.io/hedgedoc/hedgedoc:1.9.3
-    env_file:
-      - ../env.production
-      - env.production
-      - ../data/hedgedoc/env.secrets
-    environment:
-      #- CMD_CSP_ENABLE=false
-      - CMD_DB_URL=postgres://hedgedoc:password@database:5432/hedgedoc
-      - CMD_PROTOCOL_USESSL=true
-      - CMD_ALLOW_ANONYMOUS=false  # anonymous user's can't create notes
-      - CMD_ALLOW_ANONYMOUS_EDITS=true # but they can be invited to edit notes
-      - CMD_ALLOW_FREEURL=true # users can create arbitrary names
-      - CMD_EMAIL=false # only oauth logins
-      # DOMAIN and OAUTH2 variables are now in env.secret
-    volumes:
-      - ../data/hedgedoc/uploads:/hedgedoc/public/uploads
-    ports:
-      - "3000:3000"
-    restart: always
-    depends_on:
-      - database
diff --git a/hedgedoc/env.production b/hedgedoc/env.production
deleted file mode 100644
index 118615d..0000000
--- a/hedgedoc/env.production
+++ /dev/null
@@ -1,2 +0,0 @@
-CMD_OAUTH2_CLIENT_SECRET=abcdef1234
-CMD_SESSION_SECRET=abcdef1234
diff --git a/hedgedoc/keycloak.sh b/hedgedoc/keycloak.sh
new file mode 100755
index 0000000..b94337e
--- /dev/null
+++ b/hedgedoc/keycloak.sh
@@ -0,0 +1,50 @@
+#!/bin/bash -x
+# Setup the hedgedoc client connection
+
+# this might fail; we'll ignore it if we have already created it
+# https://github.com/hedgedoc/hedgedoc/issues/56
+kcadm.sh \
+	create client-scopes \
+	-r "$REALM" \
+	-f - <<EOF || echo "whatever"
+ {
+      "name": "id",
+      "protocol": "openid-connect",
+      "attributes": {
+        "include.in.token.scope": "true",
+        "display.on.consent.screen": "true"
+      },
+      "protocolMappers": [
+        {
+          "name": "id",
+          "protocol": "openid-connect",
+          "protocolMapper": "oidc-usermodel-property-mapper",
+          "consentRequired": false,
+          "config": {
+            "user.attribute": "id",
+            "id.token.claim": "true",
+            "access.token.claim": "true",
+            "jsonType.label": "String",
+            "userinfo.token.claim": "true"
+          }
+        }
+      ]
+}
+EOF
+
+client-create hedgedoc "$HEDGEDOC_HOSTNAME.$DOMAIN_NAME" "$HEDGEDOC_CLIENT_SECRET" <<EOF
+	,"defaultClientScopes": [
+		"web-origins",
+		"acr",
+		"profile",
+		"roles",
+		"id",
+		"email"
+	],
+	"optionalClientScopes": [
+		"address",
+		"phone",
+		"offline_access",
+		"microprofile-jwt"
+	]
+EOF
diff --git a/nginx/nginx/templates/docs.conf.template b/hedgedoc/nginx.conf
similarity index 90%
rename from nginx/nginx/templates/docs.conf.template
rename to hedgedoc/nginx.conf
index 930ade8..7a32b62 100644
--- a/nginx/nginx/templates/docs.conf.template
+++ b/hedgedoc/nginx.conf
@@ -4,7 +4,7 @@ map $http_upgrade $connection_upgrade {
 }
 
 server {
-	server_name ${HEDGEDOC_HOSTNAME};
+	server_name ${HEDGEDOC_HOSTNAME} ${HEDGEDOC_HOSTNAME}.${DOMAIN_NAME};
 	client_max_body_size 128m;
 
 	sendfile on;
@@ -28,7 +28,7 @@ server {
 	chunked_transfer_encoding on;
 
 	location / {
-		proxy_pass http://host.docker.internal:3000;
+		proxy_pass http://hedgedoc:3000;
 		proxy_set_header Host $host;
 		proxy_set_header X-Real-IP $remote_addr;
 		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -38,7 +38,7 @@ server {
 	# allow the fonts to be used by anything
 	location ~* \.(eot|otf|ttf|woff|woff2)$ {
 		add_header Access-Control-Allow-Origin *;
-		proxy_pass http://host.docker.internal:3000;
+		proxy_pass http://hedgedoc:3000;
 		proxy_set_header Host $host;
 		proxy_set_header X-Real-IP $remote_addr;
 		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -47,7 +47,7 @@ server {
 
 	# websocket traffic with extra headers for upgrading the connection
         location /socket.io/ {
-                proxy_pass http://host.docker.internal:3000;
+                proxy_pass http://hedgedoc:3000;
                 proxy_set_header Host $host; 
                 proxy_set_header X-Real-IP $remote_addr; 
                 proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 
diff --git a/hedgedoc/setup b/hedgedoc/setup
deleted file mode 100755
index 0255167..0000000
--- a/hedgedoc/setup
+++ /dev/null
@@ -1,69 +0,0 @@
-#!/bin/bash
-die() { echo >&2 "$@" ; exit 1 ; }
-
-DIRNAME="$(dirname $0)"
-cd "$DIRNAME"
-source ../env.production || die "no top levle env?"
-source env.production || die "no local env?"
-
-DATA="../data/hedgedoc"
-SECRETS="$DATA/env.secrets"
-
-if [ -r "$SECRETS" ]; then
-	docker-compose up -d || die "hedgedoc: unable to start"
-	exit 0
-fi
-
-docker-compose down 2>/dev/null
-
-# regenerate the client secrets
-CLIENT_SECRET="$(openssl rand -hex 20)"
-SESSION_SECRET="$(openssl rand -hex 20)"
-
-mkdir -p "$DATA/uploads"
-chmod 666 "$DATA/uploads"
-
-cat <<EOF > "$SECRETS"
-# DO NOT CHECK IN
-CMD_OAUTH2_CLIENT_SECRET=$CLIENT_SECRET
-CMD_SESSION_SECRET=$SESSION_SECRET
-CMD_DOMAIN=${HEDGEDOC_HOSTNAME}
-CMD_OAUTH2_AUTHORIZATION_URL=https://${KEYCLOAK_HOSTNAME}/realms/${REALM}/protocol/openid-connect/auth
-CMD_OAUTH2_TOKEN_URL=https://${KEYCLOAK_HOSTNAME}/realms/${REALM}/protocol/openid-connect/token
-CMD_OAUTH2_USER_PROFILE_URL=https://${KEYCLOAK_HOSTNAME}/realms/${REALM}/protocol/openid-connect/userinfo
-CMD_OAUTH2_USER_PROFILE_USERNAME_ATTR=preferred_username
-CMD_OAUTH2_USER_PROFILE_DISPLAY_NAME_ATTR=name
-CMD_OAUTH2_USER_PROFILE_EMAIL_ATTR=email
-CMD_OAUTH2_CLIENT_ID=hedgedoc
-CMD_OAUTH2_PROVIDERNAME=Keycloak
-EOF
-
-../keycloak/client-delete hedgedoc
-
-../keycloak/client-create <<EOF || die "unable to create hedgedoc client"
-{
-	"clientId": "hedgedoc",
-	"rootUrl": "https://$HEDGEDOC_HOSTNAME",
-	"adminUrl": "https://$HEDGEDOC_HOSTNAME",
-	"redirectUris": [ "https://$HEDGEDOC_HOSTNAME/*" ],
-	"webOrigins": [ "https://$HEDGEDOC_HOSTNAME" ],
-	"clientAuthenticatorType": "client-secret",
-	"secret": "$CLIENT_SECRET",
-	"defaultClientScopes": [
-		"web-origins",
-		"acr",
-		"profile",
-		"roles",
-		"id",
-		"email"
-	],
-	"optionalClientScopes": [
-		"address",
-		"phone",
-		"offline_access",
-		"microprofile-jwt"
-	]
-}
-EOF
-
-docker-compose up -d || die "hedgedoc: unable to start container"
diff --git a/keycloak.yaml b/keycloak.yaml
new file mode 100644
index 0000000..d83d08b
--- /dev/null
+++ b/keycloak.yaml
@@ -0,0 +1,63 @@
+version: '3.9'
+services:
+  keycloak-db:
+      image: mysql:5.7
+      restart: always
+      container_name: keycloak-db
+      volumes:
+        - ./data/keycloak/database:/var/lib/mysql
+      environment:
+        MYSQL_ROOT_PASSWORD: root
+        MYSQL_DATABASE: keycloak
+        MYSQL_USER: keycloak
+        MYSQL_PASSWORD: password
+
+  keycloak:
+      image: quay.io/keycloak/keycloak:18.0.0
+      restart: always
+      container_name: keycloak
+      entrypoint: /opt/keycloak/bin/kc.sh start --hostname="$${KEYCLOAK_HOSTNAME}.$${DOMAIN_NAME}" --proxy=edge
+#      healthcheck:
+#        test: ["CMD", "curl", "-f", "http://localhost:8080"]
+#        interval: 30s
+#        timeout: 10s
+#        retries: 3
+      user: "0:0" # otherwise the persistent data directory is not writable
+      env_file:
+        - env.production
+        - data/keycloak/secrets
+      environment:
+        DB_VENDOR: MYSQL
+        DB_ADDR: keycloak-db
+        DB_DATABASE: keycloak
+        DB_USER: keycloak
+        DB_PASSWORD: password
+        KEYCLOAK_ADMIN: admin
+        KEYCLOAK_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD}
+        PROXY_ADDRESS_FORWARDING: 'true'
+        # KEYCLOAK_ADMIN_PASSWORD is set in env.secrets
+      volumes:
+        - ./data/keycloak/certs:/etc/x509/https
+        - ./data/keycloak/keycloak:/opt/keycloak/data
+      depends_on:
+        - keycloak-db
+
+  keycloak-setup:
+        image: quay.io/keycloak/keycloak:18.0.0
+        profiles:
+          - setup
+        depends_on:
+          - keycloak
+        restart: never
+        env_file:
+          - env.production
+          - data/keycloak/secrets
+        entrypoint: /keycloak-setup.sh
+        volumes:
+          - ./keycloak/setup:/keycloak-setup.sh:ro
+          - ./keycloak/client-create:/bin/client-create:ro
+
+  # add the keycloak nginx configuration into the nginx volume
+  nginx:
+    volumes:
+      - ./keycloak/nginx.conf:/etc/nginx/templates/keycloak.conf.template:ro
diff --git a/keycloak/README.md b/keycloak/README.md
index 8c5278f..5b2c966 100644
--- a/keycloak/README.md
+++ b/keycloak/README.md
@@ -1,7 +1,6 @@
 # Keycloak
 
 Keycloak is the single-sign-on user authentication provider.
-You must set the `KEYCLOAK_ADMIN_PASSWORD` in the `env.secrets` file.
-This is the most important secret: it allows user accounts to be created
-for all the other services.
+In order to login to create the first account, use `admin` and
+the password stored in `data/keycloak/secrets`
 
diff --git a/keycloak/client-create b/keycloak/client-create
index 12c250d..cf6beab 100755
--- a/keycloak/client-create
+++ b/keycloak/client-create
@@ -1,20 +1,34 @@
 #!/bin/bash
 die() { echo >&2 "$@" ; exit 1 ; }
 
-DIRNAME="$(dirname $0)"
-cd "$DIRNAME"
+client_name="$1"
+hostname="$2"
+secret="$3"
 
-source ../env.production || die "no top levle env?"
-source env.production || die "no local env?"
-source "../data/keycloak/env.secrets" || die "no local secrets?"
+client_id="$(kcadm.sh get clients \
+	-r "$REALM" \
+	--fields id \
+	-q clientId="$client_name" \
+	--format csv \
+	--noquotes \
+)"
 
-docker-compose exec -T keycloak \
-  /opt/keycloak/bin/kcadm.sh \
-  create clients \
-  --server http://localhost:8080/ \
-  --user admin \
-  --realm master \
-  --password "$KEYCLOAK_ADMIN_PASSWORD" \
-  -r "$REALM" \
-  -f - \
-|| die "create client failed"
+if [ -n "$client_id" ]; then
+	kcadm.sh delete "clients/$client_id" -r "$REALM" || die "$client_id: unable to delete"
+fi
+
+# remember to add a leading , if adding extra data
+extra="$(cat -)"
+
+kcadm.sh create clients -r "$REALM"  -f - <<EOF || die "$client_id: unable to create"
+{
+	"clientId": "$client_name",
+	"rootUrl": "https://$hostname",
+	"adminUrl": "https://$hostname",
+	"redirectUris": [ "https://$hostname/*" ],
+	"webOrigins": [ "https://$hostname" ],
+	"clientAuthenticatorType": "client-secret",
+	"secret": "$secret"
+	$extra
+}
+EOF
diff --git a/keycloak/client-delete b/keycloak/client-delete
deleted file mode 100755
index 8d4f46d..0000000
--- a/keycloak/client-delete
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/bin/bash
-die() { echo >&2 "$@" ; exit 1 ; }
-
-DIRNAME="$(dirname $0)"
-cd "$DIRNAME"
-
-source ../env.production || die "no top levle env?"
-source env.production || die "no local env?"
-source "../data/keycloak/env.secrets" || die "no local secrets?"
-
-# try to get the clients by name
-CLIENT_NAME="$1"
-if [ -z "$CLIENT_NAME" ]; then
-	die "usage: $0 clientName"
-fi
-
-CLIENT_ID="$(docker-compose exec -T keycloak \
-  /opt/keycloak/bin/kcadm.sh \
-  get clients \
-  --server http://localhost:8080/ \
-  --user admin \
-  --password "$KEYCLOAK_ADMIN_PASSWORD" \
-  --realm master \
-  -r "$REALM" \
-| jq -r ".[] | select( .clientId == \"$CLIENT_NAME\" ).id")"
-
-if [ -z "$CLIENT_ID" ]; then
-	die "$CLIENT_NAME: no such client"
-fi
-
-echo "$0: $CLIENT_NAME = $CLIENT_ID"
-docker-compose exec -T keycloak \
-  /opt/keycloak/bin/kcadm.sh \
-  delete "clients/$CLIENT_ID" \
-  --server http://localhost:8080/ \
-  --user admin \
-  --realm master \
-  --password "$KEYCLOAK_ADMIN_PASSWORD" \
-  -r "$REALM" \
-  || die "$CLIENT_NAME($CLIENT_ID): unable to remove"
diff --git a/keycloak/docker-compose.yaml b/keycloak/docker-compose.yaml
deleted file mode 100644
index 62ec6e1..0000000
--- a/keycloak/docker-compose.yaml
+++ /dev/null
@@ -1,43 +0,0 @@
-version: '3'
-
-volumes:
-  mysql_data:
-      driver: local
-
-services:
-  mysql:
-      image: mysql:5.7
-      restart: always
-      volumes:
-        - ../data/keycloak/database:/var/lib/mysql
-      environment:
-        MYSQL_ROOT_PASSWORD: root
-        MYSQL_DATABASE: keycloak
-        MYSQL_USER: keycloak
-        MYSQL_PASSWORD: password
-
-  keycloak:
-      image: quay.io/keycloak/keycloak:18.0.0
-      restart: always
-      entrypoint: /opt/keycloak/bin/kc.sh start --hostname="$${KEYCLOAK_HOSTNAME}" --proxy=edge
-      user: "0:0" # otherwise the persistent data directory is not writable
-      env_file:
-        - ../env.production
-        - env.production
-        - ../data/keycloak/env.secrets
-      environment:
-        DB_VENDOR: MYSQL
-        DB_ADDR: mysql
-        DB_DATABASE: keycloak
-        DB_USER: keycloak
-        DB_PASSWORD: password
-        KEYCLOAK_ADMIN: admin
-        # KEYCLOAK_ADMIN_PASSWORD should be set in env.secrets
-        PROXY_ADDRESS_FORWARDING: 'true'
-      volumes:
-        - ../data/keycloak/certs:/etc/x509/https
-        - ../data/keycloak/keycloak:/opt/keycloak/data
-      ports:
-        - 8080:8080
-      depends_on:
-        - mysql
diff --git a/keycloak/env.production b/keycloak/env.production
deleted file mode 100644
index e69de29..0000000
diff --git a/nginx/nginx/templates/login.conf.template b/keycloak/nginx.conf
similarity index 83%
rename from nginx/nginx/templates/login.conf.template
rename to keycloak/nginx.conf
index 397b5c6..e7398b5 100644
--- a/nginx/nginx/templates/login.conf.template
+++ b/keycloak/nginx.conf
@@ -1,9 +1,9 @@
 server {
-	server_name login.${DOMAIN_NAME};
+	server_name ${KEYCLOAK_HOSTNAME} ${KEYCLOAK_HOSTNAME}.${DOMAIN_NAME};
 	client_max_body_size 128m;
 
 	location / {
-		proxy_pass http://host.docker.internal:8080;
+		proxy_pass http://keycloak:8080;
 		proxy_pass_header Set-Cookie;
 		proxy_set_header Host $host;
 		proxy_set_header X-Forwarded-For $remote_addr;
diff --git a/keycloak/setup b/keycloak/setup
deleted file mode 100755
index 15792c4..0000000
--- a/keycloak/setup
+++ /dev/null
@@ -1,119 +0,0 @@
-#!/bin/bash
-die() { echo >&2 "keycloak: ERROR: $@" ; exit 1 ; }
-info() { echo >&2 "keycloak: $@" ; }
-
-DIRNAME="$(dirname $0)"
-cd "$DIRNAME"
-source ../env.production
-source ./env.production
-source "../env.smtp" 2>/dev/null
-
-SECRETS="../data/keycloak/env.secrets"
-
-if [ -r "$SECRETS" ]; then
-	docker-compose up -d || die "keycloak: unable to start container"
-	exit 0
-fi
-
-docker-compose down 2>/dev/null
-
-KEYCLOAK_ADMIN_PASSWORD="$(openssl rand -hex 8)"
-echo "Keycloak admin password $KEYCLOAK_ADMIN_PASSWORD"
-
-mkdir -p "$(dirname "$SECRETS")"
-cat <<EOF > "$SECRETS"
-# DO NOT CHECK IN
-KEYCLOAK_ADMIN_PASSWORD=$KEYCLOAK_ADMIN_PASSWORD
-EOF
-
-docker-compose up -d || die "unable to start keycloak"
-echo "sleeping a minute while keycloak initializes..." 
-sleep 30
-
-
-info "logging into server"
-docker-compose exec keycloak \
-  /opt/keycloak/bin/kcadm.sh \
-  config credentials \
-  --server http://localhost:8080/ \
-  --user admin \
-  --password "$KEYCLOAK_ADMIN_PASSWORD" \
-  --realm master \
-|| die "unable to login"
-
-
-info "Create a new realm for '$REALM'"
-docker-compose exec keycloak \
-  /opt/keycloak/bin/kcadm.sh \
-  create realms \
-  -s "realm=$REALM" \
-  -s enabled=true \
-|| die "unable to create realm"
-
-
-# https://github.com/hedgedoc/hedgedoc/issues/56
-info "Fix up a id bug"
-docker-compose exec -T keycloak \
-  /opt/keycloak/bin/kcadm.sh \
-  create client-scopes \
-  -r "$REALM" \
-  -f - <<EOF || die "unable to create mapping"
- {
-      "name": "id",
-      "protocol": "openid-connect",
-      "attributes": {
-        "include.in.token.scope": "true",
-        "display.on.consent.screen": "true"
-      },
-      "protocolMappers": [
-        {
-          "name": "id",
-          "protocol": "openid-connect",
-          "protocolMapper": "oidc-usermodel-property-mapper",
-          "consentRequired": false,
-          "config": {
-            "user.attribute": "id",
-            "id.token.claim": "true",
-            "access.token.claim": "true",
-            "jsonType.label": "String",
-            "userinfo.token.claim": "true"
-          }
-        }
-      ]
-}
-EOF
-
-if [ -n "$SMTP_SERVER" ]; then
-	info "configuring email"
-	docker-compose exec -T keycloak \
-	  /opt/keycloak/bin/kcadm.sh update "realms/$REALM" \
-	-f - <<EOF || die "unable to configure email"
-{
-  "resetPasswordAllowed": "true",
-  "smtpServer" : {
-    "auth" : "true",
-    "starttls" : "true",
-    "user" : "$SMTP_USER",
-    "password" : "$SMTP_PASSWORD",
-    "port" : "$SMTP_PORT",
-    "host" : "$SMTP_SERVER",
-    "from" : "keycloak@$DOMAIN_NAME",
-    "fromDisplayName" : "Keycloak @ $DOMAIN_NAME",
-    "ssl" : "false"
-  }
-}
-EOF
-fi
-
-
-info "Create an admin user in realm"
-docker-compose exec -T keycloak \
-  /opt/keycloak/bin/kcadm.sh \
-  create users \
-  -o \
-  --fields id,username \
-  -r "$REALM" \
-  -s username=admin \
-  -s enabled=true \
-  -s 'credentials=[{"type":"'$KEYCLOAK_ADMIN_PASSWORD'","value":"admin","temporary":false}]' \
-|| die "$REALM: unable to create admin user"
diff --git a/mastodon.yaml b/mastodon.yaml
new file mode 100644
index 0000000..04e4c36
--- /dev/null
+++ b/mastodon.yaml
@@ -0,0 +1,198 @@
+version: '3'
+services:
+  mastodon-db:
+    image: postgres:13.4-alpine
+    restart: always
+    container_name: mastodon-db
+    #shm_size: 256mb
+#    networks:
+#      - internal_network
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', "mastodon", "-d", "mastodon_production"]
+    volumes:
+      - ./data/mastodon/database:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_USER=mastodon
+      - POSTGRES_PASSWORD=mastodon
+        #- POSTGRES_DB=mastodon_production
+    env_file:
+      - mastodon/env.production
+
+  mastodon-redis:
+    image: redis:6-alpine
+    restart: always
+    container_name: mastodon-redis
+#    networks:
+#      - internal_network
+    healthcheck:
+      test: ['CMD', 'redis-cli', 'ping']
+    env_file:
+      - mastodon/env.production
+    volumes:
+      - ./data/mastodon/redis:/data
+
+  mastodon-es:
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2
+    restart: always
+    container_name: mastodon-es
+    environment:
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - "cluster.name=es-mastodon"
+      - "discovery.type=single-node"
+      - "bootstrap.memory_lock=true"
+    env_file:
+      - mastodon/env.production
+#    networks:
+#       - internal_network
+    healthcheck:
+       test: ["CMD-SHELL", "curl --silent --fail localhost:9200/_cluster/health || exit 1"]
+    volumes:
+       - ./data/mastodon/elasticsearch:/usr/share/elasticsearch/data
+    # fixup the permissions on the data directory since they are created as root on host
+    entrypoint: ["/bin/sh", "-c", "chown -R elasticsearch:elasticsearch data && exec /usr/local/bin/docker-entrypoint.sh eswrapper"]
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+
+  mastodon:
+    image: tootsuite/mastodon
+    container_name: mastodon
+    restart: always
+    #command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 6001"
+    user: "0:0"
+    command: ["/entrypoint.sh"]
+#    networks:
+#      - external_network
+#      - internal_network
+    healthcheck:
+      # prettier-ignore
+      test: ['CMD-SHELL', 'wget -q --spider --proxy=off localhost:6001/health || exit 1']
+      #    ports:
+      #- '6001:6001'
+    depends_on:
+      - mastodon-db
+      - mastodon-redis
+      - mastodon-es
+    volumes:
+      - ./data/mastodon/system:/mastodon/public/system
+      - ./mastodon/entrypoint.sh:/entrypoint.sh:ro
+    env_file:
+      - mastodon/env.production
+    environment:
+      - WEB_DOMAIN=$MASTODON_HOSTNAME.$DOMAIN_NAME
+      - LOCAL_DOMAIN=$DOMAIN_NAME
+      - OIDC_DISPLAY_NAME=$REALM
+      - OIDC_ISSUER=https://$KEYCLOAK_HOSTNAME.$DOMAIN_NAME/realms/$REALM
+      - OIDC_REDIRECT_URI=https://$MASTODON_HOSTNAME.$DOMAIN_NAME/auth/auth/openid_connect/callback
+      - OIDC_CLIENT_SECRET=${MASTODON_CLIENT_SECRET}
+      - SECRET_KEY_BASE=${MASTODON_ADMIN_PASSWORD}
+      - OTP_SECRET=${MASTODON_SESSION_SECRET}
+      - SMTP_SERVER=$SMTP_SERVER
+      - SMTP_PORT=$SMTP_PORT
+      - SMTP_LOGIN=$SMTP_USER
+      - SMTP_PASSWORD=$SMTP_PASSWORD
+      - SMTP_FROM_ADDRESS=mastodon@$DOMAIN_NAME
+
+  mastodon-streaming:
+    image: tootsuite/mastodon
+    restart: always
+    container_name: mastodon-streaming
+    environment:
+      - WEB_DOMAIN=$MASTODON_HOSTNAME.$DOMAIN_NAME
+      - LOCAL_DOMAIN=$DOMAIN_NAME
+      - OIDC_DISPLAY_NAME=$REALM
+      - OIDC_ISSUER=https://$KEYCLOAK_HOSTNAME.$DOMAIN_NAME/realms/$REALM
+      - OIDC_REDIRECT_URI=https://$MASTODON_HOSTNAME.$DOMAIN_NAME/auth/auth/openid_connect/callback
+      - OIDC_CLIENT_SECRET=${MASTODON_CLIENT_SECRET}
+      - SECRET_KEY_BASE=${MASTODON_ADMIN_PASSWORD}
+      - OTP_SECRET=${MASTODON_SESSION_SECRET}
+      - SMTP_SERVER=$SMTP_SERVER
+      - SMTP_PORT=$SMTP_PORT
+      - SMTP_LOGIN=$SMTP_USER
+      - SMTP_PASSWORD=$SMTP_PASSWORD
+      - SMTP_FROM_ADDRESS=mastodon@$DOMAIN_NAME
+    env_file:
+      - mastodon/env.production
+    command: node ./streaming
+#    networks:
+#      - external_network
+#      - internal_network
+    volumes:
+      - ./data/mastodon/system:/mastodon/public/system
+    healthcheck:
+      # prettier-ignore
+      test: ['CMD-SHELL', 'wget -q --spider --proxy=off localhost:4000/api/v1/streaming/health || exit 1']
+    depends_on:
+      - mastodon-db
+      - mastodon-redis
+
+  mastodon-sidekiq:
+    image: tootsuite/mastodon
+    restart: always
+    container_name: mastodon-sidekiq
+    env_file:
+      - mastodon/env.production
+    environment:
+      - WEB_DOMAIN=$MASTODON_HOSTNAME.$DOMAIN_NAME
+      - LOCAL_DOMAIN=$DOMAIN_NAME
+      - OIDC_DISPLAY_NAME=$REALM
+      - OIDC_ISSUER=https://$KEYCLOAK_HOSTNAME.$DOMAIN_NAME/realms/$REALM
+      - OIDC_REDIRECT_URI=https://$MASTODON_HOSTNAME.$DOMAIN_NAME/auth/auth/openid_connect/callback
+      - OIDC_CLIENT_SECRET=${MASTODON_CLIENT_SECRET}
+      - SECRET_KEY_BASE=${MASTODON_ADMIN_PASSWORD}
+      - OTP_SECRET=${MASTODON_SESSION_SECRET}
+      - SMTP_SERVER=$SMTP_SERVER
+      - SMTP_PORT=$SMTP_PORT
+      - SMTP_LOGIN=$SMTP_USER
+      - SMTP_PASSWORD=$SMTP_PASSWORD
+      - SMTP_FROM_ADDRESS=mastodon@$DOMAIN_NAME
+    command: bundle exec sidekiq
+    depends_on:
+      - mastodon-db
+      - mastodon-redis
+#    networks:
+#      - external_network
+#      - internal_network
+    volumes:
+      - ./data/mastodon/system:/mastodon/public/system
+    healthcheck:
+      test: ['CMD-SHELL', "ps aux | grep '[s]idekiq\ 6' || false"]
+
+  ## Uncomment to enable federation with tor instances along with adding the following ENV variables
+  ## http_proxy=http://privoxy:8118
+  ## ALLOW_ACCESS_TO_HIDDEN_SERVICE=true
+  # tor:
+  #   image: sirboops/tor
+  #   networks:
+  #      - external_network
+  #      - internal_network
+  #
+  # privoxy:
+  #   image: sirboops/privoxy
+  #   volumes:
+  #     - ./priv-config:/opt/config
+  #   networks:
+  #     - external_network
+  #     - internal_network
+
+  # add the subdomain nginx configuration into the nginx volume
+  # as well as the cache directory so that nginx can send files directly from it
+  nginx:
+    volumes:
+      - ./mastodon/nginx.conf:/etc/nginx/templates/mastodon.conf.template:ro
+      - ./data/mastodon/system/cache:/mastodon/system/cache:ro
+      - ./data/mastodon/system/media_attachments:/mastodon/system/media_attachments:ro
+      - ./data/mastodon/system/accounts:/mastodon/system/accounts:ro
+
+  # add the subdomain client secrets to the keycloak-setup volume
+  keycloak-setup:
+    env_file:
+      - data/mastodon/secrets
+    volumes:
+      - ./mastodon/keycloak.sh:/keycloak-setup/mastodon.sh:ro
+
+#networks:
+#  external_network:
+#  internal_network:
+#    internal: true
diff --git a/mastodon/README.md b/mastodon/README.md
index 13794d3..3ea5519 100644
--- a/mastodon/README.md
+++ b/mastodon/README.md
@@ -1,4 +1,5 @@
 # Mastodon
 
 This is the vanilla version with Elastic Search and Single-Sign-On enabled.
-No other user accounts are allowed to join.
+No other user accounts are allowed to join - you must use the Keycloak
+server to create accounts and login.
diff --git a/mastodon/docker-compose.yaml b/mastodon/docker-compose.yaml
deleted file mode 100644
index 0861f73..0000000
--- a/mastodon/docker-compose.yaml
+++ /dev/null
@@ -1,131 +0,0 @@
-version: '3'
-services:
-  database:
-    image: postgres:13.4-alpine
-    restart: always
-    #shm_size: 256mb
-    networks:
-      - internal_network
-    healthcheck:
-      test: ['CMD', 'pg_isready', '-U', "mastodon", "-d", "mastodon_production"]
-    volumes:
-      - ../data/mastodon/database:/var/lib/postgresql/data
-    environment:
-      - POSTGRES_USER=mastodon
-      - POSTGRES_PASSWORD=mastodon
-        #- POSTGRES_DB=mastodon_production
-
-  redis:
-    image: redis:6-alpine
-    restart: always
-    networks:
-      - internal_network
-    healthcheck:
-      test: ['CMD', 'redis-cli', 'ping']
-    volumes:
-      - ../data/mastodon/redis:/data
-
-  es:
-    image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2
-    restart: always
-    environment:
-      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-      - "cluster.name=es-mastodon"
-      - "discovery.type=single-node"
-      - "bootstrap.memory_lock=true"
-    networks:
-       - internal_network
-    healthcheck:
-       test: ["CMD-SHELL", "curl --silent --fail localhost:9200/_cluster/health || exit 1"]
-    volumes:
-       - ../data/mastodon/elasticsearch:/usr/share/elasticsearch/data
-    # fixup the permissions on the data directory since they are created as root on host
-    entrypoint: /bin/sh -c "chown -R elasticsearch:elasticsearch data && /usr/local/bin/docker-entrypoint.sh eswrapper"
-    ulimits:
-      memlock:
-        soft: -1
-        hard: -1
-
-  mastodon:
-    image: tootsuite/mastodon
-    restart: always
-    env_file:
-      - ../env.production
-      - env.production
-      - ../data/mastodon/env.secrets
-    command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 6001"
-    networks:
-      - external_network
-      - internal_network
-    healthcheck:
-      # prettier-ignore
-      test: ['CMD-SHELL', 'wget -q --spider --proxy=off localhost:6001/health || exit 1']
-    ports:
-      - '6001:6001'
-    depends_on:
-      - database
-      - redis
-      - es
-    volumes:
-      - ../data/mastodon/system:/mastodon/public/system
-
-  streaming:
-    image: tootsuite/mastodon
-    restart: always
-    env_file:
-      - ../env.production
-      - env.production
-      - ../data/mastodon/env.secrets
-    command: node ./streaming
-    networks:
-      - external_network
-      - internal_network
-    healthcheck:
-      # prettier-ignore
-      test: ['CMD-SHELL', 'wget -q --spider --proxy=off localhost:4000/api/v1/streaming/health || exit 1']
-    ports:
-      - '4000:4000'
-    depends_on:
-      - database
-      - redis
-
-  sidekiq:
-    image: tootsuite/mastodon
-    restart: always
-    env_file:
-      - ../env.production
-      - env.production
-      - ../data/mastodon/env.secrets
-    command: bundle exec sidekiq
-    depends_on:
-      - database
-      - redis
-    networks:
-      - external_network
-      - internal_network
-    volumes:
-      - ../data/mastodon/system:/mastodon/public/system
-    healthcheck:
-      test: ['CMD-SHELL', "ps aux | grep '[s]idekiq\ 6' || false"]
-
-  ## Uncomment to enable federation with tor instances along with adding the following ENV variables
-  ## http_proxy=http://privoxy:8118
-  ## ALLOW_ACCESS_TO_HIDDEN_SERVICE=true
-  # tor:
-  #   image: sirboops/tor
-  #   networks:
-  #      - external_network
-  #      - internal_network
-  #
-  # privoxy:
-  #   image: sirboops/privoxy
-  #   volumes:
-  #     - ./priv-config:/opt/config
-  #   networks:
-  #     - external_network
-  #     - internal_network
-
-networks:
-  external_network:
-  internal_network:
-    internal: true
diff --git a/mastodon/entrypoint.sh b/mastodon/entrypoint.sh
new file mode 100755
index 0000000..4b0473a
--- /dev/null
+++ b/mastodon/entrypoint.sh
@@ -0,0 +1,38 @@
+#!/bin/bash -x
+
+id
+export
+pwd
+rm -f /mastodon/tmp/pids/server.pid
+
+export MASTODON_DIR=/mastodon/public/system
+export VAPID_KEY="$MASTODON_DIR/vapid_key"
+export DB_SETUP="$MASTODON_DIR/db_done"
+
+
+which rails
+chown -R mastodon:mastodon "$MASTODON_DIR"
+
+#exec su mastodon /bin/bash - <<EOF
+exec su mastodon <<EOF
+
+export PATH="$PATH:/opt/ruby/bin:/opt/node/bin:/opt/mastodon/bin"
+
+if [ ! -r "$VAPID_KEY" ]; then
+	rails mastodon:webpush:generate_vapid_key > "$VAPID_KEY" \
+	|| exit 1
+fi
+
+. "$VAPID_KEY"
+
+if [ ! -r "$DB_SETUP" ]; then
+	rails db:setup \
+	|| exit 1
+
+	touch "$DB_SETUP"
+fi
+
+exec bundle exec rails s -p 6001
+EOF
+
+
diff --git a/mastodon/env.production b/mastodon/env.production
index e99b24b..120834b 100644
--- a/mastodon/env.production
+++ b/mastodon/env.production
@@ -19,12 +19,12 @@
 
 # Redis
 # -----
-REDIS_HOST=redis
+REDIS_HOST=mastodon-redis
 REDIS_PORT=6379
 
 # PostgreSQL
 # ----------
-DB_HOST=database
+DB_HOST=mastodon-db
 DB_USER=mastodon
 DB_NAME=mastodon_production
 DB_PASS=mastodon
@@ -33,7 +33,7 @@ DB_PORT=5432
 # Elasticsearch (optional)
 # ------------------------
 ES_ENABLED=true
-ES_HOST=es
+ES_HOST=mastodon-es
 ES_PORT=9200
 # Authentication for ES (optional)
 ES_USER=elastic
@@ -67,6 +67,9 @@ ES_PASS=password
 #AWS_SECRET_ACCESS_KEY=
 #S3_ALIAS_HOST=files.example.com
 
+# Do not use sendfile since this is fronted by nginx
+RAILS_SERVE_STATIC_FILES=false
+
 # do not allow normal logins
 OMNIAUTH_ONLY=true
 
diff --git a/mastodon/keycloak.sh b/mastodon/keycloak.sh
new file mode 100755
index 0000000..389eb61
--- /dev/null
+++ b/mastodon/keycloak.sh
@@ -0,0 +1,3 @@
+#!/bin/bash -x
+
+client-create mastodon "$MASTODON_HOSTNAME.$DOMAIN_NAME" "$MASTODON_CLIENT_SECRET" </dev/null
diff --git a/mastodon/nginx.conf b/mastodon/nginx.conf
new file mode 100644
index 0000000..22f9489
--- /dev/null
+++ b/mastodon/nginx.conf
@@ -0,0 +1,154 @@
+map $http_upgrade $connection_upgrade {
+  default upgrade;
+  ''      close;
+}
+
+upstream mastodon-backend {
+    server mastodon:6001 fail_timeout=0;
+}
+
+upstream mastodon-streaming {
+    server mastodon-streaming:4000 fail_timeout=0;
+}
+
+proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=CACHE:10m inactive=7d max_size=1g;
+
+server {
+  listen 443 ssl http2;
+  server_name ${MASTODON_HOSTNAME} ${MASTODON_HOSTNAME}.${DOMAIN_NAME};
+
+  ssl_protocols TLSv1.2 TLSv1.3;
+  ssl_ciphers HIGH:!MEDIUM:!LOW:!aNULL:!NULL:!SHA;
+  ssl_prefer_server_ciphers on;
+  ssl_session_cache shared:SSL:10m;
+  ssl_session_tickets off;
+  include /etc/nginx/includes/challenge.conf;
+
+  # Uncomment these lines once you acquire a certificate:
+  # ssl_certificate     /etc/letsencrypt/live/example.com/fullchain.pem;
+  # ssl_certificate_key /etc/letsencrypt/live/example.com/privkey.pem;
+
+  ssl_certificate /etc/letsencrypt/live/${DOMAIN_NAME}/fullchain.pem;
+  ssl_certificate_key /etc/letsencrypt/live/${DOMAIN_NAME}/privkey.pem;
+
+  keepalive_timeout    70;
+  sendfile             on;
+  client_max_body_size 80m;
+
+  root /mastodon;
+
+  gzip on;
+  gzip_disable "msie6";
+  gzip_vary on;
+  gzip_proxied any;
+  gzip_comp_level 6;
+  gzip_buffers 16 8k;
+  gzip_http_version 1.1;
+  gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript image/svg+xml image/x-icon;
+
+  location / {
+    try_files $uri @proxy;
+  }
+
+  # If Docker is used for deployment and Rails serves static files,
+  # then needed must replace line `try_files $uri =404;` with `try_files $uri @proxy;`.
+  location = /sw.js {
+    add_header Cache-Control "public, max-age=604800, must-revalidate";
+    add_header Strict-Transport-Security "max-age=63072000; includeSubDomains";
+    #try_files $uri =404;
+    try_files $uri @proxy;
+  }
+
+  location ~ ^/assets/ {
+    add_header Cache-Control "public, max-age=2419200, must-revalidate";
+    add_header Strict-Transport-Security "max-age=63072000; includeSubDomains";
+    try_files $uri @proxy;
+  }
+
+  location ~ ^/avatars/ {
+    add_header Cache-Control "public, max-age=2419200, must-revalidate";
+    add_header Strict-Transport-Security "max-age=63072000; includeSubDomains";
+    try_files $uri @proxy;
+  }
+
+  location ~ ^/emoji/ {
+    add_header Cache-Control "public, max-age=2419200, must-revalidate";
+    add_header Strict-Transport-Security "max-age=63072000; includeSubDomains";
+    try_files $uri @proxy;
+  }
+
+  location ~ ^/headers/ {
+    add_header Cache-Control "public, max-age=2419200, must-revalidate";
+    add_header Strict-Transport-Security "max-age=63072000; includeSubDomains";
+    try_files $uri @proxy;
+  }
+
+  location ~ ^/packs/ {
+    add_header Cache-Control "public, max-age=2419200, must-revalidate";
+    add_header Strict-Transport-Security "max-age=63072000; includeSubDomains";
+    try_files $uri @proxy;
+  }
+
+  location ~ ^/shortcuts/ {
+    add_header Cache-Control "public, max-age=2419200, must-revalidate";
+    add_header Strict-Transport-Security "max-age=63072000; includeSubDomains";
+    try_files $uri @proxy;
+  }
+
+  location ~ ^/sounds/ {
+    add_header Cache-Control "public, max-age=2419200, must-revalidate";
+    add_header Strict-Transport-Security "max-age=63072000; includeSubDomains";
+    try_files $uri @proxy;
+  }
+
+  location ~ ^/system/ {
+    add_header Cache-Control "public, max-age=2419200, immutable";
+    add_header Strict-Transport-Security "max-age=63072000; includeSubDomains";
+    try_files $uri @proxy;
+  }
+
+  location ^~ /api/v1/streaming {
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header Proxy "";
+
+    proxy_pass http://mastodon-streaming;
+    proxy_buffering off;
+    proxy_redirect off;
+    proxy_http_version 1.1;
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection $connection_upgrade;
+
+    add_header Strict-Transport-Security "max-age=63072000; includeSubDomains";
+
+    tcp_nodelay on;
+  }
+
+  location @proxy {
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header Proxy "";
+    proxy_pass_header Server;
+
+    proxy_pass http://mastodon-backend;
+    proxy_buffering on;
+    proxy_redirect off;
+    proxy_http_version 1.1;
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection $connection_upgrade;
+
+    proxy_cache CACHE;
+    proxy_cache_valid 200 7d;
+    proxy_cache_valid 410 24h;
+    proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504;
+    add_header X-Cached $upstream_cache_status;
+
+    tcp_nodelay on;
+  }
+
+  error_page 404 500 501 502 503 504 /500.html;
+}
diff --git a/mastodon/setup b/mastodon/setup
deleted file mode 100755
index 9c535a7..0000000
--- a/mastodon/setup
+++ /dev/null
@@ -1,78 +0,0 @@
-#!/bin/bash
-die() { echo >&2 "ERROR: $@" ; exit 1 ; }
-info() { echo >&2 "$@" ; }
-
-DIRNAME="$(dirname $0)"
-cd "$DIRNAME"
-source ../env.production
-source ./env.production
-source "../env.smtp" 2>/dev/null
-
-mkdir -p ../data/mastodon/system
-chmod 777 ../data/mastodon/system
-
-SECRETS="../data/mastodon/env.secrets"
-
-if [ -r "$SECRETS" ]; then
-	docker-compose up -d || die "unable to restart mastodon"
-	exit 0
-fi
-
-# have to bring it all down before we touch the files
-docker-compose down
-
-OIDC_CLIENT_SECRET="$(openssl rand -hex 32)"
-
-# create the secrets file,
-# along with some parameters that should be in the environment
-mkdir -p "$(dirname "$SECRETS")"
-cat <<EOF > "$SECRETS"
-# DO NOT CHECK IN
-WEB_DOMAIN=$MASTODON_HOSTNAME
-LOCAL_DOMAIN=$DOMAIN_NAME
-OIDC_DISPLAY_NAME=$REALM
-OIDC_ISSUER=https://$KEYCLOAK_HOSTNAME/realms/$REALM
-OIDC_REDIRECT_URI=https://$MASTODON_HOSTNAME/auth/auth/openid_connect/callback
-OIDC_CLIENT_SECRET=$OIDC_CLIENT_SECRET
-SECRET_KEY_BASE=$(openssl rand -hex 32)
-OTP_SECRET=$(openssl rand -hex 32)
-EOF
-
-if [ -n "$SMTP_SERVER" ]; then
-	cat <<EOF >> "$SECRETS"
-SMTP_SERVER=$SMTP_SERVER
-SMTP_PORT=$SMTP_PORT
-SMTP_LOGIN=$SMTP_USER
-SMTP_PASSWORD=$SMTP_PASSWORD
-SMTP_FROM_ADDRESS=mastodon@$DOMAIN_NAME
-EOF
-fi
-
-info "mastodon: creating push keys"
-docker-compose run --rm mastodon \
-	rails mastodon:webpush:generate_vapid_key \
-	>> "$SECRETS" \
-|| die "unable to generate vapid key"
-
-info "mastodon: setting up database"
-docker-compose run --rm mastodon \
-	rails db:setup \
-|| die "unable to login"
-
-source "$SECRETS"
-
-info "mastodon: creating keycloak interface"
-../keycloak/client-delete mastodon
-../keycloak/client-create <<EOF || die "Unable to create keycloak client"
-{
-	"clientId": "mastodon",
-	"rootUrl": "https://$MASTODON_HOSTNAME/",
-	"adminUrl": "https://$MASTODON_HOSTNAME/",
-	"redirectUris": [ "https://$MASTODON_HOSTNAME/*" ],
-	"webOrigins": [ "https://$MASTODON_HOSTNAME" ],
-	"clientAuthenticatorType": "client-secret",
-	"secret": "$OIDC_CLIENT_SECRET"
-}
-EOF
-
-docker-compose up -d || die "mastodon: unable to start container"
diff --git a/nginx.yaml b/nginx.yaml
new file mode 100644
index 0000000..6fe1c76
--- /dev/null
+++ b/nginx.yaml
@@ -0,0 +1,30 @@
+version: '3.9'
+services:
+  nginx:
+#    image: nginx:1.21-alpine
+    build:
+      context: nginx
+      dockerfile: Dockerfile
+    restart: always
+    #entrypoint: /bin/sh
+    container_name: nginx
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - ./nginx/etc/includes:/etc/nginx/includes:ro
+      - ./nginx/etc/nginx.conf:/etc/nginx/nginx.conf:ro
+      - ./nginx/default.conf:/etc/nginx/templates/default.conf.template:ro
+      - ./html:/var/www/html:ro
+      - ./data/nginx/certbot/www:/var/www/certbot:ro
+      - ./data/nginx/certbot/conf:/etc/letsencrypt:rw
+      - /home:/home:ro
+    env_file:
+      - env.production
+
+  certbot:
+    image: certbot/certbot
+    container_name: certbot
+    volumes:
+      - ./data/nginx/certbot/conf:/etc/letsencrypt
+      - ./data/nginx/certbot/www:/var/www/certbot
diff --git a/nginx/Dockerfile b/nginx/Dockerfile
new file mode 100644
index 0000000..76007c8
--- /dev/null
+++ b/nginx/Dockerfile
@@ -0,0 +1,30 @@
+FROM alpine
+RUN apk update
+RUN echo "building" \
+    && apk add \
+	nginx \
+	collectd \
+	collectd-nginx \
+	nginx-mod-http-vts \
+	gettext \
+	curl \
+	openssl \
+    && mkdir -p \
+	/etc/nginx/modules-enabled \
+	/etc/nginx/conf.d \
+	/docker-entrypoint.d \
+    && ln -sf /etc/nginx/modules/10_http_vts.conf /etc/nginx/modules-enabled \
+# forward request and error logs to docker log collector
+    && ln -sf /dev/stdout /var/log/nginx/access.log \
+    && ln -sf /dev/stderr /var/log/nginx/error.log \
+    && echo "Done"
+
+COPY ["docker-entrypoint.d/*", "/docker-entrypoint.d/" ]
+COPY ["docker-entrypoint.sh", "/" ]
+
+ENTRYPOINT ["/docker-entrypoint.sh"]
+EXPOSE 80
+STOPSIGNAL SIGQUIT
+CMD ["nginx", "-g", "daemon off;"]
+
+
diff --git a/nginx/certbot-renew b/nginx/certbot-renew
deleted file mode 100755
index 45df1a2..0000000
--- a/nginx/certbot-renew
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-die() { echo >&2 "$@" ; exit 1 ; }
-
-DIRNAME="$(dirname $0)"
-cd "$DIRNAME"
-
-source ../env.production
-source ./env.production
-
-domain_args="-d $DOMAIN_NAME,$KEYCLOAK_HOSTNAME,$HEDGEDOC_HOSTNAME,$MASTODON_HOSTNAME,$NEXTCLOUD_HOSTNAME,$GRAFANA_HOSTNAME,$MATRIX_HOSTNAME,$GITEA_HOSTNAME,$MOBILIZON_HOSTNAME,$PIXELFED_HOSTNAME"
-rsa_key_size=2048
-
-set -x
-
-# move the temp live directory away if
-# this is the first time we've run anything here
-if [ ! -d "../data/certbot/conf/accounts" ]; then
-	echo "deleting temp keys"
-	rm -rf ../data/certbot/conf/live
-fi
-
-docker-compose run --rm certbot \
-  certonly \
-    --webroot \
-    --webroot-path /var/www/certbot \
-    --email "admin@$DOMAIN_NAME" \
-    --rsa-key-size "$rsa_key_size" \
-    --agree-tos \
-    --no-eff-email \
-    --force-renewal \
-    $domain_args \
-|| die "unable to renew!"
-
-docker-compose exec nginx nginx -s reload
diff --git a/nginx/nginx/templates/000-default.conf.template b/nginx/default.conf
similarity index 70%
rename from nginx/nginx/templates/000-default.conf.template
rename to nginx/default.conf
index 293d02c..da6c466 100644
--- a/nginx/nginx/templates/000-default.conf.template
+++ b/nginx/default.conf
@@ -1,6 +1,30 @@
-# Redirect *all* port 80 traffic to the same thing on port 443
+vhost_traffic_status_zone;
+
 server {
 	listen 80 default_server;
+
+	# this works on the docker container with http_stub built in
+	# only allow from localhost
+	location /nginx_status {
+		stub_status on;
+		access_log off;
+		allow 127.0.0.1;
+		deny all;
+	}
+
+	# this works with the vts module
+        location /status {
+		vhost_traffic_status_display;
+		vhost_traffic_status_display_format html;
+		access_log off;
+		#allow 127.0.0.1;
+		#deny all;
+	}
+
+	# forward certbot challenges to the certbot directory
+	include /etc/nginx/includes/challenge.conf;
+
+	# Redirect *all other* port 80 traffic to the same thing on port 443
 	location / {
 		return 301 https://$host$request_uri;
 	}
@@ -27,13 +51,16 @@ server {
 	chunked_transfer_encoding on;
 
 	# delegated Matrix server
-	location /.well-known/matrix {
-		proxy_pass https://${MATRIX_HOSTNAME};
-	}
+#	location /.well-known/matrix {
+#		proxy_pass https://${MATRIX_HOSTNAME}.${DOMAIN_NAME};
+#	}
 
 	# separate Mastodon WEB_DOMAIN and LOCAL_DOMAIN
 	location = /.well-known/host-meta {
-		return 302 https://${MASTODON_HOSTNAME}$request_uri;
+		return 302 https://${MASTODON_HOSTNAME}.${DOMAIN_NAME}$request_uri;
+	}
+	location = /.well-known/webfinger {
+		return 302 https://${MASTODON_HOSTNAME}.${DOMAIN_NAME}$request_uri;
 	}
 
 	# tilde club home directories
@@ -66,7 +93,7 @@ server {
 
 		proxy_hide_header Content-Security-Policy;
 		add_header Content-Security-Policy "script-src 'self' 'unsafe-inline' 'unsafe-eval' *.${DOMAIN_NAME}; frame-src 'self' *.${DOMAIN_NAME}; object-src 'self'; base-uri 'self' *.${DOMAIN_NAME}";
-		proxy_pass http://host.docker.internal:3000/s$request_uri;
+		proxy_pass http://hedgedoc:3000/s$request_uri;
 		proxy_cache_valid any 1m;
 
 	}
@@ -77,7 +104,7 @@ server {
 		proxy_ignore_headers Cache-Control;
 		proxy_cache_valid any 1m;
 
-		proxy_pass http://host.docker.internal:3000$request_uri;
+		proxy_pass http://hedgedoc:3000$request_uri;
 	}
 
 	listen 443 ssl default_server;
diff --git a/nginx/docker-compose.yaml b/nginx/docker-compose.yaml
deleted file mode 100644
index 11e78f8..0000000
--- a/nginx/docker-compose.yaml
+++ /dev/null
@@ -1,28 +0,0 @@
-version: '3'
-services:
-  nginx:
-    image: nginx:1.21-alpine
-    restart: always
-    ports:
-      - "80:80"
-      - "443:443"
-    volumes:
-      - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
-      - ./nginx/templates:/etc/nginx/templates:ro
-      - ./nginx/includes:/etc/nginx/includes:ro
-      - ../html:/var/www/html:ro
-      - ../data/certbot/www:/var/www/certbot:ro
-      - ../data/certbot/conf:/etc/letsencrypt:ro
-      - ../data/nginx/cache:/data/nginx/cache:rw
-      - /home:/home:ro
-    env_file:
-      -  ../env.production
-      -  env.production
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-
-  certbot:
-    image: certbot/certbot
-    volumes:
-      - ../data/certbot/conf:/etc/letsencrypt
-      - ../data/certbot/www:/var/www/certbot
diff --git a/nginx/docker-entrypoint.d/01-collectd.sh b/nginx/docker-entrypoint.d/01-collectd.sh
new file mode 100755
index 0000000..7750999
--- /dev/null
+++ b/nginx/docker-entrypoint.d/01-collectd.sh
@@ -0,0 +1,12 @@
+#!/bin/sh -x
+touch /started
+
+#cat >> /etc/collectd/collectd.conf <<EOF
+cat /etc/collectd/collectd.conf - > /tmp/conf <<EOF
+LoadPlugin nginx
+<Plugin "nginx">
+    URL "http://localhost:80/nginx_status"
+</Plugin>
+EOF
+
+#collectd
diff --git a/nginx/docker-entrypoint.d/10-createkey.sh b/nginx/docker-entrypoint.d/10-createkey.sh
new file mode 100755
index 0000000..eb718ef
--- /dev/null
+++ b/nginx/docker-entrypoint.d/10-createkey.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+
+mkdir -p /data/nginx/cache
+
+if [ -z "$DOMAIN_NAME" ]; then
+	DOMAIN_NAME="example.com"
+fi
+
+certdir="/etc/letsencrypt/live/${DOMAIN_NAME}"
+
+if [ -r "$certdir/fullchain.pem" ]; then
+	exit 0
+fi
+
+mkdir -p "$certdir"
+
+echo >&2 "$certdir: Creating temporary keys"
+openssl req \
+	-x509 \
+	-newkey rsa:2048 \
+	-keyout "$certdir/privkey.pem" \
+	-out "$certdir/fullchain.pem" \
+	-sha256 \
+	-nodes \
+	-days 365 \
+	-subj "/CN=$DOMAIN_NAME'" \
+|| exit 1
+
+echo >&2 "$certdir: Generated temporary keys -- certbot needs to request real ones"
+exit 0
+
diff --git a/nginx/docker-entrypoint.d/20-envsubst-on-templates.sh b/nginx/docker-entrypoint.d/20-envsubst-on-templates.sh
new file mode 100755
index 0000000..d0398b1
--- /dev/null
+++ b/nginx/docker-entrypoint.d/20-envsubst-on-templates.sh
@@ -0,0 +1,39 @@
+#!/bin/sh
+
+set -e
+
+ME=$(basename $0)
+
+entrypoint_log() {
+    if [ -z "${NGINX_ENTRYPOINT_QUIET_LOGS:-}" ]; then
+        echo "$@"
+    fi
+}
+
+auto_envsubst() {
+  local template_dir="${NGINX_ENVSUBST_TEMPLATE_DIR:-/etc/nginx/templates}"
+  local suffix="${NGINX_ENVSUBST_TEMPLATE_SUFFIX:-.template}"
+  local output_dir="${NGINX_ENVSUBST_OUTPUT_DIR:-/etc/nginx/conf.d}"
+  local filter="${NGINX_ENVSUBST_FILTER:-}"
+
+  local template defined_envs relative_path output_path subdir
+  defined_envs=$(printf '${%s} ' $(awk "END { for (name in ENVIRON) { print ( name ~ /${filter}/ ) ? name : \"\" } }" < /dev/null ))
+  [ -d "$template_dir" ] || return 0
+  if [ ! -w "$output_dir" ]; then
+    entrypoint_log "$ME: ERROR: $template_dir exists, but $output_dir is not writable"
+    return 0
+  fi
+  find "$template_dir" -follow -type f -name "*$suffix" -print | while read -r template; do
+    relative_path="${template#$template_dir/}"
+    output_path="$output_dir/${relative_path%$suffix}"
+    subdir=$(dirname "$relative_path")
+    # create a subdirectory where the template file exists
+    mkdir -p "$output_dir/$subdir"
+    entrypoint_log "$ME: Running envsubst on $template to $output_path"
+    envsubst "$defined_envs" < "$template" > "$output_path"
+  done
+}
+
+auto_envsubst
+
+exit 0
diff --git a/nginx/docker-entrypoint.sh b/nginx/docker-entrypoint.sh
new file mode 100755
index 0000000..e201fe6
--- /dev/null
+++ b/nginx/docker-entrypoint.sh
@@ -0,0 +1,47 @@
+#!/bin/sh
+# vim:sw=4:ts=4:et
+
+set -e
+
+entrypoint_log() {
+    if [ -z "${NGINX_ENTRYPOINT_QUIET_LOGS:-}" ]; then
+        echo "$@"
+    fi
+}
+
+if [ "$1" = "nginx" -o "$1" = "nginx-debug" ]; then
+    if /usr/bin/find "/docker-entrypoint.d/" -mindepth 1 -maxdepth 1 -type f -print -quit 2>/dev/null | read v; then
+        entrypoint_log "$0: /docker-entrypoint.d/ is not empty, will attempt to perform configuration"
+
+        entrypoint_log "$0: Looking for shell scripts in /docker-entrypoint.d/"
+        find "/docker-entrypoint.d/" -follow -type f -print | sort -V | while read -r f; do
+            case "$f" in
+                *.envsh)
+                    if [ -x "$f" ]; then
+                        entrypoint_log "$0: Sourcing $f";
+                        . "$f"
+                    else
+                        # warn on shell scripts without exec bit
+                        entrypoint_log "$0: Ignoring $f, not executable";
+                    fi
+                    ;;
+                *.sh)
+                    if [ -x "$f" ]; then
+                        entrypoint_log "$0: Launching $f";
+                        "$f"
+                    else
+                        # warn on shell scripts without exec bit
+                        entrypoint_log "$0: Ignoring $f, not executable";
+                    fi
+                    ;;
+                *) entrypoint_log "$0: Ignoring $f";;
+            esac
+        done
+
+        entrypoint_log "$0: Configuration complete; ready for start up"
+    else
+        entrypoint_log "$0: No files found in /docker-entrypoint.d/, skipping configuration"
+    fi
+fi
+
+exec "$@"
diff --git a/nginx/env.production b/nginx/env.production
deleted file mode 100644
index e69de29..0000000
diff --git a/nginx/nginx/includes/challenge.conf b/nginx/etc/includes/challenge.conf
similarity index 100%
rename from nginx/nginx/includes/challenge.conf
rename to nginx/etc/includes/challenge.conf
diff --git a/nginx/nginx/includes/options-ssl-nginx.conf b/nginx/etc/includes/options-ssl-nginx.conf
similarity index 100%
rename from nginx/nginx/includes/options-ssl-nginx.conf
rename to nginx/etc/includes/options-ssl-nginx.conf
diff --git a/nginx/nginx/includes/ssl-dhparams.pem b/nginx/etc/includes/ssl-dhparams.pem
similarity index 100%
rename from nginx/nginx/includes/ssl-dhparams.pem
rename to nginx/etc/includes/ssl-dhparams.pem
diff --git a/nginx/nginx/nginx.conf b/nginx/etc/nginx.conf
similarity index 75%
rename from nginx/nginx/nginx.conf
rename to nginx/etc/nginx.conf
index 80bcd49..2a339a7 100644
--- a/nginx/nginx/nginx.conf
+++ b/nginx/etc/nginx.conf
@@ -70,26 +70,9 @@ http {
 	include /etc/nginx/conf.d/*.conf;
 	include /etc/nginx/sites-enabled/*;
 	include /tmp/sites-enabled/*;
-}
 
+log_format main 'XXXX $http_x_forwarded_for - $remote_user [$time_local] "$host" "$request" '
+            '$status $body_bytes_sent "$http_referer" '
+            '"$http_user_agent" $request_time';
 
-#mail {
-#	# See sample authentication script at:
-#	# http://wiki.nginx.org/ImapAuthenticateWithApachePhpScript
-# 
-#	# auth_http localhost/auth.php;
-#	# pop3_capabilities "TOP" "USER";
-#	# imap_capabilities "IMAP4rev1" "UIDPLUS";
-# 
-#	server {
-#		listen     localhost:110;
-#		protocol   pop3;
-#		proxy      on;
-#	}
-# 
-#	server {
-#		listen     localhost:143;
-#		protocol   imap;
-#		proxy      on;
-#	}
-#}
+}
diff --git a/nginx/nginx/templates/pixelfed.conf.template b/nginx/nginx/templates/pixelfed.conf.template
deleted file mode 100644
index d3dbb74..0000000
--- a/nginx/nginx/templates/pixelfed.conf.template
+++ /dev/null
@@ -1,30 +0,0 @@
-server {
-	server_name ${PIXELFED_HOSTNAME};
-	client_max_body_size 128m;
-
-	sendfile on;
-	tcp_nopush on;
-	tcp_nodelay on;
-	keepalive_timeout 65;
-	types_hash_max_size 2048;
-
-	gzip on;
-	gzip_disable "msie6";
-
-	proxy_read_timeout 1800s;
-
-	location / {
-		proxy_pass http://host.docker.internal:8090;
-		proxy_set_header Host $host;
-		proxy_set_header X-Real-IP $remote_addr;
-		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-		proxy_set_header X-Forwarded-Proto $scheme;
-	}
-
-	listen 443 ssl;
-	ssl_certificate /etc/letsencrypt/live/${DOMAIN_NAME}/fullchain.pem;
-	ssl_certificate_key /etc/letsencrypt/live/${DOMAIN_NAME}/privkey.pem;
-	include /etc/nginx/includes/options-ssl-nginx.conf;
-	include /etc/nginx/includes/challenge.conf;
-	ssl_dhparam /etc/nginx/includes/ssl-dhparams.pem;
-}
diff --git a/nginx/nginx/templates/social.conf.template b/nginx/nginx/templates/social.conf.template
deleted file mode 100644
index 9170395..0000000
--- a/nginx/nginx/templates/social.conf.template
+++ /dev/null
@@ -1,41 +0,0 @@
-map $http_upgrade $connection_upgrade {
-        default upgrade;
-        ''      close;
-}
-
-server {
-	server_name social.${DOMAIN_NAME};
-	client_max_body_size 128m;
-
-	location / {
-		proxy_pass http://host.docker.internal:6001;
-		proxy_set_header Host $host;
-		proxy_set_header X-Forwarded-For $remote_addr;
-		proxy_set_header X-Forwarded-Proto https;
-	}
-
-        location /api/v1/streaming {
-                proxy_pass http://host.docker.internal:4000;
-                proxy_set_header Host $host; 
-                proxy_set_header X-Real-IP $remote_addr; 
-                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 
-                proxy_set_header X-Forwarded-Proto $scheme;
-                proxy_set_header Upgrade $http_upgrade;
-                proxy_set_header Connection $connection_upgrade;
-
-		proxy_buffering off;
-		proxy_redirect off;
-		proxy_http_version 1.1;
-		tcp_nodelay on;
-        }
-
-
-	listen 443 ssl;
-	ssl_certificate /etc/letsencrypt/live/${DOMAIN_NAME}/fullchain.pem;
-	ssl_certificate_key /etc/letsencrypt/live/${DOMAIN_NAME}/privkey.pem;
-	include /etc/nginx/includes/options-ssl-nginx.conf;
-	include /etc/nginx/includes/challenge.conf;
-	ssl_dhparam /etc/nginx/includes/ssl-dhparams.pem;
-}
-
-
diff --git a/nginx/setup b/nginx/setup
deleted file mode 100755
index 6426f22..0000000
--- a/nginx/setup
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/bash
-die() { echo >&2 "$@" ; exit 1 ; }
-
-DIRNAME="$(dirname $0)"
-cd "$DIRNAME"
-
-source ../env.production || die "no top level env"
-source env.production || die "no local env"
-
-if [ -z "${DOMAIN_NAME}" ]; then
-	die "DOMAIN_NAME not set"
-fi
-
-certdir="../data/certbot/conf/live/${DOMAIN_NAME}"
-
-if [ -r "$certdir/privkey.pem" ]; then
-	docker-compose up -d || die "nginx: unable to start"
-	exit 0
-fi
-
-mkdir -p "$certdir" || die "$certdir: unable to make"
-
-openssl req \
-	-x509 \
-	-newkey rsa:2048 \
-	-keyout "$certdir/privkey.pem" \
-	-out "$certdir/fullchain.pem" \
-	-sha256 \
-	-nodes \
-	-days 365 \
-	-subj "/CN=${DOMAIN_NAME}'" \
-|| die "$certdir/privkey.pem: unable to create temp key"
-
-docker-compose up -d || die "unable to bring up nginx"
-
-echo "SLEEPING..."
-sleep 10
-
-./certbot-renew || die "unable to create certs"
diff --git a/prometheus.yaml b/prometheus.yaml
new file mode 100644
index 0000000..c0c95ee
--- /dev/null
+++ b/prometheus.yaml
@@ -0,0 +1,18 @@
+version: '3'
+services:
+  prometheus:
+    image: prom/prometheus
+    container_name: prometheus
+    volumes:
+      - ./data/prometheus/storage:/prometheus
+      - ./prometheus/prometheus.yaml:/etc/prometheus/prometheus.yml:ro
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:latest
+    container_name: cadvisor
+    volumes:
+    - /:/rootfs:ro
+    - /var/run:/var/run:rw
+    - /sys:/sys:ro
+    - /var/lib/docker/:/var/lib/docker:ro
+
diff --git a/prometheus/prometheus.yaml b/prometheus/prometheus.yaml
new file mode 100644
index 0000000..5bf3071
--- /dev/null
+++ b/prometheus/prometheus.yaml
@@ -0,0 +1,21 @@
+global:
+  scrape_interval: 15s 
+  external_labels:
+    monitor: 'codelab-monitor'
+ 
+scrape_configs:  
+  # nginx vts data
+  - job_name: 'nginx'
+    scrape_interval: 5s
+    metrics_path: "/status/format/prometheus"
+    static_configs:
+      - targets: ['nginx:80']
+  - job_name: 'metrics'
+    scrape_interval: 5s
+    static_configs:
+      # grafana data from /metrics
+      - targets: ['dashboard:3000']
+      # host running the docker-compose 
+      - targets: ['172.17.0.1:9100']
+      # cadvisor system
+      - targets: ['cadvisor:8080']
diff --git a/start-all b/start-all
deleted file mode 100755
index cbebca4..0000000
--- a/start-all
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/bash
-die() { echo >&2 "$@" ; exit 1 ; }
-
-which jq > /dev/null || die "jq not installed?"
-which docker-compose > /dev/null || die "docker-compose not installed?"
-
-source ./env.production || die "no production env?"
-
-if [ -z "$DOMAIN_NAME" ]; then
-	die "\$DOMAIN_NAME not set; things will break"
-fi
-
-SERVICES=nginx # there is no host
-SERVICES+=\ keycloak
-SERVICES+=\ hedgedoc
-SERVICES+=\ nextcloud
-SERVICES+=\ mastodon
-SERVICES+=\ grafana
-SERVICES+=\ matrix
-SERVICES+=\ gitea
-SERVICES+=\ mobilizon
-
-HOSTS+=\ $KEYCLOAK_HOST
-HOSTS+=\ $HEDGEDOC_HOST
-HOSTS+=\ $NEXTCLOUD_HOST
-HOSTS+=\ $MASTODON_HOST
-HOSTS+=\ $GRAFANA_HOST
-HOSTS+=\ $MATRIX_HOST
-HOSTS+=\ $GITEA_HOST
-HOSTS+=\ $MOBILIZON_HOST
-
-for host in $HOSTS ; do
-	host $host > /dev/null || die "$host: DNS entry not present?"
-done
-
-for service in $SERVICES ; do
-	echo "$service: starting"
-	./$service/setup || die "$server: failed to start"
-done
diff --git a/stop-all b/stop-all
deleted file mode 100755
index 5b8e97b..0000000
--- a/stop-all
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-for file in */docker-compose.yaml ; do
-	dir="$(dirname "$file")"
-	echo "$dir"
-	( cd "$dir" ; docker-compose down )
-done

From a2b5511966de9e784ca993a333cd25a75835014c Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@hackerspace-zone.labs.lowerlayer.nl>
Date: Fri, 11 Nov 2022 13:22:04 +0000
Subject: [PATCH 02/13] prometheus: fix startup to create writable persistent
 volume

---
 prometheus.yaml          |  7 ++++++-
 prometheus/entrypoint.sh | 11 +++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)
 create mode 100755 prometheus/entrypoint.sh

diff --git a/prometheus.yaml b/prometheus.yaml
index c0c95ee..6ce8dce 100644
--- a/prometheus.yaml
+++ b/prometheus.yaml
@@ -2,13 +2,18 @@ version: '3'
 services:
   prometheus:
     image: prom/prometheus
+    restart: always
     container_name: prometheus
+    user: root
     volumes:
-      - ./data/prometheus/storage:/prometheus
+      - ./data/prometheus/storage:/prometheus:rw
       - ./prometheus/prometheus.yaml:/etc/prometheus/prometheus.yml:ro
+      - ./prometheus/entrypoint.sh:/entrypoint.sh:ro
+    entrypoint: ["/entrypoint.sh"]
 
   cadvisor:
     image: gcr.io/cadvisor/cadvisor:latest
+    restart: always
     container_name: cadvisor
     volumes:
     - /:/rootfs:ro
diff --git a/prometheus/entrypoint.sh b/prometheus/entrypoint.sh
new file mode 100755
index 0000000..fe60e34
--- /dev/null
+++ b/prometheus/entrypoint.sh
@@ -0,0 +1,11 @@
+#!/bin/sh -x
+
+chmod 777 /prometheus
+exec su -s /bin/sh nobody <<EOF
+exec /bin/prometheus \
+	--config.file=/etc/prometheus/prometheus.yml \
+	--web.console.libraries=/etc/prometheus/console_libraries \
+	--web.console.templates=/etc/prometheus/consoles
+EOF
+
+# --storage.local.path=/prometheus \

From 03086c7a06908e359a100282eaf984a4bfbc4b3e Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@hackerspace-zone.labs.lowerlayer.nl>
Date: Fri, 11 Nov 2022 14:10:46 +0000
Subject: [PATCH 03/13] grafana: provision prometheus and some sample
 dashboards out of the box

---
 grafana.yaml                                  |    4 +-
 grafana/dashboards/docker.json                | 1862 +++++++++++++++++
 grafana/dashboards/nginx.json                 | 1518 ++++++++++++++
 .../provisioning/dashboards/dashboards.yaml   |   15 +
 .../provisioning/datasources/prometheus.yaml  |   13 +
 5 files changed, 3411 insertions(+), 1 deletion(-)
 create mode 100644 grafana/dashboards/docker.json
 create mode 100644 grafana/dashboards/nginx.json
 create mode 100644 grafana/provisioning/dashboards/dashboards.yaml
 create mode 100644 grafana/provisioning/datasources/prometheus.yaml

diff --git a/grafana.yaml b/grafana.yaml
index 24cea0c..c3312a2 100644
--- a/grafana.yaml
+++ b/grafana.yaml
@@ -23,7 +23,9 @@ services:
     # reset the admin password on every run, since otherwise it defaults to admin/admin
     entrypoint: ["sh", "-c", "grafana-cli admin reset-admin-password ${GRAFANA_ADMIN_PASSWORD} && /run.sh"]
     volumes:
-      - ./data/grafana:/var/lib/grafana
+      - ./data/grafana/data:/var/lib/grafana
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/etc/grafana/dashboards:ro
     restart: always
     #    ports:
     #  - 3000:3000
diff --git a/grafana/dashboards/docker.json b/grafana/dashboards/docker.json
new file mode 100644
index 0000000..f0ccb26
--- /dev/null
+++ b/grafana/dashboards/docker.json
@@ -0,0 +1,1862 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "A simple overview of the most important Docker host and container metrics. (cAdvisor/Prometheus)",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "gnetId": 10619,
+  "graphTooltip": 1,
+  "id": 7,
+  "iteration": 1668173784265,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 4,
+        "x": 0,
+        "y": 0
+      },
+      "id": 22,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "8.5.1",
+      "targets": [
+        {
+          "bucketAggs": [
+            {
+              "field": "@timestamp",
+              "id": "2",
+              "settings": {
+                "interval": "auto",
+                "min_doc_count": 0,
+                "trimEdges": 0
+              },
+              "type": "date_histogram"
+            }
+          ],
+          "expr": "count(container_last_seen{instance=~\"$node:$port\",job=~\"$job\",image!=\"\"})",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "metrics": [
+            {
+              "field": "select field",
+              "id": "1",
+              "type": "count"
+            }
+          ],
+          "query": "count(container_last_seen{instance=~\"$node:$port\",job=~\"$job\",image!=\"\"})",
+          "refId": "A",
+          "timeField": "@timestamp"
+        }
+      ],
+      "title": "Running containers",
+      "transparent": true,
+      "type": "stat"
+    },
+    {
+      "aliasColors": {
+        "{id=\"/\",instance=\"cadvisor:8080\",job=\"prometheus\"}": "#BA43A9"
+      },
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "editable": true,
+      "error": false,
+      "fill": 3,
+      "fillGradient": 0,
+      "grid": {},
+      "gridPos": {
+        "h": 6,
+        "w": 8,
+        "x": 4,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 5,
+      "isNew": true,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": true,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(container_cpu_system_seconds_total[1m]))",
+          "hide": true,
+          "intervalFactor": 2,
+          "legendFormat": "a",
+          "refId": "B",
+          "step": 120
+        },
+        {
+          "expr": "sum(rate(container_cpu_system_seconds_total{name=~\".+\"}[1m]))",
+          "hide": true,
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "nur container",
+          "refId": "F",
+          "step": 10
+        },
+        {
+          "expr": "sum(rate(container_cpu_system_seconds_total{id=\"/\"}[1m]))",
+          "hide": true,
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "nur docker host",
+          "metric": "",
+          "refId": "A",
+          "step": 20
+        },
+        {
+          "expr": "sum(rate(process_cpu_seconds_total[$interval])) * 100",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "host",
+          "metric": "",
+          "refId": "C",
+          "step": 4
+        },
+        {
+          "expr": "sum(rate(container_cpu_system_seconds_total{name=~\".+\"}[1m])) + sum(rate(container_cpu_system_seconds_total{id=\"/\"}[1m])) + sum(rate(process_cpu_seconds_total[1m]))",
+          "hide": true,
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "D",
+          "step": 120
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "CPU Usage on Node",
+      "tooltip": {
+        "msResolution": true,
+        "shared": true,
+        "sort": 0,
+        "value_type": "cumulative"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": false,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "percent",
+          "label": "",
+          "logBase": 1,
+          "max": 120,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {
+        "Belegete Festplatte": "#BF1B00",
+        "Free Disk Space": "#7EB26D",
+        "Used Disk Space": "#BF1B00",
+        "{}": "#BF1B00"
+      },
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "editable": true,
+      "error": false,
+      "fill": 4,
+      "fillGradient": 0,
+      "grid": {},
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 12,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 13,
+      "isNew": true,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 3,
+      "links": [],
+      "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": true,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "5qpBRfD4k"
+          },
+          "expr": "node_filesystem_avail_bytes{device=\"/dev/vda1\"}",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{mountpoint}}",
+          "refId": "A",
+          "step": 4
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "5qpBRfD4k"
+          },
+          "expr": "node_filesystem_size_bytes{device=\"/dev/vda1\"} - node_filesystem_avail_bytes{device=\"/dev/vda1\"}",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "Used Disk Space",
+          "refId": "B",
+          "step": 4
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Free and Used Disk Space on Node",
+      "tooltip": {
+        "msResolution": true,
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": false,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "label": "",
+          "logBase": 1,
+          "min": 0,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {
+        "Available Memory": "#7EB26D",
+        "Unavailable Memory": "#BF1B00"
+      },
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "editable": true,
+      "error": false,
+      "fill": 4,
+      "fillGradient": 0,
+      "grid": {},
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 16,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 20,
+      "isNew": true,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 3,
+      "links": [],
+      "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": true,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "5qpBRfD4k"
+          },
+          "expr": "container_memory_rss{name=~\".+\"}",
+          "format": "time_series",
+          "hide": true,
+          "intervalFactor": 2,
+          "legendFormat": "{{__name__}}",
+          "refId": "D",
+          "step": 30
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "5qpBRfD4k"
+          },
+          "expr": "sum(container_memory_rss{name=~\".+\"})",
+          "format": "time_series",
+          "hide": true,
+          "intervalFactor": 2,
+          "legendFormat": "{{__name__}}",
+          "refId": "A",
+          "step": 20
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "5qpBRfD4k"
+          },
+          "expr": "container_memory_usage_bytes{name=~\".+\"}",
+          "format": "time_series",
+          "hide": true,
+          "intervalFactor": 2,
+          "legendFormat": "{{name}}",
+          "refId": "B",
+          "step": 20
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "5qpBRfD4k"
+          },
+          "expr": "container_memory_rss{id=\"/\"}",
+          "format": "time_series",
+          "hide": true,
+          "intervalFactor": 2,
+          "legendFormat": "{{__name__}}",
+          "refId": "C",
+          "step": 30
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "5qpBRfD4k"
+          },
+          "expr": "sum(container_memory_rss)",
+          "format": "time_series",
+          "hide": true,
+          "intervalFactor": 2,
+          "legendFormat": "{{__name__}}",
+          "refId": "E",
+          "step": 30
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "5qpBRfD4k"
+          },
+          "expr": "node_memory_Buffers",
+          "format": "time_series",
+          "hide": true,
+          "intervalFactor": 2,
+          "legendFormat": "node_memory_Dirty",
+          "refId": "N",
+          "step": 30
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "5qpBRfD4k"
+          },
+          "expr": "node_memory_MemFree",
+          "format": "time_series",
+          "hide": true,
+          "intervalFactor": 2,
+          "legendFormat": "{{__name__}}",
+          "refId": "F",
+          "step": 30
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "5qpBRfD4k"
+          },
+          "expr": "node_memory_MemAvailable",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "Available Memory",
+          "refId": "H",
+          "step": 4
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "5qpBRfD4k"
+          },
+          "expr": "node_memory_MemTotal - node_memory_MemAvailable",
+          "format": "time_series",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "Unavailable Memory",
+          "refId": "G",
+          "step": 4
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "5qpBRfD4k"
+          },
+          "expr": "node_memory_Inactive",
+          "format": "time_series",
+          "hide": true,
+          "intervalFactor": 2,
+          "legendFormat": "{{__name__}}",
+          "refId": "I",
+          "step": 30
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "5qpBRfD4k"
+          },
+          "expr": "node_memory_KernelStack",
+          "format": "time_series",
+          "hide": true,
+          "intervalFactor": 2,
+          "legendFormat": "{{__name__}}",
+          "refId": "J",
+          "step": 30
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "5qpBRfD4k"
+          },
+          "expr": "node_memory_Active",
+          "format": "time_series",
+          "hide": true,
+          "intervalFactor": 2,
+          "legendFormat": "{{__name__}}",
+          "refId": "K",
+          "step": 30
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "5qpBRfD4k"
+          },
+          "expr": "node_memory_MemTotal - (node_memory_Active + node_memory_MemFree + node_memory_Inactive)",
+          "format": "time_series",
+          "hide": true,
+          "intervalFactor": 2,
+          "legendFormat": "Unknown",
+          "refId": "L",
+          "step": 40
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "5qpBRfD4k"
+          },
+          "expr": "node_memory_MemFree + node_memory_Inactive ",
+          "format": "time_series",
+          "hide": true,
+          "intervalFactor": 2,
+          "legendFormat": "{{__name__}}",
+          "refId": "M",
+          "step": 30
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "5qpBRfD4k"
+          },
+          "expr": "container_memory_rss{name=~\".+\"}",
+          "format": "time_series",
+          "hide": true,
+          "intervalFactor": 2,
+          "legendFormat": "{{__name__}}",
+          "refId": "O",
+          "step": 30
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "5qpBRfD4k"
+          },
+          "expr": "node_memory_Inactive + node_memory_MemFree + node_memory_MemAvailable",
+          "format": "time_series",
+          "hide": true,
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "P",
+          "step": 40
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Available Memory on Node",
+      "tooltip": {
+        "msResolution": true,
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": false,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "label": "",
+          "logBase": 1,
+          "max": 4200000000,
+          "min": 0,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "editable": true,
+      "error": false,
+      "fill": 1,
+      "fillGradient": 0,
+      "grid": {},
+      "gridPos": {
+        "h": 4,
+        "w": 2,
+        "x": 20,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 3,
+      "isNew": true,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "5qpBRfD4k"
+          },
+          "expr": "sum(rate(node_disk_bytes_read[$interval])) by (device)",
+          "intervalFactor": 2,
+          "legendFormat": "OUT on /{{device}}",
+          "metric": "node_disk_bytes_read",
+          "refId": "A",
+          "step": 4
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "5qpBRfD4k"
+          },
+          "expr": "sum(rate(node_disk_bytes_written[$interval])) by (device)",
+          "intervalFactor": 2,
+          "legendFormat": "IN on /{{device}}",
+          "metric": "",
+          "refId": "B",
+          "step": 4
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "5qpBRfD4k"
+          },
+          "expr": "",
+          "intervalFactor": 2,
+          "refId": "C"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Disk I/O on Node",
+      "tooltip": {
+        "msResolution": true,
+        "shared": true,
+        "sort": 0,
+        "value_type": "cumulative"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": false,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "Bps",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {
+        "node_load15": "#CCA300"
+      },
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "editable": true,
+      "error": false,
+      "fill": 0,
+      "fillGradient": 0,
+      "grid": {},
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 4
+      },
+      "hiddenSeries": false,
+      "id": 4,
+      "isNew": true,
+      "legend": {
+        "avg": false,
+        "current": true,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "{__name__=~\"^node_load.*\"}",
+          "intervalFactor": 2,
+          "legendFormat": "{{__name__}}",
+          "metric": "node",
+          "refId": "A",
+          "step": 2
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "System Load on Node",
+      "tooltip": {
+        "msResolution": true,
+        "shared": true,
+        "sort": 0,
+        "value_type": "cumulative"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {
+        "SENT": "#BF1B00"
+      },
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "editable": true,
+      "error": false,
+      "fill": 5,
+      "fillGradient": 0,
+      "grid": {},
+      "gridPos": {
+        "h": 4,
+        "w": 12,
+        "x": 0,
+        "y": 6
+      },
+      "hiddenSeries": false,
+      "id": 19,
+      "isNew": true,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.1",
+      "pointradius": 1,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(container_network_receive_bytes_total{id=\"/\"}[$interval])) by (id)",
+          "intervalFactor": 2,
+          "legendFormat": "RECEIVED",
+          "refId": "A",
+          "step": 4
+        },
+        {
+          "expr": "- sum(rate(container_network_transmit_bytes_total{id=\"/\"}[$interval])) by (id)",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "SENT",
+          "refId": "B",
+          "step": 4
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Network Traffic on Node",
+      "tooltip": {
+        "msResolution": true,
+        "shared": true,
+        "sort": 0,
+        "value_type": "cumulative"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": false,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "editable": true,
+      "error": false,
+      "fill": 5,
+      "fillGradient": 0,
+      "grid": {},
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 10
+      },
+      "hiddenSeries": false,
+      "id": 1,
+      "isNew": true,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": true,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(container_cpu_usage_seconds_total{name=~\".+\"}[$interval])) by (name) * 100",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "{{name}}",
+          "metric": "container_cp",
+          "refId": "F",
+          "step": 2
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "CPU Usage per Container (Stacked)",
+      "tooltip": {
+        "msResolution": true,
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "percent",
+          "label": "",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "editable": true,
+      "error": false,
+      "fill": 3,
+      "fillGradient": 0,
+      "grid": {},
+      "gridPos": {
+        "h": 6,
+        "w": 12,
+        "x": 12,
+        "y": 12
+      },
+      "hiddenSeries": false,
+      "id": 10,
+      "isNew": true,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": true,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(container_memory_rss{name=~\".+\"}) by (name)",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{name}}",
+          "refId": "A",
+          "step": 2
+        },
+        {
+          "expr": "container_memory_usage_bytes{name=~\".+\"}",
+          "hide": true,
+          "intervalFactor": 2,
+          "legendFormat": "{{name}}",
+          "refId": "B",
+          "step": 240
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Memory Usage per Container (Stacked)",
+      "tooltip": {
+        "msResolution": true,
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "label": "",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "editable": true,
+      "error": false,
+      "fill": 1,
+      "fillGradient": 0,
+      "grid": {},
+      "gridPos": {
+        "h": 6,
+        "w": 12,
+        "x": 0,
+        "y": 18
+      },
+      "hiddenSeries": false,
+      "id": 9,
+      "isNew": true,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "hideEmpty": false,
+        "hideZero": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(container_network_transmit_bytes_total{name=~\".+\"}[$interval])) by (name)",
+          "intervalFactor": 2,
+          "legendFormat": "{{name}}",
+          "refId": "A",
+          "step": 2
+        },
+        {
+          "expr": "rate(container_network_transmit_bytes_total{id=\"/\"}[$interval])",
+          "hide": true,
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "B",
+          "step": 10
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Sent Network Traffic per Container",
+      "tooltip": {
+        "msResolution": true,
+        "shared": true,
+        "sort": 0,
+        "value_type": "cumulative"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "Bps",
+          "label": "",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 10,
+          "max": 8,
+          "min": 0,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "editable": true,
+      "error": false,
+      "fill": 3,
+      "fillGradient": 0,
+      "grid": {},
+      "gridPos": {
+        "h": 6,
+        "w": 12,
+        "x": 12,
+        "y": 18
+      },
+      "hiddenSeries": false,
+      "id": 11,
+      "isNew": true,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": true,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "container_memory_rss{name=~\".+\"}",
+          "hide": true,
+          "intervalFactor": 2,
+          "legendFormat": "{{name}}",
+          "refId": "A",
+          "step": 20
+        },
+        {
+          "expr": "container_memory_usage_bytes{name=~\".+\"}",
+          "hide": true,
+          "intervalFactor": 2,
+          "legendFormat": "{{name}}",
+          "refId": "B",
+          "step": 20
+        },
+        {
+          "expr": "sum(container_memory_cache{name=~\".+\"}) by (name)",
+          "hide": false,
+          "intervalFactor": 2,
+          "legendFormat": "{{name}}",
+          "refId": "C",
+          "step": 2
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Cached Memory per Container (Stacked)",
+      "tooltip": {
+        "msResolution": true,
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "editable": true,
+      "error": false,
+      "fill": 1,
+      "fillGradient": 0,
+      "grid": {},
+      "gridPos": {
+        "h": 6,
+        "w": 12,
+        "x": 0,
+        "y": 24
+      },
+      "hiddenSeries": false,
+      "id": 8,
+      "isNew": true,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "nullPointMode": "null as zero",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(container_network_receive_bytes_total{name=~\".+\"}[$interval])) by (name)",
+          "intervalFactor": 2,
+          "legendFormat": "{{name}}",
+          "refId": "A",
+          "step": 2
+        },
+        {
+          "expr": "- rate(container_network_transmit_bytes_total{name=~\".+\"}[$interval])",
+          "hide": true,
+          "intervalFactor": 2,
+          "legendFormat": "{{name}}",
+          "refId": "B",
+          "step": 10
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Received Network Traffic per Container",
+      "tooltip": {
+        "msResolution": true,
+        "shared": true,
+        "sort": 0,
+        "value_type": "cumulative"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "Bps",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "columns": [
+        {
+          "text": "Avg",
+          "value": "avg"
+        }
+      ],
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "editable": true,
+      "error": false,
+      "fontSize": "100%",
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 24
+      },
+      "hideTimeOverride": false,
+      "id": 18,
+      "isNew": true,
+      "links": [],
+      "pageSize": 100,
+      "scroll": true,
+      "showHeader": true,
+      "sort": {
+        "col": 0,
+        "desc": true
+      },
+      "styles": [
+        {
+          "align": "auto",
+          "dateFormat": "YYYY-MM-DD HH:mm:ss",
+          "pattern": "Time",
+          "type": "date"
+        },
+        {
+          "align": "auto",
+          "colors": [
+            "rgba(245, 54, 54, 0.9)",
+            "rgba(237, 129, 40, 0.89)",
+            "rgba(50, 172, 45, 0.97)"
+          ],
+          "decimals": 2,
+          "pattern": "/.*/",
+          "thresholds": [],
+          "type": "number",
+          "unit": "short"
+        }
+      ],
+      "targets": [
+        {
+          "expr": "cadvisor_version_info",
+          "intervalFactor": 2,
+          "legendFormat": "cAdvisor Version: {{cadvisorVersion}}",
+          "refId": "A",
+          "step": 2
+        },
+        {
+          "expr": "prometheus_build_info",
+          "intervalFactor": 2,
+          "legendFormat": "Prometheus Version: {{version}}",
+          "refId": "B",
+          "step": 2
+        },
+        {
+          "expr": "node_exporter_build_info",
+          "intervalFactor": 2,
+          "legendFormat": "Node-Exporter Version: {{version}}",
+          "refId": "C",
+          "step": 2
+        },
+        {
+          "expr": "cadvisor_version_info",
+          "intervalFactor": 2,
+          "legendFormat": "Docker Version: {{dockerVersion}}",
+          "refId": "D",
+          "step": 2
+        },
+        {
+          "expr": "cadvisor_version_info",
+          "intervalFactor": 2,
+          "legendFormat": "Host OS Version: {{osVersion}}",
+          "refId": "E",
+          "step": 2
+        },
+        {
+          "expr": "cadvisor_version_info",
+          "intervalFactor": 2,
+          "legendFormat": "Host Kernel Version: {{kernelVersion}}",
+          "refId": "F",
+          "step": 2
+        }
+      ],
+      "transform": "timeseries_aggregations",
+      "type": "table-old"
+    }
+  ],
+  "refresh": false,
+  "schemaVersion": 36,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "auto": true,
+        "auto_count": 50,
+        "auto_min": "50s",
+        "current": {
+          "selected": false,
+          "text": "auto",
+          "value": "$__auto_interval_interval"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "Interval",
+        "multi": false,
+        "name": "interval",
+        "options": [
+          {
+            "selected": true,
+            "text": "auto",
+            "value": "$__auto_interval_interval"
+          },
+          {
+            "selected": false,
+            "text": "30s",
+            "value": "30s"
+          },
+          {
+            "selected": false,
+            "text": "1m",
+            "value": "1m"
+          },
+          {
+            "selected": false,
+            "text": "2m",
+            "value": "2m"
+          },
+          {
+            "selected": false,
+            "text": "3m",
+            "value": "3m"
+          },
+          {
+            "selected": false,
+            "text": "5m",
+            "value": "5m"
+          },
+          {
+            "selected": false,
+            "text": "7m",
+            "value": "7m"
+          },
+          {
+            "selected": false,
+            "text": "10m",
+            "value": "10m"
+          },
+          {
+            "selected": false,
+            "text": "30m",
+            "value": "30m"
+          },
+          {
+            "selected": false,
+            "text": "1h",
+            "value": "1h"
+          },
+          {
+            "selected": false,
+            "text": "6h",
+            "value": "6h"
+          },
+          {
+            "selected": false,
+            "text": "12h",
+            "value": "12h"
+          },
+          {
+            "selected": false,
+            "text": "1d",
+            "value": "1d"
+          },
+          {
+            "selected": false,
+            "text": "7d",
+            "value": "7d"
+          },
+          {
+            "selected": false,
+            "text": "14d",
+            "value": "14d"
+          },
+          {
+            "selected": false,
+            "text": "30d",
+            "value": "30d"
+          }
+        ],
+        "query": "30s,1m,2m,3m,5m,7m,10m,30m,1h,6h,12h,1d,7d,14d,30d",
+        "refresh": 2,
+        "skipUrlSync": false,
+        "type": "interval"
+      },
+      {
+        "current": {
+          "selected": false,
+          "text": "metrics",
+          "value": "metrics"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "5qpBRfD4k"
+        },
+        "definition": "label_values(container_cpu_user_seconds_total, job)",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Job",
+        "multi": false,
+        "name": "job",
+        "options": [],
+        "query": {
+          "query": "label_values(container_cpu_user_seconds_total, job)",
+          "refId": "Prometheus-job-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": "All",
+        "current": {
+          "selected": false,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "5qpBRfD4k"
+        },
+        "definition": "label_values(container_cpu_user_seconds_total{job=~\"$job\"}, instance)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Host:",
+        "multi": true,
+        "name": "node",
+        "options": [],
+        "query": {
+          "query": "label_values(container_cpu_user_seconds_total{job=~\"$job\"}, instance)",
+          "refId": "Prometheus-node-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "/([^:]+):.*/",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "current": {
+          "isNone": true,
+          "selected": false,
+          "text": "None",
+          "value": ""
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "5qpBRfD4k"
+        },
+        "definition": "label_values(container_cpu_user_seconds_total{instance=~\"$node:(.*)\"}, instance)",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Port",
+        "multi": false,
+        "name": "port",
+        "options": [],
+        "query": {
+          "query": "label_values(container_cpu_user_seconds_total{instance=~\"$node:(.*)\"}, instance)",
+          "refId": "Prometheus-port-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "/[^:]+:(.*)/",
+        "skipUrlSync": false,
+        "sort": 3,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": "All",
+        "current": {
+          "selected": false,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "5qpBRfD4k"
+        },
+        "definition": "label_values(container_cpu_user_seconds_total{instance=~\"10.115.220.32:8080|10.115.220.33:8080\", name!=\"cadvisor\"}, name)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Prod",
+        "multi": true,
+        "name": "Prod",
+        "options": [],
+        "query": {
+          "query": "label_values(container_cpu_user_seconds_total{instance=~\"10.115.220.32:8080|10.115.220.33:8080\", name!=\"cadvisor\"}, name)",
+          "refId": "Prometheus-Prod-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": "All",
+        "current": {
+          "selected": false,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "5qpBRfD4k"
+        },
+        "definition": "label_values(container_cpu_user_seconds_total{instance=~\"10.115.222.21:8080|10.115.222.22:8080\", name!=\"cadvisor\"}, name)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "NonProd",
+        "multi": true,
+        "name": "NonProd",
+        "options": [],
+        "query": {
+          "query": "label_values(container_cpu_user_seconds_total{instance=~\"10.115.222.21:8080|10.115.222.22:8080\", name!=\"cadvisor\"}, name)",
+          "refId": "Prometheus-NonProd-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-15m",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "browser",
+  "title": "Docker Container & Host Metrics",
+  "uid": "4dMaCsRZz",
+  "version": 4,
+  "weekStart": ""
+}
\ No newline at end of file
diff --git a/grafana/dashboards/nginx.json b/grafana/dashboards/nginx.json
new file mode 100644
index 0000000..648a564
--- /dev/null
+++ b/grafana/dashboards/nginx.json
@@ -0,0 +1,1518 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "limit": 100,
+        "name": "Annotations & Alerts",
+        "showIn": 0,
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "nginx_vts_status grafana graph",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "gnetId": 9785,
+  "graphTooltip": 0,
+  "id": 5,
+  "iteration": 1668174400091,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 4,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 17,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(nginx_vts_main_connections{instance=~\"$Instance\", status=~\"active|writing|reading|waiting\"}) by (status)",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{status}}",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Server Connections",
+      "tooltip": {
+        "msResolution": false,
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 3,
+        "x": 12,
+        "y": 0
+      },
+      "id": 9,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "8.5.1",
+      "targets": [
+        {
+          "expr": "sum(irate(nginx_vts_main_connections{status=\"active\"}[1m]))",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "active",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 3,
+        "x": 15,
+        "y": 0
+      },
+      "id": 11,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "8.5.1",
+      "targets": [
+        {
+          "expr": "sum(irate(nginx_vts_main_connections{status=\"writing\"}[1m]))",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "writing",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 3,
+        "x": 18,
+        "y": 0
+      },
+      "id": 13,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "8.5.1",
+      "targets": [
+        {
+          "expr": "sum(irate(nginx_vts_main_connections{status=\"reading\"}[1m]))",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "read",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "match": "null",
+                "result": {
+                  "text": "N/A"
+                }
+              },
+              "type": "special"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 3,
+        "x": 21,
+        "y": 0
+      },
+      "id": 12,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "none",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "8.5.1",
+      "targets": [
+        {
+          "expr": "sum(irate(nginx_vts_main_connections{status=\"waiting\"}[1m]))",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "waiting",
+      "type": "stat"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 6,
+        "w": 6,
+        "x": 0,
+        "y": 4
+      },
+      "hiddenSeries": false,
+      "id": 21,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(irate(nginx_vts_server_requests_total{instance=~\"$Instance\", host=~\"$Host\", code!=\"total\"}[5m])) by (code)",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "{{ code }}",
+          "metric": "nginx_server_requests",
+          "refId": "A",
+          "step": 240
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Server Requests",
+      "tooltip": {
+        "msResolution": false,
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "description": "This one is providing aggregated error codes, but it's still possible to graph these per upstream.",
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 6,
+        "w": 6,
+        "x": 6,
+        "y": 4
+      },
+      "hiddenSeries": false,
+      "id": 25,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(irate(nginx_vts_upstream_requests_total{instance=~\"$Instance\", upstream=~\"^$Upstream$\", backend=~\"^$Backend$\", code!=\"total\"}[5m])) by (code)",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "{{ code }}",
+          "metric": "nginx_upstream_requests",
+          "refId": "A",
+          "step": 240
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Upstream Requests",
+      "tooltip": {
+        "msResolution": false,
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 6,
+        "w": 6,
+        "x": 12,
+        "y": 4
+      },
+      "hiddenSeries": false,
+      "id": 15,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": true,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": false,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(irate(nginx_vts_upstream_requests_total{backend=~\"$Backend\", instance=~\"$Instance\", code!=\"total\"} [1m])) by (code)",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "{{code}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Request delta/sec (BACKEND)",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": 0,
+          "format": "short",
+          "label": "Rate",
+          "logBase": 2,
+          "min": "1",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 6,
+        "w": 6,
+        "x": 18,
+        "y": 4
+      },
+      "hiddenSeries": false,
+      "id": 5,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": true,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": false,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(irate(nginx_vts_filter_requests_total{filter=~\"country::$Host\", filter_name=~\"$Country\", instance=~\"$Instance\", direction!=\"total\"} [1m])) by (direction)",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "{{direction}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Request delta/sec (FILTER)",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": 0,
+          "format": "short",
+          "label": "Rate",
+          "logBase": 2,
+          "min": "1",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 24,
+        "x": 0,
+        "y": 10
+      },
+      "hiddenSeries": false,
+      "id": 2,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": false,
+        "hideEmpty": true,
+        "hideZero": true,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(irate(nginx_vts_filter_request_seconds{filter=~\"country::$Host\", filter_name=~\"$Country\", instance=~\"$Instance\"} [1m])) by (filter_name) * 1000",
+          "format": "time_series",
+          "instant": false,
+          "intervalFactor": 1,
+          "legendFormat": "{{filter_name}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [
+        {
+          "colorMode": "ok",
+          "fill": true,
+          "line": true,
+          "op": "lt",
+          "value": 250,
+          "yaxis": "left"
+        },
+        {
+          "colorMode": "warning",
+          "fill": true,
+          "line": true,
+          "op": "lt",
+          "value": 750,
+          "yaxis": "left"
+        },
+        {
+          "colorMode": "critical",
+          "fill": true,
+          "line": true,
+          "op": "gt",
+          "value": 750,
+          "yaxis": "left"
+        }
+      ],
+      "timeRegions": [],
+      "title": "Response times (FILTER)",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "dtdurationms",
+          "logBase": 2,
+          "max": "5000",
+          "min": "200",
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 6,
+        "w": 8,
+        "x": 0,
+        "y": 17
+      },
+      "hiddenSeries": false,
+      "id": 14,
+      "legend": {
+        "alignAsTable": true,
+        "avg": false,
+        "current": true,
+        "max": false,
+        "min": false,
+        "rightSide": true,
+        "show": false,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(irate(nginx_vts_filter_bytes_total{filter=~\"country::$Host\", filter_name=~\"$Country\", instance=~\"$Instance\"} [1m])) by (direction)",
+          "format": "time_series",
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "{{direction}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "bandwith delta/sec (FILTER)",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": 0,
+          "format": "decbytes",
+          "label": "Rate",
+          "logBase": 1,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 6,
+        "w": 8,
+        "x": 8,
+        "y": 17
+      },
+      "hiddenSeries": false,
+      "id": 23,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(irate(nginx_vts_server_bytes_total{instance=~\"$Instance\", host=~\"$Host\"}[5m])) by (direction)",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{ direction }}",
+          "metric": "nginx_server_bytes",
+          "refId": "A",
+          "step": 240
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Server Bytes",
+      "tooltip": {
+        "msResolution": false,
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "bytes",
+          "logBase": 1,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 6,
+        "w": 8,
+        "x": 16,
+        "y": 17
+      },
+      "hiddenSeries": false,
+      "id": 27,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(irate(nginx_vts_upstream_bytes_total{instance=~\"$Instance\", upstream=~\"^$Upstream$\", backend=~\"^$Backend$\"}[5m])) by (direction)",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "{{ direction }}",
+          "metric": "nginx_upstream_bytes",
+          "refId": "A",
+          "step": 240
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Upstream Bytes",
+      "tooltip": {
+        "msResolution": false,
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 23
+      },
+      "hiddenSeries": false,
+      "id": 29,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(nginx_vts_upstream_response_seconds{instance=~\"$Instance\", upstream=~\"^$Upstream$\", backend=~\"^$Backend$\"}) by (backend)",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "{{ backend }}",
+          "metric": "nginx_upstream_response",
+          "refId": "A",
+          "step": 120
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Upstream Backend Response",
+      "tooltip": {
+        "msResolution": false,
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 23
+      },
+      "hiddenSeries": false,
+      "id": 19,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(irate(nginx_vts_server_cache_total{instance=~\"$Instance\", host=~\"$Host\"}[5m])) by (status)",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 2,
+          "legendFormat": "{{ status }}",
+          "metric": "nginx_server_cache",
+          "refId": "A",
+          "step": 240
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Server Cache",
+      "tooltip": {
+        "msResolution": false,
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "logBase": 1,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 36,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "5qpBRfD4k"
+        },
+        "definition": "nginx_vts_filter_bytes_total",
+        "hide": 0,
+        "includeAll": true,
+        "multi": true,
+        "name": "Country",
+        "options": [],
+        "query": {
+          "query": "nginx_vts_filter_bytes_total",
+          "refId": "Prometheus-Country-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "/.*filter_name=\"([^\"]*).*/",
+        "skipUrlSync": false,
+        "sort": 5,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "current": {
+          "selected": false,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "5qpBRfD4k"
+        },
+        "definition": "label_values(nginx_vts_server_bytes_total, instance)",
+        "hide": 0,
+        "includeAll": true,
+        "multi": false,
+        "name": "Instance",
+        "options": [],
+        "query": {
+          "query": "label_values(nginx_vts_server_bytes_total, instance)",
+          "refId": "Prometheus-Instance-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "current": {
+          "selected": false,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "5qpBRfD4k"
+        },
+        "definition": "label_values(nginx_vts_server_requests_total{instance=~\"$Instance\"}, host)",
+        "hide": 0,
+        "includeAll": true,
+        "multi": false,
+        "name": "Host",
+        "options": [],
+        "query": {
+          "query": "label_values(nginx_vts_server_requests_total{instance=~\"$Instance\"}, host)",
+          "refId": "Prometheus-Host-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": false,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "5qpBRfD4k"
+        },
+        "definition": "label_values(nginx_vts_upstream_requests_total{instance=~\"$Instance\"}, upstream)",
+        "hide": 0,
+        "includeAll": true,
+        "multi": false,
+        "name": "Upstream",
+        "options": [],
+        "query": {
+          "query": "label_values(nginx_vts_upstream_requests_total{instance=~\"$Instance\"}, upstream)",
+          "refId": "Prometheus-Upstream-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": false,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "5qpBRfD4k"
+        },
+        "definition": "label_values(nginx_vts_upstream_requests_total{instance=~\"$Instance\", upstream=~\"$Upstream\"}, backend)",
+        "hide": 0,
+        "includeAll": true,
+        "multi": false,
+        "name": "Backend",
+        "options": [],
+        "query": {
+          "query": "label_values(nginx_vts_upstream_requests_total{instance=~\"$Instance\", upstream=~\"$Upstream\"}, backend)",
+          "refId": "Prometheus-Backend-Variable-Query"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "",
+  "title": "Nginx VTS",
+  "uid": "ZQAsi-Xiz",
+  "version": 1,
+  "weekStart": ""
+}
\ No newline at end of file
diff --git a/grafana/provisioning/dashboards/dashboards.yaml b/grafana/provisioning/dashboards/dashboards.yaml
new file mode 100644
index 0000000..ec78138
--- /dev/null
+++ b/grafana/provisioning/dashboards/dashboards.yaml
@@ -0,0 +1,15 @@
+apiVersion: 1
+
+providers:
+  - name: 'hackerspace-zone dashboards'
+    orgId: 1
+    folder: ''
+    folderUid: ''
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 10
+    allowUiUpdates: false
+    options:
+      path: /etc/grafana/dashboards
+      foldersFromFilesStructure: true
+
diff --git a/grafana/provisioning/datasources/prometheus.yaml b/grafana/provisioning/datasources/prometheus.yaml
new file mode 100644
index 0000000..fdd9738
--- /dev/null
+++ b/grafana/provisioning/datasources/prometheus.yaml
@@ -0,0 +1,13 @@
+apiVersion: 1
+datasources:
+  - name: Prometheus
+    version: 2
+    orgId: 1
+    uid: 5qpBRfD4k
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    basicAuth: false
+    isDefault: true
+    jsonData: 
+        httpMethod: POST

From e20e00a3c2dd98f0bfded89949eb554fa8177c66 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@hackerspace-zone.labs.lowerlayer.nl>
Date: Fri, 11 Nov 2022 17:53:45 +0000
Subject: [PATCH 04/13] keycloak: fix setup to use a shell script

---
 keycloak.yaml                |  8 ++++++--
 keycloak/entrypoint-setup.sh | 19 +++++++++++++++++++
 keycloak/mail-setup.sh       | 27 +++++++++++++++++++++++++++
 3 files changed, 52 insertions(+), 2 deletions(-)
 create mode 100755 keycloak/entrypoint-setup.sh
 create mode 100755 keycloak/mail-setup.sh

diff --git a/keycloak.yaml b/keycloak.yaml
index d83d08b..35151f0 100644
--- a/keycloak.yaml
+++ b/keycloak.yaml
@@ -42,6 +42,9 @@ services:
       depends_on:
         - keycloak-db
 
+  # all of the various subdomains can install files in
+  # /keycloak-setup/ to be executed during the setup phase
+  # to enable their clients using the client-create tool
   keycloak-setup:
         image: quay.io/keycloak/keycloak:18.0.0
         profiles:
@@ -52,9 +55,10 @@ services:
         env_file:
           - env.production
           - data/keycloak/secrets
-        entrypoint: /keycloak-setup.sh
+        entrypoint: /entrypoint.sh
         volumes:
-          - ./keycloak/setup:/keycloak-setup.sh:ro
+          - ./keycloak/entrypoint-setup.sh:/entrypoint.sh:ro
+          - ./keycloak/mail-setup.sh:/keycloak-setup/mail-setup.sh:ro
           - ./keycloak/client-create:/bin/client-create:ro
 
   # add the keycloak nginx configuration into the nginx volume
diff --git a/keycloak/entrypoint-setup.sh b/keycloak/entrypoint-setup.sh
new file mode 100755
index 0000000..051bff2
--- /dev/null
+++ b/keycloak/entrypoint-setup.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+export PATH=/opt/keycloak/bin:$PATH
+
+# perform an authentication as admin so that all other scripts can
+# use the cached credentials
+
+kcadm.sh \
+	config credentials \
+	--server http://keycloak:8080/ \
+	--user admin \
+	--password "$KEYCLOAK_ADMIN_PASSWORD" \
+	--realm master \
+|| exit 1
+
+for file in /keycloak-setup/* ; do
+	echo >&2 "$file: running setup"
+	$file || exit 1
+done
diff --git a/keycloak/mail-setup.sh b/keycloak/mail-setup.sh
new file mode 100755
index 0000000..6e8c716
--- /dev/null
+++ b/keycloak/mail-setup.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+if [ -z "$SMTP_SERVER" ]; then
+	exit 0
+fi
+
+echo >&2 "*** configuring email to use $SMTP_SERVER"
+/opt/keycloak/bin/kcadm.sh update \
+	"realms/$REALM" \
+	-f - <<EOF || exit 1
+{
+  "resetPasswordAllowed": "true",
+  "smtpServer" : {
+    "auth" : "true",
+    "starttls" : "true",
+    "user" : "$SMTP_USER",
+    "password" : "$SMTP_PASSWORD",
+    "port" : "$SMTP_PORT",
+    "host" : "$SMTP_SERVER",
+    "from" : "keycloak@$DOMAIN_NAME",
+    "fromDisplayName" : "Keycloak @ $DOMAIN_NAME",
+    "ssl" : "false"
+  }
+}
+EOF
+
+exit 0

From 1d1b19dced87661c88ef4c911a787d094baa47cf Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@hackerspace-zone.labs.lowerlayer.nl>
Date: Fri, 11 Nov 2022 17:54:26 +0000
Subject: [PATCH 05/13] mastodon: silent startup

---
 mastodon/entrypoint.sh | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/mastodon/entrypoint.sh b/mastodon/entrypoint.sh
index 4b0473a..bae2cb8 100755
--- a/mastodon/entrypoint.sh
+++ b/mastodon/entrypoint.sh
@@ -1,19 +1,13 @@
 #!/bin/bash -x
 
-id
-export
-pwd
 rm -f /mastodon/tmp/pids/server.pid
 
 export MASTODON_DIR=/mastodon/public/system
 export VAPID_KEY="$MASTODON_DIR/vapid_key"
 export DB_SETUP="$MASTODON_DIR/db_done"
 
-
-which rails
 chown -R mastodon:mastodon "$MASTODON_DIR"
 
-#exec su mastodon /bin/bash - <<EOF
 exec su mastodon <<EOF
 
 export PATH="$PATH:/opt/ruby/bin:/opt/node/bin:/opt/mastodon/bin"

From 7a37d33ff2b98fcf349ac26976ce54b848dd2d3b Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@hackerspace-zone.labs.lowerlayer.nl>
Date: Fri, 11 Nov 2022 17:55:46 +0000
Subject: [PATCH 06/13] matrix: federation works and setup seems to do the
 right thing

---
 Makefile                            |  5 ++
 matrix.yaml                         | 56 ++++++++++++++++++
 matrix/10-envsubst-config.sh        |  5 ++
 matrix/README.md                    |  3 +
 matrix/config.sample.json           | 53 +++++++++++++++++
 matrix/docker-compose.yaml          | 27 ---------
 matrix/element-config.json.template | 73 ------------------------
 matrix/entrypoint-synapse.sh        | 62 ++++++++++++++++++++
 matrix/env.production               |  1 -
 matrix/keycloak.sh                  |  4 ++
 matrix/nginx.conf                   | 71 +++++++++++++++++++++++
 matrix/setup                        | 88 -----------------------------
 nginx.yaml                          |  1 +
 nginx/default.conf                  | 12 +++-
 nginx/etc/nginx.conf                |  3 +
 15 files changed, 272 insertions(+), 192 deletions(-)
 create mode 100644 matrix.yaml
 create mode 100755 matrix/10-envsubst-config.sh
 create mode 100644 matrix/README.md
 create mode 100644 matrix/config.sample.json
 delete mode 100644 matrix/docker-compose.yaml
 delete mode 100644 matrix/element-config.json.template
 create mode 100755 matrix/entrypoint-synapse.sh
 delete mode 100644 matrix/env.production
 create mode 100755 matrix/keycloak.sh
 create mode 100644 matrix/nginx.conf
 delete mode 100755 matrix/setup

diff --git a/Makefile b/Makefile
index 85a4244..0d7e613 100644
--- a/Makefile
+++ b/Makefile
@@ -4,6 +4,7 @@ MODULES += hedgedoc
 MODULES += grafana
 MODULES += prometheus
 MODULES += mastodon
+MODULES += matrix
 #MODULES += pixelfed
 
 include env.production
@@ -36,6 +37,10 @@ mastodon-shell:
 	$(DOCKER) exec mastodon bash
 mastodon-streaming-shell:
 	$(DOCKER) exec mastodon-streaming bash
+matrix-shell:
+	$(DOCKER) exec matrix-synapse bash
+matrix-logs:
+	$(DOCKER) logs -f matrix-synapse
 nginx-build: data/nginx/secrets
 	$(DOCKER) build nginx
 
diff --git a/matrix.yaml b/matrix.yaml
new file mode 100644
index 0000000..e62999e
--- /dev/null
+++ b/matrix.yaml
@@ -0,0 +1,56 @@
+version: '3'
+services:
+# the default synpase uses a sqlite database; this should be fixed at somepoint
+#  matrix-db:
+#    image: postgres:13.4-alpine
+#    restart: unless-stopped
+#    volumes:
+#     - ./data/matrix/db:/var/lib/postgresql/data
+#    environment:
+#     - POSTGRES_DB=synapse
+#     - POSTGRES_USER=synapse
+#     - POSTGRES_PASSWORD=STRONGPASSWORD
+     
+  matrix-element:
+    image: vectorim/element-web:latest
+    restart: unless-stopped
+    container_name: matrix-element
+    env_file:
+     - env.production
+    volumes:
+      - ./matrix/10-envsubst-config.sh:/docker-entrypoint.d/10-envsubst-config.sh:ro
+      - ./matrix/config.sample.json:/app/config.sample.json:ro
+    depends_on:
+      - matrix-synapse
+#    ports:
+#      - "5000:80"
+
+  matrix-synapse:
+    image: matrixdotorg/synapse:latest
+    restart: unless-stopped
+    container_name: matrix-synapse
+    volumes:
+      - ./data/matrix/synapse:/data
+      - ./matrix/entrypoint-synapse.sh:/entrypoint.sh:ro
+    entrypoint: ["/entrypoint.sh"]
+    env_file:
+      - env.production
+    environment:
+      - MATRIX_CLIENT_SECRET=${MATRIX_CLIENT_SECRET}
+    depends_on:
+      - keycloak
+      - nginx
+#    ports:
+#      - "5008:8008"
+
+  # add the nginx configuration into the nginx volume
+  nginx:
+    volumes:
+      - ./matrix/nginx.conf:/etc/nginx/templates/matrix.conf.template:ro
+
+  # add the client secrets to the keycloak-setup volume
+  keycloak-setup:
+    env_file:
+      - data/matrix/secrets
+    volumes:
+      - ./matrix/keycloak.sh:/keycloak-setup/matrix.sh:ro
diff --git a/matrix/10-envsubst-config.sh b/matrix/10-envsubst-config.sh
new file mode 100755
index 0000000..6abd845
--- /dev/null
+++ b/matrix/10-envsubst-config.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+echo >&2 "**** Configuring for $DOMAIN_NAME"
+envsubst < /app/config.sample.json > /app/config.json
+head /app/config.json
diff --git a/matrix/README.md b/matrix/README.md
new file mode 100644
index 0000000..f1ee362
--- /dev/null
+++ b/matrix/README.md
@@ -0,0 +1,3 @@
+# Matrix/Element chat
+
+The sample config for the JSON comes from 8891698745897388db037ea8692937edc199630c on vector-im/element-web
diff --git a/matrix/config.sample.json b/matrix/config.sample.json
new file mode 100644
index 0000000..fccb437
--- /dev/null
+++ b/matrix/config.sample.json
@@ -0,0 +1,53 @@
+{
+    "default_server_config": {
+        "m.homeserver": {
+            "base_url": "https://${MATRIX_HOSTNAME}.${DOMAIN_NAME}",
+            "server_name": "${DOMAIN_NAME}"
+        },
+        "m.identity_server": {
+            "base_url": "https://vector.im"
+        }
+    },
+    "disable_custom_urls": false,
+    "disable_guests": false,
+    "disable_login_language_selector": false,
+    "disable_3pid_login": false,
+    "brand": "Element",
+    "integrations_ui_url": "https://scalar.vector.im/",
+    "integrations_rest_url": "https://scalar.vector.im/api",
+    "integrations_widgets_urls": [
+        "https://scalar.vector.im/_matrix/integrations/v1",
+        "https://scalar.vector.im/api",
+        "https://scalar-staging.vector.im/_matrix/integrations/v1",
+        "https://scalar-staging.vector.im/api",
+        "https://scalar-staging.riot.im/scalar/api"
+    ],
+    "bug_report_endpoint_url": "https://element.io/bugreports/submit",
+    "uisi_autorageshake_app": "element-auto-uisi",
+    "default_country_code": "GB",
+    "show_labs_settings": false,
+    "features": { },
+    "default_federate": true,
+    "default_theme": "light",
+    "room_directory": {
+        "servers": [
+            "matrix.org"
+        ]
+    },
+    "enable_presence_by_hs_url": {
+        "https://matrix.org": false,
+        "https://matrix-client.matrix.org": false
+    },
+    "setting_defaults": {
+        "breadcrumbs": true
+    },
+    "jitsi": {
+        "preferred_domain": "meet.element.io"
+    },
+    "element_call": {
+        "url": "https://call.element.io",
+        "participant_limit": 8,
+        "brand": "Element Call"
+    },
+    "map_style_url": "https://api.maptiler.com/maps/streets/style.json?key=fU3vlMsMn4Jb6dnEIFsx"
+}
diff --git a/matrix/docker-compose.yaml b/matrix/docker-compose.yaml
deleted file mode 100644
index ba42dfc..0000000
--- a/matrix/docker-compose.yaml
+++ /dev/null
@@ -1,27 +0,0 @@
-version: '3'
-services:
-  postgres:
-    image: postgres:13.4-alpine
-    restart: unless-stopped
-    volumes:
-     - ../data/matrix/postgresdata:/var/lib/postgresql/data
-    environment:
-     - POSTGRES_DB=synapse
-     - POSTGRES_USER=synapse
-     - POSTGRES_PASSWORD=STRONGPASSWORD
-     
-  element:
-    image: vectorim/element-web:latest
-    restart: unless-stopped
-    volumes:
-      - ../data/matrix/element-config.json:/app/config.json
-    ports:
-      - "5000:80"
-
-  synapse:
-    image: matrixdotorg/synapse:latest
-    restart: unless-stopped
-    volumes:
-     - ../data/matrix/synapse:/data
-    ports:
-      - "5008:8008"
diff --git a/matrix/element-config.json.template b/matrix/element-config.json.template
deleted file mode 100644
index baef3cb..0000000
--- a/matrix/element-config.json.template
+++ /dev/null
@@ -1,73 +0,0 @@
-{
-     "default_server_config": {
-        "m.homeserver": {
-            "base_url": "https://${MATRIX_HOSTNAME}",
-            "server_name": "${DOMAIN_NAME}"
-        },
-        "m.identity_server": {
-            "base_url": "https://vector.im"
-        }
-    },
-    "brand": "Element",
-    "integrations_ui_url": "https://scalar.vector.im/",
-    "integrations_rest_url": "https://scalar.vector.im/api",
-    "integrations_widgets_urls": [
-        "https://scalar.vector.im/_matrix/integrations/v1",
-        "https://scalar.vector.im/api",
-        "https://scalar-staging.vector.im/_matrix/integrations/v1",
-        "https://scalar-staging.vector.im/api",
-        "https://scalar-staging.riot.im/scalar/api"
-    ],
-    "hosting_signup_link": "https://element.io/matrix-services?utm_source=element-web&utm_medium=web",
-    "bug_report_endpoint_url": "https://element.io/bugreports/submit",
-    "uisi_autorageshake_app": "element-auto-uisi",
-    "showLabsSettings": true,
-    "piwik": {
-        "url": "https://piwik.riot.im/",
-        "siteId": 1,
-        "policyUrl": "https://element.io/cookie-policy"
-    },
-    "roomDirectory": {
-        "servers": [
-            "matrix.org",
-            "gitter.im",
-            "libera.chat"
-        ]
-    },
-    "enable_presence_by_hs_url": {
-        "https://matrix.org": false,
-        "https://matrix-client.matrix.org": false
-    },
-    "terms_and_conditions_links": [
-        {
-            "url": "https://element.io/privacy",
-            "text": "Privacy Policy"
-        },
-        {
-            "url": "https://element.io/cookie-policy",
-            "text": "Cookie Policy"
-        }
-    ],
-    "hostSignup": {
-      "brand": "Element Home",
-      "cookiePolicyUrl": "https://element.io/cookie-policy",
-      "domains": [
-          "matrix.org"
-      ],
-      "privacyPolicyUrl": "https://element.io/privacy",
-      "termsOfServiceUrl": "https://element.io/terms-of-service",
-      "url": "https://ems.element.io/element-home/in-app-loader"
-    },
-    "sentry": {
-        "dsn": "https://029a0eb289f942508ae0fb17935bd8c5@sentry.matrix.org/6",
-        "environment": "develop"
-    },
-    "posthog": {
-        "projectApiKey": "phc_Jzsm6DTm6V2705zeU5dcNvQDlonOR68XvX2sh1sEOHO",
-        "apiHost": "https://posthog.element.io"
-    },
-    "features": {
-        "feature_spotlight": true
-    },
-    "map_style_url": "https://api.maptiler.com/maps/streets/style.json?key=fU3vlMsMn4Jb6dnEIFsx"
-}
diff --git a/matrix/entrypoint-synapse.sh b/matrix/entrypoint-synapse.sh
new file mode 100755
index 0000000..353484e
--- /dev/null
+++ b/matrix/entrypoint-synapse.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+# This is the custom startup script for the synpase server
+
+# fix up the Element client config to have the correct hostname
+# based on the environment variables
+#export DOMAIN_NAME MATRIX_HOSTNAME
+#envsubst < "element-config.json.template" > "$DATA/element-config.json"
+
+HOMESERVER_YAML="/data/homeserver.yaml"
+
+if [ ! -r "$HOMESERVER_YAML" ]; then
+	echo >&2 "***** Configuring the home server for $DOMAIN_NAME *****"
+
+	export SYNAPSE_SERVER_NAME="$DOMAIN_NAME"
+	export SYNAPSE_REPORT_STATS="no"
+
+	/start.py generate \
+	|| exit 1
+
+	echo >&2 "***** Adding OIDC provider *****"
+	cat <<EOF >> "$HOMESERVER_YAML"
+#
+# added by hackerspace-zone setup scripts
+#
+suppress_key_server_warning: true
+web_client_location: https://${MATRIX_HOSTNAME}.${DOMAIN_NAME}
+public_baseurl: https://${MATRIX_HOSTNAME}.${DOMAIN_NAME}
+oidc_providers:
+  - idp_id: keycloak
+    idp_name: "Keycloak"
+    issuer: "https://${KEYCLOAK_HOSTNAME}.${DOMAIN_NAME}/realms/${REALM}"
+    client_id: "matrix"
+    client_secret: "${MATRIX_CLIENT_SECRET}"
+    scopes: ["openid", "profile"]
+    user_mapping_provider:
+      config:
+        localpart_template: "{{ user.preferred_username }}"
+        display_name_template: "{{ user.name }}"
+EOF
+
+fi
+
+if ! grep -q '^  smtp_host:' && [ -n "$SMTP_SERVER" ]; then
+	echo >&2 "***** Adding SMTP setup to yaml"
+	cat <<EOF >> "$HOMESERVER_YAML"
+#
+# added by hackerspace-zone setup scripts
+#
+email:
+  smtp_host: ${SMTP_SERVER}
+  smtp_port: ${SMTP_PORT}
+  smtp_user: "${SMTP_USER}"
+  smtp_pass: "${SMTP_PASSWORD}"
+  require_transport_security: true
+  notif_from: "%(app)s matrix homeserver <noreply@${DOMAIN_NAME}>"
+  app_name: ${DOMAIN_NAME}
+EOF
+fi
+
+# hack to let keycloak startup
+sleep 5
+exec /start.py
diff --git a/matrix/env.production b/matrix/env.production
deleted file mode 100644
index 85b953c..0000000
--- a/matrix/env.production
+++ /dev/null
@@ -1 +0,0 @@
-# variables
diff --git a/matrix/keycloak.sh b/matrix/keycloak.sh
new file mode 100755
index 0000000..8b74049
--- /dev/null
+++ b/matrix/keycloak.sh
@@ -0,0 +1,4 @@
+#!/bin/bash -x
+# Setup the OAuth client connection
+
+client-create matrix "$MATRIX_HOSTNAME.$DOMAIN_NAME" "$MATRIX_CLIENT_SECRET" </dev/null
diff --git a/matrix/nginx.conf b/matrix/nginx.conf
new file mode 100644
index 0000000..5ee4671
--- /dev/null
+++ b/matrix/nginx.conf
@@ -0,0 +1,71 @@
+map $http_upgrade $connection_upgrade {
+        default upgrade;
+        ''      close;
+}
+
+server {
+	server_name ${MATRIX_HOSTNAME} ${MATRIX_HOSTNAME}.${DOMAIN_NAME};
+	client_max_body_size 128m;
+
+	sendfile on;
+	tcp_nopush on;
+	tcp_nodelay on;
+	keepalive_timeout 65;
+	types_hash_max_size 2048;
+	#include /etc/nginx/mime.types;
+	#default_type application/octet-stream;
+
+	gzip on;
+	gzip_disable "msie6";
+
+	proxy_read_timeout 1800s;
+
+	# required to avoid HTTP 411: see Issue #1486 (https://github.com/dotcloud/docker/issues/1486)
+	chunked_transfer_encoding on;
+
+	location / {
+		proxy_pass http://matrix-element:80;
+		proxy_set_header Host $host;
+		proxy_set_header X-Real-IP $remote_addr;
+		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+		proxy_set_header X-Forwarded-Proto $scheme;
+	}
+
+	location ~ ^(/_matrix|/_synapse/client) {
+		# note: do not add a path (even a single /) after the port in `proxy_pass`,
+		# otherwise nginx will canonicalise the URI and cause signature verification
+		# errors.
+		proxy_pass http://matrix-synapse:8008;
+		proxy_set_header X-Forwarded-For $remote_addr;
+		proxy_set_header X-Forwarded-Proto $scheme;
+		proxy_set_header Host $host;
+
+		# Nginx by default only allows file uploads up to 1M in size
+		# Increase client_max_body_size to match max_upload_size defined in homeserver.yaml
+		client_max_body_size 50M;
+	}
+
+	# serve the static content for the well known files
+	location /.well-known/matrix/server {
+		default_type application/json;
+		return 200 '{"m.server": "${MATRIX_HOSTNAME}.${DOMAIN_NAME}:443"}';
+	}
+
+	location /.well-known/matrix/client {
+		default_type application/json;
+		return 200 '{"m.homeserver":{"base_url": "https://${MATRIX_HOSTNAME}.${DOMAIN_NAME}"}}';  
+	}
+
+	# The federation port is also enabled, although it can also go through 443
+	listen 8448 ssl http2 default_server;
+	#listen [::]:8448 ssl http2 default_server;
+
+	# For the user connection
+	listen 443 ssl http2;
+
+	ssl_certificate /etc/letsencrypt/live/${DOMAIN_NAME}/fullchain.pem;
+	ssl_certificate_key /etc/letsencrypt/live/${DOMAIN_NAME}/privkey.pem;
+	include /etc/nginx/includes/options-ssl-nginx.conf;
+	include /etc/nginx/includes/challenge.conf;
+	ssl_dhparam /etc/nginx/includes/ssl-dhparams.pem;
+}
diff --git a/matrix/setup b/matrix/setup
deleted file mode 100755
index 4e6dc0e..0000000
--- a/matrix/setup
+++ /dev/null
@@ -1,88 +0,0 @@
-#!/bin/bash
-die() { echo >&2 "matrix: ERROR $@" ; exit 1 ; }
-info() { echo >&2 "matrix: $@" ; }
-
-DIRNAME="$(dirname $0)"
-cd "$DIRNAME"
-source ../env.production || die "no top levle env?"
-source ../env.smtp 2>/dev/null 
-source env.production || die "no local env?"
-
-DATA="../data/matrix"
-SYNAPSE_DIR="$DATA/synapse"
-HOMESERVER_YAML="$SYNAPSE_DIR/homeserver.yaml"
-if [  -r "$HOMESERVER_YAML" ]; then
-	docker-compose up -d || die "matrix: unable to restart"
-	exit 0
-fi
-
-docker-compose down 2>/dev/null
-mkdir -p "$DATA"
-
-# fix up the Element client config to have the correct hostname
-# based on the environment variables
-export DOMAIN_NAME MATRIX_HOSTNAME
-envsubst < "element-config.json.template" > "$DATA/element-config.json"
-
-
-# This will create a *delegated* matrix server,
-# where the "servername" is just the top level domain,
-# but it is hosted on "matrix.DOMAIN_NAME".
-# the syntax here is confusing and it is not clear in
-# the docs *which* have to be updated.
-docker-compose run \
-	--rm \
-	-e SYNAPSE_SERVER_NAME="$DOMAIN_NAME" \
-	-e SYNAPSE_REPORT_STATS="no" \
-	synapse generate \
-|| die "unable to generate synapse config"
-
-MATRIX_CLIENT_SECRET="$(openssl rand -hex 20)"
-
-cat <<EOF >> "$HOMESERVER_YAML"
-web_client_location: https://${MATRIX_HOSTNAME}/
-public_baseurl: https://${MATRIX_HOSTNAME}/
-oidc_providers:
-  - idp_id: keycloak
-    idp_name: "KeyCloak"
-    issuer: "https://${KEYCLOAK_HOSTNAME}/realms/${REALM}"
-    client_id: "synapse"
-    client_secret: "${MATRIX_CLIENT_SECRET}"
-    scopes: ["openid", "profile"]
-    user_mapping_provider:
-      config:
-        localpart_template: "{{ user.preferred_username }}"
-        display_name_template: "{{ user.name }}"
-EOF
-
-if [ -n "$SMTP_SERVER" ]; then
-	info "configuring email"
-	cat <<EOF >> "$HOMESERVER_YAML"
-email:
-  smtp_host: ${SMTP_SERVER}
-  smtp_port: ${SMTP_PORT}
-  smtp_user: "${SMTP_USER}"
-  smtp_pass: "${SMTP_PASSWORD}"
-  require_transport_security: true
-  notif_from: "%(app)s matrix homeserver <noreply@${DOMAIN_NAME}>"
-  app_name: ${DOMAIN_NAME}
-EOF
-fi
-
-
-../keycloak/client-delete 'synapse' 2>/dev/null
-
-../keycloak/client-create << EOF || die "unable to create client id"
-{
-	"clientId": "synapse",
-	"rootUrl": "https://$MATRIX_HOSTNAME/",
-	"adminUrl": "https://$MATRIX_HOSTNAME/",
-	"redirectUris": [ "https://$MATRIX_HOSTNAME/*" ],
-	"webOrigins": [ "https://$MATRIX_HOSTNAME" ],
-	"clientAuthenticatorType": "client-secret",
-	"secret": "$MATRIX_CLIENT_SECRET"
-}
-EOF
-
-
-docker-compose up -d || die "matrix: unable to start container"
diff --git a/nginx.yaml b/nginx.yaml
index 6fe1c76..4e42ad7 100644
--- a/nginx.yaml
+++ b/nginx.yaml
@@ -11,6 +11,7 @@ services:
     ports:
       - "80:80"
       - "443:443"
+      - "8448:8448"
     volumes:
       - ./nginx/etc/includes:/etc/nginx/includes:ro
       - ./nginx/etc/nginx.conf:/etc/nginx/nginx.conf:ro
diff --git a/nginx/default.conf b/nginx/default.conf
index da6c466..9953951 100644
--- a/nginx/default.conf
+++ b/nginx/default.conf
@@ -51,9 +51,10 @@ server {
 	chunked_transfer_encoding on;
 
 	# delegated Matrix server
-#	location /.well-known/matrix {
-#		proxy_pass https://${MATRIX_HOSTNAME}.${DOMAIN_NAME};
-#	}
+	location /.well-known/matrix {
+		#return 302 https://${MATRIX_HOSTNAME}.${DOMAIN_NAME}$request_uri;
+		proxy_pass https://${MATRIX_HOSTNAME}.${DOMAIN_NAME}$request_uri;
+	}
 
 	# separate Mastodon WEB_DOMAIN and LOCAL_DOMAIN
 	location = /.well-known/host-meta {
@@ -63,6 +64,11 @@ server {
 		return 302 https://${MASTODON_HOSTNAME}.${DOMAIN_NAME}$request_uri;
 	}
 
+	# OpenID services
+	location = /.well-known/openid-configuration {
+		return 302 https://${KEYCLOAK_HOSTNAME}.${DOMAIN_NAME}$request_uri;
+	}
+
 	# tilde club home directories
 	location ~ ^/~(.+?)(/.*)?$ {
 		alias /home/$1/public_html$2;
diff --git a/nginx/etc/nginx.conf b/nginx/etc/nginx.conf
index 2a339a7..7cbedce 100644
--- a/nginx/etc/nginx.conf
+++ b/nginx/etc/nginx.conf
@@ -16,6 +16,9 @@ http {
 	# Basic Settings
 	##
 
+	# docker container resolver
+	resolver 127.0.0.11 ipv6=off;
+
 	sendfile on;
 	tcp_nopush on;
 	tcp_nodelay on;

From 2e670279a6e18385316cf15fed3f23803a015248 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@hackerspace-zone.labs.lowerlayer.nl>
Date: Fri, 11 Nov 2022 19:28:09 +0000
Subject: [PATCH 07/13] matrix: enable synapse metrics for prometheus

---
 grafana/dashboards/synapse.json | 4937 +++++++++++++++++++++++++++++++
 matrix/entrypoint-synapse.sh    |    6 +
 prometheus/prometheus.yaml      |    5 +
 3 files changed, 4948 insertions(+)
 create mode 100644 grafana/dashboards/synapse.json

diff --git a/grafana/dashboards/synapse.json b/grafana/dashboards/synapse.json
new file mode 100644
index 0000000..06655d3
--- /dev/null
+++ b/grafana/dashboards/synapse.json
@@ -0,0 +1,4937 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "uid": "$datasource"
+        },
+        "enable": false,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "limit": 100,
+        "name": "Annotations & Alerts",
+        "showIn": 0,
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "Synapse dashboard from https://github.com/matrix-org/synapse/blob/master/contrib/grafana/synapse.json",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "gnetId": 10046,
+  "graphTooltip": 0,
+  "id": 3,
+  "iteration": 1668194752622,
+  "links": [
+    {
+      "asDropdown": true,
+      "icon": "external link",
+      "keepTime": true,
+      "tags": [
+        "matrix"
+      ],
+      "title": "Dashboards",
+      "type": "dashboards"
+    }
+  ],
+  "liveNow": false,
+  "panels": [
+    {
+      "collapsed": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 73,
+      "panels": [],
+      "title": "Overview",
+      "type": "row"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "uid": "$datasource"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 1
+      },
+      "hiddenSeries": false,
+      "id": 75,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "rate(process_cpu_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "{{job}}-{{index}} ",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "CPU usage",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "percentunit",
+          "logBase": 1,
+          "max": "1",
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "cards": {
+        "cardPadding": 0
+      },
+      "color": {
+        "cardColor": "#b4ff00",
+        "colorScale": "sqrt",
+        "colorScheme": "interpolateSpectral",
+        "exponent": 0.5,
+        "mode": "spectrum"
+      },
+      "dataFormat": "tsbuckets",
+      "datasource": {
+        "uid": "$datasource"
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 1
+      },
+      "heatmap": {},
+      "hideZeroBuckets": false,
+      "highlightCards": true,
+      "id": 85,
+      "legend": {
+        "show": false
+      },
+      "links": [],
+      "reverseYBuckets": false,
+      "targets": [
+        {
+          "expr": "sum(rate(synapse_http_server_response_time_seconds_bucket{servlet='RoomSendEventRestServlet',instance=\"$instance\"}[$bucket_size])) by (le)",
+          "format": "heatmap",
+          "intervalFactor": 1,
+          "legendFormat": "{{le}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Event Send Time",
+      "tooltip": {
+        "show": true,
+        "showHistogram": false
+      },
+      "type": "heatmap",
+      "xAxis": {
+        "show": true
+      },
+      "yAxis": {
+        "format": "s",
+        "logBase": 2,
+        "show": true
+      },
+      "yBucketBound": "auto"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "uid": "$datasource"
+      },
+      "editable": true,
+      "error": false,
+      "fill": 1,
+      "fillGradient": 0,
+      "grid": {},
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 10
+      },
+      "hiddenSeries": false,
+      "id": 33,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 2,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(synapse_storage_events_persisted_events{instance=\"$instance\"}[$bucket_size])) without (job,index)",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "",
+          "refId": "A",
+          "step": 20,
+          "target": ""
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Events Persisted",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "cumulative"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "hertz",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 17
+      },
+      "id": 54,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "editable": true,
+          "error": false,
+          "fill": 0,
+          "fillGradient": 0,
+          "grid": {},
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 18
+          },
+          "hiddenSeries": false,
+          "id": 34,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "8.5.1",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": true,
+          "targets": [
+            {
+              "expr": "process_resident_memory_bytes{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}} {{index}}",
+              "refId": "A",
+              "step": 20,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "timeRegions": [],
+          "title": "Memory",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "bytes",
+              "logBase": 1,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 18
+          },
+          "hiddenSeries": false,
+          "id": 37,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "8.5.1",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [
+            {
+              "alias": "/max$/",
+              "color": "#890F02",
+              "fill": 0,
+              "legend": false
+            }
+          ],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "process_open_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}}",
+              "refId": "A",
+              "step": 20
+            },
+            {
+              "expr": "process_max_fds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "format": "time_series",
+              "hide": true,
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} max",
+              "refId": "B",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeRegions": [],
+          "title": "Open FDs",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "none",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 25
+          },
+          "hiddenSeries": false,
+          "id": 50,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "8.5.1",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(python_twisted_reactor_tick_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_twisted_reactor_tick_time_count[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeRegions": [],
+          "title": "Avg reactor tick time",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "s",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "description": "Shows the time in which the given percentage of reactor ticks completed, over the sampled timespan",
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 25
+          },
+          "hiddenSeries": false,
+          "id": 105,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "8.5.1",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "histogram_quantile(0.99, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} 99%",
+              "refId": "A",
+              "step": 20
+            },
+            {
+              "expr": "histogram_quantile(0.95, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} 95%",
+              "refId": "B"
+            },
+            {
+              "expr": "histogram_quantile(0.90, rate(python_twisted_reactor_tick_time_bucket{index=~\"$index\",instance=\"$instance\",job=~\"$job\"}[$bucket_size]))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} 90%",
+              "refId": "C"
+            },
+            {
+              "expr": "",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "refId": "D"
+            }
+          ],
+          "thresholds": [],
+          "timeRegions": [],
+          "title": "Reactor tick quantiles",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "s",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "fill": 0,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 32
+          },
+          "hiddenSeries": false,
+          "id": 53,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "8.5.1",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "min_over_time(up{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeRegions": [],
+          "title": "Up",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 32
+          },
+          "hiddenSeries": false,
+          "id": 49,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "8.5.1",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [
+            {
+              "alias": "/^up/",
+              "legend": false,
+              "yaxis": 2
+            }
+          ],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "scrape_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeRegions": [],
+          "title": "Prometheus scrape time",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "s",
+              "logBase": 1,
+              "min": "0",
+              "show": true
+            },
+            {
+              "decimals": 0,
+              "format": "none",
+              "label": "",
+              "logBase": 1,
+              "max": "0",
+              "min": "-1",
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "fillGradient": 0,
+          "grid": {},
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 39
+          },
+          "hiddenSeries": false,
+          "id": 5,
+          "legend": {
+            "alignAsTable": false,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "8.5.1",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [
+            {
+              "alias": "/user/"
+            },
+            {
+              "alias": "/system/"
+            }
+          ],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(process_cpu_system_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} system ",
+              "metric": "",
+              "refId": "B",
+              "step": 20
+            },
+            {
+              "expr": "rate(process_cpu_user_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} user",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [
+            {
+              "colorMode": "custom",
+              "line": true,
+              "lineColor": "rgba(216, 200, 27, 0.27)",
+              "op": "gt",
+              "value": 0.5
+            },
+            {
+              "colorMode": "custom",
+              "line": true,
+              "lineColor": "rgba(234, 112, 112, 0.22)",
+              "op": "gt",
+              "value": 0.8
+            }
+          ],
+          "timeRegions": [],
+          "title": "CPU",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "percentunit",
+              "label": "",
+              "logBase": 1,
+              "max": "1.2",
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        }
+      ],
+      "title": "Process info",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 18
+      },
+      "id": 56,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "decimals": 1,
+          "fill": 1,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 47
+          },
+          "id": 40,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_storage_events_persisted_by_source_type{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{type}}",
+              "refId": "D"
+            }
+          ],
+          "thresholds": [],
+          "title": "Events/s Local vs Remote",
+          "tooltip": {
+            "shared": true,
+            "sort": 2,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "label": "",
+              "logBase": 1,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "decimals": 1,
+          "fill": 1,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 47
+          },
+          "id": 46,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_storage_events_persisted_by_event_type{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+              "format": "time_series",
+              "instant": false,
+              "intervalFactor": 2,
+              "legendFormat": "{{type}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "title": "Events/s by Type",
+          "tooltip": {
+            "shared": false,
+            "sort": 2,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "logBase": 1,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {
+            "irc-freenode (local)": "#EAB839"
+          },
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "decimals": 1,
+          "fill": 1,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 54
+          },
+          "id": 44,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_storage_events_persisted_by_origin{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{origin_entity}} ({{origin_type}})",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "title": "Events/s by Origin",
+          "tooltip": {
+            "shared": false,
+            "sort": 2,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "logBase": 1,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "decimals": 1,
+          "fill": 1,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 54
+          },
+          "id": 45,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum(rate(synapse_storage_events_persisted_events_sep{job=~\"$job\",index=~\"$index\", type=\"m.room.member\",instance=\"$instance\"}[$bucket_size])) by (origin_type, origin_entity)",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{origin_entity}} ({{origin_type}})",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "title": "Memberships/s by Origin",
+          "tooltip": {
+            "shared": true,
+            "sort": 2,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "logBase": 1,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        }
+      ],
+      "title": "Event persist rates",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 19
+      },
+      "id": 57,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "editable": true,
+          "error": false,
+          "fill": 2,
+          "grid": {},
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 62
+          },
+          "id": 4,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": true,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_http_server_requests_received{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [
+            {
+              "colorMode": "custom",
+              "fill": true,
+              "fillColor": "rgba(216, 200, 27, 0.27)",
+              "op": "gt",
+              "value": 100
+            },
+            {
+              "colorMode": "custom",
+              "fill": true,
+              "fillColor": "rgba(234, 112, 112, 0.22)",
+              "op": "gt",
+              "value": 250
+            }
+          ],
+          "title": "Request Count by arrival time",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 62
+          },
+          "id": 32,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_http_server_requests_received{instance=\"$instance\",job=~\"$job\",index=~\"$index\",method!=\"OPTIONS\"}[$bucket_size]) and topk(10,synapse_http_server_requests_received{instance=\"$instance\",job=~\"$job\",method!=\"OPTIONS\"})",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{method}} {{servlet}} {{job}}-{{index}}",
+              "refId": "A",
+              "step": 20,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "title": "Top 10 Request Counts",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "editable": true,
+          "error": false,
+          "fill": 2,
+          "grid": {},
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 70
+          },
+          "id": 23,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": true,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_http_server_response_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_response_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [
+            {
+              "colorMode": "custom",
+              "fill": true,
+              "fillColor": "rgba(216, 200, 27, 0.27)",
+              "op": "gt",
+              "value": 100,
+              "yaxis": "left"
+            },
+            {
+              "colorMode": "custom",
+              "fill": true,
+              "fillColor": "rgba(234, 112, 112, 0.22)",
+              "op": "gt",
+              "value": 250,
+              "yaxis": "left"
+            }
+          ],
+          "title": "Total CPU Usage by Endpoint",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "percentunit",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "editable": true,
+          "error": false,
+          "fill": 2,
+          "grid": {},
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 70
+          },
+          "id": 52,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": true,
+            "max": false,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "(rate(synapse_http_server_response_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_http_server_response_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) / rate(synapse_http_server_response_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [
+            {
+              "colorMode": "custom",
+              "fill": true,
+              "fillColor": "rgba(216, 200, 27, 0.27)",
+              "op": "gt",
+              "value": 100
+            },
+            {
+              "colorMode": "custom",
+              "fill": true,
+              "fillColor": "rgba(234, 112, 112, 0.22)",
+              "op": "gt",
+              "value": 250
+            }
+          ],
+          "title": "Average CPU Usage by Endpoint",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "s",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 78
+          },
+          "id": 7,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_http_server_response_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "title": "DB Usage by endpoint",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "percentunit",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "editable": true,
+          "error": false,
+          "fill": 2,
+          "grid": {},
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 78
+          },
+          "id": 47,
+          "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": false,
+            "hideEmpty": false,
+            "hideZero": true,
+            "max": true,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_http_server_response_time_seconds_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])/rate(synapse_http_server_response_time_seconds_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\",tag!=\"incremental_sync\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}} {{tag}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "title": "Non-sync avg response time",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "s",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 86
+          },
+          "id": 103,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "topk(10,synapse_http_server_in_flight_requests_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"})",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} {{method}} {{servlet}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "title": "Requests in flight",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        }
+      ],
+      "title": "Requests",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 20
+      },
+      "id": 97,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 49
+          },
+          "id": 99,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_background_process_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} {{name}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "title": "CPU usage by background jobs",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "percentunit",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 12,
+            "y": 49
+          },
+          "id": 101,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_background_process_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) +  rate(synapse_background_process_db_sched_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} {{name}}",
+              "refId": "A"
+            },
+            {
+              "expr": "",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "refId": "B"
+            }
+          ],
+          "thresholds": [],
+          "title": "DB usage by background jobs (including scheduling time)",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "percentunit",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        }
+      ],
+      "title": "Background jobs",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 21
+      },
+      "id": 81,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 64
+          },
+          "id": 79,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_federation_client_sent_transactions{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "txn rate",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "title": "Outgoing federation transaction rate",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 12,
+            "y": 64
+          },
+          "id": 83,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_federation_server_received_pdus{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "pdus",
+              "refId": "A"
+            },
+            {
+              "expr": "rate(synapse_federation_server_received_edus{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "edus",
+              "refId": "B"
+            }
+          ],
+          "thresholds": [],
+          "title": "Incoming PDU/EDU rate",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        }
+      ],
+      "title": "Federation",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 22
+      },
+      "id": 60,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "fill": 1,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 65
+          },
+          "id": 51,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_push_httppusher_http_pushes_processed{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "processed {{job}}",
+              "refId": "A",
+              "step": 20
+            },
+            {
+              "expr": "rate(synapse_push_httppusher_http_pushes_failed{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "failed {{job}}",
+              "refId": "B",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "title": "HTTP Push rate",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        }
+      ],
+      "title": "Pushes",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 23
+      },
+      "id": 58,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 24
+          },
+          "hiddenSeries": false,
+          "id": 48,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "8.5.1",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_storage_schedule_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_schedule_time_count[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeRegions": [],
+          "title": "Avg time waiting for db conn",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "s",
+              "label": "",
+              "logBase": 1,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "description": "Shows the time in which the given percentage of database queries were scheduled, over the sampled timespan",
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 24
+          },
+          "hiddenSeries": false,
+          "id": 104,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "8.5.1",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "histogram_quantile(0.99, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "{{job}} {{index}} 99%",
+              "refId": "A",
+              "step": 20
+            },
+            {
+              "expr": "histogram_quantile(0.95, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}} {{index}} 95%",
+              "refId": "B"
+            },
+            {
+              "expr": "histogram_quantile(0.90, rate(synapse_storage_schedule_time_bucket{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}} {{index}} 90%",
+              "refId": "C"
+            }
+          ],
+          "thresholds": [],
+          "timeRegions": [],
+          "title": "Db scheduling time quantiles",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "s",
+              "label": "",
+              "logBase": 1,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "editable": true,
+          "error": false,
+          "fill": 0,
+          "fillGradient": 0,
+          "grid": {},
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 31
+          },
+          "hiddenSeries": false,
+          "id": 10,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "8.5.1",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "topk(10, rate(synapse_storage_transaction_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{desc}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeRegions": [],
+          "title": "Top DB transactions by txn rate",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "logBase": 1,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "fillGradient": 0,
+          "grid": {},
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 31
+          },
+          "hiddenSeries": false,
+          "id": 11,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "hideEmpty": true,
+            "hideZero": true,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "8.5.1",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": true,
+          "steppedLine": true,
+          "targets": [
+            {
+              "expr": "rate(synapse_storage_transaction_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} {{desc}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "timeRegions": [],
+          "title": "Top DB transactions by total txn time",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "percentunit",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        }
+      ],
+      "title": "Database",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 24
+      },
+      "id": 59,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
+          "gridPos": {
+            "h": 13,
+            "w": 12,
+            "x": 0,
+            "y": 60
+          },
+          "id": 12,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_util_metrics_block_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\",block_name!=\"wrapped_request_handler\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{block_name}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "title": "Total CPU Usage by Block",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "percentunit",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
+          "gridPos": {
+            "h": 13,
+            "w": 12,
+            "x": 12,
+            "y": 60
+          },
+          "id": 26,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "(rate(synapse_util_metrics_block_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds[$bucket_size])) / rate(synapse_util_metrics_block_count[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{block_name}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "title": "Average CPU Time per Block",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "ms",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
+          "gridPos": {
+            "h": 13,
+            "w": 12,
+            "x": 0,
+            "y": 73
+          },
+          "id": 13,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\",block_name!=\"wrapped_request_handler\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}} {{block_name}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "title": "Total DB Usage by Block",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "percentunit",
+              "logBase": 1,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
+          "gridPos": {
+            "h": 13,
+            "w": 12,
+            "x": 12,
+            "y": 73
+          },
+          "id": 27,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{block_name}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "title": "Average Database Time per Block",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "ms",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
+          "gridPos": {
+            "h": 13,
+            "w": 12,
+            "x": 0,
+            "y": 86
+          },
+          "id": 28,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{block_name}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "title": "Average Transactions per Block",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "none",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
+          "gridPos": {
+            "h": 13,
+            "w": 12,
+            "x": 12,
+            "y": 86
+          },
+          "id": 25,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": false,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_util_metrics_block_time_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_count[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{block_name}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "title": "Average Wallclock Time per Block",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "ms",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        }
+      ],
+      "title": "Per-block metrics",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 25
+      },
+      "id": 61,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "decimals": 2,
+          "editable": true,
+          "error": false,
+          "fill": 0,
+          "grid": {},
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 68
+          },
+          "id": 1,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "hideEmpty": true,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_util_caches_cache:hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])/rate(synapse_util_caches_cache:total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{name}} {{job}}-{{index}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "title": "Cache Hit Ratio",
+          "tooltip": {
+            "msResolution": true,
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "percentunit",
+              "label": "",
+              "logBase": 1,
+              "max": "1",
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 68
+          },
+          "id": 8,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "synapse_util_caches_cache:size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{name}} {{job}}-{{index}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "title": "Cache Size",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "logBase": 1,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 0,
+            "y": 78
+          },
+          "id": 38,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_util_caches_cache:total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{name}} {{job}}-{{index}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "title": "Cache request rate",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "rps",
+              "logBase": 1,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "fill": 1,
+          "gridPos": {
+            "h": 10,
+            "w": 12,
+            "x": 12,
+            "y": 78
+          },
+          "id": 39,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "topk(10, rate(synapse_util_caches_cache:total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]) - rate(synapse_util_caches_cache:hits{job=\"$job\",instance=\"$instance\"}[$bucket_size]))",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{name}} {{job}}-{{index}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "title": "Top 10 cache misses",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "rps",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 88
+          },
+          "id": 65,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_util_caches_cache:evicted_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{name}} {{job}}-{{index}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "title": "Cache eviction rate",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "transparent": false,
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "label": "entries / second",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        }
+      ],
+      "title": "Caches",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 26
+      },
+      "id": 62,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 27
+          },
+          "id": 91,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": true,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[10m])",
+              "format": "time_series",
+              "instant": false,
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} gen {{gen}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "title": "Total GC time by bucket (10m smoothing)",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "percentunit",
+              "logBase": 1,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "decimals": 3,
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {},
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 12,
+            "y": 27
+          },
+          "id": 21,
+          "legend": {
+            "alignAsTable": true,
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null as zero",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(python_gc_time_sum{instance=\"$instance\",job=~\"$job\"}[$bucket_size])/rate(python_gc_time_count[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}} {{index}} gen {{gen}} ",
+              "refId": "A",
+              "step": 20,
+              "target": ""
+            }
+          ],
+          "thresholds": [],
+          "title": "Average GC Time Per Collection",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "s",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 36
+          },
+          "id": 89,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "hideEmpty": true,
+            "hideZero": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "python_gc_counts{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} gen {{gen}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "title": "Currently allocated objects",
+          "tooltip": {
+            "shared": false,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 12,
+            "y": 36
+          },
+          "id": 93,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(python_gc_unreachable_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} gen {{gen}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "title": "Object counts per collection",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 45
+          },
+          "id": 95,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(python_gc_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} gen {{gen}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "title": "GC frequency",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "cards": {
+            "cardPadding": 0
+          },
+          "color": {
+            "cardColor": "#b4ff00",
+            "colorScale": "sqrt",
+            "colorScheme": "interpolateSpectral",
+            "exponent": 0.5,
+            "min": 0,
+            "mode": "spectrum"
+          },
+          "dataFormat": "tsbuckets",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "5qpBRfD4k"
+          },
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 12,
+            "y": 45
+          },
+          "heatmap": {},
+          "highlightCards": true,
+          "id": 87,
+          "legend": {
+            "show": true
+          },
+          "links": [],
+          "targets": [
+            {
+              "expr": "sum(rate(python_gc_time_bucket[$bucket_size])) by (le)",
+              "format": "heatmap",
+              "intervalFactor": 1,
+              "legendFormat": "{{le}}",
+              "refId": "A"
+            }
+          ],
+          "title": "GC durations",
+          "tooltip": {
+            "show": true,
+            "showHistogram": false
+          },
+          "type": "heatmap",
+          "xAxis": {
+            "show": true
+          },
+          "yAxis": {
+            "format": "s",
+            "logBase": 1,
+            "show": true
+          },
+          "yBucketBound": "auto"
+        }
+      ],
+      "title": "GC",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 27
+      },
+      "id": 63,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "fill": 1,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 97
+          },
+          "id": 2,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_replication_tcp_resource_user_sync{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "user started/stopped syncing",
+              "refId": "A",
+              "step": 20
+            },
+            {
+              "expr": "rate(synapse_replication_tcp_resource_federation_ack{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "federation ack",
+              "refId": "B",
+              "step": 20
+            },
+            {
+              "expr": "rate(synapse_replication_tcp_resource_remove_pusher{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "remove pusher",
+              "refId": "C",
+              "step": 20
+            },
+            {
+              "expr": "rate(synapse_replication_tcp_resource_invalidate_cache{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "invalidate cache",
+              "refId": "D",
+              "step": 20
+            },
+            {
+              "expr": "rate(synapse_replication_tcp_resource_user_ip_cache{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "user ip cache",
+              "refId": "E",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "title": "Rate of events on replication master",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "fill": 1,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 97
+          },
+          "id": 41,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "rate(synapse_replication_tcp_resource_stream_updates{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{stream_name}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "title": "Outgoing stream updates",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "fill": 1,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 104
+          },
+          "id": 42,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum (rate(synapse_replication_tcp_protocol_inbound_commands{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{command}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "title": "Rate of incoming commands",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "fill": 1,
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 104
+          },
+          "id": 43,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum (rate(synapse_replication_tcp_protocol_outbound_commands{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])) without (name, conn_id)",
+              "format": "time_series",
+              "intervalFactor": 2,
+              "legendFormat": "{{job}}-{{index}} {{command}}",
+              "refId": "A",
+              "step": 20
+            }
+          ],
+          "thresholds": [],
+          "title": "Rate of outgoing commands",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        }
+      ],
+      "title": "Replication",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "5qpBRfD4k"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 28
+      },
+      "id": 69,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 29
+          },
+          "id": 67,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": " synapse_event_persisted_position{instance=\"$instance\",job=\"synapse\"}  - ignoring(index, job, name) group_right() synapse_event_processing_positions{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} ",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "title": "Event processing lag",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": "events",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "uid": "$datasource"
+          },
+          "fill": 1,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 12,
+            "y": 29
+          },
+          "id": 71,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "time()*1000-synapse_event_processing_last_ts{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}",
+              "format": "time_series",
+              "hide": false,
+              "intervalFactor": 1,
+              "legendFormat": "{{job}}-{{index}} {{name}}",
+              "refId": "B"
+            }
+          ],
+          "thresholds": [],
+          "title": "Age of last processed event",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "ms",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        }
+      ],
+      "title": "Event processing loop positions",
+      "type": "row"
+    }
+  ],
+  "refresh": "1m",
+  "schemaVersion": 36,
+  "style": "dark",
+  "tags": [
+    "matrix"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "Prometheus",
+          "value": "Prometheus"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "allFormat": "glob",
+        "auto": true,
+        "auto_count": 100,
+        "auto_min": "30s",
+        "current": {
+          "selected": false,
+          "text": "auto",
+          "value": "$__auto_interval_bucket_size"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "Bucket Size",
+        "multi": false,
+        "multiFormat": "glob",
+        "name": "bucket_size",
+        "options": [
+          {
+            "selected": true,
+            "text": "auto",
+            "value": "$__auto_interval_bucket_size"
+          },
+          {
+            "selected": false,
+            "text": "30s",
+            "value": "30s"
+          },
+          {
+            "selected": false,
+            "text": "1m",
+            "value": "1m"
+          },
+          {
+            "selected": false,
+            "text": "2m",
+            "value": "2m"
+          },
+          {
+            "selected": false,
+            "text": "5m",
+            "value": "5m"
+          },
+          {
+            "selected": false,
+            "text": "10m",
+            "value": "10m"
+          },
+          {
+            "selected": false,
+            "text": "15m",
+            "value": "15m"
+          }
+        ],
+        "query": "30s,1m,2m,5m,10m,15m",
+        "refresh": 2,
+        "skipUrlSync": false,
+        "type": "interval"
+      },
+      {
+        "current": {
+          "selected": false,
+          "text": "matrix-synapse:9000",
+          "value": "matrix-synapse:9000"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "$datasource"
+        },
+        "definition": "",
+        "hide": 0,
+        "includeAll": false,
+        "multi": false,
+        "name": "instance",
+        "options": [],
+        "query": {
+          "query": "label_values(synapse_util_metrics_block_ru_utime_seconds, instance)",
+          "refId": "Prometheus-instance-Variable-Query"
+        },
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allFormat": "regex wildcard",
+        "allValue": "",
+        "current": {
+          "selected": false,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "$datasource"
+        },
+        "definition": "",
+        "hide": 0,
+        "hideLabel": false,
+        "includeAll": true,
+        "label": "Job",
+        "multi": true,
+        "multiFormat": "regex values",
+        "name": "job",
+        "options": [],
+        "query": {
+          "query": "label_values(synapse_util_metrics_block_ru_utime_seconds, job)",
+          "refId": "Prometheus-job-Variable-Query"
+        },
+        "refresh": 2,
+        "refresh_on_load": false,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "allFormat": "regex wildcard",
+        "allValue": ".*",
+        "current": {
+          "selected": false,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "$datasource"
+        },
+        "definition": "",
+        "hide": 0,
+        "hideLabel": false,
+        "includeAll": true,
+        "label": "",
+        "multi": true,
+        "multiFormat": "regex values",
+        "name": "index",
+        "options": [],
+        "query": {
+          "query": "label_values(synapse_util_metrics_block_ru_utime_seconds, index)",
+          "refId": "Prometheus-index-Variable-Query"
+        },
+        "refresh": 2,
+        "refresh_on_load": false,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 3,
+        "tagValuesQuery": "",
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {
+    "now": true,
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "",
+  "title": "Synapse",
+  "uid": "000000012",
+  "version": 1,
+  "weekStart": ""
+}
\ No newline at end of file
diff --git a/matrix/entrypoint-synapse.sh b/matrix/entrypoint-synapse.sh
index 353484e..d09d26d 100755
--- a/matrix/entrypoint-synapse.sh
+++ b/matrix/entrypoint-synapse.sh
@@ -22,6 +22,7 @@ if [ ! -r "$HOMESERVER_YAML" ]; then
 #
 # added by hackerspace-zone setup scripts
 #
+enable_metrics: true
 suppress_key_server_warning: true
 web_client_location: https://${MATRIX_HOSTNAME}.${DOMAIN_NAME}
 public_baseurl: https://${MATRIX_HOSTNAME}.${DOMAIN_NAME}
@@ -38,6 +39,11 @@ oidc_providers:
         display_name_template: "{{ user.name }}"
 EOF
 
+# enable promemtheus metrics on a local-only port
+# since this is in docker, it won't be exposed to any outside connections
+	echo >&2 "**** Enabling prometheus metrics ****"
+	sed -i -e '/^listeners:/a\  - port: 9000\n    type: metrics\n' "$HOMESERVER_YAML"
+
 fi
 
 if ! grep -q '^  smtp_host:' && [ -n "$SMTP_SERVER" ]; then
diff --git a/prometheus/prometheus.yaml b/prometheus/prometheus.yaml
index 5bf3071..4601020 100644
--- a/prometheus/prometheus.yaml
+++ b/prometheus/prometheus.yaml
@@ -19,3 +19,8 @@ scrape_configs:
       - targets: ['172.17.0.1:9100']
       # cadvisor system
       - targets: ['cadvisor:8080']
+  - job_name: "synapse"
+    scrape_interval: 15s
+    metrics_path: "/_synapse/metrics"
+    static_configs:
+      - targets: ["matrix-synapse:9000"]

From 0fd744370b90d78bd86f6a20f43e6d4ce9609fd2 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@hackerspace-zone.labs.lowerlayer.nl>
Date: Fri, 11 Nov 2022 22:30:10 +0000
Subject: [PATCH 08/13] nextcloud: works with sso and has a better
 setup/install script on first run

---
 Makefile                                      |  3 +
 env.production                                |  1 +
 nextcloud.yaml                                | 48 +++++++++++
 nextcloud/docker-compose.yaml                 | 36 --------
 nextcloud/env.production                      |  1 -
 nextcloud/keycloak.sh                         |  4 +
 .../nginx.conf                                |  6 +-
 nextcloud/setup                               | 82 -------------------
 nextcloud/setup.sh                            | 52 ++++++++++++
 9 files changed, 110 insertions(+), 123 deletions(-)
 create mode 100644 nextcloud.yaml
 delete mode 100644 nextcloud/docker-compose.yaml
 delete mode 100644 nextcloud/env.production
 create mode 100755 nextcloud/keycloak.sh
 rename nginx/nginx/templates/cloud.conf.template => nextcloud/nginx.conf (91%)
 delete mode 100755 nextcloud/setup
 create mode 100755 nextcloud/setup.sh

diff --git a/Makefile b/Makefile
index 0d7e613..2405b76 100644
--- a/Makefile
+++ b/Makefile
@@ -5,6 +5,7 @@ MODULES += grafana
 MODULES += prometheus
 MODULES += mastodon
 MODULES += matrix
+MODULES += nextcloud
 #MODULES += pixelfed
 
 include env.production
@@ -41,6 +42,8 @@ matrix-shell:
 	$(DOCKER) exec matrix-synapse bash
 matrix-logs:
 	$(DOCKER) logs -f matrix-synapse
+nextcloud-logs:
+	$(DOCKER) logs -f nextcloud
 nginx-build: data/nginx/secrets
 	$(DOCKER) build nginx
 
diff --git a/env.production b/env.production
index 1d807cb..3330897 100644
--- a/env.production
+++ b/env.production
@@ -24,3 +24,4 @@ PROMETHEUS_HOSTNAME=metrics
 AUTH_URL=https://${KEYCLOAK_HOSTNAME}.${DOMAIN_NAME}/realms/${REALM}/protocol/openid-connect/auth
 TOKEN_URL=https://${KEYCLOAK_HOSTNAME}.${DOMAIN_NAME}/realms/${REALM}/protocol/openid-connect/token
 USERINFO_URL=https://${KEYCLOAK_HOSTNAME}.${DOMAIN_NAME}/realms/${REALM}/protocol/openid-connect/userinfo
+LOGOUT_URL=https://${KEYCLOAK_HOSTNAME}.${DOMAIN_NAME}/realms/${REALM}/protocol/openid-connect/logout
diff --git a/nextcloud.yaml b/nextcloud.yaml
new file mode 100644
index 0000000..9de5c77
--- /dev/null
+++ b/nextcloud.yaml
@@ -0,0 +1,48 @@
+version: "3"
+
+services:
+  nextcloud-db:
+    image: postgres:13.4-alpine
+    container_name: nextcloud-db
+    restart: always
+    environment:
+      - POSTGRES_USER=nextcloud
+      - POSTGRES_PASSWORD=nextcloud
+      - POSTGRES_DB=nextcloud
+    volumes:
+      - ./data/nextcloud/database:/var/lib/postgresql/data
+
+  nextcloud:
+    image: nextcloud:25.0.1-apache
+    container_name: nextcloud
+    restart: always
+    env_file:
+      - env.production
+    environment:
+      POSTGRES_HOST: nextcloud-db
+      POSTGRES_DB: nextcloud
+      POSTGRES_USER: nextcloud
+      POSTGRES_PASSWORD: nextcloud
+      OVERWRITEPROTOCOL: https
+      NEXTCLOUD_ADMIN_USER: admin
+      NEXTCLOUD_ADMIN_PASSWORD: ${NEXTCLOUD_ADMIN_PASSWORD}
+      NEXTCLOUD_CLIENT_SECRET: ${NEXTCLOUD_CLIENT_SECRET}
+      NEXTCLOUD_TRUSTED_DOMAINS: ${NEXTCLOUD_HOSTNAME}.${DOMAIN_NAME}
+    volumes:
+      - ./data/nextcloud/nextcloud:/var/www/html
+      - ./nextcloud/setup.sh:/setup.sh:ro
+    depends_on:
+      - nextcloud-db
+    entrypoint: ["/setup.sh"]
+
+  # add the nginx configuration into the nginx volume
+  nginx:
+    volumes:
+      - ./nextcloud/nginx.conf:/etc/nginx/templates/nextcloud.conf.template:ro
+
+  # add the grafana client secrets to the keycloak-setup volume
+  keycloak-setup:
+    env_file:
+      - data/nextcloud/secrets
+    volumes:
+      - ./nextcloud/keycloak.sh:/keycloak-setup/nextcloud.sh:ro
diff --git a/nextcloud/docker-compose.yaml b/nextcloud/docker-compose.yaml
deleted file mode 100644
index cf80a5a..0000000
--- a/nextcloud/docker-compose.yaml
+++ /dev/null
@@ -1,36 +0,0 @@
-version: "3"
-
-services:
-  database:
-    image: postgres:13.4-alpine
-    restart: always
-    environment:
-      - POSTGRES_USER=nextcloud
-      - POSTGRES_PASSWORD=nextcloud
-      - POSTGRES_DB=nextcloud
-    volumes:
-      - ../data/nextcloud/database:/var/lib/postgresql/data
-
-  nextcloud:
-    image: nextcloud:23.0.4
-    restart: always
-    ports:
-      - 9000:80
-    env_file:
-      - ../env.production
-      - env.production
-      - ../data/nextcloud/env.secrets
-    environment:
-      POSTGRES_HOST: database
-      POSTGRES_DB: nextcloud
-      POSTGRES_USER: nextcloud
-      POSTGRES_PASSWORD: nextcloud
-      OVERWRITEPROTOCOL: https
-      NEXTCLOUD_ADMIN_USER: admin
-      # NEXTCLOUD_ADMIN_PASSWORD in env.secrets
-      # NEXTCLOUD_TRUSTED_DOMAINS also set in env.secrets
-    volumes:
-      - ../data/nextcloud/nextcloud:/var/www/html
-    depends_on:
-      - database
-
diff --git a/nextcloud/env.production b/nextcloud/env.production
deleted file mode 100644
index 5fccdb9..0000000
--- a/nextcloud/env.production
+++ /dev/null
@@ -1 +0,0 @@
-# non-secret nextcloud config
diff --git a/nextcloud/keycloak.sh b/nextcloud/keycloak.sh
new file mode 100755
index 0000000..cce90b6
--- /dev/null
+++ b/nextcloud/keycloak.sh
@@ -0,0 +1,4 @@
+#!/bin/bash -x
+# Setup the OAuth client connection
+
+client-create nextcloud "$NEXTCLOUD_HOSTNAME.$DOMAIN_NAME" "$NEXTCLOUD_CLIENT_SECRET" </dev/null
diff --git a/nginx/nginx/templates/cloud.conf.template b/nextcloud/nginx.conf
similarity index 91%
rename from nginx/nginx/templates/cloud.conf.template
rename to nextcloud/nginx.conf
index 306003a..3276a21 100644
--- a/nginx/nginx/templates/cloud.conf.template
+++ b/nextcloud/nginx.conf
@@ -1,5 +1,5 @@
 server {
-	server_name ${NEXTCLOUD_HOSTNAME};
+	server_name ${NEXTCLOUD_HOSTNAME} ${NEXTCLOUD_HOSTNAME}.${DOMAIN_NAME};
 	client_max_body_size 128m;
 
 	sendfile on;
@@ -27,7 +27,7 @@ server {
 	}
 
 	location / {
-		proxy_pass http://host.docker.internal:9000;
+		proxy_pass http://nextcloud;
 		proxy_set_header Host $host;
 		proxy_set_header X-Real-IP $remote_addr;
 		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -42,5 +42,3 @@ server {
 	ssl_dhparam /etc/nginx/includes/ssl-dhparams.pem;
 
 }
-
-
diff --git a/nextcloud/setup b/nextcloud/setup
deleted file mode 100755
index fda0251..0000000
--- a/nextcloud/setup
+++ /dev/null
@@ -1,82 +0,0 @@
-#!/bin/bash
-die() { echo >&2 "$@" ; exit 1 ; }
-
-DIRNAME="$(dirname $0)"
-cd "$DIRNAME"
-source ../env.production || die "no top level env?"
-source env.production || die "no local env?"
-
-SECRETS="../data/nextcloud/env.secrets"
-if [ -r "$SECRETS" ]; then
-	docker-compose up -d || die "nextcloud: unable to start"
-	exit 0
-fi
-
-docker-compose down 2>/dev/null
-
-NEXTCLOUD_CLIENT_SECRET="$(openssl rand -hex 32)"
-NEXTCLOUD_ADMIN_PASSWORD="$(openssl rand -hex 6)"
-
-echo "Generating secrets: admin password $NEXTCLOUD_ADMIN_PASSWORD"
-mkdir -p "$(dirname "$SECRETS")"
-cat <<EOF > "$SECRETS"
-# Do not check in!
-NEXTCLOUD_ADMIN_PASSWORD=$NEXTCLOUD_ADMIN_PASSWORD
-NEXTCLOUD_TRUSTED_DOMAINS=$NEXTCLOUD_HOSTNAME
-NEXTCLOUD_CLIENT_SECRET=$NEXTCLOUD_CLIENT_SECRET
-EOF
-
-BASE="https://$KEYCLOAK_HOSTNAME/realms/$REALM/protocol/openid-connect"
-PROVIDER="$(jq -c . <<EOF
-{
-	"custom_oidc": [
-		{
-			"name":		"keycloak",
-			"title":	"Keycloak",
-			"clientId":	"nextcloud",
-			"clientSecret":	"$NEXTCLOUD_CLIENT_SECRET",
-			"authorizeUrl":	"$BASE/auth",
-			"tokenUrl":	"$BASE/token",
-			"userInfoUrl":	"$BASE/userinfo",
-			"logoutUrl":	"$BASE/logout",
-			"scope":	"openid",
-			"groupsClaim":	"roles",
-			"style":	"keycloak",
-			"displayNameClaim": "",
-			"defaultGroup":	""
-		}
-	]
-}
-EOF
-)"
-
-
-docker-compose up -d || die "unable to bring up docker"
-
-# wait for the nextcloud instance to be responsive
-# TODO: how to find out if it is ready?
-echo "Sleeping a minute while nextcloud installs"
-sleep 60
-
-
-docker-compose exec -u www-data -T nextcloud bash -x <<EOF || die "unable to configure sociallogin"
-./occ app:install calendar
-./occ app:install sociallogin
-./occ config:app:set sociallogin prevent_create_email_exists --value=1 || exit 1
-./occ config:app:set sociallogin update_profile_on_login --value=1 || exit 1
-./occ config:app:set sociallogin custom_providers --value='$PROVIDER' || exit 1
-EOF
-
-../keycloak/client-delete 'nextcloud' || echo "client did not exist?"
-
-../keycloak/client-create << EOF || die "unable to create client id"
-{
-	"clientId": "nextcloud",
-	"rootUrl": "https://$NEXTCLOUD_HOSTNAME/",
-	"adminUrl": "https://$NEXTCLOUD_HOSTNAME/",
-	"redirectUris": [ "https://$NEXTCLOUD_HOSTNAME/*" ],
-	"webOrigins": [ "https://$NEXTCLOUD_HOSTNAME" ],
-	"clientAuthenticatorType": "client-secret",
-	"secret": "$NEXTCLOUD_CLIENT_SECRET"
-}
-EOF
diff --git a/nextcloud/setup.sh b/nextcloud/setup.sh
new file mode 100755
index 0000000..3eb63fd
--- /dev/null
+++ b/nextcloud/setup.sh
@@ -0,0 +1,52 @@
+#!/bin/bash -x
+
+SERVER="apache2-foreground"
+CANARY="/var/www/html/.installed"
+if [ -r "$CANARY" ]; then
+	exec "/entrypoint.sh" "$SERVER"
+fi
+
+echo >&2 "**** installing nextcloud"
+NEXTCLOUD_UPDATE=1 bash /entrypoint.sh date || exit 1
+
+echo >&2 "***** Setting up nextcloud for ${DOMAIN_NAME}"
+occ() { su -p www-data -s /bin/sh -c "php /var/www/html/occ $*" ; }
+#occ maintenance:install || exit 1
+
+PROVIDER="$(cat <<EOF
+{
+	"custom_oidc": [
+		{
+			"name":		"keycloak",
+			"title":	"Keycloak",
+			"clientId":	"nextcloud",
+			"clientSecret":	"$NEXTCLOUD_CLIENT_SECRET",
+			"authorizeUrl":	"$AUTH_URL",
+			"tokenUrl":	"$TOKEN_URL",
+			"userInfoUrl":	"$USERINFO_URL",
+			"logoutUrl":	"$LOGOUT_URL",
+			"scope":	"openid",
+			"groupsClaim":	"roles",
+			"style":	"keycloak",
+			"displayNameClaim": "",
+			"defaultGroup":	""
+		}
+	]
+}
+EOF
+)"
+
+for app in calendar sociallogin; do
+	if [ ! -r "$CANARY.$app" ]; then
+		echo >&2 "installing app $app"
+		occ app:install $app || exit 1
+		touch "$CANARY.$app"
+	fi
+done
+
+occ config:app:set sociallogin prevent_create_email_exists --value=1 || exit 1
+occ config:app:set sociallogin update_profile_on_login --value=1 || exit 1
+occ config:app:set sociallogin custom_providers --value=\'$PROVIDER\' || exit 1
+
+touch "$CANARY"
+exec "/entrypoint.sh" "$SERVER"

From 78fb46c35d7122d88960d0624ac09479832149e2 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@hackerspace-zone.labs.lowerlayer.nl>
Date: Fri, 11 Nov 2022 23:33:20 +0000
Subject: [PATCH 09/13] keycloak: rename setup container

---
 keycloak.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/keycloak.yaml b/keycloak.yaml
index 35151f0..ad6cc6b 100644
--- a/keycloak.yaml
+++ b/keycloak.yaml
@@ -47,6 +47,7 @@ services:
   # to enable their clients using the client-create tool
   keycloak-setup:
         image: quay.io/keycloak/keycloak:18.0.0
+        container_name: keycloak-setup
         profiles:
           - setup
         depends_on:

From da94dbd1d5633f4cfa0dd304f5d373182cff9fd6 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@hackerspace-zone.labs.lowerlayer.nl>
Date: Fri, 11 Nov 2022 23:34:26 +0000
Subject: [PATCH 10/13] mobilizon: work with the new single docker environment,
 perform setup

---
 Makefile                                      |  1 +
 mobilizon.yaml                                | 62 +++++++++++++++++++
 mobilizon/docker-compose.yml                  | 26 --------
 mobilizon/env.production                      | 24 -------
 mobilizon/keycloak.sh                         |  4 ++
 .../nginx.conf                                |  6 +-
 mobilizon/setup                               | 62 -------------------
 7 files changed, 70 insertions(+), 115 deletions(-)
 create mode 100644 mobilizon.yaml
 delete mode 100644 mobilizon/docker-compose.yml
 delete mode 100644 mobilizon/env.production
 create mode 100755 mobilizon/keycloak.sh
 rename nginx/nginx/templates/events.conf.template => mobilizon/nginx.conf (80%)
 delete mode 100755 mobilizon/setup

diff --git a/Makefile b/Makefile
index 2405b76..d9ba2b0 100644
--- a/Makefile
+++ b/Makefile
@@ -6,6 +6,7 @@ MODULES += prometheus
 MODULES += mastodon
 MODULES += matrix
 MODULES += nextcloud
+MODULES += mobilizon
 #MODULES += pixelfed
 
 include env.production
diff --git a/mobilizon.yaml b/mobilizon.yaml
new file mode 100644
index 0000000..df56731
--- /dev/null
+++ b/mobilizon.yaml
@@ -0,0 +1,62 @@
+version: "3"
+
+services:
+  mobilizon:
+    image: framasoft/mobilizon
+    container_name: mobilizon
+    restart: always
+    volumes:
+      - ./data/mobilizon/uploads:/var/lib/mobilizon/uploads
+      - ./mobilizon/config.exs:/etc/mobilizon/config.exs:ro
+    environment:
+      - KEYCLOAK_HOSTNAME=${KEYCLOAK_HOSTNAME}.${DOMAIN_NAME}
+      - REALM=${REALM}
+      - MOBILIZON_INSTANCE_NAME=${MOBILIZON_HOSTNAME}.${DOMAIN_NAME}
+      - MOBILIZON_INSTANCE_HOST=${MOBILIZON_HOSTNAME}.${DOMAIN_NAME}
+      - MOBILIZON_INSTANCE_SECRET_KEY_BASE=${MOBILIZON_ADMIN_PASSWORD}
+      - MOBILIZON_INSTANCE_SECRET_KEY=${MOBILIZON_SESSION_SECRET}
+      - MOBILIZON_CLIENT_SECRET=${MOBILIZON_CLIENT_SECRET}
+      - MOBILIZON_INSTANCE_EMAIL=events@${DOMAIN_NAME}
+      - MOBILIZON_REPLY_EMAIL=noreply@${DOMAIN_NAME}
+      - MOBILIZON_SMTP_SERVER=${SMTP_SERVER}
+      - MOBILIZON_SMTP_PORT=${SMTP_PORT}
+      - MOBILIZON_SMTP_USERNAME=${SMTP_USER}
+      - MOBILIZON_SMTP_PASSWORD=${SMTP_PASSWORD}
+      - MOBILIZON_SMTP_SSL=true
+      - MOBILIZON_DATABASE_USERNAME=mobilizon
+      - MOBILIZON_DATABASE_PASSWORD=mobilizon
+      - MOBILIZON_DATABASE_DBNAME=mobilizon
+      - MOBILIZON_DATABASE_HOST=mobilizon-db
+      - MOBILIZON_INSTANCE_REGISTRATIONS_OPEN=false
+      - MOBILIZON_INSTANCE_PORT=7000
+    user: root
+    entrypoint:
+      - "/bin/sh"
+      - "-c"
+      - "chmod 777 /var/lib/mobilizon/uploads && exec su -p nobody -s /bin/sh /docker-entrypoint.sh"
+
+#    ports:
+#      - "7000:7000"
+
+  mobilizon-db:
+    image: postgis/postgis:13-3.1
+    container_name: mobilizon-db
+    restart: always
+    volumes:
+      - ./data/mobilizon/db:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_USER=mobilizon
+      - POSTGRES_PASSWORD=mobilizon
+      - POSTGRES_DB=mobilizon
+
+  # add the nginx configuration into the nginx volume
+  nginx:
+    volumes:
+      - ./mobilizon/nginx.conf:/etc/nginx/templates/mobilizon.conf.template:ro
+
+  # add the client secrets to the keycloak-setup volume
+  keycloak-setup:
+    env_file:
+      - data/mobilizon/secrets
+    volumes:
+      - ./mobilizon/keycloak.sh:/keycloak-setup/mobilizon.sh:ro
diff --git a/mobilizon/docker-compose.yml b/mobilizon/docker-compose.yml
deleted file mode 100644
index 4f359ff..0000000
--- a/mobilizon/docker-compose.yml
+++ /dev/null
@@ -1,26 +0,0 @@
-version: "3"
-
-services:
-  mobilizon:
-    image: framasoft/mobilizon
-    restart: always
-    env_file:
-      - ../env.production
-      - ./env.production
-      - ../data/mobilizon/env.secrets
-    volumes:
-      - ../data/mobilizon/uploads:/var/lib/mobilizon/uploads
-      - ./config.exs:/etc/mobilizon/config.exs:ro
-      # - ${PWD}/GeoLite2-City.mmdb:/var/lib/mobilizon/geo_db/GeoLite2-City.mmdb
-    ports:
-      - "7000:7000"
-
-  db:
-    image: postgis/postgis:13-3.1
-    restart: always
-    volumes:
-      - ../data/mobilizon/db:/var/lib/postgresql/data
-    environment:
-      - POSTGRES_USER=mobilizon
-      - POSTGRES_PASSWORD=mobilizon
-      - POSTGRES_DB=mobilizon
diff --git a/mobilizon/env.production b/mobilizon/env.production
deleted file mode 100644
index 4f67db1..0000000
--- a/mobilizon/env.production
+++ /dev/null
@@ -1,24 +0,0 @@
-# Database settings
-POSTGRES_USER=mobilizon
-POSTGRES_PASSWORD=changethis
-POSTGRES_DB=mobilizon
-MOBILIZON_DATABASE_USERNAME=mobilizon
-MOBILIZON_DATABASE_PASSWORD=mobilizon
-MOBILIZON_DATABASE_DBNAME=mobilizon
-MOBILIZON_DATABASE_HOST=db
-
-
-# Instance configuration
-MOBILIZON_INSTANCE_REGISTRATIONS_OPEN=false
-MOBILIZON_INSTANCE_PORT=7000
-
-MOBILIZON_INSTANCE_EMAIL=noreply@mobilizon.lan
-MOBILIZON_REPLY_EMAIL=contact@mobilizon.lan
-
-# Email settings
-MOBILIZON_SMTP_SERVER=localhost
-MOBILIZON_SMTP_PORT=25
-MOBILIZON_SMTP_HOSTNAME=localhost
-MOBILIZON_SMTP_USERNAME=noreply@mobilizon.lan
-MOBILIZON_SMTP_PASSWORD=password
-MOBILIZON_SMTP_SSL=false
diff --git a/mobilizon/keycloak.sh b/mobilizon/keycloak.sh
new file mode 100755
index 0000000..994117d
--- /dev/null
+++ b/mobilizon/keycloak.sh
@@ -0,0 +1,4 @@
+#!/bin/bash -x
+# Setup the OAuth client connection
+
+client-create mobilizon "$MOBILIZON_HOSTNAME.$DOMAIN_NAME" "$MOBILIZON_CLIENT_SECRET" </dev/null
diff --git a/nginx/nginx/templates/events.conf.template b/mobilizon/nginx.conf
similarity index 80%
rename from nginx/nginx/templates/events.conf.template
rename to mobilizon/nginx.conf
index b3dec83..afbb1dd 100644
--- a/nginx/nginx/templates/events.conf.template
+++ b/mobilizon/nginx.conf
@@ -1,5 +1,5 @@
 server {
-	server_name ${MOBILIZON_HOSTNAME};
+	server_name ${MOBILIZON_HOSTNAME} ${MOBILIZON_HOSTNAME}.${DOMAIN_NAME};
 	client_max_body_size 128m;
 
 	sendfile on;
@@ -14,7 +14,7 @@ server {
 	proxy_read_timeout 1800s;
 
 	location / {
-		proxy_pass http://host.docker.internal:7000;
+		proxy_pass http://mobilizon:7000;
 		proxy_set_header Host $host;
 		proxy_set_header X-Real-IP $remote_addr;
 		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -23,7 +23,7 @@ server {
 
 	# force login with OIDC
 	location /login {
-		return 302 https://${MOBILIZON_HOSTNAME}/auth/keycloak;
+		return 302 https://${MOBILIZON_HOSTNAME}.${DOMAIN_NAME}/auth/keycloak;
 	}
 
 	listen 443 ssl;
diff --git a/mobilizon/setup b/mobilizon/setup
deleted file mode 100755
index b3d49dc..0000000
--- a/mobilizon/setup
+++ /dev/null
@@ -1,62 +0,0 @@
-#!/bin/bash
-die() { echo >&2 "mobilizon: $@" ; exit 1 ; }
-
-DIRNAME="$(dirname $0)"
-cd "$DIRNAME"
-source ../env.production || die "no top level env?"
-source env.production || die "no local env?"
-source ../env.smtp 2>/dev/null
-
-DATA="../data/mobilizon"
-SECRETS="$DATA/env.secrets"
-
-if [ -r "$SECRETS" ]; then
-	docker-compose up -d || die "unable to start"
-	exit 0
-fi
-
-docker-compose down 2>/dev/null
-
-CLIENT_SECRET="$(openssl rand -hex 20)"
-
-mkdir -p "$DATA/uploads"
-chmod 777 "$DATA/uploads"
-
-mkdir -p "$(dirname "$SECRETS")"
-cat <<EOF > "$SECRETS"
-# DO NOT CHECK IN
-MOBILIZON_INSTANCE_NAME=${DOMAIN_NAME}
-MOBILIZON_INSTANCE_HOST=${MOBILIZON_HOSTNAME}
-MOBILIZON_INSTANCE_SECRET_KEY_BASE=$(openssl rand -hex 20)
-MOBILIZON_INSTANCE_SECRET_KEY=$(openssl rand -hex 20)
-MOBILIZON_CLIENT_SECRET=${CLIENT_SECRET}
-EOF
-
-if [ -n "$SMTP_SERVER" ]; then
-	cat <<EOF >> "$SECRETS"
-MOBILIZON_INSTANCE_EMAIL=events@${DOMAIN_NAME}
-MOBILIZON_REPLY_EMAIL=noreply@${DOMAIN_NAME}
-MOBILIZON_SMTP_SERVER=${SMTP_SERVER}
-MOBILIZON_SMTP_PORT=${SMTP_PORT}
-MOBILIZON_SMTP_USERNAME=${SMTP_USER}
-MOBILIZON_SMTP_PASSWORD=${SMTP_PASSWORD}
-EOF
-fi
-
-../keycloak/client-delete mobilizon
-
-../keycloak/client-create <<EOF || die "unable to create client"
-{
-	"clientId": "mobilizon",
-	"rootUrl": "https://$MOBILIZON_HOSTNAME",
-	"adminUrl": "https://$MOBILIZON_HOSTNAME",
-	"redirectUris": [ "https://$MOBILIZON_HOSTNAME/*" ],
-	"webOrigins": [ "https://$MOBILIZON_HOSTNAME" ],
-	"clientAuthenticatorType": "client-secret",
-	"secret": "$CLIENT_SECRET"
-}
-EOF
-
-docker-compose up -d || die "unable to start container"
-
-

From 154c7208c3186bc49277be28a1219be7fc5907e0 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@hackerspace-zone.labs.lowerlayer.nl>
Date: Sun, 13 Nov 2022 18:58:14 +0000
Subject: [PATCH 11/13] README: updated install instructions

---
 README.md | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index 7243285..c6baf12 100644
--- a/README.md
+++ b/README.md
@@ -4,17 +4,13 @@ Infrastructure for the self-hosted, single-sign-on, community-run services.
 
 * Set the domain name in `env.production`
 * Create the DNS entries in the domain for `login`, `cloud`, `matrix`, `dashboard`, `docs` and maybe more.
-* Install dependencies:
+* Install dependencies (note that `docker-compose 1.25` breaks environment variables as we use them):
 
 ```
-apt install jq docker-compose
-apt install prometheus
+apt install python3-pip prometheus
+pip3 install docker-compose
 ```
 
-* Setup each of the services. `keycloak` and `nginx` are required to start the others:
-
-```
-./keycloak/setup
-./nginx/setup
-./start-all
-```
+* `make run` to startup all of the containers
+* `make keycloak-setup` to setup all of the OIDC links
+* `make down` to stop everything

From fcd2565b47a41aa04d223e1cc3fc32221f12d595 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@hackerspace-zone.labs.lowerlayer.nl>
Date: Sun, 13 Nov 2022 18:58:46 +0000
Subject: [PATCH 12/13] wireguard: instructions for setting up proxy

---
 wireguard/README.md       | 29 +++++++++++++++++++++++++++++
 wireguard/wg0-proxy.conf  | 33 +++++++++++++++++++++++++++++++++
 wireguard/wg0-server.conf | 28 ++++++++++++++++++++++++++++
 3 files changed, 90 insertions(+)
 create mode 100644 wireguard/README.md
 create mode 100644 wireguard/wg0-proxy.conf
 create mode 100644 wireguard/wg0-server.conf

diff --git a/wireguard/README.md b/wireguard/README.md
new file mode 100644
index 0000000..cdabf97
--- /dev/null
+++ b/wireguard/README.md
@@ -0,0 +1,29 @@
+# Wireguard proxy setup
+
+This is for a server that is inside of a firewall or behind a NAT gateway
+that doesn't have a static IP address.  A cheap $6/month DigitalOcean droplet
+can be created that will route *all* internet traffic to the server, allowing
+it to change IP.
+
+* On both proxy and the server:
+
+```
+sudo apt install wireguard-tools net-tools
+wg genkey \
+| sudo tee /etc/wireguard/wg0.key \
+| wg pubkey \
+| sudo tee /etc/wireguard/wg0.pub
+sudo chmod -R go-rwx /etc/wireguard
+```
+
+* Copy `wireguard/wg0-proxy.conf` to `/etc/wireguard/wg0.conf` on the proxy
+* On the **proxy** edit `/etc/wireguard/wg0.conf`:
+  * Change `${SERVER_PUBKEY}` to the public key that was output on the server
+
+* Copy `wireguard/wg0-server.conf` to `/etc/wireguard/wg0.conf` on the server.
+* On the **server** edit `/etc/wireguard/wg0.conf`:
+  * Change `${PROXY_IP}` to the public IP address of the proxy (two places)
+  * Change `${PROXY_PUBKEY}` to the public key output on the proxy (two places)
+  * Change `${SERVER_GW}` to the gateway address used to reach the internet from the server
+
+* On both machines run `sudo wg-quick up /etc/wireguard/wg0.conf`
diff --git a/wireguard/wg0-proxy.conf b/wireguard/wg0-proxy.conf
new file mode 100644
index 0000000..e7bdbc5
--- /dev/null
+++ b/wireguard/wg0-proxy.conf
@@ -0,0 +1,33 @@
+[Interface]
+Address = 192.168.4.1/24
+ListenPort = 51820
+
+PostUp = wg set %i private-key /etc/wireguard/%i.key
+
+# Enable IP masquerading for the remote host
+PostUp = echo 1 > /proc/sys/net/ipv4/ip_forward
+PostUp = iptables -A FORWARD -i %i -j ACCEPT
+PostUp = iptables -A FORWARD -o %i -j ACCEPT
+PostUp = iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
+
+# accept the wireguard connection
+PostUp = iptables -t nat -A PREROUTING -i eth0 -p udp --dport 51820 -j ACCEPT
+
+# redirect ssh to port 23
+PostUp = iptables -t nat -A PREROUTING -i eth0 -p tcp --dport 23 -j REDIRECT --to-port 22
+
+# redirect *all* traffic to the wg tunnel
+PostUp = iptables -t nat -A PREROUTING -i eth0 -p all -j DNAT --to-destination 192.168.4.2
+
+# Tear down the proxy
+PostDown = iptables -D FORWARD -i %i -j ACCEPT
+PostDown = iptables -D FORWARD -o %i -j ACCEPT
+PostDown = iptables -t nat -D POSTROUTING -o eth0 -j MASQUERADE
+
+PostDown = iptables -t nat -D PREROUTING -i eth0 -p udp --dport 51820 -j ACCEPT
+PostDown = iptables -t nat -D PREROUTING -i eth0 -p tcp --dport 23 -j REDIRECT -to-port 22
+PostDown = iptables -t nat -D PREROUTING -i eth0 -p all -j DNAT --to-destination 192.168.4.2
+
+[Peer]
+PublicKey = ${SERVER_PUBKEY}
+AllowedIPs = 192.168.4.2/32
diff --git a/wireguard/wg0-server.conf b/wireguard/wg0-server.conf
new file mode 100644
index 0000000..9e65fba
--- /dev/null
+++ b/wireguard/wg0-server.conf
@@ -0,0 +1,28 @@
+# wg0-server.conf
+#
+# This is the configuration for the server hidden behind the wireguard proxy.
+# It routes all internet traffic via the proxy, with the exception of traffic
+# to the proxy itself.  It is still accessible on the local network.
+#
+# When moving this to a new machine:
+# * Update the PostUp route so that the proxy address has an explicit route via the local gateway
+# * Update the PownDown to delete the explicit route and restore the default gw
+# * Update the Peer PublicKey and Endpoint with the proxy key and address
+#
+[Interface]
+PostUp = wg set %i private-key /etc/wireguard/%i.key
+Address = 192.168.4.2/24
+
+# Delete the default gateway and add an explicit route for the wireguard tunnel
+PostUp = route add ${PROXY_IP} gw ${SERVER_GW} || echo "wrong route"
+PostUp = route del default || echo "no default"
+PostUp = route add default gw 192.168.4.1
+
+PostDown = route del ${PROXY_IP}
+PostDown = route add default gw ${SERVER_GW}
+
+[Peer]
+PublicKey = ${PROXY_PUBKEY}
+Endpoint = ${PROXY_IP}:51820
+AllowedIPs = 0.0.0.0/0
+PersistentKeepalive = 25

From 083a1f37624921883b4b9dd6880697a1cf11db75 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@hackerspace-zone.labs.lowerlayer.nl>
Date: Sun, 13 Nov 2022 20:34:50 +0000
Subject: [PATCH 13/13] gitea: hacked to work with the new format

---
 Makefile                                      |  7 ++
 gitea.yaml                                    | 69 +++++++++++++++++++
 gitea/README.md                               |  5 +-
 gitea/docker-compose.yaml                     | 46 -------------
 gitea/env.production                          |  7 --
 gitea/{add-ssh-user => host-setup.sh}         |  0
 gitea/keycloak.sh                             |  4 ++
 .../git.conf.template => gitea/nginx.conf     | 14 ++--
 gitea/setup                                   | 69 -------------------
 gitea/setup.sh                                | 36 ++++++++++
 10 files changed, 126 insertions(+), 131 deletions(-)
 create mode 100644 gitea.yaml
 delete mode 100644 gitea/docker-compose.yaml
 delete mode 100644 gitea/env.production
 rename gitea/{add-ssh-user => host-setup.sh} (100%)
 create mode 100755 gitea/keycloak.sh
 rename nginx/nginx/templates/git.conf.template => gitea/nginx.conf (75%)
 delete mode 100755 gitea/setup
 create mode 100755 gitea/setup.sh

diff --git a/Makefile b/Makefile
index d9ba2b0..638055f 100644
--- a/Makefile
+++ b/Makefile
@@ -7,6 +7,7 @@ MODULES += mastodon
 MODULES += matrix
 MODULES += nextcloud
 MODULES += mobilizon
+MODULES += gitea
 #MODULES += pixelfed
 
 include env.production
@@ -66,6 +67,12 @@ data/%/secrets:
 	echo >>$@ "export $(GET_MODULE)_CLIENT_SECRET=$(call RAND,20)"
 	echo >>$@ "export $(GET_MODULE)_SESSION_SECRET=$(call RAND,20)"
 
+data/gitea/secrets: data/gitea/host-setup.done
+data/gitea/host-setup.done:
+	sudo ./gitea/host-setup.sh
+	mkdir -p $(dir $@)
+	touch $@
+
 keycloak-setup: secrets-setup
 	$(DOCKER) run keycloak-setup
 
diff --git a/gitea.yaml b/gitea.yaml
new file mode 100644
index 0000000..c2ea9db
--- /dev/null
+++ b/gitea.yaml
@@ -0,0 +1,69 @@
+# gitea requires ssh access from the host machine, which needs special setup
+# In order to create the git user and auth keys, you need to run:
+#
+#     sudo gitea/setup.sh
+#
+version: "3"
+
+services:
+  gitea:
+    image: gitea/gitea:1.17.3
+    container_name: gitea
+    env_file:
+      - ./env.production
+    environment:
+      - USER_UID=2222 # must match git user on host system
+      - USER_GID=2222
+      - GITEA_CLIENT_SECRET=${GITEA_CLIENT_SECRET}
+      - GITEA_ADMIN_PASSWORD=${GITEA_ADMIN_PASSWORD}
+      - GITEA__database__DB_TYPE=postgres
+      - GITEA__database__HOST=gitea-db:5432
+      - GITEA__database__NAME=gitea
+      - GITEA__database__USER=gitea
+      - GITEA__database__PASSWD=gitea
+      - GITEA__oauth2_client__ENABLE_AUTO_REGISTRATION=true
+      - GITEA__openid__ENABLE_OPENID_SIGNIN=true
+      - GITEA__openid__ENABLE_OPENID_SIGNUP=false
+      - GITEA__service__DISABLE_REGISTRATION=true
+      - GITEA__service__ALLOW_ONLY_EXTERNAL_REGISTRATION=true
+      - GITEA__repository__DEFAULT_BRANCH=main
+      - GITEA__server__ROOT_URL=https://${GITEA_HOSTNAME}.${DOMAIN_NAME}/
+      - GITEA__server__SSH_DOMAIN=${GITEA_HOSTNAME}.${DOMAIN_NAME}
+      - GITEA__security__SECRET_KEY=${GITEA_SESSION_SECRET}
+      - GITEA__security__INSTALL_LOCK=true
+    entrypoint: ["/setup.sh"]
+    volumes:
+      - ./gitea/setup.sh:/setup.sh:ro
+      - ./data/gitea:/data
+      - /etc/timezone:/etc/timezone:ro
+      - /etc/localtime:/etc/localtime:ro
+      - /home/git/.ssh/:/data/git/.ssh
+    ports:
+#      - "3030:3000"
+      - "2222:22"  # route host port 2222 to container port 22 for inbound ssh
+    restart: always
+    depends_on:
+      - gitea-db
+ 
+  gitea-db:
+    image: postgres:13.4-alpine
+    container_name: gitea-db
+    restart: always
+    environment:
+      - POSTGRES_USER=gitea
+      - POSTGRES_PASSWORD=gitea
+      - POSTGRES_DB=gitea
+    volumes:
+      - ./data/gitea/postgres:/var/lib/postgresql/data
+
+  # add the gitea nginx configuration into the nginx volume
+  nginx:
+    volumes:
+      - ./gitea/nginx.conf:/etc/nginx/templates/gitea.conf.template:ro
+
+  # add the gitea client secrets to the keycloak-setup volume
+  keycloak-setup:
+    env_file:
+      - data/gitea/secrets
+    volumes:
+      - ./gitea/keycloak.sh:/keycloak-setup/gitea.sh:ro
diff --git a/gitea/README.md b/gitea/README.md
index ef1b0be..2930b65 100644
--- a/gitea/README.md
+++ b/gitea/README.md
@@ -1,3 +1,6 @@
 # gitea
 
-OIDC setup is now automated
+There is still a sudo step that has to happen on the host.
+
+OIDC setup is now automated in the container.
+
diff --git a/gitea/docker-compose.yaml b/gitea/docker-compose.yaml
deleted file mode 100644
index 0fee41d..0000000
--- a/gitea/docker-compose.yaml
+++ /dev/null
@@ -1,46 +0,0 @@
-version: "3"
-
-networks:
-  gitea:
-    external: false
-
-services:
-  gitea:
-    image: gitea/gitea:1.16.6
-    env_file:
-      - ../env.production
-      - env.production
-      - ../data/gitea/env.secrets
-    environment:
-      - USER_UID=2222 # must match git user on host system
-      - USER_GID=2222
-      - GITEA__database__DB_TYPE=postgres
-      - GITEA__database__HOST=db:5432
-      - GITEA__database__NAME=gitea
-      - GITEA__database__USER=gitea
-      - GITEA__database__PASSWD=gitea
-    networks:
-      - gitea
-    volumes:
-      - ../data/gitea:/data
-      - /etc/timezone:/etc/timezone:ro
-      - /etc/localtime:/etc/localtime:ro
-      - /home/git/.ssh/:/data/git/.ssh
-    ports:
-      - "3030:3000"
-      - "2222:22"
-    restart: always
-    depends_on:
-      - db
- 
-  db:
-    image: postgres:13.4-alpine
-    restart: always
-    environment:
-      - POSTGRES_USER=gitea
-      - POSTGRES_PASSWORD=gitea
-      - POSTGRES_DB=gitea
-    volumes:
-      - ../data/gitea/postgres:/var/lib/postgresql/data
-    networks:
-      - gitea
diff --git a/gitea/env.production b/gitea/env.production
deleted file mode 100644
index 70b7682..0000000
--- a/gitea/env.production
+++ /dev/null
@@ -1,7 +0,0 @@
-# gitea config for keycloak integration
-# only allow open id sign-in, turn off all other registrations
-GITEA__openid__ENABLE_OPENID_SIGNIN=true
-GITEA__openid__ENABLE_OPENID_SIGNUP=false
-#GITEA__service__DISABLE_REGISTRATION=true
-GITEA__service__ALLOW_ONLY_EXTERNAL_REGISTRATION=true
-GITEA__repository__DEFAULT_BRANCH=main
diff --git a/gitea/add-ssh-user b/gitea/host-setup.sh
similarity index 100%
rename from gitea/add-ssh-user
rename to gitea/host-setup.sh
diff --git a/gitea/keycloak.sh b/gitea/keycloak.sh
new file mode 100755
index 0000000..dc2341b
--- /dev/null
+++ b/gitea/keycloak.sh
@@ -0,0 +1,4 @@
+#!/bin/bash -x
+# Setup the gitea client connection
+
+client-create gitea "$GITEA_HOSTNAME.$DOMAIN_NAME" "$GITEA_CLIENT_SECRET" </dev/null
diff --git a/nginx/nginx/templates/git.conf.template b/gitea/nginx.conf
similarity index 75%
rename from nginx/nginx/templates/git.conf.template
rename to gitea/nginx.conf
index 5c96a3a..25671e8 100644
--- a/nginx/nginx/templates/git.conf.template
+++ b/gitea/nginx.conf
@@ -1,5 +1,5 @@
 server {
-	server_name ${GITEA_HOSTNAME};
+	server_name ${GITEA_HOSTNAME} ${GITEA_HOSTNAME}.${DOMAIN_NAME};
 	client_max_body_size 128m;
 
 	sendfile on;
@@ -14,17 +14,17 @@ server {
 	proxy_read_timeout 1800s;
 
 	location / {
-		proxy_pass http://host.docker.internal:3030;
+		proxy_pass http://gitea:3000;
 		proxy_set_header Host $host;
 		proxy_set_header X-Real-IP $remote_addr;
 		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 		proxy_set_header X-Forwarded-Proto $scheme;
 	}
 
-  # force login with OIDC
-  location /user/login {
-    return 302 https://${GITEA_HOSTNAME}/user/oauth2/keycloak;
-  }
+	# force login with OIDC
+	location /user/login {
+		return 302 https://${GITEA_HOSTNAME}.${DOMAIN_NAME}/user/oauth2/keycloak;
+	}
 
 	listen 443 ssl;
 	ssl_certificate /etc/letsencrypt/live/${DOMAIN_NAME}/fullchain.pem;
@@ -33,5 +33,3 @@ server {
 	include /etc/nginx/includes/challenge.conf;
 	ssl_dhparam /etc/nginx/includes/ssl-dhparams.pem;
 }
-
-
diff --git a/gitea/setup b/gitea/setup
deleted file mode 100755
index 67d22be..0000000
--- a/gitea/setup
+++ /dev/null
@@ -1,69 +0,0 @@
-#!/bin/bash
-die() { echo >&2 "gitea: ERROR $*" ; exit 1 ; }
-info() { echo >&2 "gitea: $*" ; }
-
-DIRNAME="$(dirname $0)"
-cd "$DIRNAME"
-
-source ../env.production || die "no top level environment"
-source ./env.production || die "no local environment"
-
-DATA="../data/gitea"
-SECRETS="$DATA/env.secrets"
-INI="$DATA/gitea/conf/app.ini"
-
-if [ -r "$SECRETS" ]; then
-	docker-compose up -d || die "unable to start"
-	exit 0
-fi
-
-./add-ssh-user || die "unable to add ssh user"
-
-GITEA_CLIENT_SECRET="$(openssl rand -hex 32)"
-GITEA_ADMIN_PASSWORD="$(openssl rand -hex 8)"
-
-info "creating new secrets $SECRETS"
-
-mkdir -p "$DATA"
-cat <<EOF > "$SECRETS"
-# DO NOT CHECK IN
-GITEA_CLIENT_SECRET=$GITEA_CLIENT_SECRET
-GITEA_ADMIN_PASSWORD=$GITEA_ADMIN_PASSWORD
-GITEA__server__ROOT_URL=https://$GITEA_HOSTNAME/
-GITEA__server__SSH_DOMAIN=$GITEA_HOSTNAME
-GITEA__security__INSTALL_LOCK=true
-GITEA__security__SECRET_KEY=$(openssl rand -hex 32)
-EOF
-
-
-docker-compose down 2>/dev/null
-
-../keycloak/client-delete gitea 2>/dev/null
-../keycloak/client-create <<EOF || die "unable to create gitea client"
-{
-	"clientId": "gitea",
-	"rootUrl": "https://$GITEA_HOSTNAME",
-	"adminUrl": "https://$GITEA_HOSTNAME",
-	"redirectUris": [ "https://$GITEA_HOSTNAME/*" ],
-	"webOrigins": [ "https://$GITEA_HOSTNAME" ],
-	"clientAuthenticatorType": "client-secret",
-	"secret": "$GITEA_CLIENT_SECRET"
-}
-EOF
-
-docker-compose up -d || die "unable to start container"
-
-info "waiting for startup..."
-sleep 5
-
-info "adding oauth login"
-docker-compose exec -u git gitea \
-	gitea admin auth add-oauth \
-	--name "keycloak" \
-	--provider "openidConnect" \
-	--key "gitea" \
-	--secret "$GITEA_CLIENT_SECRET" \
-	--auto-discover-url "https://${KEYCLOAK_HOSTNAME}/realms/${REALM}/.well-known/openid-configuration" \
-	--group-claim-name "groups" \
-	--admin-group "admin" \
-|| die "unable to add oauth interface"
diff --git a/gitea/setup.sh b/gitea/setup.sh
new file mode 100755
index 0000000..091bd35
--- /dev/null
+++ b/gitea/setup.sh
@@ -0,0 +1,36 @@
+#!/bin/bash -x
+# This is *container* setup for the OIDC stuff
+
+export APP_NAME="${DOMAIN_NAME} Gitea"
+OIDC_CANARY="/data/oidc.done"
+
+if [ -r "$OIDC_CANARY" ]; then
+	# based on https://github.com/go-gitea/gitea/blob/main/Dockerfile
+	exec "/usr/bin/entrypoint" "/bin/s6-svscan"  "/etc/s6";
+fi
+
+
+# We have to do some setup, so start things and wait for the config
+# file to appear so that we can edit it.
+"/usr/bin/entrypoint" "/bin/s6-svscan"  "/etc/s6" &
+
+echo >&2 "*** Sleeping for setup"
+sleep 30
+
+echo >&2 "*** Adding OIDC login for $DOMAIN_NAME"
+su -s /bin/sh git <<EOF || exit 1
+gitea admin auth add-oauth \
+	--name "keycloak" \
+	--provider "openidConnect" \
+	--key "gitea" \
+	--secret "${GITEA_CLIENT_SECRET}" \
+	--auto-discover-url "https://${KEYCLOAK_HOSTNAME}.${DOMAIN_NAME}/realms/${REALM}/.well-known/openid-configuration" \
+	--group-claim-name "groups" \
+	--admin-group "admin" \
+
+EOF
+
+touch "${OIDC_CANARY}"
+
+echo >&2 "*** Done, maybe it works?"
+exit 0