Self-Hosted Cloud on a VPS: Fedora Server + MicroShift + GitOps

sed -i 's/^#Port 22/Port 222/' /etc/ssh/sshd_config
grep -q "^PermitRootLogin" /etc/ssh/sshd_config \
  && sed -i 's/^PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config \
  || echo "PermitRootLogin prohibit-password" >> /etc/ssh/sshd_config
grep -q "^PasswordAuthentication" /etc/ssh/sshd_config \
  && sed -i 's/^PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config \
  || echo "PasswordAuthentication no" >> /etc/ssh/sshd_config

# SELinux: allow sshd on port 222
dnf install -y policycoreutils-python-utils
semanage port -a -t ssh_port_t -p tcp 222

# Copy your authorized_keys, then:
systemctl restart sshd

dnf install -y etckeeper
etckeeper init
etckeeper commit "initial fedora server setup"

dnf install -y \
  vim-enhanced htop jq \
  podman btrfs-progs \
  fail2ban wireguard-tools \
  snapper btrbk \
  dnf5-automatic \
  policycoreutils-python-utils

cat > /etc/sysctl.d/99-microshift.conf <<'EOF'
fs.inotify.max_user_watches = 524288
fs.inotify.max_user_instances = 16384
EOF

cat > /etc/sysctl.d/90-redis.conf <<'EOF'
vm.overcommit_memory = 1
EOF

cat > /etc/sysctl.d/90-wireguard.conf <<'EOF'
net.ipv4.ip_forward = 1
EOF

sysctl --system

install -m 600 /dev/stdin /etc/telegramrc <<'EOF'
TOKEN=<YOUR_BOT_TOKEN>
CHATID=<YOUR_CHAT_ID>
EOF

#!/bin/bash
# Usage: sendtelegram.sh [-c configfile] [-t token] [-i chatid] [-m message]

while getopts ":c:t:i:p:m:v" opt; do
    case "$opt" in
        c) CONFIGFILE=$OPTARG ;;  t) TOKEN_ARG=$OPTARG ;;
        i) CHATID_ARG=$OPTARG ;;  m) TEXT=$OPTARG ;;
        p) PARSEMODE_ARG=$OPTARG ;; v) VERBOSE=1 ;;
    esac
done

if [ -n "$CONFIGFILE" ]; then . "$CONFIGFILE"
elif [ -f /etc/telegramrc ]; then . /etc/telegramrc; fi

if [ -n "$TOKEN_ARG" ]; then TOKEN=$TOKEN_ARG; fi
if [ -n "$CHATID_ARG" ]; then CHATID=$CHATID_ARG; fi

URL="https://api.telegram.org/bot$TOKEN/sendMessage"
CMDARGS="chat_id=$CHATID&disable_web_page_preview=1&text=$TEXT"
[ -n "${PARSEMODE_ARG:-}" ] && CMDARGS="${CMDARGS}&parse_mode=$PARSEMODE_ARG"
curl -s --max-time 10 -d "$CMDARGS" "$URL" > /dev/null

install -m 755 sendtelegram.sh /usr/local/bin/sendtelegram.sh
sendtelegram.sh -m "Base system ready"

cat > /etc/dnf/automatic.conf <<'EOF'
[commands]
upgrade_type = default
random_sleep = 0
download_updates = yes
apply_updates = yes
reboot = when-needed
reboot_command = "shutdown -r +5 'Reboot after automatic update'"

[emitters]
emit_via = stdio
EOF

systemctl enable --now dnf5-automatic.timer

systemctl mask passim

etckeeper commit "base system: ssh/222, packages, sysctl, telegram, dnf-automatic"
snapper -c root create -d "basis-system-fertig"

mkdir -p /mnt/btrfs-top
mount -o subvolid=5 /dev/vda3 /mnt/btrfs-top

for sv in var home var_log var_cache var_tmp \
          var_lib_microshift var_lib_pvc var_lib_containers var_lib_kubelet \
          data btrbk_snapshots; do
  btrfs subvolume create /mnt/btrfs-top/$sv
done

umount /mnt/btrfs-top

mkdir -p /var/lib/{microshift,pvc,containers,kubelet} /data /mnt/btrfs-top
systemctl daemon-reload && mount -a

semanage fcontext -a -t container_file_t "/var/lib/pvc(/.*)?"
restorecon -Rv /var/lib/pvc

# Root
snapper -c root create-config /
snapper -c root set-config \
  TIMELINE_CREATE=yes TIMELINE_CLEANUP=yes \
  TIMELINE_LIMIT_HOURLY=24 TIMELINE_LIMIT_DAILY=8 \
  TIMELINE_LIMIT_WEEKLY=5 TIMELINE_LIMIT_MONTHLY=0 TIMELINE_LIMIT_YEARLY=0 \
  NUMBER_CLEANUP=yes NUMBER_LIMIT=20

# All other subvolumes — same retention
for CFG_SUBVOL in \
  "home:/home" "data:/data" \
  "var_lib_pvc:/var/lib/pvc" "var_lib_microshift:/var/lib/microshift" \
  "var:/var" "var_log:/var/log" "var_lib_containers:/var/lib/containers"; do
  CFG="${CFG_SUBVOL%%:*}"
  SUBVOL="${CFG_SUBVOL##*:}"
  snapper -c "$CFG" create-config "$SUBVOL"
  snapper -c "$CFG" set-config \
    TIMELINE_CREATE=yes TIMELINE_CLEANUP=yes \
    TIMELINE_LIMIT_HOURLY=24 TIMELINE_LIMIT_DAILY=8 \
    TIMELINE_LIMIT_WEEKLY=5 TIMELINE_LIMIT_MONTHLY=0 TIMELINE_LIMIT_YEARLY=0 \
    NUMBER_CLEANUP=yes NUMBER_LIMIT=20
done

systemctl enable --now snapper-timeline.timer snapper-cleanup.timer

cat > /usr/local/bin/snap-all <<'EOF'
#!/bin/bash
set -euo pipefail
DESC="${1:?Verwendung: snap-all <beschreibung>}"
for cfg in root home data var_lib_pvc var_lib_microshift var var_log var_lib_containers; do
  printf "  %-22s ... " "$cfg"
  snapper -c "$cfg" create --cleanup-algorithm number --description "$DESC"
  echo "ok"
done
EOF
chmod 755 /usr/local/bin/snap-all

dnf install -y make gettext
git clone https://github.com/Antynea/grub-btrfs /root/git/grub-btrfs
cd /root/git/grub-btrfs && make install

# Fedora-specific paths
sed -i \
  -e 's|#GRUB_BTRFS_GRUB_DIRNAME=.*|GRUB_BTRFS_GRUB_DIRNAME="/boot/grub2"|' \
  -e 's|#GRUB_BTRFS_SCRIPT_CHECK=.*|GRUB_BTRFS_SCRIPT_CHECK=grub2-script-check|' \
  /etc/default/grub-btrfs/config

systemctl enable --now grub-btrfsd.service
grub2-mkconfig -o /boot/grub2/grub.cfg

cat > /etc/btrbk/btrbk.conf <<'EOF'
timestamp_format        long
# Local: keep only the latest snapshot as parent reference for incremental send
# snapper handles local rollback, btrbk local snapshots are minimal
snapshot_preserve_min   latest
snapshot_preserve       2h 1d 0w
# Remote (Pi): full retention history
target_preserve_min     1h
target_preserve         24h 8d 5w

ssh_identity            /root/.ssh/id_ed25519

volume /mnt/btrfs-top
  snapshot_dir  btrbk_snapshots

  subvolume root
  subvolume var
  subvolume home
  subvolume var_log
  subvolume var_lib_pvc
  subvolume var_lib_microshift
  subvolume var_lib_containers
  subvolume data

  target send-receive   ssh://backupuser@10.0.0.12/backup/btrfs/server
EOF

systemctl enable --now btrbk.timer

cat > /usr/local/sbin/boot-backup.sh <<'EOF'
#!/bin/bash
set -euo pipefail
DEST=/var/lib/boot-backup
mkdir -p "$DEST/boot" "$DEST/efi"
rsync -aAX --delete /boot/     "$DEST/boot/"
rsync -aAX --delete /boot/efi/ "$DEST/efi/"
sfdisk --dump /dev/vda > "$DEST/partition-table.sfdisk"
sgdisk --backup="$DEST/partition-table.sgdisk" /dev/vda
EOF
chmod 755 /usr/local/sbin/boot-backup.sh

# systemd timer (daily + after every dnf5-automatic run)
# See full unit files in the repo
systemctl enable --now boot-backup.timer

# FedoraServer zone — remove defaults, add only what's needed
firewall-cmd --permanent --zone=FedoraServer --remove-service=cockpit
firewall-cmd --permanent --zone=FedoraServer --remove-service=dhcpv6-client

# Custom SSH service on port 222
firewall-cmd --permanent --new-service=myssh
firewall-cmd --permanent --service=myssh --add-port=222/tcp
firewall-cmd --permanent --zone=FedoraServer --add-service=myssh
firewall-cmd --permanent --zone=FedoraServer --remove-service=ssh

# HTTP(S) for ingress
firewall-cmd --permanent --zone=FedoraServer --add-port=80/tcp
firewall-cmd --permanent --zone=FedoraServer --add-port=443/tcp

# Mail ports
for p in 25 587 993; do
  firewall-cmd --permanent --zone=FedoraServer --add-port=${p}/tcp
done

# WireGuard
firewall-cmd --permanent --zone=FedoraServer --add-port=51820/udp

# acme-dns (DNS-01 challenge server)
firewall-cmd --permanent --zone=FedoraServer --add-port=53/tcp
firewall-cmd --permanent --zone=FedoraServer --add-port=53/udp

# NAT for WireGuard clients
firewall-cmd --permanent --zone=FedoraServer --add-masquerade

# internal zone — wg0 interface
firewall-cmd --permanent --zone=internal --add-interface=wg0
for p in 25 53 80 222 443 587 993 6443 8000 10250; do
  firewall-cmd --permanent --zone=internal --add-port=${p}/tcp
done
firewall-cmd --permanent --zone=internal --add-port=53/udp

# trusted zone — Pod CIDR
firewall-cmd --permanent --zone=trusted --add-source=10.42.0.0/16
firewall-cmd --permanent --zone=trusted --add-source=169.254.169.1/32
firewall-cmd --permanent --zone=trusted --add-port=30000-32767/tcp

# WireGuard clients → Internet (NAT)
firewall-cmd --permanent --new-policy=wg-to-internet
firewall-cmd --permanent --policy=wg-to-internet --add-ingress-zone=internal
firewall-cmd --permanent --policy=wg-to-internet --add-egress-zone=FedoraServer
firewall-cmd --permanent --policy=wg-to-internet --set-target=ACCEPT

# WireGuard clients → Pods (via HAProxy DNAT)
firewall-cmd --permanent --new-policy=wg-to-cluster
firewall-cmd --permanent --policy=wg-to-cluster --add-ingress-zone=internal
firewall-cmd --permanent --policy=wg-to-cluster --add-egress-zone=trusted
firewall-cmd --permanent --policy=wg-to-cluster --set-target=ACCEPT

firewall-cmd --reload

cd /etc/wireguard && umask 077
wg genkey | tee server.key | wg pubkey > server.pub

[Interface]
Address    = 10.0.0.1/24
ListenPort = 51820
PrivateKey = <CONTENTS OF server.key>

PostUp   = iptables -A FORWARD -i %i -j ACCEPT; iptables -A FORWARD -o %i -j ACCEPT
PostDown = iptables -D FORWARD -i %i -j ACCEPT; iptables -D FORWARD -o %i -j ACCEPT

[Peer]
# Example client
PublicKey  = <client-pubkey>
AllowedIPs = 10.0.0.2/32

# Add one [Peer] block per device

chmod 600 /etc/wireguard/wg0.conf
systemctl enable --now wg-quick@wg0

cat > /etc/fail2ban/jail.local <<'EOF'
[DEFAULT]
bantime  = 86400
findtime = 259200
maxretry = 3
backend  = auto
action   = firewallcmd-rich-rules[actiontype=<multiport>]

[sshd]
enabled  = true
port     = 222
logpath  = %(sshd_log)s
EOF

systemctl enable --now fail2ban

kubectl get pvc -n mailstack mail-logs -o jsonpath='{.spec.volumeName}'

[postfix]
enabled  = true
port     = smtp,submission
backend  = polling
logpath  = /var/lib/pvc/<PVC_UUID>_mailstack_mail-logs/postfix.log

[postfix-sasl]
enabled  = true
port     = smtp,submission
backend  = polling
logpath  = /var/lib/pvc/<PVC_UUID>_mailstack_mail-logs/postfix.log

[dovecot]
enabled  = true
port     = imaps
backend  = polling
logpath  = /var/lib/pvc/<PVC_UUID>_mailstack_mail-logs/dovecot.log

[postfix-sasl-unknown]
enabled        = true
filter         = postfix-sasl-unknown
action         = firewallcmd-rich-rules[actiontype=<allports>]
ignorecommand  = /usr/local/bin/fail2ban-sasl-mysql-check.sh <ip>
backend        = polling
logpath        = /var/lib/pvc/<PVC_UUID>_mailstack_mail-logs/postfix.log
maxretry       = 1
bantime        = 604800
findtime       = 86400

[sshd-unknown]
enabled        = true
port           = 222
backend        = systemd
maxretry       = 1
bantime        = 604800

[dovecot-unknown]
enabled        = true
port           = imaps
filter         = dovecot
backend        = polling
logpath        = /var/lib/pvc/<PVC_UUID>_mailstack_mail-logs/dovecot.log
maxretry       = 1
bantime        = 604800

[postfix-rcpt-unknown]
enabled        = true
port           = smtp,submission
filter         = postfix
backend        = polling
logpath        = /var/lib/pvc/<PVC_UUID>_mailstack_mail-logs/postfix.log
maxretry       = 3
findtime       = 3600
bantime        = 604800

[INCLUDES]
before = common.conf

[Definition]
failregex = warning: \S+\[<HOST>\](?::\d+)?: SASL \S+ authentication failed: .*, sasl_username=<F-USER>\S+</F-USER>
ignoreregex =
datepattern = %%b %%d %%H:%%M:%%S
              {^LN-BEG}

#!/bin/bash
IP="$1"
LOG="/var/lib/pvc/<PVC_UUID>_mailstack_mail-logs/postfix.log"
source /etc/fail2ban/.mysql-sasl-check.conf

USER=$(grep -F "[$IP]" "$LOG" | grep "SASL.*authentication failed" | \
    grep -oP 'sasl_username=\K\S+' | tail -1)

[ -z "$USER" ] && exit 1
echo "$USER" | grep -qP '^[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}$' || exit 1

SAFE_USER="${USER//\'/\'\'}"
COUNT=$(mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASS" "$MYSQL_DB" -sN \
    -e "SELECT COUNT(*) FROM mailbox WHERE username='$SAFE_USER' AND active=1" 2>/dev/null)

[ "$COUNT" = "1" ] && exit 0 || exit 1

chmod 755 /usr/local/bin/fail2ban-sasl-mysql-check.sh

# MYSQL_HOST: kubectl get svc -n mailstack maildb -o jsonpath='{.spec.clusterIP}'
install -m 600 /dev/stdin /etc/fail2ban/.mysql-sasl-check.conf <<'EOF'
MYSQL_HOST=<maildb-cluster-ip>
MYSQL_USER=postfix
MYSQL_PASS=<postfix-db-password>
MYSQL_DB=postfix
EOF

systemctl restart fail2ban

cat > /usr/local/bin/unban-fail2ban-clients.sh <<'EOF'
#!/bin/bash
for JAIL in $(fail2ban-client status | awk -F: '/Jail list/ {print $2}' | tr -d ',\t'); do
  for IP in $(fail2ban-client status "$JAIL" | awk -F: '/Banned IP list/ {print $2}'); do
    case "$IP" in
      10.0.0.*) fail2ban-client set "$JAIL" unbanip "$IP" ;;
    esac
  done
done
EOF
chmod 755 /usr/local/bin/unban-fail2ban-clients.sh

# /etc/systemd/system/unban-fail2ban-clients.timer
[Unit]
Description=Hourly unban of WireGuard IPs
[Timer]
OnCalendar=hourly
Persistent=true
[Install]
WantedBy=timers.target

systemctl enable --now unban-fail2ban-clients.timer

dnf copr enable -y @redhat-et/microshift
dnf install -y microshift openshift-clients   # installs kubectl + oc

install -m 644 /dev/stdin /etc/microshift/config.yaml <<'EOF'
network:
  cniPlugin: "none"
storage:
  driver: none
  optionalCsiComponents:
    - none
apiServer:
  logLevel: Warning
auditLog:
  maxFileSize: 200
  maxFiles: 3
  maxFileAge: 7
EOF

install -m 644 /dev/stdin /etc/microshift/config.d/02-kindnet-paths.yaml <<'EOF'
manifests:
  kustomizePaths:
  - /usr/lib/microshift/manifests
  - /usr/lib/microshift/manifests.d/000-microshift-kube-proxy
  - /etc/microshift/manifests
  - /etc/microshift/manifests.d/*
EOF

# In the DaemonSet env section:
- name: POD_SUBNET
  value: "10.42.0.0/16"

systemctl enable --now microshift

export KUBECONFIG=/var/lib/microshift/resources/kubeadmin/kubeconfig
echo 'export KUBECONFIG=/var/lib/microshift/resources/kubeadmin/kubeconfig' >> /root/.bashrc
echo 'alias k=kubectl' >> /root/.bashrc

kubectl get pods -A -w

ssh-keygen -t ed25519 -C "fedora-server" -f /root/.ssh/id_ed25519 -N ""
cat /root/.ssh/id_ed25519.pub
# → Add as Deploy Key (with write access) to github.com/youruser/homelab

cat > /etc/microshift/manifests/flux-scc.yaml <<'EOF'
apiVersion: v1
kind: Namespace
metadata:
  name: flux-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: flux-source-controller-scc
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:openshift:scc:privileged
subjects:
  - kind: ServiceAccount
    name: source-controller
    namespace: flux-system
---
# Repeat for: kustomize-controller, helm-controller, notification-controller
EOF

systemctl restart microshift

curl -s https://fluxcd.io/install.sh | FLUX_VERSION=2.8.6 bash -s /usr/local/bin

flux bootstrap git \
  --url=ssh://git@github.com/youruser/homelab \
  --branch=main \
  --path=configuration/ \
  --private-key-file=/root/.ssh/id_ed25519 \
  --silent

cd /data && git clone git@github.com:youruser/homelab.git .

kubectl create secret generic telegram-secret -n flux-system \
  --from-literal=token="$(awk -F= '/^TOKEN=/{print $2}' /etc/telegramrc)"

# Alert.spec.exclusionList:
- ".*Reconciliation in progress.*"
- ".*retry budget.*"

ssh-keygen -t ed25519 -f /root/.ssh/github-actions -N "" -C "gh-actions-snapshot"

KEY=$(cat /root/.ssh/github-actions.pub)
echo "command=\"/usr/local/bin/pre-flux-snapshot.sh\",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty $KEY" \
  >> /root/.ssh/authorized_keys

cat > /data/scripts/pre-flux-snapshot.sh <<'EOF'
#!/bin/bash
export KUBECONFIG=/var/lib/microshift/resources/kubeadmin/kubeconfig
flux suspend ks --all
snapper -c root create -d "vor-flux-update-$(date +%Y%m%d-%H%M%S)"
flux resume ks --all
EOF
chmod 755 /data/scripts/pre-flux-snapshot.sh
ln -s /data/scripts/pre-flux-snapshot.sh /usr/local/bin/pre-flux-snapshot.sh

{
  "extends": ["config:base"],
  "automergeType": "pr",
  "prHourlyLimit": 2,
  "packageRules": [
    {
      "matchUpdateTypes": ["patch", "minor"],
      "automerge": true,
      "minimumReleaseAge": "3 days"
    },
    {
      "matchUpdateTypes": ["major"],
      "automerge": false
    }
  ]
}

POD=$(kubectl get pod -n acme-dns -l app=acme-dns -o name)
for DOMAIN in example.com example.net; do
  kubectl exec -n acme-dns "$POD" -- curl -s -X POST http://localhost:8081/register \
    -H 'Content-Type: application/json' -d "{}"
done
# Save the JSON output — contains username, password, fulldomain per domain

install -m 600 <json-file> /etc/acme-dns/acmedns.json
kubectl create secret generic acmedns-credentials -n cert-manager \
  --from-file=acmedns.json=/etc/acme-dns/acmedns.json

spec:
  secretName: wildcard-tls
  issuerRef:
    name: letsencrypt-prod
    kind: ClusterIssuer
  dnsNames:
    - "*.example.com"
    - example.com
    - "*.example.net"
    - example.net

cat > /usr/local/bin/sync-wildcard-tls.sh <<'EOF'
#!/bin/bash
set -euo pipefail
export KUBECONFIG=/var/lib/microshift/resources/kubeadmin/kubeconfig

NAMESPACES=(pihole vaultwarden nextcloud paperless collabora mailstack homepage)

TMP=$(mktemp); trap 'rm -f "$TMP"' EXIT
kubectl get secret wildcard-tls -n cert-manager -o json > "$TMP"

for NS in "${NAMESPACES[@]}"; do
  jq --arg ns "$NS" '
    .metadata.name = ($ns + "-tls")
    | .metadata.namespace = $ns
    | del(.metadata.resourceVersion, .metadata.uid,
          .metadata.creationTimestamp, .metadata.ownerReferences,
          .metadata.managedFields)
  ' "$TMP" | kubectl apply -f -
done
EOF
chmod 755 /usr/local/bin/sync-wildcard-tls.sh

# systemd timer: every Monday 00:00
systemctl enable --now sync-wildcard-tls.timer
/usr/local/bin/sync-wildcard-tls.sh   # run immediately

# configuration/pihole/deployment.yaml (excerpt)
strategy:
  type: Recreate    # prevents Pending state when hostPort: 53 is already bound
containers:
- name: pihole
  image: docker.io/pihole/pihole:latest
  env:
  - name: FTLCONF_dns_upstreams
    value: "8.8.8.8;8.8.4.4"
  - name: FTLCONF_dns_listeningMode
    value: "ALL"
  - name: FTLCONF_webserver_api_password
    valueFrom:
      secretKeyRef:
        name: pihole-password
        key: FTLCONF_webserver_api_password
  ports:
  - containerPort: 53
    hostPort: 53
    hostIP: 10.0.0.1        # bind to WireGuard interface only
    protocol: UDP
    name: dns-udp
  - containerPort: 53
    hostPort: 53
    hostIP: 10.0.0.1
    protocol: TCP
    name: dns-tcp
  - containerPort: 80
    name: http
  securityContext:
    privileged: true        # required for FTL (NET_ADMIN)
  volumeMounts:
  - name: pihole-data
    mountPath: /etc/pihole
  - name: dnsmasq-custom
    mountPath: /etc/dnsmasq.d/99-custom.conf
    subPath: 99-custom.conf

# configuration/pihole/ingress.yaml
metadata:
  annotations:
    route.openshift.io/termination: "edge"
    haproxy.router.openshift.io/ip_whitelist: "10.0.0.0/24 <CORP_IP_RANGE>"
spec:
  tls:
  - hosts: [dnsconfig.example.com]
    secretName: pihole-tls
  rules:
  - host: dnsconfig.example.com

# configuration/pihole/scc.yaml — required for privileged workloads
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: pihole-privileged-scc
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:openshift:scc:privileged
subjects:
- kind: ServiceAccount
  name: default
  namespace: pihole

# configuration/vaultwarden/deployment.yaml (excerpt)
containers:
- name: vaultwarden
  image: docker.io/vaultwarden/server:1.35.7
  env:
  - name: WEBSOCKET_ENABLED
    value: "true"
  - name: ROCKET_ADDRESS
    value: "0.0.0.0"
  resources:
    limits:
      memory: 100Mi
  volumeMounts:
  - name: data
    mountPath: /data
  strategy:
    type: Recreate   # important: only one writer at a time

# configuration/vaultwarden/ingress.yaml (excerpt)
annotations:
  haproxy.router.openshift.io/ip_whitelist: "10.0.0.0/24 <CORP_IP_RANGE>"
  haproxy.router.openshift.io/timeout: "300s"

# configuration/nextcloud/deployment.yaml (excerpt)
containers:
- name: nextcloud
  image: docker.io/nextcloud:33-apache
  env:
  - name: MYSQL_HOST
    value: mariadb
  - name: MYSQL_DATABASE
    value: owncloud
  - name: MYSQL_USER
    value: owncloud
  - name: MYSQL_PASSWORD
    valueFrom:
      secretKeyRef:
        name: nextcloud-secret
        key: MYSQL_PASSWORD
  - name: NEXTCLOUD_TRUSTED_DOMAINS
    value: cloud.example.com
  - name: REDIS_HOST
    value: redis
  - name: PHP_UPLOAD_LIMIT
    value: 10G
  - name: PHP_MEMORY_LIMIT
    value: 1G
  - name: APACHE_DISABLE_REWRITE_IP
    value: "1"
  - name: TRUSTED_PROXIES
    value: 10.42.0.0/16    # pod CIDR — required for HAProxy reverse-proxy headers
  resources:
    limits:
      memory: 2Gi

# configuration/nextcloud/ingress.yaml
annotations:
  haproxy.router.openshift.io/timeout: "300s"
  haproxy.router.openshift.io/proxy-body-size: "10g"
  haproxy.router.openshift.io/hsts_header: max-age=15552000;includeSubDomains;preload

# configuration/nextcloud/cronjob.yaml
---
apiVersion: batch/v1
kind: CronJob
metadata:
  name: nextcloud-cron
spec:
  schedule: "*/5 * * * *"
  concurrencyPolicy: Forbid
  jobTemplate:
    spec:
      template:
        spec:
          securityContext:
            runAsUser: 33    # www-data
            runAsGroup: 33
          containers:
          - name: nextcloud-cron
            image: docker.io/nextcloud:33-apache
            command: ["php", "-f", "/var/www/html/cron.php"]
            volumeMounts:
            - {name: html, mountPath: /var/www/html}
            - {name: data, mountPath: /var/www/html/data}
---
apiVersion: batch/v1
kind: CronJob
metadata:
  name: nextcloud-app-update
spec:
  schedule: "0 3 * * *"
  concurrencyPolicy: Forbid
  jobTemplate:
    spec:
      template:
        spec:
          securityContext:
            runAsUser: 33
            runAsGroup: 33
          containers:
          - name: nextcloud-app-update
            image: docker.io/nextcloud:33-apache
            command: ["php", "-f", "/var/www/html/occ", "app:update", "--all"]

# configuration/mailstack/postfix-deployment.yaml (excerpt)
strategy:
  type: Recreate
template:
  spec:
    hostNetwork: true
    dnsPolicy: ClusterFirstWithHostNet
    containers:
    - name: postfix
      image: docker.io/boky/postfix:latest
      command: ["/usr/sbin/postfix", "-c", "/etc/postfix", "start-fg"]
      securityContext:
        privileged: true
      volumeMounts:
      - {name: config, mountPath: /etc/postfix/main.cf, subPath: main.cf}
      - {name: tls,    mountPath: /etc/ssl/mail,         readOnly: true}
      - {name: mail-logs, mountPath: /var/log/mail}
    - name: log-tailer   # sidecar: tails postfix.log to stdout for kubectl logs
      image: docker.io/library/alpine:3
      command: ["/bin/sh", "-c", "tail -F /var/log/mail/postfix.log"]
      volumeMounts:
      - {name: mail-logs, mountPath: /var/log/mail}
    - name: logrotate   # sidecar: daily rotation of postfix.log
      image: docker.io/library/alpine:3
      command: ["/bin/sh", "-c"]
      args: ["apk add logrotate -q && while true; do logrotate ... || { echo 'ERROR'; exit 1; }; sleep 86400; done"]
      volumeMounts:
      - {name: mail-logs, mountPath: /var/log/mail}

# configuration/mailstack/dovecot-deployment.yaml (excerpt)
strategy:
  type: Recreate
template:
  spec:
    hostNetwork: true
    dnsPolicy: ClusterFirstWithHostNet
    containers:
    - name: dovecot
      image: docker.io/dovecot/dovecot:latest-root
      ports:
      - {containerPort: 993}   # metadata only — hostNetwork binds to host stack directly
      securityContext:
        privileged: true
      volumeMounts:
      - {name: config,    mountPath: /etc/dovecot/conf.d/10-auth.conf, subPath: 10-auth.conf}
      - {name: db-secret, mountPath: /etc/dovecot/conf.d/01-db-connection.conf,
         subPath: 01-db-connection.conf, readOnly: true}   # MySQL credentials as secret file
      - {name: vmail,     mountPath: /home/vmail}
      - {name: tls,       mountPath: /etc/dovecot/ssl, readOnly: true}
      - {name: mail-logs, mountPath: /var/log/mail}
    - name: log-tailer   # sidecar: tails dovecot.log to stdout for kubectl logs
      image: docker.io/library/alpine:3
      command: ["/bin/sh", "-c", "tail -F /var/log/mail/dovecot.log"]
      volumeMounts:
      - {name: mail-logs, mountPath: /var/log/mail}

# configuration/mailstack/rspamd-deployment.yaml (excerpt)
metadata:
  annotations:
    prometheus.io/scrape: "true"    # Alloy picks this up for Grafana Cloud
    prometheus.io/port: "11334"
    prometheus.io/path: "/metrics"
containers:
- name: rspamd
  image: docker.io/rspamd/rspamd:4.0.1
  securityContext:
    privileged: true
  ports:
  - {containerPort: 11332, name: milter, hostPort: 11332}
  - {containerPort: 11334, name: controller}
  resources:
    requests:
      memory: 128Mi
    limits:
      memory: 768Mi    # raised from 512Mi — startup spike causes OOMKill on node reboot
  livenessProbe:
    httpGet: {path: /ping, port: 11334}   # use httpGet, NOT tcpSocket
  readinessProbe:
    httpGet: {path: /ping, port: 11334}
  volumeMounts:
  - {name: config,    mountPath: /etc/rspamd/local.d}
  - {name: dkim-keys, mountPath: /etc/rspamd/dkim, readOnly: true}

- name: freshclam
  image: docker.io/clamav/clamav:latest
  command: ["/bin/sh", "-c"]
  args: ["while true; do freshclam; sleep 3600; done"]

#!/bin/bash
export KUBECONFIG=/var/lib/microshift/resources/kubeadmin/kubeconfig
source /etc/telegramrc

send() {
  curl -s -X POST "https://api.telegram.org/bot${TOKEN}/sendMessage" \
    -d chat_id="${CHATID}" -d parse_mode="HTML" \
    --data-urlencode text="$1" > /dev/null
}

# WireGuard peer status
WG_PEERS=$(wg show wg0 | grep -c "latest handshake")
WG_RECENT=$(wg show wg0 | awk '/latest handshake/ {
  if ($0 ~ /second/ || $0 ~ /minute/) count++
  else if ($0 ~ /hour/) { match($0, /([0-9]+) hour/, a); if (a[1]+0 < 2) count++ }
} END{print count+0}')

# Kubernetes pod health
NOT_RUNNING=$(kubectl get pods -A --no-headers 2>/dev/null \
  | grep -v -E "Running|Completed" | wc -l)
NOT_RUNNING_LIST=$(kubectl get pods -A --no-headers 2>/dev/null \
  | grep -v -E "Running|Completed" \
  | awk '{print $1"/"$2" ("$4")"}' | head -10)
TOTAL_PODS=$(kubectl get pods -A --no-headers 2>/dev/null | grep -c "Running")

# Mail stats (last 24h from postfix logs)
MAIL_IN=$(kubectl logs -n mailstack deploy/mail-postfix --since=24h 2>/dev/null \
  | grep -c "postfix/lmtp.*status=sent" || echo 0)
MAIL_OUT=$(kubectl logs -n mailstack deploy/mail-postfix --since=24h 2>/dev/null \
  | grep -c "postfix/smtp.*status=sent" || echo 0)
MAIL_REJECT=$(kubectl logs -n mailstack deploy/mail-postfix --since=24h 2>/dev/null \
  | grep -c "NOQUEUE: reject" || echo 0)
QUEUE_RAW=$(kubectl exec -n mailstack deploy/mail-postfix -- postqueue -p 2>/dev/null \
  | tail -1)
QUEUE=$(echo "$QUEUE_RAW" | grep -q "empty" && echo "leer" || echo "$QUEUE_RAW")

# fail2ban
BANNED_COUNT=0
for jail in $(fail2ban-client status 2>/dev/null \
  | grep "Jail list:" | sed 's/.*Jail list:\s*//' | tr ', ' '\n' | grep -v '^$'); do
  n=$(fail2ban-client status "$jail" 2>/dev/null \
    | grep "Currently banned:" | awk '{print $NF}')
  BANNED_COUNT=$((BANNED_COUNT + ${n:-0}))
done

DISK_FREE=$(df -h / | awk 'NR==2 {print $4 " von " $2 " frei (" $5 " belegt)"}')

# Build status icons
[ "$WG_RECENT" -ge 1 ] \
  && WG_STATUS="Aktiv: ${WG_RECENT}/${WG_PEERS} Peers (Handshake unter 2h)" \
  && WG_ICON="OK" \
  || { WG_STATUS="Keine aktiven Peers!"; WG_ICON="WARN"; }

[ "$NOT_RUNNING" -eq 0 ] \
  && K8S_STATUS="${TOTAL_PODS} Pods Running" && K8S_ICON="OK" \
  || { K8S_STATUS="${NOT_RUNNING} Pod(s) nicht Running:\n${NOT_RUNNING_LIST}"; K8S_ICON="WARN"; }

[ "$BANNED_COUNT" -gt 0 ] && F2B_ICON="WARN" || F2B_ICON="OK"

send "System-Check $(date '+%d.%m.%Y %H:%M')

WireGuard [${WG_ICON}]
${WG_STATUS}

Kubernetes [${K8S_ICON}]
${K8S_STATUS}

Mail letzte 24h:
  Eingehend zugestellt: ${MAIL_IN}
  Ausgehend gesendet:   ${MAIL_OUT}
  Rejects:              ${MAIL_REJECT}
  Queue:                ${QUEUE}

Fail2ban [${F2B_ICON}]
  Gebannte Clients:     ${BANNED_COUNT}

Speicher:
  ${DISK_FREE}"

# /etc/systemd/system/system-check.timer
[Unit]
Description=System Check täglich 08:00
[Timer]
OnCalendar=*-*-* 08:00:00
Persistent=false
[Install]
WantedBy=timers.target

#!/bin/bash
export KUBECONFIG=/var/lib/microshift/resources/kubeadmin/kubeconfig
NAMESPACES=(pihole vaultwarden nextcloud paperless collabora mailstack homepage)

for NS in "${NAMESPACES[@]}"; do
  export NS
  kubectl get secret wildcard-tls -n cert-manager -o json | python3 -c "
import json, sys, os
s = json.load(sys.stdin)
ns = os.environ['NS']
s['metadata']['namespace'] = ns
s['metadata']['name'] = ns + '-tls'
for k in ['resourceVersion','uid','creationTimestamp','managedFields',
          'annotations','ownerReferences','labels']:
    s['metadata'].pop(k, None)
print(json.dumps(s))
" | kubectl apply -f -
  echo "Synced wildcard-tls -> ${NS}/${NS}-tls"
done

#!/bin/bash
export KUBECONFIG=/var/lib/microshift/resources/kubeadmin/kubeconfig
DESCRIPTION="vor-flux-update-$(date +%Y%m%d-%H%M%S)"
flux suspend ks --all -n flux-system 2>/dev/null
snap-all "$DESCRIPTION"
flux resume ks --all -n flux-system 2>/dev/null
echo "Snapshots erstellt: $DESCRIPTION"

#!/bin/bash
set -euo pipefail
BTRBK_CONF=/etc/btrbk/btrbk.conf
BTRFS_TOP=/mnt/btrfs-top

EXCLUDED=(btrbk_snapshots var_cache var_tmp var_lib_kubelet)
EXCLUDED_PATTERNS=("root-snap-pre-*")

MOUNTED_HERE=0
if ! mountpoint -q "$BTRFS_TOP" 2>/dev/null; then
  mount "$BTRFS_TOP"; MOUNTED_HERE=1
fi
trap '[[ $MOUNTED_HERE -eq 1 ]] && umount "$BTRFS_TOP" 2>/dev/null || true' EXIT

mapfile -t CURRENT_SUBVOLS < <(
  btrfs subvolume list "$BTRFS_TOP" \
    | awk '$7 == 5 && $NF !~ /\// { print $NF }' | sort)
mapfile -t CONFIGURED < <(
  grep -E '^\s+subvolume\s+' "$BTRBK_CONF" | awk '{print $2}' | sort)

is_excluded() {
  local sv="$1"
  for ex in "${EXCLUDED[@]}"; do [[ "$sv" == "$ex" ]] && return 0; done
  for pat in "${EXCLUDED_PATTERNS[@]}"; do [[ "$sv" == $pat ]] && return 0; done
  return 1
}

MISSING_BTRBK=()
MISSING_SNAPPER=()
for sv in "${CURRENT_SUBVOLS[@]}"; do
  is_excluded "$sv" && continue
  configured=0
  for conf in "${CONFIGURED[@]}"; do [[ "$sv" == "$conf" ]] && configured=1 && break; done
  [[ $configured -eq 0 ]] && MISSING_BTRBK+=("$sv")
  [[ -f "/etc/snapper/configs/$sv" ]] || MISSING_SNAPPER+=("$sv")
done

OVERALL_EXIT=0
[[ ${#MISSING_BTRBK[@]} -gt 0 ]] && { echo "WARN btrbk: ${MISSING_BTRBK[*]}"; OVERALL_EXIT=1; }
[[ ${#MISSING_SNAPPER[@]} -gt 0 ]] && { echo "WARN snapper: ${MISSING_SNAPPER[*]}"; OVERALL_EXIT=1; }
[[ $OVERALL_EXIT -eq 0 ]] && echo "OK: Alle Subvolumes in btrbk und snapper konfiguriert."
exit $OVERALL_EXIT

flux suspend ks --all -n flux-system
snap-all "vor-flux-update-$(date +%Y%m%d-%H%M%S)"
flux resume ks --all -n flux-system

check-backup-btrfs-subvolumes.sh
# → OK: Alle relevanten Subvolumes sind in btrbk.conf konfiguriert.
# → OK: Alle relevanten Subvolumes haben eine snapper-Config.

# See recent commits
etckeeper vcs log --oneline -10

# Restore a single file
etckeeper vcs checkout HEAD~1 -- /etc/microshift/config.yaml

snapper rollback   # sets snapshot as new default subvolume
reboot             # boots into the restored, writable system

snapper -c root list
snapper -c root rollback 116   # rolls back root subvolume
reboot

# For any subvolume, the snapshots are here:
# /var/lib/pvc/.snapshots/<nr>/snapshot/
# /home/.snapshots/<nr>/snapshot/
# etc.

# Example: restore a PVC directory from snapshot 42
snapper -c var_lib_pvc list
kubectl scale deploy vaultwarden -n vaultwarden --replicas=0
cp -a /var/lib/pvc/.snapshots/42/snapshot/<pvc-dir>/. /var/lib/pvc/<pvc-dir>/
kubectl scale deploy vaultwarden -n vaultwarden --replicas=1

mount /mnt/btrfs-top
btrbk list snapshots   # find the right snapshot name

systemctl stop microshift

# Save current state
btrfs subvolume snapshot /mnt/btrfs-top/var_lib_pvc \
  /mnt/btrfs-top/btrbk_snapshots/var_lib_pvc.recovery-backup

# Replace with snapshot
umount /var/lib/pvc
btrfs subvolume delete /mnt/btrfs-top/var_lib_pvc
btrfs subvolume snapshot \
  /mnt/btrfs-top/btrbk_snapshots/var_lib_pvc.20260515T0200 \
  /mnt/btrfs-top/var_lib_pvc
mount /var/lib/pvc

systemctl start microshift
umount /mnt/btrfs-top

dnf install -y btrfs-progs rsync gdisk

# Fetch partition table from Pi backup
scp mko@10.0.0.12:/backup/btrfs/server/var/<snapshot>/lib/boot-backup/partition-table.sfdisk /tmp/

# Restore (same disk size)
sfdisk /dev/vda < /tmp/partition-table.sfdisk

mkfs.vfat -F32 /dev/vda1
mkfs.xfs -f /dev/vda2
xfs_admin -U bafddb4f-cda4-4aae-9d4b-b75da20e3680 /dev/vda2

mkfs.btrfs -f \
  -L "fedora_v220241250910305995" \
  -U 9017b7d7-c4e6-45a3-9c04-48f25cd640fd \
  /dev/vda3

mount -o subvolid=5 /dev/vda3 /mnt/restore

for sv in root var home var_log var_lib_microshift var_lib_pvc var_lib_containers data; do
  SNAP=$(ssh mko@10.0.0.12 "ls /backup/btrfs/server/${sv}/ | sort | tail -1")
  ssh mko@10.0.0.12 "btrfs send /backup/btrfs/server/${sv}/${SNAP}" \
    | btrfs receive /mnt/restore/
  # Convert read-only snapshot to writable subvolume
  btrfs subvolume snapshot /mnt/restore/${SNAP} /mnt/restore/${sv}_new
  btrfs subvolume delete /mnt/restore/${SNAP}
  mv /mnt/restore/${sv}_new /mnt/restore/${sv}
done

mount /dev/vda2 /mnt/boot
mount /dev/vda1 /mnt/efi
rsync -aAX /mnt/restore/var/lib/boot-backup/boot/ /mnt/boot/
rsync -aAX /mnt/restore/var/lib/boot-backup/efi/  /mnt/efi/

mount -o subvol=root /dev/vda3 /mnt/sysroot
# ... mount remaining subvolumes ...
for d in dev dev/pts proc sys run; do mount --bind /$d /mnt/sysroot/$d; done
chroot /mnt/sysroot grub2-install /dev/vda
chroot /mnt/sysroot grub2-mkconfig -o /boot/grub2/grub.cfg

touch /mnt/sysroot/.autorelabel   # trigger SELinux relabel on first boot
umount -R /mnt/sysroot
reboot