bin/stack: revamp utility commands

.config/shell/server

@@ -22,7 +22,7 @@ stack() {
 	cmdname=${FUNCNAME:-$0}
 	local pod_suffix='-\(0\|[0-f]\+\)'
 	if test $# -lt 1; then
-		builtin cd "$STACKSPIN"
+		builtin cd "$STACKSPIN" || cd /mnt/b/media/backups/servers/stackspin/2310_stackspin
 		echo "Usage: $cmdname <COMMAND> [args...]"
 		echo "Stackspin commands: select, sso, user, push"
 		echo "Kubepod commands: pod, exec, app, shell, ls, logs, upload"
@@ -43,23 +43,18 @@ stack() {
 		echo Selected Stackspin cluster "$_cluster_name" with IP "$_cluster_ip"
 		echo "$_cluster_name" >"$_stackspin_cluster_cache"
 		#test "$PWD" = "$HOME" && builtin cd "$STACKSPIN"
-        . $STACKSPIN/env/bin/activate
+		test -d "$STACKSPIN" && . $STACKSPIN/env/bin/activate
 		;;
-    (flux)
+	(sso) "$cmdname" exec dashboard-backend -- flask "$@";;
-        kubectl apply -k "$CLUSTER_DIR"
-        flux reconcile -n flux-system kustomization velero
-        flux get -A kustomizations --no-header | awk -F' ' '{system("flux reconcile -n " $1 " kustomization " $2)}'
-        ;;
-    (edit)
-        app=$1
-        kubectl edit configmap -n flux-system stackspin-$app-kustomization-variables
-        flux reconcile kustomization $app
-        flux reconcile helmrelease -n stackspin-apps $app
-        ;;
-	(sso) "$cmdname" exec dashboard --container backend -- flask "$@";;
 	(users)
-		if test $# -gt 0
+		if test "$1" = "delete"
-		then for arg
+		then shift
+			for arg
+			do "$cmdname" user delete "$arg"
+			done
+		elif test $# -gt 0
+		then
+			for arg
 			do "$cmdname" user show $arg
 			done
 		else "$cmdname" users $("$cmdname" user list | sed 's|.*<\(.*\)>.*|\1|')
@@ -68,23 +63,73 @@ stack() {
 		if test "$1" = "init"
 		then mail="$2"
 			shift 2
-			"$cmdname" user create "$mail"
+			"$cmdname" user create "$mail" &&
-			"$cmdname" user update "$mail" name "$*"
+			"$cmdname" user update "$mail" name "$*" &&
 			echo "Initialized user '$*' with email '$mail'"
 		else "$cmdname" sso cli "$command" "$@"
 		fi;;
+	(invite) (
+		# Mail invitation to new users
+		export mail=$1
+		export name=${2:-$(echo $mail | sed -E 's/(.*)\.(.*)@.*/\u\1 \u\2/' )}
+		#echo "$mail,$name"
+		stack user init "$mail" "$name"
+		stack-invite
+		);;
 	(push)
-		git commit -a "$@"
+		test -f "$1" && $EDITOR "$1"
+		# Allow force: https://open.greenhost.net/xeruf/stackspout/-/settings/repository#js-protected-branches-settings
+		git commit "$@"
 		git push &&
 		flux reconcile source git -n flux-system "$(basename $(git rev-parse --show-toplevel))"
-		flux reconcile kustomization -n flux-system "$(basename $(git rev-parse --show-toplevel))";;
+		flux reconcile kustomization -n flux-system "$(basename $(git rev-parse --show-toplevel))"
+		;;
+	# FLUX
+	(flux)
+		case "$1" in
+		(env) # Apply changes to .flux.env
+			kubectl apply -k "$CLUSTER_DIR"
+			flux reconcile -n flux-system kustomization velero
+			flux get -A kustomizations --no-header | awk -F' ' '{system("flux reconcile -n " $1 " kustomization " $2)}'
+			;;
+		esac
+		;;
+	(reconcile)
+		app=$1
+		namespace=${2:-stackspout}
+		if flux suspend helmrelease -n $namespace $app
+		then flux resume helmrelease -n $namespace $app
+		else flux suspend helmrelease -n stackspin-apps $app
+			 flux resume helmrelease -n stackspin-apps $app
+		fi
+	    flux suspend kustomization $app
+	    flux resume kustomization $app
+		;;
+	(edit)
+		# Edit the URL for an application
+		app=$1
+		kubectl edit configmap -n flux-system stackspin-$app-kustomization-variables
+		"$0" reconcile $app
+		;;
 	# Velero
+	(restic)
+		(
+		namespace=stackspin
+		case $1 in (-n|--namespace) namespace=$2; shift 2;; esac
+		source $CLUSTER_DIR/.flux.env || exit $?
+		export RESTIC_REPOSITORY="s3:${backup_s3_url}/${backup_s3_bucket}/${backup_s3_prefix}/restic/$namespace"
+		export AWS_ACCESS_KEY_ID="${backup_s3_aws_access_key_id}"
+		export AWS_SECRET_ACCESS_KEY="${backup_s3_aws_secret_access_key}"
+		export RESTIC_PASSWORD="$(kubectl get secret -n velero velero-repo-credentials -o jsonpath='{.data.repository-password}' | base64 -d)"
+		restic "$@"
+		)
+		;;
 	(backup)
 		backupname=$(date +%y%m%d.%H%m)
 		velero create backup $backupname --exclude-namespaces velero --wait
 		velero backup logs $backupname;;
 	(restore)
-        test $# -lt 2 && echo "$0 $command <backup> <app>" >&2 && return 1
+		test $# -lt 2 && echo "$0 $command <backup> <app> [namespace]" >&2 && return 1
 		backup=$1; app=$2
 		namespace=${3:-stackspin-apps} # TODO automatically handle stackspout apps
 		restore="${backup}-$app-$(date +%s)"
@@ -93,8 +138,9 @@ stack() {
 			hr="$kust-database"
 			namespace=stackspin
 		else hr="$app"
+			 kust="$app"
 		fi
-        flux suspend kustomization ${kust:-$app}
+		flux suspend kustomization $kust
 		flux suspend helmrelease -n $namespace $hr
 		kubectl delete all -n $namespace -l stackspin.net/backupSet=$app
 		kubectl delete secret -n $namespace -l stackspin.net/backupSet=$app
@@ -107,25 +153,48 @@ stack() {
 		echo "Press enter if backup is ready to resume flux resources:"
 		read
 		test $app = dashboard && kubectl delete secret -n stackspin hydra && flux reconcile helmrelease -n stackspin hydra
-        flux resume helmrelease -n $namespace $hr
+		flux resume helmrelease -n $namespace $hr # TODO timeout
-        flux resume kustomization ${kust:-$app}
+		flux resume kustomization $kust
+		;;
+	(restore-pvc)
+		test $# -lt 1 && echo "$0 $command <app> [dir]" >&2 && return 1
+		local app=$1
+		if test -d "$2"
+		then dir="$2"
+			target=$(ssh "$_cluster_name" find /var/lib/Stackspin/local-storage/ -maxdepth 1 -name "*$app")
+			test -z "$target" && echo "No target found for ${app}" && return 1
+			ssh "$_cluster_name" mv -v "$target" "$target.$(date +%s)"
+			rsync --links --hard-links --times --recursive --info=progress2,remove,symsafe,flist,del --human-readable "$dir/" "$_cluster_name:$target/"
+		else
+			for vol in $(ls -d pvc*$app* | cut -d_ -f3 | sort)
+			do "$cmdname" restore-pvc $vol $(find -maxdepth 1 -name "*$vol")
+			done
+		fi
 		;;
 	# KUBE
 	# app clis
 	(occ) "$cmdname" exec nc-nextcloud -c nextcloud -it -- su www-data -s /bin/bash -c "php $command $*";;
-	(vikunja) local pod=${2:-vikunja}
+	(vikunja*)
+		local pod=$command
 		case "$1" in
 		(dump|export) cd "$PROJECTS/vikunja"
-			"$cmdname" exec "$pod" -c api -- sh -c 'rm -f *.zip && ./vikunja dump >/dev/null && ls --color -lAhF >&2 && cat *.zip' >"$pod-dump_$(date +%F).zip"
+			"$cmdname" exec "$pod-api" -- sh -c 'rm -f *.zip && ./vikunja dump >/dev/null && ls --color -lAhF >&2 && cat *.zip' >"$pod-dump_$(date +%F).zip"
 			;;
-		(restore) "$cmdname" upload "$pod" "$3" -c api
+		(restore)
-			"$cmdname" exec "$pod" -c api -it -- ./vikunja restore "$3"
+			if ! test -f "$2"
+			then echo "Usage: $0 vikunja[suffix] restore <file>" >&2
+				 return 2
+			fi
+			file=$2
+			"$cmdname" upload "$pod-api" "$file"
+			"$cmdname" exec "$pod-api" -it -- ./vikunja restore "$file"
 			;;
-		(psql) kubectl exec -it -n $("$cmdname" pod "$pod-postgresql") -- sh -c "PGPASSWORD=$(kubectl get secret --namespace stackspout $pod-postgresql -o jsonpath='{.data.postgresql-password}' | base64 --decode) psql -h localhost -U vikunja -p 5432 vikunja";;
+		(psql) kubectl exec -it -n $("$cmdname" pod "$pod-postgresql") -- sh -c "PGPASSWORD=$(kubectl get secret --namespace stackspout $pod-postgresql -o jsonpath='{.data.password}' | base64 --decode) psql -h localhost -U vikunja -p 5432 vikunja";;
-		(*) echo "Unknown Subcommand";;
+		(*) echo "Unknown $command subcommand";;
 		esac
 		;;
-	(maria) app=$1
+	(maria)
+		app=$1
 		pw="$(kubectl get secret -n flux-system stackspin-$app-variables --template '{{.data.mariadb_password}}' | base64 -d 2>/dev/null ||
 			kubectl get secret -n flux-system stackspin-$app-variables --template "{{.data.${app}_mariadb_password}}" | base64 -d)"
 		case $app in
@@ -133,7 +202,13 @@ stack() {
 			(wordpress) n=wordpress-database;;
 			(*) n=$app-mariadb;;
 		esac
-		"$cmdname" exec $n -it -- env "MYSQL_PWD=$pw" mysql -u $app "$@";;
+		"$cmdname" exec $n -it -- env "MYSQL_PWD=$pw" mysql -u $app "$@"
+		;;
+	(mariar)
+		name="$1-mariadb"
+		shift
+		"$cmdname" exec "$name" -it -- env "MYSQL_PWD=$(kubectl get secret -n $(kubectl get secret --all-namespaces -o=custom-columns=S:.metadata.namespace,N:.metadata.name --no-headers | grep --color=never -- "$name") -o jsonpath='{.data.mariadb-root-password}' | base64 -d)" mysql -u root "$@"
+		;;
 	# high-level
 	(shell)
 		container=$1
@@ -174,7 +249,12 @@ stack() {
 		done;;
 	(pod)
 		test $# -gt 0 && local podname=$1 && shift
-		kubectl get pods --all-namespaces --field-selector="status.phase=Running" -o=custom-columns=S:.metadata.namespace,N:.metadata.name --no-headers "$@" | grep --color=never -- "$podname";;
+		if ! kubectl get pods --all-namespaces --field-selector="status.phase=Running" -o=custom-columns=S:.metadata.namespace,N:.metadata.name --no-headers "$@" | grep --color=never -- "$podname"
+		then code=$?
+			echo "No pod found for $podname" >&2
+			return $code
+		fi
+		;;
 	# stackspin bare
 	(*) if which "$cmdname-$command" >/dev/null 2>&1
 		then "$cmdname-$command" "$@"

.local/bin/scripts/stack-helm

@@ -0,0 +1,21 @@
+#!/bin/sh -e
+# Emulate helm repo adding for easy command copy-pasting
+cd "$STACKSPIN/../stackspout"
+cmd=$1
+shift
+case "$cmd" in
+(install) true;;
+(repo) shift;;
+(*) echo 'Unknown command!'>&2 && exit 2;;
+esac
+name=$1
+url=$2
+
+echo "apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: HelmRepository
+metadata:
+  name: $name
+  namespace: flux-system
+spec:
+  interval: 60m
+  url: $url" | tee "infrastructure/sources/$name.yaml"

.local/bin/scripts/stack-invite

@@ -0,0 +1,5 @@
+#!/bin/sh
+#for app in wordpress nextcloud velero vikunja ninja
+#do stack user setrole
+#done
+pass business/ftt/invite | envsubst | ssh nc-iridion sudo sendmail -v "$mail"

.local/bin/scripts/stack-template

@@ -0,0 +1,266 @@
+#!/bin/sh -e
+if test $# -lt 1; then
+  echo "You should be in the root apps folder."
+  echo "Usage: $0 <app> [subdomain] [repo] [namespace]"
+  exit 1
+fi
+
+app=$1
+subdomain=${2:-$app}
+repo=${3:-$app}
+namespace=${4:-stackspout}
+
+cat <<EOF >>"$subdomain-kustomization.yaml"
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+kind: Kustomization
+metadata:
+  name: add-${subdomain}
+  namespace: flux-system
+spec:
+  interval: 10m
+  prune: true
+  path: ./apps/${subdomain}
+  sourceRef:
+    kind: GitRepository
+    name: ${namespace}
+EOF
+
+if test "$(basename "$PWD")" != "${subdomain}"
+then mkdir -p "${subdomain}"
+     cd "${subdomain}"
+fi
+
+# Values
+
+cat <<EOF >>"$app-kustomization.yaml"
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+kind: Kustomization
+metadata:
+  name: ${app}
+  namespace: flux-system
+spec:
+  interval: 5m
+  retryInterval: 2m
+  timeout: 10m
+  wait: true
+  prune: true
+  path: ./apps/${subdomain}/${app}
+  sourceRef:
+    kind: GitRepository
+    name: ${namespace}
+  dependsOn:
+    - name: flux
+    - name: local-path-provisioner
+    - name: ${app}-secrets
+    - name: nginx
+    - name: single-sign-on
+  postBuild:
+    substituteFrom:
+      - kind: Secret
+        name: stackspin-cluster-variables
+      - kind: ConfigMap
+        name: stackspin-${app}-kustomization-variables
+      - kind: Secret
+        name: stackspin-${app}-variables
+      # OIDC
+      - kind: Secret
+        name: stackspin-${app}-oauth-variables
+      - kind: ConfigMap
+        name: stackspin-single-sign-on-kustomization-variables
+EOF
+
+if mkdir "$app"
+then
+cat <<EOF >"$app/$app-oauth-client.yaml"
+apiVersion: hydra.ory.sh/v1alpha1
+kind: OAuth2Client
+metadata:
+  name: $app-oauth-client
+  # Has to live in the same namespace as the stackspin-$app-oauth-variables secret
+  namespace: flux-system
+spec:
+  # TODO copied from wekan: https://github.com/wekan/wekan/wiki/Keycloak
+  grantTypes:
+    - authorization_code
+    - refresh_token
+    - client_credentials
+    - implicit
+  responseTypes:
+    - id_token
+    - code
+  scope: "openid profile email stackspin_roles"
+  secretName: stackspin-$app-oauth-variables
+  #redirectUris:
+  #  - https://\${${app}_domain}/oauth/openid/
+  #tokenEndpointAuthMethod: client_secret_post
+EOF
+cat <<EOF >"$app/$app-release.yaml"
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: $app
+  namespace: $namespace
+spec:
+  releaseName: $app
+  chart:
+    spec:
+      chart: $app
+      version: 1.0 # TODO
+      sourceRef:
+        kind: HelmRepository
+        name: $repo
+        namespace: flux-system
+  interval: 5m
+  valuesFrom:
+    - kind: ConfigMap
+      name: stackspin-$app-values
+      optional: false
+    # Allow overriding values by ConfigMap or Secret
+    - kind: ConfigMap
+      name: stackspin-$app-override
+      optional: true
+    - kind: Secret
+      name: stackspin-$app-override
+      optional: true
+EOF
+cat <<EOF >"$app/$app-values-configmap.yaml"
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: stackspin-$app-values
+  namespace: $namespace
+data:
+  values.yaml: |
+    # TODO verify structure matches chart
+    commonLabels:
+      stackspin.net/backupSet: "${app}"
+    podLabels:
+      stackspin.net/backupSet: "${app}"
+    # TODO Configure PVC for data & database including backup labels
+    podAnnotations:
+      backup.velero.io/backup-volumes: "data"
+    persistence:
+      enabled: true
+      existingClaim: "${app}-data"
+
+    ingress:
+      enabled: true
+      # Elaborate style
+      annotations:
+        kubernetes.io/tls-acme: "true"
+      hosts:
+        - host: "\${${app}_domain}"
+          paths:
+             - path: /
+               pathType: Prefix
+      tls:
+        - secretName: $app-tls
+          hosts:
+            - "\${${app}_domain}"
+      # Bitnami style
+      hostname: "\${${app}_domain}"
+      tls: true
+      certManager: true
+    # TODO Adjust $app Mailing config
+    #    mailer:
+    #      enabled: "\${outgoing_mail_enabled}"
+    #      host: "\${outgoing_mail_smtp_host}"
+    #      port: "\${outgoing_mail_smtp_port}"
+    #      username: "\${outgoing_mail_smtp_user}"
+    #      password: "\${outgoing_mail_smtp_password}"
+    #      fromemail: "\${outgoing_mail_from_address}"
+    # TODO Adjust $app OpenID Connect Single Sign-On Configuration
+    #    - name: Stackspin
+    #      key: "\${client_id}"
+    #      secret: "\${client_secret}"
+    #      issuer: "https://\${hydra_domain}"
+    #      autoDiscoverUrl: 'https://\${hydra_domain}/.well-known/openid-configuration'
+EOF
+cat <<EOF >"$app/$app-pvc.yaml"
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: $app-data
+  namespace: $namespace
+  labels:
+    stackspin.net/backupSet: "$app"
+spec:
+  accessModes:
+    - ReadWriteOnce
+  volumeMode: Filesystem
+  resources:
+    requests:
+      storage: 2Gi
+  storageClassName: local-path
+EOF
+fi
+
+# Secrets
+
+cat <<EOF >>"$app-secrets-kustomization.yaml"
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+kind: Kustomization
+metadata:
+  name: ${app}-secrets
+  namespace: flux-system
+spec:
+  interval: 5m
+  timeout: 4m
+  wait: true
+  prune: true
+  path: ./apps/${subdomain}/${app}-secrets
+  sourceRef:
+    kind: GitRepository
+    name: ${namespace}
+  dependsOn:
+    - name: flux
+    - name: secrets-controller
+  postBuild:
+    substituteFrom:
+      - kind: Secret
+        name: stackspin-cluster-variables
+EOF
+if mkdir "$app-secrets"
+then
+cat <<EOF >"$app-secrets/$app-kustomization-variables.yaml"
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: stackspin-$app-kustomization-variables
+  namespace: flux-system
+data:
+  ${app}_domain: ${subdomain}.\${domain}
+EOF
+cat <<EOF >>"$app-secrets/$app-variables.yaml"
+---
+apiVersion: secretgenerator.mittwald.de/v1alpha1
+kind: StringSecret
+metadata:
+  name: stackspin-$app-variables
+  namespace: flux-system
+spec:
+  fields:
+  - fieldname: password
+EOF
+cat <<EOF >"$app-secrets/$app-oauth-secret.yaml"
+---
+apiVersion: secretgenerator.mittwald.de/v1alpha1
+kind: StringSecret
+metadata:
+  name: stackspin-$app-oauth-variables
+  namespace: flux-system
+spec:
+  data:
+    client_id: $app
+  fields:
+  - fieldName: client_secret
+    length: "32"
+EOF
+fi
+
+../generate-kustomizations.sh .
+echo "TODO: Obtain chart version, check configmap, adjust secrets" >&2
+exec $SHELL

.local/bin/server/stack-template

@@ -1,134 +0,0 @@
-#!/bin/sh -e
-if test $# -lt 1; then
-  echo "You should be in the root apps folder."
-  echo "Usage: $0 <app> [subdomain] [repo] [namespace]"
-  exit 1
-fi
-
-app=$1
-subdomain=${2:-$app}
-repo=${3:-$app}
-namespace=${4:-stackspout}
-
-if test "$(basename "$PWD")" != "$subdomain"
-then mkdir -p "$subdomain" && cd "$subdomain"
-fi
-
-cat <<EOF >$app-oauth-client.yaml
-apiVersion: hydra.ory.sh/v1alpha1
-kind: OAuth2Client
-metadata:
-  name: $app-oauth-client
-  # Has to live in the same namespace as the stackspin-$app-oauth-variables secret
-  namespace: flux-system
-spec:
-  # TODO copied from wekan: https://github.com/wekan/wekan/wiki/Keycloak
-  grantTypes:
-    - authorization_code
-    - refresh_token
-    - client_credentials
-    - implicit
-  responseTypes:
-    - id_token
-    - code
-  scope: "openid profile email stackspin_roles"
-  secretName: stackspin-$app-oauth-variables
-  #redirectUris:
-  #  - https://$subdomain.\${domain}/oauth/openid/
-  #tokenEndpointAuthMethod: client_secret_post
-EOF
-
-cat <<EOF >$app-release.yaml
-apiVersion: helm.toolkit.fluxcd.io/v2beta1
-kind: HelmRelease
-metadata:
-  name: $app
-  namespace: $namespace
-spec:
-  releaseName: $app
-  chart:
-    spec:
-      chart: $app
-      version: # TODO
-      sourceRef:
-        kind: HelmRepository
-        name: $repo
-        namespace: flux-system
-  interval: 5m
-  valuesFrom:
-    - kind: ConfigMap
-      name: stackspin-$app-values
-      optional: false
-    # Allow overriding values by ConfigMap or Secret
-    - kind: ConfigMap
-      name: stackspin-$app-override
-      optional: true
-    - kind: Secret
-      name: stackspin-$app-override
-      optional: true
-EOF
-
-cat <<EOF >$app-values-configmap.yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: stackspin-$app-values
-  namespace: $namespace
-data:
-  values.yaml: |
-    # TODO verify structure matches chart
-    ingress:
-      enabled: true
-      # Elaborate style
-      annotations:
-        kubernetes.io/tls-acme: "true"
-      hosts:
-        - host: "$subdomain.\${domain}"
-          paths:
-             - path: /
-               pathType: Prefix
-      tls:
-        - secretName: $app-tls
-          hosts:
-            - "$subdomain.\${domain}"
-      # Bitnami style
-      hostname: "$subdomain.\${domain}"
-      tls: true
-      certManager: true
-    # TODO Configure PVC for data & database
-    # TODO Adjust $app Mailing config
-    #    mailer:
-    #      enabled: "\${outgoing_mail_enabled}"
-    #      host: "\${outgoing_mail_smtp_host}"
-    #      port: "\${outgoing_mail_smtp_port}"
-    #      username: "\${outgoing_mail_smtp_user}"
-    #      password: "\${outgoing_mail_smtp_password}"
-    #      fromemail: "\${outgoing_mail_from_address}"
-    # TODO Adjust $app OpenID Connect Single Sign-On Configuration
-    #    - name: Stackspin
-    #      key: "\${client_id}"
-    #      secret: "\${client_secret}"
-    #      autoDiscoverUrl: 'https://sso.\${domain}/.well-known/openid-configuration'
-EOF
-
-cat <<EOF >$app-pvc.yaml
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: $app-data
-  namespace: $namespace
-  labels:
-    stackspin.net/backupSet: "$app"
-spec:
-  accessModes:
-    - ReadWriteOnce
-  volumeMode: Filesystem
-  resources:
-    requests:
-      storage: 2Gi
-  storageClassName: local-path
-EOF
-
-ls -l
-echo "To do: Obtain chart version, check configmap, create oauth secrets if needed" >&2
-exec $SHELL