Fix broken dependencies in helm chart and allow using existing secrets in the chart (#18941)

* Add ability to specify an existing Secret (#18139)

Closes #18139

* Allow using secrets with external postgres

* Upgrade CronJob to batch/v1

* Allow using redis.auth.existingSecret

* Helmignore mastodon-*.tgz for easy local development

* Upgrade helm dependencies

* Upgrade postgresql to 11

* Allow putting SMTP password into a secret

* Add optional login to SMTP secret

This to allow setting LOGIN either in values.yaml or
in the secret.

* Switch to bitnami charts full archive

This prevents older versions from disappearing, see
https://github.com/bitnami/charts/issues/10539 for
full context.

Co-authored-by: Ted Tramonte <ted.tramonte@gmail.com>

chart/.helmignore

@@ -21,3 +21,4 @@
 .idea/
 *.tmproj
 .vscode/
+mastodon-*.tgz

chart/Chart.lock

@@ -1,12 +1,12 @@
 dependencies:
 - name: elasticsearch
-  repository: https://charts.bitnami.com/bitnami
+  repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
-  version: 15.10.3
+  version: 19.0.1
 - name: postgresql
-  repository: https://charts.bitnami.com/bitnami
+  repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
-  version: 8.10.14
+  version: 11.1.3
 - name: redis
-  repository: https://charts.bitnami.com/bitnami
+  repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
-  version: 10.9.0
+  version: 16.13.2
-digest: sha256:f5c57108f7768fd16391c1a050991c7809f84a640cca308d7d24d87379d04000
+digest: sha256:17ea58a3264aa22faff18215c4269f47dabae956d0df273c684972f356416193
-generated: "2021-08-05T08:01:01.457727804Z"
+generated: "2022-08-08T21:44:18.0195364+02:00"

chart/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.2.1
+version: 2.0.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
@@ -24,13 +24,13 @@ appVersion: 3.3.0
 
 dependencies:
   - name: elasticsearch
-    version: 15.10.3
+    version: 19.0.1
-    repository: https://charts.bitnami.com/bitnami
+    repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
     condition: elasticsearch.enabled
   - name: postgresql
-    version: 8.10.14
+    version: 11.1.3
-    repository: https://charts.bitnami.com/bitnami
+    repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
     condition: postgresql.enabled
   - name: redis
-    version: 10.9.0
+    version: 16.13.2
-    repository: https://charts.bitnami.com/bitnami
+    repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami

chart/templates/_helpers.tpl

@@ -77,3 +77,53 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- define "mastodon.postgresql.fullname" -}}
 {{- printf "%s-%s" .Release.Name "postgresql" | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
+
+{{/*
+Get the mastodon secret.
+*/}}
+{{- define "mastodon.secretName" -}}
+{{- if .Values.mastodon.secrets.existingSecret }}
+    {{- printf "%s" (tpl .Values.mastodon.secrets.existingSecret $) -}}
+{{- else -}}
+    {{- printf "%s" (include "common.names.fullname" .) -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Get the postgresql secret.
+*/}}
+{{- define "mastodon.postgresql.secretName" -}}
+{{- if (and (or .Values.postgresql.enabled .Values.postgresql.postgresqlHostname) .Values.postgresql.auth.existingSecret) }}
+    {{- printf "%s" (tpl .Values.postgresql.auth.existingSecret $) -}}
+{{- else if .Values.postgresql.enabled -}}
+    {{- printf "%s-postgresql" (tpl .Release.Name $) -}}
+{{- else -}}
+    {{- printf "%s" (include "common.names.fullname" .) -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Get the redis secret.
+*/}}
+{{- define "mastodon.redis.secretName" -}}
+{{- if .Values.redis.auth.existingSecret }}
+    {{- printf "%s" (tpl .Values.redis.auth.existingSecret $) -}}
+{{- else if .Values.redis.existingSecret }}
+    {{- printf "%s" (tpl .Values.redis.existingSecret $) -}}
+{{- else -}}
+    {{- printf "%s-redis" (tpl .Release.Name $) -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return true if a mastodon secret object should be created
+*/}}
+{{- define "mastodon.createSecret" -}}
+{{- if (or
+    (and .Values.mastodon.s3.enabled (not .Values.mastodon.s3.existingSecret))
+    (not .Values.mastodon.secrets.existingSecret )
+    (and (not .Values.postgresql.enabled) (not .Values.postgresql.auth.existingSecret))
+    ) -}}
+    {{- true -}}
+{{- end -}}
+{{- end -}}

chart/templates/configmap-env.yaml

@@ -10,14 +10,14 @@ data:
   {{- else }}
   DB_HOST: {{ .Values.postgresql.postgresqlHostname }}
   {{- end }}
-  DB_NAME: {{ .Values.postgresql.postgresqlDatabase }}
+  DB_NAME: {{ .Values.postgresql.auth.database }}
   DB_POOL: {{ .Values.mastodon.sidekiq.concurrency | quote }}
   DB_PORT: "5432"
-  DB_USER: {{ .Values.postgresql.postgresqlUsername }}
+  DB_USER: {{ .Values.postgresql.auth.username }}
   DEFAULT_LOCALE: {{ .Values.mastodon.locale }}
   {{- if .Values.elasticsearch.enabled }}
   ES_ENABLED: "true"
-  ES_HOST: {{ template "mastodon.elasticsearch.fullname" . }}-master
+  ES_HOST: {{ template "mastodon.elasticsearch.fullname" . }}-master-hl
   ES_PORT: "9200"
   {{- end }}
   LOCAL_DOMAIN: {{ .Values.mastodon.local_domain }}

chart/templates/cronjob-media-remove.yaml

@@ -1,5 +1,5 @@
 {{ if .Values.mastodon.cron.removeMedia.enabled }}
-apiVersion: batch/v1beta1
+apiVersion: batch/v1
 kind: CronJob
 metadata:
   name: {{ include "mastodon.fullname" . }}-media-remove
@@ -49,21 +49,17 @@ spec:
                 - configMapRef:
                     name: {{ include "mastodon.fullname" . }}-env
                 - secretRef:
-                    name: {{ template "mastodon.fullname" . }}
+                    name: {{ template "mastodon.secretName" . }}
               env:
                 - name: "DB_PASS"
                   valueFrom:
                     secretKeyRef:
-                      {{- if .Values.postgresql.enabled }}
+                      name: {{ template "mastodon.postgresql.secretName" . }}
-                      name: {{ .Release.Name }}-postgresql
+                      key: password
-                      {{- else }}
-                      name: {{ template "mastodon.fullname" . }}
-                      {{- end }}
-                      key: postgresql-password
                 - name: "REDIS_PASSWORD"
                   valueFrom:
                     secretKeyRef:
-                      name: {{ .Release.Name }}-redis
+                      name: {{ template "mastodon.redis.secretName" . }}
                       key: redis-password
                 - name: "PORT"
                   value: {{ .Values.mastodon.web.port | quote }}

chart/templates/deployment-sidekiq.yaml

@@ -70,22 +70,31 @@ spec:
             - configMapRef:
                 name: {{ include "mastodon.fullname" . }}-env
             - secretRef:
-                name: {{ template "mastodon.fullname" . }}
+                name: {{ template "mastodon.secretName" . }}
           env:
             - name: "DB_PASS"
               valueFrom:
                 secretKeyRef:
-                  {{- if .Values.postgresql.enabled }}
+                  name: {{ template "mastodon.postgresql.secretName" . }}
-                  name: {{ .Release.Name }}-postgresql
+                  key: password
-                  {{- else }}
-                  name: {{ template "mastodon.fullname" . }}
-                  {{- end }}
-                  key: postgresql-password
             - name: "REDIS_PASSWORD"
               valueFrom:
                 secretKeyRef:
-                  name: {{ .Release.Name }}-redis
+                  name: {{ template "mastodon.redis.secretName" . }}
                   key: redis-password
+            {{- if .Values.mastodon.smtp.existingSecret }}
+            - name: "SMTP_LOGIN"
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.mastodon.smtp.existingSecret }}
+                  key: login
+                  optional: true
+            - name: "SMTP_PASSWORD"
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.mastodon.smtp.existingSecret }}
+                  key: password
+            {{- end -}}
           {{- if (not .Values.mastodon.s3.enabled) }}
           volumeMounts:
             - name: assets

chart/templates/deployment-streaming.yaml

@@ -43,16 +43,12 @@ spec:
             - name: "DB_PASS"
               valueFrom:
                 secretKeyRef:
-                  {{- if .Values.postgresql.enabled }}
+                  name: {{ template "mastodon.postgresql.secretName" . }}
-                  name: {{ .Release.Name }}-postgresql
+                  key: password
-                  {{- else }}
-                  name: {{ template "mastodon.fullname" . }}
-                  {{- end }}
-                  key: postgresql-password
             - name: "REDIS_PASSWORD"
               valueFrom:
                 secretKeyRef:
-                  name: {{ .Release.Name }}-redis
+                  name: {{ template "mastodon.redis.secretName" . }}
                   key: redis-password
             - name: "PORT"
               value: {{ .Values.mastodon.streaming.port | quote }}

chart/templates/deployment-web.yaml

@@ -56,21 +56,17 @@ spec:
             - configMapRef:
                 name: {{ include "mastodon.fullname" . }}-env
             - secretRef:
-                name: {{ template "mastodon.fullname" . }}
+                name: {{ template "mastodon.secretName" . }}
           env:
             - name: "DB_PASS"
               valueFrom:
                 secretKeyRef:
-                  {{- if .Values.postgresql.enabled }}
+                  name: {{ template "mastodon.postgresql.secretName" . }}
-                  name: {{ .Release.Name }}-postgresql
+                  key: password
-                  {{- else }}
-                  name: {{ template "mastodon.fullname" . }}
-                  {{- end }}
-                  key: postgresql-password
             - name: "REDIS_PASSWORD"
               valueFrom:
                 secretKeyRef:
-                  name: {{ .Release.Name }}-redis
+                  name: {{ template "mastodon.redis.secretName" . }}
                   key: redis-password
             - name: "PORT"
               value: {{ .Values.mastodon.web.port | quote }}

chart/templates/job-assets-precompile.yaml

@@ -50,21 +50,17 @@ spec:
             - configMapRef:
                 name: {{ include "mastodon.fullname" . }}-env
             - secretRef:
-                name: {{ template "mastodon.fullname" . }}
+                name: {{ template "mastodon.secretName" . }}
           env:
             - name: "DB_PASS"
               valueFrom:
                 secretKeyRef:
-                  {{- if .Values.postgresql.enabled }}
+                  name: {{ template "mastodon.postgresql.secretName" . }}
-                  name: {{ .Release.Name }}-postgresql
+                  key: password
-                  {{- else }}
-                  name: {{ template "mastodon.fullname" . }}
-                  {{- end }}
-                  key: postgresql-password
             - name: "REDIS_PASSWORD"
               valueFrom:
                 secretKeyRef:
-                  name: {{ .Release.Name }}-redis
+                  name: {{ template "mastodon.redis.secretName" . }}
                   key: redis-password
             - name: "PORT"
               value: {{ .Values.mastodon.web.port | quote }}

chart/templates/job-chewy-upgrade.yaml

@@ -51,21 +51,17 @@ spec:
             - configMapRef:
                 name: {{ include "mastodon.fullname" . }}-env
             - secretRef:
-                name: {{ template "mastodon.fullname" . }}
+                name: {{ template "mastodon.secretName" . }}
           env:
             - name: "DB_PASS"
               valueFrom:
                 secretKeyRef:
-                  {{- if .Values.postgresql.enabled }}
+                  name: {{ template "mastodon.postgresql.secretName" . }}
-                  name: {{ .Release.Name }}-postgresql
+                  key: password
-                  {{- else }}
-                  name: {{ template "mastodon.fullname" . }}
-                  {{- end }}
-                  key: postgresql-password
             - name: "REDIS_PASSWORD"
               valueFrom:
                 secretKeyRef:
-                  name: {{ .Release.Name }}-redis
+                  name: {{ template "mastodon.redis.secretName" . }}
                   key: redis-password
             - name: "PORT"
               value: {{ .Values.mastodon.web.port | quote }}

chart/templates/job-create-admin.yaml

@@ -56,21 +56,17 @@ spec:
             - configMapRef:
                 name: {{ include "mastodon.fullname" . }}-env
             - secretRef:
-                name: {{ template "mastodon.fullname" . }}
+                name: {{ template "mastodon.secretName" . }}
           env:
             - name: "DB_PASS"
               valueFrom:
                 secretKeyRef:
-                  {{- if .Values.postgresql.enabled }}
+                  name: {{ template "mastodon.postgresql.secretName" . }}
-                  name: {{ .Release.Name }}-postgresql
+                  key: password
-                  {{- else }}
-                  name: {{ template "mastodon.fullname" . }}
-                  {{- end }}
-                  key: postgresql-password
             - name: "REDIS_PASSWORD"
               valueFrom:
                 secretKeyRef:
-                  name: {{ .Release.Name }}-redis
+                  name: {{ template "mastodon.redis.secretName" . }}
                   key: redis-password
             - name: "PORT"
               value: {{ .Values.mastodon.web.port | quote }}

chart/templates/job-db-migrate.yaml

@@ -50,21 +50,17 @@ spec:
             - configMapRef:
                 name: {{ include "mastodon.fullname" . }}-env
             - secretRef:
-                name: {{ template "mastodon.fullname" . }}
+                name: {{ template "mastodon.secretName" . }}
           env:
             - name: "DB_PASS"
               valueFrom:
                 secretKeyRef:
-                  {{- if .Values.postgresql.enabled }}
+                  name: {{ template "mastodon.postgresql.secretName" . }}
-                  name: {{ .Release.Name }}-postgresql
+                  key: password
-                  {{- else }}
-                  name: {{ template "mastodon.fullname" . }}
-                  {{- end }}
-                  key: postgresql-password
             - name: "REDIS_PASSWORD"
               valueFrom:
                 secretKeyRef:
-                  name: {{ .Release.Name }}-redis
+                  name: {{ template "mastodon.redis.secretName" . }}
                   key: redis-password
             - name: "PORT"
               value: {{ .Values.mastodon.web.port | quote }}

chart/templates/secrets.yaml

@@ -1,3 +1,4 @@
+{{- if (include "mastodon.createSecret" .) }}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -7,9 +8,12 @@ metadata:
 type: Opaque
 data:
   {{- if .Values.mastodon.s3.enabled }}
+  {{- if not .Values.mastodon.s3.existingSecret }}
   AWS_ACCESS_KEY_ID: "{{ .Values.mastodon.s3.access_key | b64enc }}"
   AWS_SECRET_ACCESS_KEY: "{{ .Values.mastodon.s3.access_secret | b64enc }}"
   {{- end }}
+  {{- end }}
+  {{- if not .Values.mastodon.secrets.existingSecret }}
   {{- if not (empty .Values.mastodon.secrets.secret_key_base) }}
   SECRET_KEY_BASE: "{{ .Values.mastodon.secrets.secret_key_base | b64enc }}"
   {{- else }}
@@ -30,6 +34,10 @@ data:
   {{- else }}
   VAPID_PUBLIC_KEY: {{ required "vapid.public_key is required" .Values.mastodon.secrets.vapid.public_key }}
   {{- end }}
-  {{- if not .Values.postgresql.enabled }}
-  postgresql-password: "{{ .Values.postgresql.postgresqlPassword | b64enc }}"
   {{- end }}
+  {{- if not .Values.postgresql.enabled }}
+  {{- if not .Values.postgresql.auth.existingSecret }}
+  postgresql-password: "{{ .Values.postgresql.auth.password | b64enc }}"
+  {{- end }}
+  {{- end }}
+{{- end -}}

chart/values.yaml

@@ -48,6 +48,9 @@ mastodon:
     enabled: false
     access_key: ""
     access_secret: ""
+    # you can also specify the name of an existing Secret
+    # with keys AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
+    existingSecret: ""
     bucket: ""
     endpoint: https://us-east-1.linodeobjects.com
     hostname: us-east-1.linodeobjects.com
@@ -61,6 +64,10 @@ mastodon:
     vapid:
       private_key: ""
       public_key: ""
+    # you can also specify the name of an existing Secret
+    # with keys SECRET_KEY_BASE and OTP_SECRET and
+    # VAPID_PRIVATE_KEY and VAPID_PUBLIC_KEY
+    existingSecret: ""
   sidekiq:
     concurrency: 25
   smtp:
@@ -70,13 +77,16 @@ mastodon:
     domain:
     enable_starttls_auto: true
     from_address: notifications@example.com
-    login:
     openssl_verify_mode: peer
-    password:
     port: 587
     reply_to:
     server: smtp.mailgun.org
     tls: false
+    login:
+    password:
+    # you can also specify the name of an existing Secret
+    # with the keys login and password
+    existingSecret:
   streaming:
     port: 4000
     # this should be set manually since os.cpus() returns the number of CPUs on
@@ -127,18 +137,26 @@ postgresql:
   # must match those of that external postgres instance
   enabled: true
   # postgresqlHostname: preexisting-postgresql
-  postgresqlDatabase: mastodon_production
+  auth:
-  # you must set a password; the password generated by the postgresql chart will
+    database: mastodon_production
-  # be rotated on each upgrade:
+    username: postgres
-  # https://github.com/bitnami/charts/tree/master/bitnami/postgresql#upgrade
+    # you must set a password; the password generated by the postgresql chart will
-  postgresqlPassword: ""
+    # be rotated on each upgrade:
-  postgresqlUsername: postgres
+    # https://github.com/bitnami/charts/tree/master/bitnami/postgresql#upgrade
+    password: ""
+    # you can also specify the name of an existing Secret
+    # with a key of postgres-password set to the password you want
+    existingSecret: ""
 
 # https://github.com/bitnami/charts/tree/master/bitnami/redis#parameters
 redis:
   # you must set a password; the password generated by the redis chart will be
   # rotated on each upgrade:
   password: ""
+  # you can also specify the name of an existing Secret
+  # with a key of redis-password set to the password you want
+  # auth:
+    # existingSecret: ""
 
 service:
   type: ClusterIP