Merge pull request #3 from diffgram/improve-azure-install

Improve data validations for installs on azure

templates/default/deployment.yaml

@@ -20,12 +20,14 @@ spec:
       - name: diffgramsecret
       {{ end }}
       volumes:
+      {{ if eq .Values.diffgramSettings.DIFFGRAM_STATIC_STORAGE_PROVIDER "gcp"}}
       - name: service-account-credentials-volume
         secret:
           secretName: gcp-service-account-credentials
           items:
           - key: sa_json
             path: sa_credentials.json
+      {{ end }}
       initContainers:
       - name: check-db-ready
         image: postgres:9.6.5
@@ -51,10 +53,12 @@ spec:
         name: diffgram-default
         ports:
         - containerPort: 8080
+      {{ if eq .Values.diffgramSettings.DIFFGRAM_STATIC_STORAGE_PROVIDER "gcp"}}
         volumeMounts:
         - name: service-account-credentials-volume
           mountPath: /etc/gcp
           readOnly: true
+      {{ end }}
         envFrom:
         - configMapRef:
             name: diffgram-default-configmap

templates/hooks/configmap_db_migrations.yaml

@@ -3,7 +3,7 @@ kind: ConfigMap
 metadata:
   annotations:
     "helm.sh/hook": pre-install,pre-upgrade, pre-rollback
-    "helm.sh/hook-weight": "1" # we use a smaller weight so it's created before the job
+    "helm.sh/hook-weight": "-3" # we use a smaller weight so it's created before the job
     "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded
   name: db-migrations-configmap
 data:

templates/hooks/database_pre_install.yaml

@@ -28,12 +28,17 @@ spec:
       - name: db-migrations-pull-secret
       {{ end }}
       volumes:
+      {{ if eq .Values.diffgramSettings.DIFFGRAM_STATIC_STORAGE_PROVIDER "gcp"}}
       - name: service-account-credentials-volume-hook
         secret:
           secretName: gcp-service-account-credentials-hook
           items:
           - key: sa_json
             path: sa_credentials.json
+      {{ end }}
+      - name: postgres-storage
+        persistentVolumeClaim:
+          claimName: postgres-pv-claim
       containers:
       {{ if eq .Values.diffgramEdition "enterprise"}}
       - image: gcr.io/diffgram-enterprise/default:{{ .Values.diffgramVersion }}
@@ -43,14 +48,16 @@ spec:
       {{ end }}
         imagePullPolicy: Always
         name: pre-upgrade-alembic-hook
+      {{ if eq .Values.diffgramSettings.DIFFGRAM_STATIC_STORAGE_PROVIDER "gcp"}}
         volumeMounts:
-        - name: service-account-credentials-volume-hook
+        - name: service-account-credentials-volume
           mountPath: /etc/gcp
           readOnly: true
+      {{ end }}
         envFrom:
         - configMapRef:
             name: db-migrations-configmap
         - secretRef:
             name:  db-migrations-secret
         # The actual migrations command
-        command: ["sh","-c", "cd shared; export PYTHONPATH=/app; pip install sqlalchemy-utils==0.36.6;python /app/play_and_scripts/scripts/create_database.py; alembic upgrade head"]
+        command: ["sh","-c", "cd shared; export PYTHONPATH=/app; pip install sqlalchemy-utils==0.36.6;python /app/play_and_scripts/scripts/create_database.py; alembic upgrade head"]

templates/hooks/db_service_migrations.yaml

@@ -40,52 +40,6 @@ spec:
 status:
   loadBalancer: {}
 {{ end }}
-
-{{ if eq .Values.dbSettings.dbProvider "local"}}
-apiVersion: "apps/v1"
-kind: "Deployment"
-metadata:
-  annotations:
-    # This is what defines this resource as a hook. Without this line, the
-    # job is considered part of the release.
-    "helm.sh/hook": pre-install, pre-upgrade, pre-rollback
-    "helm.sh/hook-weight": "0"
-    "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded
-  name: "postgres-hook"
-  namespace: "default"
-  labels:
-    app: "postgres-hook"
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: "postgres"
-  template:
-    metadata:
-      labels:
-        app: "postgres"
-    spec:
-      containers:
-      - name: "postgres"
-        image: "postgres:9.6.2"
-        env:
-        - name: "POSTGRES_DB"
-          value: {{ .Values.dbSettings.dbName }}
-        - name: "POSTGRES_USER"
-          value: {{ .Values.dbSettings.dbUser }}
-        - name: "POSTGRES_PASSWORD"
-          value: {{ .Values.dbSettings.dbPassword }}
-        ports:
-          - containerPort: 5432
-            name: postgres
-        volumeMounts:
-          - name: postgres-storage
-            mountPath: /var/lib/postgresql/db-data
-      volumes:
-        - name: postgres-storage
-          persistentVolumeClaim:
-            claimName: postgres-pv-claim
-{{ end }}
 {{ if eq .Values.dbSettings.dbProvider "local"}}
 apiVersion: v1
 kind: Service
@@ -94,12 +48,12 @@ metadata:
     # This is what defines this resource as a hook. Without this line, the
     # job is considered part of the release.
     "helm.sh/hook": pre-install, pre-upgrade, pre-rollback
-    "helm.sh/hook-weight": "0"
+    "helm.sh/hook-weight": "1"
     "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded
   name: diffgram-postgres-hook
 spec:
   ports:
     - port: 5432
   selector:
-    app: diffgram-postgres-hook
+    app: postgres
 {{ end }}

templates/hooks/secret_gcp_service_account_hook.yaml

@@ -1,3 +1,4 @@
+{{ if eq .Values.diffgramSettings.DIFFGRAM_STATIC_STORAGE_PROVIDER "gcp"}}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -9,4 +10,5 @@ metadata:
 type: Opaque
 data:
   # This is the JSON file encoded in base64. It will be mounted as a volume on the container.
-  sa_json: {{ .Values.diffgramSecrets.SERVICE_ACCOUNT_JSON_B64 }}
+  sa_json: {{ .Values.diffgramSecrets.SERVICE_ACCOUNT_JSON_B64 }}
+{{ end }}

templates/hooks/secret_gcr_db_migrations.yaml

@@ -1,3 +1,4 @@
+{{ if eq .Values.diffgramEdition "enterprise"}}
 apiVersion: v1
 data:
   .dockerconfigjson: {{ .Values.imagePullCredentials.gcrCredentials }}
@@ -21,3 +22,4 @@ metadata:
   name: db-migrations-pull-secret
   namespace: default
 type: kubernetes.io/dockerconfigjson
+{{ end }}

templates/hooks/secrets_db_migrations.yaml

@@ -4,7 +4,7 @@ kind: Secret
 metadata:
   annotations:
     "helm.sh/hook": pre-install, pre-upgrade, pre-rollback
-    "helm.sh/hook-weight": "1"
+    "helm.sh/hook-weight": "-3"
     "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded
   name: db-migrations-secret
 type: Opaque
@@ -17,7 +17,7 @@ stringData:
   SECRET_KEY: {{ .Values.diffgramSecrets.SECRET_KEY }}
   INTER_SERVICE_SECRET: {{ .Values.diffgramSecrets.INTER_SERVICE_SECRET }}
   {{ if eq .Values.dbSettings.dbProvider "local"}}
-  DATABASE_URL:  "postgresql+psycopg2://{{ .Values.dbSettings.dbUser }}:{{ .Values.dbSettings.dbPassword }}@diffgram-postgres/{{ .Values.dbSettings.dbName }}"
+  DATABASE_URL:  "postgresql+psycopg2://{{ .Values.dbSettings.dbUser }}:{{ .Values.dbSettings.dbPassword }}@diffgram-postgres-hook/{{ .Values.dbSettings.dbName }}"
   {{ end }}
   {{ if eq .Values.dbSettings.dbProvider "rds"}}
   DATABASE_URL:  "postgresql+psycopg2://{{ .Values.dbSettings.dbUser }}:{{ .Values.dbSettings.dbPassword }}@{{ .Values.dbSettings.rdsEndpoint }}/{{ .Values.dbSettings.dbName }}"
@@ -26,4 +26,4 @@ stringData:
   DATABASE_URL:  "postgresql+psycopg2://{{ .Values.dbSettings.dbUser }}:{{ .Values.dbSettings.dbPassword }}@{{ .Values.dbSettings.azureSqlEndpoint }}/{{ .Values.dbSettings.dbName }}"
   {{ end }}
   USER_PASSWORDS_SECRET:  {{ .Values.diffgramSecrets.USER_PASSWORDS_SECRET }}
-  DIFFGRAM_AZURE_CONNECTION_STRING:  {{ .Values.diffgramSecrets.DIFFGRAM_AZURE_CONNECTION_STRING }}
+  DIFFGRAM_AZURE_CONNECTION_STRING:  {{ .Values.diffgramSecrets.DIFFGRAM_AZURE_CONNECTION_STRING }}

templates/postgres/deployment.yaml

@@ -1,11 +1,19 @@
-{{ if eq .Values.dbSettings.dbProvider "local"}}
+{{- $postgresdeploy := (lookup "apps/v1" "Deployment" .Release.Namespace (printf "postgres")) }}
+{{ if eq .Values.dbSettings.dbProvider "local" }}
+{{ if not $postgresdeploy }}
 apiVersion: "apps/v1"
 kind: "Deployment"
 metadata:
   name: "postgres"
   namespace: "default"
   labels:
     app: "postgres"
+  annotations:
+    # This is what defines this resource as a hook. Without this line, the
+    # job is considered part of the release.
+    "helm.sh/hook": pre-install
+    "helm.sh/hook-weight": "-3"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
 spec:
   replicas: 1
   selector:
@@ -36,4 +44,5 @@ spec:
         - name: postgres-storage
           persistentVolumeClaim:
             claimName: postgres-pv-claim
+{{ end }}
 {{ end }}

templates/postgres/volumeclaim.yaml

@@ -1,10 +1,19 @@
+{{- $mypvc := (lookup "v1" "PersistentVolumeClaim" .Release.Namespace (printf "postgres-pv-claim")) }}
+{{ if eq .Values.dbSettings.dbProvider "local" }}
+{{ if not $mypvc}}
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
   name: postgres-pv-claim
+  annotations:
+    "helm.sh/resource-policy": keep
+    "helm.sh/hook": "pre-install"
+    "helm.sh/hook-weight": "-5"
 spec:
   accessModes:
-    - ReadWriteOnce
+  - ReadWriteOnce
   resources:
     requests:
-      storage: {{ .Values.dbSettings.storageAmount }}
+      storage: {{ .Values.dbSettings.storageAmount }}
+{{ end }}
+{{ end }}

templates/secret-gcr.yaml

@@ -1,3 +1,4 @@
+{{ if eq .Values.diffgramEdition "enterprise"}}
 apiVersion: v1
 data:
   .dockerconfigjson: {{ .Values.imagePullCredentials.gcrCredentials }}
@@ -16,3 +17,4 @@ metadata:
   name: diffgramsecret
   namespace: default
 type: kubernetes.io/dockerconfigjson
+{{ end }}

templates/service_account_secret.yaml

@@ -1,8 +1,10 @@
+{{ if eq .Values.diffgramSettings.DIFFGRAM_STATIC_STORAGE_PROVIDER "gcp"}}
 apiVersion: v1
 kind: Secret
 metadata:
   name: gcp-service-account-credentials
 type: Opaque
 data:
   # This is the JSON file encoded in base64. It will be mounted as a volume on the container.
-  sa_json: {{ .Values.diffgramSecrets.SERVICE_ACCOUNT_JSON_B64 }}
+  sa_json: {{ .Values.diffgramSecrets.SERVICE_ACCOUNT_JSON_B64 }}
+{{ end }}

templates/walrus/deployment.yaml

@@ -20,12 +20,14 @@ spec:
       - name: diffgramsecret
       {{ end }}
       volumes:
+      {{ if eq .Values.diffgramSettings.DIFFGRAM_STATIC_STORAGE_PROVIDER "gcp"}}
       - name: service-account-credentials-volume
         secret:
           secretName: gcp-service-account-credentials
           items:
           - key: sa_json
             path: sa_credentials.json
+      {{ end }}
       - name: empty-dir
         emptyDir: {}
       initContainers:
@@ -57,11 +59,12 @@ spec:
         name: diffgram-walrus
         ports:
         - containerPort: 8080
+      {{ if eq .Values.diffgramSettings.DIFFGRAM_STATIC_STORAGE_PROVIDER "gcp"}}
         volumeMounts:
         - name: service-account-credentials-volume
           mountPath: /etc/gcp
           readOnly: true
-
+      {{ end }}
         envFrom:
         - configMapRef:
             name: diffgram-walrus-configmap

values.yaml

@@ -10,6 +10,8 @@ diffgramVersion: latest
 diffgramEdition: opencore
 
 # Set this to your public domain where you want diffgram to be.
+# This must be a domain name and not a public IP address.
+# The chart will generate TLS certificates for the provided domain if useCertManager is 'true'
 diffgramDomain: example.com
 
 # Set this to true if you want to use cert manager for TLS certificates generation.
@@ -59,6 +61,9 @@ diffgramSettings:
   ML__CLOUD_STORAGE_BUCKET: diffgram-testing
   GOOGLE_APPLICATION_CREDENTIALS: /etc/gcp/sa_credentials.json # Check the volume in deployment.yaml and service_account_secret.yaml
 
+  # Set this value if you want to use GCP as your storage. Put your json service account encoded in base 64
+  SERVICE_ACCOUNT_JSON_B64: none
+
   SERVICE_ACCOUNT: sa_credentials.json
 
 imagePullCredentials:
@@ -90,8 +95,8 @@ frontendService:
 walrusService:
   numReplicas: 1
   requests:
-    cpu: "16.0"
-    memory: "48G"
+    cpu: "4.0"
+    memory: "16G"
   limits:
-    cpu: "16.0"
-    memory: "48G"
+    cpu: "4.0"
+    memory: "16G"