fix: queries and other quality of life improvements

Author: zeidlitz

helpers/helpers.go

@@ -19,8 +19,12 @@ import (
 var templateFS embed.FS
 var configPath string
 var output string
+var withDB bool
+var withXml bool
 
 type TemplateData struct {
+	WithDB                       bool
+	WithXml                      bool
 	JobName                      string
 	JobReleaseLabel              string
 	JobArgs                      string
@@ -81,10 +85,14 @@ func init() {
 	cmd.AddCommand(renderCmd)
 	renderCmd.Flags().StringVarP(&configPath, "config", "c", "config.yaml", "Path to configuration file")
 	renderCmd.Flags().StringVarP(&output, "output", "o", "job.yaml", "Path to write the rendered file to")
+	renderCmd.Flags().BoolVarP(&withDB, "database", "d", false, "Render manifest with database values included")
+	renderCmd.Flags().BoolVarP(&withXml, "xml", "x", false, "Render manifest with xml volumes included")
 }
 
 func createTemplateData(cfg *config.Config) (TemplateData, error) {
 	templateData := &TemplateData{
+		WithDB:                       withDB,
+		WithXml:                      withXml,
 		JobName:                      strings.ToLower(strings.ReplaceAll(cfg.DatasetFolder, "_", "-")),
 		JobReleaseLabel:              "sda",
 		JobArgs:                      fmt.Sprintf("[\"job\", \"%d\"]", cfg.ExpectedNrFiles),

helpers/templates/job.template.yaml

@@ -50,6 +50,7 @@ spec:
               value: {{ .MailSmptHost }}
             - name: MAIL_SMTP_PORT
               value: "{{ .MailSmptPort }}"
+            {{- if .WithDB }}
             - name: DB_HOST
               valueFrom:
                 secretKeyRef:
@@ -91,6 +92,7 @@ spec:
               value: "/.secrets/tls/tls.crt"
             - name: DB_CLIENT_KEY
               value: "/.secrets/tls/tls.key"
+          {{- end }}
           securityContext:
             allowPrivilegeEscalation: false
             runAsNonRoot: true
@@ -103,17 +105,21 @@ spec:
           volumeMounts:
             - name: data-volume
               mountPath: /data
+            {{- if .WithXml }}
             - name: xml-data-{{ .JobName }}
               mountPath: /data/xml
+            {{- end }}
             - name: tls
               mountPath: /.secrets/tls/
       volumes:
         - name: data-volume
           emptyDir: {}
+        {{- if .WithXml }}
         - name: xml-data-{{ .JobName }}
           secret:
             defaultMode: 288
             secretName: {{ .MailXmlSecretName }}
+        {{- end }}
         - name: tls
           secret:
             defaultMode: 288

internal/client/client.go

@@ -61,7 +61,7 @@ func New(cfg *config.Config) (*Client, error) {
 	return client, nil
 }
 
-func (c *Client) GetUsersFilesWithPrefix() ([]byte, error) {
+func (c *Client) GetUsersFilesWithPrefix() ([]models.FileInfo, error) {
 	basePath := fmt.Sprintf("users/%s/files", c.userID)
 
 	u, err := url.Parse(basePath)
@@ -73,7 +73,17 @@ func (c *Client) GetUsersFilesWithPrefix() ([]byte, error) {
 	q.Set("path_prefix", c.datasetFolder)
 	u.RawQuery = q.Encode()
 
-	return c.doRequest("GET", u.String(), nil)
+	respBody, err := c.doRequest("GET", u.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	var files []models.FileInfo
+	err = json.Unmarshal(respBody, &files)
+	if err != nil {
+		return nil, err
+	}
+	return files, err
 }
 
 func (c *Client) GetUsersFiles() ([]models.FileInfo, error) {

internal/client/client_interface.go

@@ -6,6 +6,7 @@ import (
 
 type APIClient interface {
 	GetUsersFiles() ([]models.FileInfo, error)
+	GetUsersFilesWithPrefix() ([]models.FileInfo, error)
 	PostFileIngest([]byte) ([]byte, error)
 	PostFileAccession(payload []byte) ([]byte, error)
 }

internal/dataset/dataset.go

@@ -43,13 +43,10 @@ var datasetCmd = &cobra.Command{
 			return err
 		}
 
-		resp, err := api.GetUsersFilesWithPrefix()
-		if err != nil {
-			return err
-		}
-
 		var files []models.FileInfo
-		if err := json.Unmarshal(resp, &files); err != nil {
+
+		files, err = api.GetUsersFilesWithPrefix()
+		if err != nil {
 			return err
 		}
 

internal/ingest/ingest.go

@@ -32,7 +32,7 @@ var ingestCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
-		files, err := api.GetUsersFiles()
+		files, err := api.GetUsersFilesWithPrefix()
 		if err != nil {
 			return err
 		}
@@ -52,7 +52,7 @@ func init() {
 }
 
 func Run(api client.APIClient, datasetFolder string, userID string, expectedFiles int) (int, error) {
-	files, err := api.GetUsersFiles()
+	files, err := api.GetUsersFilesWithPrefix()
 	if err != nil {
 		return 0, err
 	}