chore: Update embedder queue to handle all types

Author: HaruHunab1320

apps/embedder/src/server/api/packs/[id]/loaders/[loaderId].get.ts

@@ -3,7 +3,11 @@ import { embedderDb, Loader } from '@magickml/embedder-db-pg'
 import { and, eq } from 'drizzle-orm'
 import { createLoader } from '@magickml/embedder-loaders-core'
 import { Storage } from '@google-cloud/storage'
-import { LoaderWithChunks, LoaderSchema } from '@magickml/embedder-schemas'
+import {
+  LoaderWithChunks,
+  LoaderSchema,
+  LoaderType,
+} from '@magickml/embedder-schemas'
 
 const storage = new Storage({
   projectId: process.env.GOOGLE_CLOUD_PROJECT_ID,
@@ -26,6 +30,15 @@ async function generateV4ReadSignedUrl(bucketName: string, fileName: string) {
   return url
 }
 
+function isValidUrl(string: string) {
+  try {
+    new URL(string)
+    return true
+  } catch (_) {
+    return false
+  }
+}
+
 export default defineEventHandler(async event => {
   const packId = z.string().parse(event.context.params?.id)
   const loaderId = z.string().parse(event.context.params?.loaderId)
@@ -45,26 +58,48 @@ export default defineEventHandler(async event => {
   }
 
   if (loader.status === 'completed') {
-    let url
-
-    // Generate signed URL for private bucket
-    try {
-      url = await generateV4ReadSignedUrl(
-        process.env.GOOGLE_PRIVATE_BUCKET_NAME || '',
-        (loader.config as any).filePathOrUrl || ''
-      )
-    } catch (error) {
-      console.error('Error getting loader:', error)
+    let sourceField: string | undefined
+    let needsSignedUrl = false
+
+    switch (loader.type as LoaderType) {
+      case 'pdf':
+      case 'docx':
+      case 'excel':
+      case 'ppt':
+        sourceField = 'filePathOrUrl'
+        needsSignedUrl = true
+        break
+      case 'web':
+        sourceField = 'urlOrContent'
+        break
+      case 'youtube':
+        sourceField = 'videoIdOrUrl'
+        break
+      case 'sitemap':
+        sourceField = 'url'
+        break
+      // Add other cases as needed
     }
 
-    let config = loader.config
-
-    if (url) {
-      url = (loader.config as any).filePathOrUrl
+    let url = sourceField ? (loader.config as any)[sourceField] : undefined
+
+    if (needsSignedUrl && url && !isValidUrl(url)) {
+      try {
+        url = await generateV4ReadSignedUrl(
+          process.env.GOOGLE_PRIVATE_BUCKET_NAME || '',
+          url
+        )
+      } catch (error) {
+        console.error('Error generating signed URL:', error)
+        // Handle the error gracefully
+      }
+    }
 
+    let config = { ...loader.config }
+    if (sourceField && url) {
       config = {
-        ...(loader.config as any),
-        ...(url && { filePathOrUrl: url }),
+        ...config,
+        [sourceField]: url,
       }
     }