feat: improve global variables(time, file, dataset)

packages/global/common/file/tools.ts

@@ -1,7 +1,7 @@
 import { detect } from 'jschardet';
 import { documentFileType } from './constants';
 import { ChatFileTypeEnum } from '../../core/chat/constants';
-import { type UserChatItemValueItemType } from '../../core/chat/type';
+import { type UserChatItemFileItemType } from '../../core/chat/type';
 import * as fs from 'fs';
 
 export const formatFileSize = (bytes: number): string => {
@@ -36,7 +36,7 @@ export const detectFileEncodingByPath = async (path: string) => {
 };
 
 // Url => user upload file type
-export const parseUrlToFileType = (url: string): UserChatItemValueItemType['file'] | undefined => {
+export const parseUrlToFileType = (url: string): UserChatItemFileItemType | undefined => {
   if (typeof url !== 'string') return;
 
   // Handle base64 image
@@ -74,13 +74,13 @@ export const parseUrlToFileType = (url: string): UserChatItemValueItemType['file
     // Default to image type for non-document files
     return {
       type: ChatFileTypeEnum.image,
-      name: filename || 'null.png',
+      name: filename || 'null',
       url
     };
   } catch (error) {
     return {
-      type: ChatFileTypeEnum.image,
-      name: 'invalid.png',
+      type: ChatFileTypeEnum.file,
+      name: url,
       url
     };
   }

packages/global/core/app/constants.ts

@@ -77,3 +77,36 @@ export const defaultFileExtensionTypes = {
   canSelectCustomFileExtension: []
 };
 export type FileExtensionKeyType = keyof typeof defaultFileExtensionTypes;
+export const getUploadFileType = ({
+  canSelectFile,
+  canSelectImg,
+  canSelectVideo,
+  canSelectAudio,
+  canSelectCustomFileExtension,
+  customFileExtensionList
+}: {
+  canSelectFile?: boolean;
+  canSelectImg?: boolean;
+  canSelectVideo?: boolean;
+  canSelectAudio?: boolean;
+  canSelectCustomFileExtension?: boolean;
+  customFileExtensionList?: string[];
+}) => {
+  const types: string[] = [];
+  if (canSelectFile) {
+    types.push(...defaultFileExtensionTypes.canSelectFile);
+  }
+  if (canSelectImg) {
+    types.push(...defaultFileExtensionTypes.canSelectImg);
+  }
+  if (canSelectVideo) {
+    types.push(...defaultFileExtensionTypes.canSelectVideo);
+  }
+  if (canSelectAudio) {
+    types.push(...defaultFileExtensionTypes.canSelectAudio);
+  }
+  if (canSelectCustomFileExtension && customFileExtensionList) {
+    types.push(...customFileExtensionList);
+  }
+  return types.join(', ');
+};

packages/global/core/app/type.d.ts

@@ -170,7 +170,7 @@ export type SettingAIDataType = {
 };
 
 // variable
-export type VariableItemType = {
+export type VariableItemType = AppFileSelectConfigType & {
   // id: string;
   key: string;
   label: string;
@@ -189,13 +189,8 @@ export type VariableItemType = {
   min?: number;
   // select
   list?: { label: string; value: string }[];
-  // file
-  canSelectFile?: boolean;
-  canSelectImg?: boolean;
-  maxFiles?: number;
   // timeSelect
   timeGranularity?: 'second' | 'minute' | 'hour' | 'day';
-  timeType?: 'point' | 'range';
   timeRangeStart?: string;
   timeRangeEnd?: string;
 
@@ -241,7 +236,7 @@ export type AppAutoExecuteConfigType = {
 };
 // File
 export type AppFileSelectConfigType = {
-  maxFiles: number;
+  maxFiles?: number;
   canSelectFile?: boolean;
   customPdfParse?: boolean;
   canSelectImg?: boolean;

packages/global/core/chat/type.d.ts

@@ -51,17 +51,18 @@ export type ChatWithAppSchema = Omit<ChatSchemaType, 'appId'> & {
 };
 
 /* --------- chat item ---------- */
+export type UserChatItemFileItemType = {
+  type: `${ChatFileTypeEnum}`;
+  name?: string;
+  key?: string;
+  url: string;
+};
 export type UserChatItemValueItemType = {
   type: ChatItemValueTypeEnum.text | ChatItemValueTypeEnum.file;
   text?: {
     content: string;
   };
-  file?: {
-    type: `${ChatFileTypeEnum}`;
-    name?: string;
-    key?: string;
-    url: string;
-  };
+  file?: UserChatItemFileItemType;
 };
 export type UserChatItemType = {
   obj: ChatRoleEnum.Human;

packages/global/core/workflow/constants.ts

@@ -328,7 +328,7 @@ export enum VariableInputEnum {
   password = 'password',
   file = 'file',
 
-  modelSelect = 'modelSelect',
+  llmSelect = 'llmSelect',
   datasetSelect = 'datasetSelect',
 
   custom = 'custom',
@@ -386,20 +386,26 @@ export const variableConfigs: VariableConfigType[][] = [
       label: i18nT('common:core.workflow.inputType.switch'),
       value: VariableInputEnum.switch,
       defaultValueType: WorkflowIOValueTypeEnum.boolean
+    },
+    {
+      icon: 'core/workflow/inputType/timePointSelect',
+      label: i18nT('common:core.workflow.inputType.timePointSelect'),
+      value: VariableInputEnum.timePointSelect,
+      defaultValueType: WorkflowIOValueTypeEnum.string
+    },
+    {
+      icon: 'core/workflow/inputType/timeRangeSelect',
+      label: i18nT('common:core.workflow.inputType.timeRangeSelect'),
+      value: VariableInputEnum.timeRangeSelect,
+      defaultValueType: WorkflowIOValueTypeEnum.arrayString
+    },
+    {
+      icon: 'core/workflow/inputType/model',
+      label: i18nT('common:core.workflow.inputType.modelSelect'),
+      value: VariableInputEnum.llmSelect,
+      defaultValueType: WorkflowIOValueTypeEnum.string
     }
     // {
-    //   icon: 'core/workflow/inputType/timePointSelect',
-    //   label: i18nT('common:core.workflow.inputType.timePointSelect'),
-    //   value: VariableInputEnum.timePointSelect,
-    //   defaultValueType: WorkflowIOValueTypeEnum.string
-    // },
-    // {
-    //   icon: 'core/workflow/inputType/timeRangeSelect',
-    //   label: i18nT('common:core.workflow.inputType.timeRangeSelect'),
-    //   value: VariableInputEnum.timeRangeSelect,
-    //   defaultValueType: WorkflowIOValueTypeEnum.arrayString
-    // }
-    // {
     //   icon: 'core/workflow/inputType/file',
     //   label: i18nT('common:core.workflow.inputType.file'),
     //   value: VariableInputEnum.file,
@@ -410,14 +416,14 @@ export const variableConfigs: VariableConfigType[][] = [
   //   {
   //     icon: 'core/workflow/inputType/model',
   //     label: i18nT('common:core.workflow.inputType.modelSelect'),
-  //     value: VariableInputEnum.modelSelect,
+  //     value: VariableInputEnum.llmSelect,
   //     defaultValueType: WorkflowIOValueTypeEnum.string
   //   },
   //   {
   //     icon: 'core/workflow/inputType/dataset',
   //     label: i18nT('common:core.workflow.inputType.datasetSelect'),
   //     value: VariableInputEnum.datasetSelect,
-  //     defaultValueType: WorkflowIOValueTypeEnum.arrayString
+  //     defaultValueType: WorkflowIOValueTypeEnum.selectDataset
   //   }
   // ],
   [

packages/global/core/workflow/runtime/utils.ts

@@ -367,7 +367,7 @@ export const checkNodeRunStatus = ({
 
   // Classify edges
   const { commonEdges, recursiveEdgeGroups } = splitNodeEdges(node);
-  console.log(JSON.stringify({ commonEdges, recursiveEdgeGroups }, null, 2));
+
   // Entry
   if (commonEdges.length === 0 && recursiveEdgeGroups.length === 0) {
     return 'run';

packages/global/core/workflow/utils.ts

@@ -262,7 +262,7 @@ export const appData2FlowNodeIO = ({
           [VariableInputEnum.switch]: [FlowNodeInputTypeEnum.switch],
           [VariableInputEnum.password]: [FlowNodeInputTypeEnum.password],
           [VariableInputEnum.file]: [FlowNodeInputTypeEnum.fileSelect],
-          [VariableInputEnum.modelSelect]: [FlowNodeInputTypeEnum.selectLLMModel],
+          [VariableInputEnum.llmSelect]: [FlowNodeInputTypeEnum.selectLLMModel],
           [VariableInputEnum.datasetSelect]: [FlowNodeInputTypeEnum.selectDataset],
           [VariableInputEnum.internal]: [FlowNodeInputTypeEnum.hidden],
           [VariableInputEnum.custom]: [FlowNodeInputTypeEnum.input, FlowNodeInputTypeEnum.reference]

packages/service/core/workflow/dispatch/index.ts

@@ -43,7 +43,7 @@ import type { ChatNodeUsageType } from '@fastgpt/global/support/wallet/bill/type
 import { addLog } from '../../../common/system/log';
 import { surrenderProcess } from '../../../common/system/tools';
 import type { DispatchFlowResponse, WorkflowDebugResponse } from './type';
-import { rewriteRuntimeWorkFlow, removeSystemVariable } from './utils';
+import { rewriteRuntimeWorkFlow, runtimeSystemVar2StoreType } from './utils';
 import { getHandleId } from '@fastgpt/global/core/workflow/utils';
 import { callbackMap } from './constants';
 import { anyValueDecrypt } from '../../../common/secret/utils';
@@ -79,7 +79,16 @@ export async function dispatchWorkFlow({
   concatUsage,
   ...data
 }: Props & WorkflowUsageProps): Promise<DispatchFlowResponse> {
-  const { res, stream, runningUserInfo, runningAppInfo, lastInteractive, histories, query } = data;
+  const {
+    res,
+    stream,
+    runningUserInfo,
+    runningAppInfo,
+    lastInteractive,
+    histories,
+    query,
+    chatConfig
+  } = data;
 
   await checkTeamAIPoints(runningUserInfo.teamId);
   const [{ timezone, externalProvider }, newUsageId] = await Promise.all([
@@ -141,14 +150,15 @@ export async function dispatchWorkFlow({
   }
 
   // Get default variables
+
   const defaultVariables = {
     ...externalProvider.externalWorkflowVariables,
-    ...getSystemVariables({
+    ...(await getSystemVariables({
       ...data,
       query,
       histories,
       timezone
-    })
+    }))
   };
 
   // Init some props
@@ -211,11 +221,11 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
       [DispatchNodeResponseKeyEnum.runTimes]: 1,
       [DispatchNodeResponseKeyEnum.assistantResponses]: [],
       [DispatchNodeResponseKeyEnum.toolResponses]: null,
-      newVariables: removeSystemVariable(
+      [DispatchNodeResponseKeyEnum.newVariables]: runtimeSystemVar2StoreType({
         variables,
-        externalProvider.externalWorkflowVariables,
-        data.chatConfig?.variables
-      ),
+        removeObj: externalProvider.externalWorkflowVariables,
+        userVariablesConfigs: data.chatConfig?.variables
+      }),
       durationSeconds: 0
     };
   }
@@ -983,12 +993,6 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
     });
   }
 
-  const encryptedNewVariables = removeSystemVariable(
-    variables,
-    externalProvider.externalWorkflowVariables,
-    data.chatConfig?.variables
-  );
-
   return {
     flowResponses: workflowQueue.chatResponses,
     flowUsages: workflowQueue.chatNodeUsages,
@@ -999,7 +1003,11 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
       workflowQueue.chatAssistantResponse
     ),
     [DispatchNodeResponseKeyEnum.toolResponses]: workflowQueue.toolRunResponse,
-    [DispatchNodeResponseKeyEnum.newVariables]: encryptedNewVariables,
+    [DispatchNodeResponseKeyEnum.newVariables]: runtimeSystemVar2StoreType({
+      variables,
+      removeObj: externalProvider.externalWorkflowVariables,
+      userVariablesConfigs: data.chatConfig?.variables
+    }),
     [DispatchNodeResponseKeyEnum.memories]:
       Object.keys(workflowQueue.system_memories).length > 0
         ? workflowQueue.system_memories
@@ -1009,7 +1017,7 @@ export const runWorkflow = async (data: RunWorkflowProps): Promise<DispatchFlowR
 };
 
 /* get system variable */
-const getSystemVariables = ({
+const getSystemVariables = async ({
   timezone,
   runningAppInfo,
   chatId,
@@ -1020,30 +1028,29 @@ const getSystemVariables = ({
   variables
 }: Props & {
   timezone: string;
-}): SystemVariablesType => {
+}): Promise<SystemVariablesType> => {
   // Get global variables(Label -> key; Key -> key)
   const variablesConfig = chatConfig?.variables || [];
 
-  const variablesMap = variablesConfig.reduce<Record<string, any>>((acc, item) => {
+  const variablesMap: Record<string, any> = {};
+  for await (const item of variablesConfig) {
     // For internal variables, ignore external input and use default value
     if (item.type === VariableInputEnum.password) {
       const val = variables[item.label] || variables[item.key] || item.defaultValue;
       const actualValue = anyValueDecrypt(val);
-      acc[item.key] = valueTypeFormat(actualValue, item.valueType);
+      variablesMap[item.key] = valueTypeFormat(actualValue, item.valueType);
     }
-
     // API
     else if (variables[item.label] !== undefined) {
-      acc[item.key] = valueTypeFormat(variables[item.label], item.valueType);
+      variablesMap[item.key] = valueTypeFormat(variables[item.label], item.valueType);
     }
     // Web
     else if (variables[item.key] !== undefined) {
-      acc[item.key] = valueTypeFormat(variables[item.key], item.valueType);
+      variablesMap[item.key] = valueTypeFormat(variables[item.key], item.valueType);
     } else {
-      acc[item.key] = valueTypeFormat(item.defaultValue, item.valueType);
+      variablesMap[item.key] = valueTypeFormat(item.defaultValue, item.valueType);
     }
-    return acc;
-  }, {});
+  }
 
   return {
     ...variablesMap,

packages/service/core/workflow/dispatch/tools/runUpdateVar.ts

@@ -11,7 +11,7 @@ import {
 } from '@fastgpt/global/core/workflow/runtime/utils';
 import { type TUpdateListItem } from '@fastgpt/global/core/workflow/template/system/variableUpdate/type';
 import { type ModuleDispatchProps } from '@fastgpt/global/core/workflow/runtime/type';
-import { removeSystemVariable } from '../utils';
+import { runtimeSystemVar2StoreType } from '../utils';
 import { isValidReferenceValue } from '@fastgpt/global/core/workflow/utils';
 import { valueTypeFormat } from '@fastgpt/global/core/workflow/runtime/utils';
 
@@ -92,11 +92,11 @@ export const dispatchUpdateVariable = async (props: Props): Promise<Response> =>
   if (!runningAppInfo.isChildApp) {
     workflowStreamResponse?.({
       event: SseResponseEventEnum.updateVariables,
-      data: removeSystemVariable(
+      data: runtimeSystemVar2StoreType({
         variables,
-        externalProvider.externalWorkflowVariables,
-        chatConfig?.variables
-      )
+        removeObj: externalProvider.externalWorkflowVariables,
+        userVariablesConfigs: chatConfig?.variables
+      })
     });
   }